diff --git a/client/container_preparation/Dockerfile b/client/container_preparation/Dockerfile index 5429bfa..4172d01 100644 --- a/client/container_preparation/Dockerfile +++ b/client/container_preparation/Dockerfile @@ -16,11 +16,11 @@ WORKDIR /container_preparation # Install necessary packages RUN apk add --no-cache \ - git=2.47.2-r0 \ - curl=8.12.1-r1 \ - jq=1.7.1-r0 \ + git=2.49.1-r0 \ + curl=8.14.1-r2 \ + jq=1.8.1-r0 \ build-base=0.5-r3 \ - libffi-dev=3.4.7-r0 + libffi-dev=3.4.8-r0 # Install spire-agent RUN curl -LsSf -o spire-1.9.0-linux-amd64-musl.tar.gz https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz && \ tar xvf spire-1.9.0-linux-amd64-musl.tar.gz && \ diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh index 354c846..4f6308b 100755 --- a/client/container_preparation/entrypoint.sh +++ b/client/container_preparation/entrypoint.sh @@ -6,6 +6,7 @@ # Default values for arguments docker_path="/var/run/docker.sock" docker_host_path=${docker_path} +sif_host_path="" # Argument parser, arguments for both container preparation and key shipping should be handled here. parse_args() { @@ -59,6 +60,10 @@ parse_args() { docker_host_path="$2" shift 2 ;; + --sif-host-path) + sif_host_path="$2" + shift 2 + ;; -h | --help) python3 ./prepare_container.py --help python3 ./utils/ship_a_key.py --help @@ -89,16 +94,17 @@ end_entrypoint() { else echo "Cleaning everything before leaving ..." rm -rf /tmp/data - if [ "${1}" -ne 0 ] ; then + # On failure, preserve agent logs and config for debugging before removing them + if [ "${2}" -ne 0 ] ; then timestamp=$(date +%s) mv /tmp/agent.log "/tmp/agent.log-${timestamp}" || true mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true fi for suffix in log conf sock ; do - rm "/tmp/agent.${suffix}" + rm -f "/tmp/agent.${suffix}" done rm -f /tmp/keys - rm /tmp/dataset_info.yaml + rm -f /tmp/dataset_info.yaml kill "$1" fi exit "$2" @@ -124,12 +130,20 @@ if [ -n "$encrypted" ]; then python3 ./utils/spawn_agent.py --config "$config" >/dev/null 2>/dev/null & spire_agent_pid=$! -fi + # Wait for socket to be created + sleep 2 + until [ -e /tmp/agent.sock ]; do + if ! ps "$spire_agent_pid" >/dev/null 2>&1; then + echo "spire agent died, aborting" + end_entrypoint "$spire_agent_pid" 1 + fi + printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 5 seconds ${NC}" + sleep 5 + done -ps "$spire_agent_pid" >/dev/null || ( - echo "spire agent died, aborting" - end_entrypoint "$spire_agent_pid" 1 -) + # Make socket accessible to all processes + chmod 777 /tmp/agent.sock +fi # ## [END] Perform node attestation @@ -142,9 +156,17 @@ printf "%b\n" "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Run co # if [ -z "$encrypted" ]; then - python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1 + if [ -n "$sif_host_path" ]; then + python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" --sif-host-path "${sif_host_path}" || end_entrypoint "$spire_agent_pid" 1 + else + python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1 + fi else - python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1 + if [ -n "$sif_host_path" ]; then + python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" --sif-host-path "${sif_host_path}" || end_entrypoint "$spire_agent_pid" 1 + else + python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1 + fi fi # diff --git a/client/container_preparation/lib/image_build.py b/client/container_preparation/lib/image_build.py index 97f3210..c0d0351 100644 --- a/client/container_preparation/lib/image_build.py +++ b/client/container_preparation/lib/image_build.py @@ -100,18 +100,23 @@ def build_prepared_image(base_image_tag, docker_client: DockerClient): def create_sif_image( prepared_image_tag, destination_path, + destination_host_path, docker_client: DockerClient, docker_socket_path: str, docker_host_socket_path: str, encrypted=False, + force=True, + verbose=False, ): """Creates an Apptainer SIF image (encrypted or not) using the sd-container/build_env out of a prepared OCI image. Args: prepared_image_tag (string): The OCI Image tag (i.e sd-container/prepared_cowsay) - destination_path (string): The path where the .sif image will be written (i.e /tmp => /tmp/prepared_cowsay.sif) + destination_path (string): The path where the .sif image will be written from this container's perspective (i.e /tmp => /tmp/prepared_cowsay.sif) + destination_host_path (string): The path on the host machine where the .sif image will be written (for nested container volume mounts) encrypted (bool, optional): Wether or not to encrypt the container, keys are generated relatively to the current path, and are called "keys, keys.pub". Defaults to False. + force (bool, optional): Force overwrite of existing SIF image. Defaults to True. """ # Check that the build environment exists build_env_exists = check_build_env_exists(docker_client=docker_client) @@ -127,59 +132,67 @@ def create_sif_image( prepared_image_name = prepared_image_tag.split("/")[-1].split(":")[0] # Composing the final command to run (the build env consists in apptainer build + the command var) - command = f"/output/{prepared_image_name}.sif docker-daemon://{prepared_image_tag}" + force_flag = "--force " if force else "" + command = f"{force_flag}/output/{prepared_image_name}.sif docker-daemon://{prepared_image_tag}" # Composing the dictionary of volumes (docker socket for image reading, output directory for result image) + # Use destination_host_path for the volume mount (host perspective), not destination_path (container perspective) volumes = [ f"{docker_host_socket_path}:{docker_socket_path}", - f"{destination_path}:/output", + f"{destination_host_path}:/output", "/etc/passwd:/etc/passwd", "/etc/group:/etc/group", ] # Run the secured container build if not encrypted: - docker_client.containers.run( + output = docker_client.containers.run( image=build_env_image_tag, command=command, volumes=volumes, remove=True, user=os.getuid(), group_add=[f"{os.stat(docker_socket_path).st_gid}"], + stderr=True, ) + if verbose: + print(f"Apptainer build output: {output.decode('utf-8')}") # Keeping the if/else for easy code rollback if encrypted containers can be used (that's why we don't have only one "docker_client.containers.run()") else: - docker_client.containers.run( + output = docker_client.containers.run( image=build_env_image_tag, command=command, volumes=volumes, remove=True, user=os.getuid(), group_add=[f"{os.stat(docker_socket_path).st_gid}"], + stderr=True, ) + if verbose: + print(f"Apptainer build output: {output.decode('utf-8')}") # Generate necessary keys to encrypt the SIF file sif_decryption_key = x25519.Identity.generate() sif_enryption_key = sif_decryption_key.to_public() # Encrypt the SIF image - with open(f"/tmp/{prepared_image_name}.sif", "rb") as inputfile: + with open(f"{destination_path}/{prepared_image_name}.sif", "rb") as inputfile: encrypted = encrypt(inputfile.read(), [sif_enryption_key]) # Write the encrypted SIF image to the encrypted SIF file - open(f"/tmp/encrypted_{prepared_image_name}.sif", "wb+").write(encrypted) + open(f"{destination_path}/encrypted_{prepared_image_name}.sif", "wb+").write(encrypted) # Remove the unencrypted image - os.remove(f"/tmp/{prepared_image_name}.sif") + os.remove(f"{destination_path}/{prepared_image_name}.sif") print( - f"SIF image encrypted, written to /tmp/encrypted_{prepared_image_name}.sif" + f"SIF image encrypted, written to {destination_path}/encrypted_{prepared_image_name}.sif" ) - open(f"/tmp/keys", "w+").write(str(sif_decryption_key)) + open(f"{destination_path}/keys", "w+").write(str(sif_decryption_key)) - print(f"Encryption : Keys written to /tmp/keys") + print(f"Encryption : Keys written to {destination_path}/keys") # Commenting out the encrypted container runtime part since we can't run them atm # # Define pem path diff --git a/client/container_preparation/prepare_container.py b/client/container_preparation/prepare_container.py index 6eae713..5892904 100644 --- a/client/container_preparation/prepare_container.py +++ b/client/container_preparation/prepare_container.py @@ -11,9 +11,11 @@ # For future usage base_image = arguments.base_oci_image sif_path = arguments.sif_path + sif_host_path = arguments.sif_host_path if arguments.sif_host_path else arguments.sif_path encrypted = arguments.encrypted docker_socket_path = arguments.docker_path docker_host_socket_path = arguments.docker_host_path + force = arguments.force # Checking the access to the docker socket - avoid waiting for an error to kill the program print(f"Checking the access to docker socket at {docker_socket_path}") @@ -42,10 +44,12 @@ create_sif_image( prepared_image_tag, destination_path=sif_path, + destination_host_path=sif_host_path, docker_client=docker_client, docker_socket_path=docker_socket_path, docker_host_socket_path=docker_host_socket_path, encrypted=encrypted, + force=force, ) print(f"{sif_path}/{prepared_image_tag.split(':')[0]}.sif successfully created") diff --git a/client/container_preparation/tools/cli/cli.py b/client/container_preparation/tools/cli/cli.py index 470819c..da47cf1 100644 --- a/client/container_preparation/tools/cli/cli.py +++ b/client/container_preparation/tools/cli/cli.py @@ -37,8 +37,21 @@ def parse_arguments() -> argparse.ArgumentParser: parser.add_argument( "--docker-host-path", type=str, - help="Path to the docker socket (default : /var/run/docker.sock)", + help="Path to the docker socket on the host (default : /var/run/docker.sock)", default="/var/run/docker.sock", ) + parser.add_argument( + "--sif-host-path", + type=str, + help="Path to the SIF directory on the host (default : same as --sif-path). Use this when running in a container with volume mounts", + default=None, + ) + parser.add_argument( + "--force", + "-f", + action="store_true", + help="Force overwrite existing SIF image (default : True)", + default=True, + ) return parser.parse_args() diff --git a/client/data_preparation/entrypoint.sh b/client/data_preparation/entrypoint.sh index 79c3371..080fd45 100755 --- a/client/data_preparation/entrypoint.sh +++ b/client/data_preparation/entrypoint.sh @@ -70,9 +70,17 @@ parse_args() { end_entrypoint() { echo "Cleaning everything before leaving ..." rm -rf /tmp/data - rm /tmp/agent* + # On failure, preserve agent logs and config for debugging before removing them + if [ "${2}" -ne 0 ]; then + timestamp=$(date +%s) + mv /tmp/agent.log "/tmp/agent.log-${timestamp}" || true + mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true + fi + for suffix in log conf sock; do + rm -f "/tmp/agent.${suffix}" + done rm -f /tmp/keys - rm /tmp/dataset_info.yaml + rm -f /tmp/dataset_info.yaml kill "$1" exit "$2" } @@ -97,15 +105,20 @@ printf "%b\n" "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Registering python3 ./utils/spawn_agent.py --config "$config" >/dev/null 2>/dev/null & spire_agent_pid=$! +# Wait for socket to be created +sleep 2 until [ -e /tmp/agent.sock ]; do - printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 10 seconds ${NC}" - sleep 10 - if ! pgrep -f "$spire_agent_pid" > /dev/null; then - echo "spire agent died, aborting" - end_entrypoint "$spire_agent_pid" 1 - fi + if ! ps "$spire_agent_pid" >/dev/null 2>&1; then + echo "spire agent died, aborting" + end_entrypoint "$spire_agent_pid" 1 + fi + printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 5 seconds ${NC}" + sleep 5 done +# Make socket accessible to all processes +chmod 777 /tmp/agent.sock + # ## [END] Perform node attestation # diff --git a/client/job_preparation/prepare_job.py b/client/job_preparation/prepare_job.py index 11a0daf..c7856cb 100644 --- a/client/job_preparation/prepare_job.py +++ b/client/job_preparation/prepare_job.py @@ -8,6 +8,7 @@ from utils.ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command from utils.conf.client.conf import parse_configuration from time import sleep +from threading import Thread from pyrage import x25519 if __name__ == "__main__": @@ -88,48 +89,43 @@ jobid = stdout.read().decode().split(" ")[-1].replace("\n", "") print(f"Job successfully created with id {jobid}") - # If user asked to follow the logs of the job - if options.follow: - # Follow the job's output - command = f"cd {options.workdir} ; touch {options.job_name}.out ; tail -f -n 0 {options.job_name}.out" - stdin, stdout, stderr = ssh_run_command(ssh_client, command, True) - stdin.close() + if options.follow or options.verbose: + # If verbose, stream job log output in a background thread with indentation + if options.verbose: + verbose_client = ssh_connect(options.username) + command = f"cd {options.workdir} ; touch {options.job_name}.out ; tail -f -n 0 {options.job_name}.out" + _, log_stdout, _ = ssh_run_command(verbose_client, command, True) - # Keep read alive - for line in iter(stdout.readline, ""): - print(line, end="") + def stream_log(): + for line in iter(log_stdout.readline, ""): + print(f" {line}", end="") - # If user doesn't follow job's logs - else: - # Still, follow the job's status at fixed interval - print( - f"Waiting for the job to run. You can now exit this script if needed, outputs will be available in {options.workdir}/output when finished" - ) + Thread(target=stream_log, daemon=True).start() - # Specific output format squeue command to parse informations about submitted job + # Poll squeue until the job completes command = f"squeue -o '%A;%u;%T' | grep {options.username} | grep {jobid}" - - # Run first status update stdin, stdout, stderr = ssh_run_command(ssh_client, command, True) job_status = "" stdout = stdout.read().decode().replace("\n", "") stdin.close() - # While job runs (while it has an entry in squeue) while stdout != "": - # If status has changed if job_status != stdout.split(";")[-1]: job_status = stdout.split(";")[-1] - print(f"Job's state changed, job is now in state : {job_status}") - - # Wait + print(f"Job status: {job_status}") sleep(10) - - # Run status update stdin, stdout, stderr = ssh_run_command(ssh_client, command, True) stdout = stdout.read().decode().replace("\n", "") stdin.close() - # Job is over - print(f"Job finished, please find outputs in {options.workdir}/output") + print(f"Job finished, outputs available in {options.workdir}/output") + + if options.verbose: + verbose_client.close() + + else: + print( + f"Outputs will be available in {options.workdir}/output when the job finishes" + ) + ssh_client.close() diff --git a/client/job_preparation/utils/cli/cli.py b/client/job_preparation/utils/cli/cli.py index 2ad71b9..6594eca 100644 --- a/client/job_preparation/utils/cli/cli.py +++ b/client/job_preparation/utils/cli/cli.py @@ -109,7 +109,14 @@ def parse_arguments() -> argparse.Namespace: "--follow", "-f", action="store_true", - help="Follow job's output (default : False)", + help="Follow job status, polling squeue until the job completes (default: False)", + default=False, + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Stream job log output in addition to status updates, implies --follow (default: False)", default=False, ) diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template index f3c5120..d6632c6 100644 --- a/client/job_preparation/utils/sbatch.template +++ b/client/job_preparation/utils/sbatch.template @@ -28,6 +28,8 @@ # Cleanup cleanup() { + echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} === spire-agent log ===" + cat /tmp/agent.log 2>/dev/null || echo "(no agent log found)" rm -r /tmp/data rm /tmp/agent* kill "$1" @@ -80,11 +82,18 @@ if ! which gocryptfs ; then rm -r gocryptfs* fi +if ! which spire-agent ; then + mkdir -p bin + curl -O -L https://github.com/spiffe/spire/releases/download/v1.9.6/spire-1.9.6-linux-amd64-musl.tar.gz || exit 1 + tar xvf spire-1.9.6-linux-amd64-musl.tar.gz + mv spire-1.9.6/bin/spire-agent ./bin/ + rm -r spire-1.9.6* +fi echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Running agent registration" # Spawn spire-agent -python3 ./utils/spawn_agent.py --config ~/.config/hpcs-client.conf -cn > $WORKING_DIRECTORY/agent.log 2> $WORKING_DIRECTORY/agent.log & +python3 -u ./utils/spawn_agent.py --config ~/.config/hpcs-client.conf -cn > $WORKING_DIRECTORY/agent.log 2>&1 & spire_agent_pid=$! # Wait until agent runs properly @@ -94,14 +103,23 @@ do sleep 10 done +# Make socket accessible to all processes +chmod 777 /tmp/agent.sock + echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Starting container setup" echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Decrypting container image" echo "Getting SVID from workload API ..." -# Get SVID from workload -svid=$(spire-agent api fetch jwt -audience TESTING -socketPath /tmp/agent.sock -spiffeID APPLICATION_SPIFFEID | head -n2 | tail -n 1 | tr -d ' ' | tr -d '\t' ) +# Retry SVID fetch — entries may take a few seconds to sync after attestation +svid="" +for i in 1 2 3 4 5; do + svid=$(spire-agent api fetch jwt -audience TESTING -socketPath /tmp/agent.sock -spiffeID APPLICATION_SPIFFEID 2>/dev/null | head -n2 | tail -n 1 | tr -d ' ' | tr -d '\t') + [ -n "$svid" ] && break + echo "SVID not yet available, retrying in 5 seconds (attempt $i/5) ..." + sleep 5 +done echo "Logging in to the vault ..." @@ -118,7 +136,7 @@ echo "$data_key" > /tmp/container_key echo "Decrypting container image ..." # Decrypt the container image -age --decrypt -i /tmp/container_key -o $WORKING_DIRECTORY/app.sif APPLICATION_PATH || exit 1 +age --decrypt -i /tmp/container_key -o $WORKING_DIRECTORY/app.sif APPLICATION_PATH || cleanup $spire_agent_pid 1 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Creating encrypted volumes" diff --git a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml index 4a59f55..dd3a94c 100644 --- a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml +++ b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml @@ -3,6 +3,8 @@ kind: ConfigMap metadata: name: hpcs-server data: + digests: | +{{ .Values.server.digests | indent 4 }} hpcs-server.conf: | [spire-server] address = {{ .Values.server.spire.address }} diff --git a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml index 8bc8a24..d0c7a33 100644 --- a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml +++ b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml @@ -28,6 +28,10 @@ spec: - name: hpcs-server-configs mountPath: /tmp/ readOnly: false + - name: hpcs-server-configs + mountPath: /server/tools/digests + subPath: digests + readOnly: true - name: hpcs-spire-sockets mountPath: /var/run/sockets readOnly: false diff --git a/k8s/hpcs-stack/charts/hpcs/values.yaml b/k8s/hpcs-stack/charts/hpcs/values.yaml index 12eb15f..10d6ff7 100644 --- a/k8s/hpcs-stack/charts/hpcs/values.yaml +++ b/k8s/hpcs-stack/charts/hpcs/values.yaml @@ -4,6 +4,7 @@ image: # tag: pullPolicy: IfNotPresent server: + digests: "" port: 10080 spire: address: localhost diff --git a/server/app.py b/server/app.py index 857c07e..043c1a7 100644 --- a/server/app.py +++ b/server/app.py @@ -63,12 +63,19 @@ # Defining the trust domain (SPIRE Trust Domain) trust_domain = configuration["spire-server"]["trust-domain"] -# Perform vault login, to be able to run later operations against vault -hvac_client = vault_login( - configuration["vault"]["url"], - get_server_identity_JWT(), - configuration["vault"]["server-role"], -) + +def get_vault_client(): + """Get a fresh Vault client with a new JWT token. + + JWT SVIDs expire after 1h and Vault tokens expire after 24h. + This function fetches a fresh JWT from SPIRE and authenticates with Vault, + ensuring we always have a valid token for Vault operations. + """ + return vault_login( + configuration["vault"]["url"], + get_server_identity_JWT(), + configuration["vault"]["server-role"], + ) # Dummy endpoint that handles the registration of compute nodes. @@ -116,8 +123,11 @@ async def handle_client_registration(): client_id = request.headers.get("Remote-Addr") client_id = hashlib.sha256(client_id.encode()).hexdigest()[0:9] + # Get fresh Vault client to avoid token expiration issues + vault_client = get_vault_client() + # Write a policy to the vault to authorize the client to write secrets - write_client_policy(hvac_client, f"client_{client_id}") + write_client_policy(vault_client, f"client_{client_id}") # Create spiffeID out of this client id agent_spiffeID = SpiffeId(f"spiffe://{trust_domain}/c/{client_id}") @@ -135,7 +145,7 @@ async def handle_client_registration(): workload_spiffeID = SpiffeId(f"spiffe://{trust_domain}/c/{client_id}/workload") # Write the role bound to the workload's spiffeID - write_client_role(hvac_client, f"client_{client_id}", workload_spiffeID) + write_client_role(vault_client, f"client_{client_id}", workload_spiffeID) # For each authorized container preparation process (Here, a list of docker container_preparation image names) for digest in get_build_env_image_digests(): @@ -233,8 +243,25 @@ async def handle_workload_creation(): # Compute node's agent spiffeID parentID = SpiffeId(f"spiffe://{trust_domain}/h/{compute_node}") - # For each user - if data["users"] != None: + # For each user — use uid selectors if provided (more reliable on HPC systems + # where static binaries can't do LDAP lookups for unix:user resolution) + uids = data.get("uids") + if uids is not None: + for uid in uids: + selectors = [f"unix:uid:{uid}"] + result = entry_create(parentID, spiffeID, selectors) + if result == None or not result.stderr.decode().find( + "similar entry already exists" + ): + return { + "success": False, + "message": f"error occured while registering a workload : {str(spiffeID)}. Registered selectors available in compute_nodes_added subobject.", + "clientID": client_id, + "spiffeID": str(spiffeID), + "compute_nodes_added": compute_nodes_added, + } + users_added.append(uid) + elif data["users"] != None: for user in data["users"]: # Create proper selector @@ -289,11 +316,14 @@ async def handle_workload_creation(): compute_nodes_added[compute_node]["users"] = users_added compute_nodes_added[compute_node]["groups"] = groups_added + # Get fresh Vault client to avoid token expiration issues + vault_client = get_vault_client() + # Generate and create a policy that gives read-only access to the application's secret - write_user_policy(hvac_client, f"client_{client_id}", data["secret"]) + write_user_policy(vault_client, f"client_{client_id}", data["secret"]) # Generate and create a role bound to the policy and to the spiffeID - write_user_role(hvac_client, f"client_{client_id}", data["secret"], spiffeID) + write_user_role(vault_client, f"client_{client_id}", data["secret"], spiffeID) # Success return { diff --git a/server/tools/digests b/server/tools/digests index b32e66d..74122ff 100644 --- a/server/tools/digests +++ b/server/tools/digests @@ -1,3 +1,9 @@ -ghcr.io/cscfi/hpcs/data-prep:dev-20251022 -ghcr.io/cscfi/hpcs/container-prep:dev-20251022 -ghcr.io/cscfi/hpcs/job-prep:dev-20251022 +sha256:98bf31bf00d59897e489ef24c5cead47e0a28ddf884a7568e9def07b78df7076 +sha256:74b6e0bf8024d5e35876b4199a004a32e25e4cf327ee3f76b35a6ec5885fe989 +sha256:1b2c346699b177ac40231d73a5631e1821ce32ec3d859bb2896cfae6f7f0c360 +sha256:4bf6770b8833b56bc18c19235c6b1cff1ca9907ba0b163467ecefdbc5ec2a9b7 +sha256:f7caf92037d7ec44f12d9a9c8c1e1f119290f797d72aa25c0dcaa33763f2cbd0 +sha256:81caef82d0d14c49473d2fefebafe1ba2da43d70e405bb65dacd4e16ab0e616f +sha256:1a0bb841735829bae5974afec9538e3db6180b9aff27acda04924c3e5cd7d267 +sha256:caee83226b7853960fec665a24ded5c8559bbced6de359ced8db6837c4f079a8 +sha256:b0274cd2632456d192bbd617d06320aacffcfbb56ae43baec84418bca6692860 diff --git a/server/tools/evict.sh b/server/tools/evict.sh new file mode 100755 index 0000000..f054c02 --- /dev/null +++ b/server/tools/evict.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# +# evict.sh — Evict SPIRE agents or delete SPIRE entries from the hpcs-stack deployment. +# +# Usage: +# evict.sh agents [--all | --jq-selector ] [--verbose] +# evict.sh entries [--all | --jq-selector ] [--verbose] +# +# --all Select all agents/entries +# --jq-selector A jq expression producing one ID per line from the JSON list output. +# For agents: input is `spire-server agent list -output json` +# expression should yield agent path, e.g. '.agents[].id.path' +# For entries: input is `spire-server entry show -output json` +# expression should yield entry ID, e.g. '.entries[].id' +# --verbose Print each successful eviction/deletion, and any failures with detail +# +# Examples: +# evict.sh agents --all --verbose +# evict.sh entries --all +# evict.sh entries --jq-selector '.entries[] | select(.selectors[].value | startswith("user:")) | .id' --verbose + +SPIRE_POD="spire-server-0" +SPIRE_CONTAINER="spire-server" +SPIRE_NAMESPACE="hpcs" +SPIRE_SOCKET="/tmp/spire-server/private/api.sock" +SPIRE_BIN="/opt/spire/bin/spire-server" +TRUST_DOMAIN="hpcs" + +spire_exec() { + kubectl exec -i "$SPIRE_POD" -c "$SPIRE_CONTAINER" -n "$SPIRE_NAMESPACE" -- \ + "$SPIRE_BIN" "$@" -socketPath "$SPIRE_SOCKET" +} + +usage() { + echo "Usage: $0 [--all | --jq-selector ] [--verbose]" + exit 1 +} + +verb="" +all=false +jq_selector="" +verbose=false + +while [ "$#" -gt 0 ]; do + case "$1" in + agents|entries) + verb="$1" + shift + ;; + --all) + all=true + shift + ;; + --jq-selector) + jq_selector="$2" + shift 2 + ;; + --verbose) + verbose=true + shift + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +if [ -z "$verb" ]; then + echo "Error: verb required (agents or entries)" + usage +fi + +if [ "$all" = false ] && [ -z "$jq_selector" ]; then + echo "Error: specify --all or --jq-selector" + usage +fi + +if [ "$verb" = "agents" ]; then + if [ -z "$jq_selector" ]; then + jq_selector='.agents[].id.path' + fi + + items=$(spire_exec agent list -output json | jq -r "$jq_selector" | tr -d ' \t') + + if [ -z "$items" ]; then + echo "No agents matched." + exit 0 + fi + + for agent_path in $items; do + spiffe_id="spiffe://${TRUST_DOMAIN}${agent_path}" + result=$(spire_exec agent evict -spiffeID "$spiffe_id" 2>&1) + if [ $? -eq 0 ]; then + [ "$verbose" = true ] && echo "Evicted agent: $spiffe_id" + else + echo "Failed to evict agent: $spiffe_id" + [ "$verbose" = true ] && echo "$result" + fi + done + +elif [ "$verb" = "entries" ]; then + if [ -z "$jq_selector" ]; then + jq_selector='.entries[].id' + fi + + items=$(spire_exec entry show -output json | jq -r "$jq_selector" | tr -d ' \t') + + if [ -z "$items" ]; then + echo "No entries matched." + exit 0 + fi + + for entry_id in $items; do + result=$(spire_exec entry delete -entryID "$entry_id" 2>&1) + if [ $? -eq 0 ]; then + [ "$verbose" = true ] && echo "Deleted entry: $entry_id" + else + echo "Failed to delete entry: $entry_id" + [ "$verbose" = true ] && echo "$result" + fi + done +fi diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py index 36967dd..e946ab9 100644 --- a/utils/ship_a_key.py +++ b/utils/ship_a_key.py @@ -1,6 +1,54 @@ import argparse from re import search from os import path + + +def expand_nodelist(nodelist_expr: str) -> list: + """Expand a Slurm nodelist expression into individual node names. + + Supports formats like: + nid003044 + nid[003044-003046] + nid[005124,005126,005128-005133,005136-005153] + nid[003044-003046],other005 + """ + nodes = [] + + # Split on top-level commas (i.e. not inside brackets) + parts = [] + depth = 0 + current = [] + for char in nodelist_expr: + if char == "[": + depth += 1 + current.append(char) + elif char == "]": + depth -= 1 + current.append(char) + elif char == "," and depth == 0: + parts.append("".join(current)) + current = [] + else: + current.append(char) + if current: + parts.append("".join(current)) + + for part in parts: + if "[" in part: + prefix, rest = part.split("[", 1) + rest = rest.rstrip("]") + for item in rest.split(","): + if "-" in item: + start, end = item.split("-", 1) + width = len(start) # preserve zero-padding + for i in range(int(start), int(end) + 1): + nodes.append(f"{prefix}{str(i).zfill(width)}") + else: + nodes.append(f"{prefix}{item}") + else: + nodes.append(part) + + return nodes from pyspiffe.workloadapi import default_jwt_source, default_workload_api_client from pyspiffe.spiffe_id import spiffe_id from pyspiffe.exceptions import SpiffeIdError @@ -9,7 +57,7 @@ from vault.vault_utils import vault_login, write_secret import yaml from hashlib import sha512 -from ssh_utils import ssh_connect, ssh_copy_file +from ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command from conf.client.conf import parse_configuration # Provide client_id from cli$ @@ -139,7 +187,7 @@ def validate_options(options: argparse.ArgumentParser): # Parse compute, check that their names are correctly formed (using UNIX regex's) compute_nodes = options.compute_nodes if compute_nodes != None: - compute_nodes = options.compute_nodes.split(",") + compute_nodes = expand_nodelist(options.compute_nodes) for compute_node in compute_nodes: if not search( "^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$", compute_node @@ -188,7 +236,7 @@ def validate_options(options: argparse.ArgumentParser): def create_authorized_workloads( - SVID: JwtSvid, secret, url, users, groups, compute_nodes + SVID: JwtSvid, secret, url, users, groups, compute_nodes, uids=None ): """Create workloads that are authorized to access to a secret @@ -200,6 +248,7 @@ def create_authorized_workloads( users ([str]): List of users in UNIX format, comma separated groups ([str]): List of groups in UNIX format, comma separated compute_nodes ([str]): List of compute nodes UNIX format, comma separated + uids ([str]): List of numeric UIDs resolved from the HPC system, used instead of usernames for workload attestation Returns: Data returned by the server (spiffeID to access the secret, this client's id, the path to the secret created workloads have access to) @@ -213,6 +262,7 @@ def create_authorized_workloads( "users": users, "groups": groups, "compute_nodes": compute_nodes, + "uids": uids, } # POST request @@ -260,6 +310,22 @@ def create_authorized_workloads( username, ) = validate_options(options) + # Create SSH connection early so we can resolve UIDs from the HPC system + ssh_client = ssh_connect(username=username) + + # Resolve UIDs for each user via SSH — the unix workload attestor on compute nodes + # uses static binaries that can't do LDAP lookups, so only produces unix:uid selectors + uids = None + if users is not None: + uids = [] + for user in users: + _, stdout, _ = ssh_run_command(ssh_client, f"id -u {user}") + uid = stdout.read().decode().strip() + if not uid.isdigit(): + print(f"Error: could not resolve UID for user '{user}' on the supercomputer") + exit(1) + uids.append(uid) + # Create the workloadAPI access jwt_workload_api = default_jwt_source.DefaultJwtSource( spiffe_socket_path=f"unix://{socketpath}", @@ -278,6 +344,7 @@ def create_authorized_workloads( users, groups, compute_nodes, + uids, ) # Login to the vault using client's certificate @@ -300,11 +367,9 @@ def create_authorized_workloads( # Compute file's checksum checksum = sha512(open(options.data_path, "rb").read()).hexdigest() - # Create SSH connection - ssh_client = ssh_connect(username=username) - - # Ship data via SSH - ssh_copy_file(ssh_client, data_path, options.data_path_at_rest) + # Ensure destination directory exists, then ship data via SSH + ssh_run_command(ssh_client, f"mkdir -p {options.data_path_at_rest}") + ssh_copy_file(ssh_client, data_path, f"{options.data_path_at_rest}/{data_path.split('/')[-1]}".replace("//", "/")) # Write datasets info object dataset_info_object = { @@ -331,7 +396,7 @@ def create_authorized_workloads( ) print( - f"Data and info file were shipped to te supercomputer. Info about the dataset are available at {options.data_path_at_rest}/{secret_name}.info.yaml" + f"Data and info file were shipped to the supercomputer. Info about the dataset are available at {options.data_path_at_rest.rstrip('/')}/{secret_name}.info.yaml" ) ssh_client.close() diff --git a/utils/ssh_utils.py b/utils/ssh_utils.py index 5b0f35e..845f961 100644 --- a/utils/ssh_utils.py +++ b/utils/ssh_utils.py @@ -1,5 +1,5 @@ from paramiko.client import SSHClient -from paramiko import SSHException, AutoAddPolicy, RSAKey +from paramiko import SSHException, AutoAddPolicy, RSAKey, PasswordRequiredException from scp import SCPClient # Hostname and port configuration @@ -30,7 +30,15 @@ def ssh_connect(username: str) -> SSHClient: # Probably running in a container except SSHException: - pkey = RSAKey.from_private_key_file("/tmp/.ssh/id_rsa") + try: + pkey = RSAKey.from_private_key_file("/tmp/.ssh/id_rsa") + except PasswordRequiredException: + import getpass + + p = getpass.getpass( + prompt="Please enter your ssh key decryption password\n" + ) + pkey = RSAKey.from_private_key_file("/tmp/.ssh/id_rsa", p) client.connect( host, port,