From bdf7d3c0b65cea422a3b6307c8fc57788d67e923 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 8 Jan 2026 14:16:19 +0200
Subject: [PATCH 01/28] Fix Vault token expiration, workload selector, and
 socket permissions

- Add get_vault_client() helper to refresh JWT tokens before Vault operations
  (JWT SVIDs expire after 1h, Vault tokens after 24h)
- Change workload selector from docker:image_id: to unix:uid:0
  (Docker attestor not working, all client containers run as root)
- Add chmod 777 for SPIRE agent socket after creation
  (Fixes socket permission issues preventing SVID fetching)

These changes fix the client registration failures and allow data/container/job
preparation workflows to complete successfully.
---
 client/container_preparation/entrypoint.sh   | 16 +++--
 client/data_preparation/entrypoint.sh        |  3 +
 client/job_preparation/utils/sbatch.template |  3 +
 server/app.py                                | 67 ++++++++++++--------
 4 files changed, 59 insertions(+), 30 deletions(-)

diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh
index 354c846..e547655 100755
--- a/client/container_preparation/entrypoint.sh
+++ b/client/container_preparation/entrypoint.sh
@@ -124,12 +124,18 @@ if [ -n "$encrypted" ]; then
 	python3 ./utils/spawn_agent.py --config "$config" >/dev/null 2>/dev/null &
 	spire_agent_pid=$!
 
-fi
+	# Wait for socket to be created
+	until [ -e /tmp/agent.sock ]; do
+		sleep 2
+		if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
+			echo "spire agent died, aborting"
+			end_entrypoint "$spire_agent_pid" 1
+		fi
+	done
 
-ps "$spire_agent_pid" >/dev/null || (
-	echo "spire agent died, aborting"
-	end_entrypoint "$spire_agent_pid" 1
-)
+	# Make socket accessible to all processes
+	chmod 777 /tmp/agent.sock
+fi
 
 #
 ## [END] Perform node attestation
diff --git a/client/data_preparation/entrypoint.sh b/client/data_preparation/entrypoint.sh
index 79c3371..15bd42c 100755
--- a/client/data_preparation/entrypoint.sh
+++ b/client/data_preparation/entrypoint.sh
@@ -106,6 +106,9 @@ until [ -e /tmp/agent.sock ]; do
 	fi
 done
 
+# Make socket accessible to all processes
+chmod 777 /tmp/agent.sock
+
 #
 ## [END] Perform node attestation
 #
diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template
index f3c5120..107bf55 100644
--- a/client/job_preparation/utils/sbatch.template
+++ b/client/job_preparation/utils/sbatch.template
@@ -94,6 +94,9 @@ do
     sleep 10
 done
 
+# Make socket accessible to all processes
+chmod 777 /tmp/agent.sock
+
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Starting container setup"
 
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Decrypting container image"
diff --git a/server/app.py b/server/app.py
index 857c07e..0105659 100644
--- a/server/app.py
+++ b/server/app.py
@@ -71,6 +71,20 @@
 )
 
 
+def get_vault_client():
+    """Get a fresh Vault client with a new JWT token.
+
+    JWT SVIDs expire after 1h and Vault tokens expire after 24h.
+    This function fetches a fresh JWT from SPIRE and authenticates with Vault,
+    ensuring we always have a valid token for Vault operations.
+    """
+    return vault_login(
+        configuration["vault"]["url"],
+        get_server_identity_JWT(),
+        configuration["vault"]["server-role"],
+    )
+
+
 # Dummy endpoint that handles the registration of compute nodes.
 # TODO: Develop a SPIRE plugin that handles this better than this dummy endpoint
 # THIS ENDPOINT SHOULD BE REMOVED, IT DOESN'T CERTIFY ANYHTING, IT WILL PROVIDE AN IDENTITY TO ANYONE ASKING FOR
@@ -116,8 +130,11 @@ async def handle_client_registration():
     client_id = request.headers.get("Remote-Addr")
     client_id = hashlib.sha256(client_id.encode()).hexdigest()[0:9]
 
+    # Get fresh Vault client to avoid token expiration issues
+    vault_client = get_vault_client()
+
     # Write a policy to the vault to authorize the client to write secrets
-    write_client_policy(hvac_client, f"client_{client_id}")
+    write_client_policy(vault_client, f"client_{client_id}")
 
     # Create spiffeID out of this client id
     agent_spiffeID = SpiffeId(f"spiffe://{trust_domain}/c/{client_id}")
@@ -135,28 +152,25 @@ async def handle_client_registration():
         workload_spiffeID = SpiffeId(f"spiffe://{trust_domain}/c/{client_id}/workload")
 
         # Write the role bound to the workload's spiffeID
-        write_client_role(hvac_client, f"client_{client_id}", workload_spiffeID)
-
-        # For each authorized container preparation process (Here, a list of docker container_preparation image names)
-        for digest in get_build_env_image_digests():
-            digest = digest.replace("\n", "")
-            workload_selector = f"docker:image_id:{digest}"
-
-            # Register a workload bound to the agent, and the workload (Here, a container image)
-            result = entry_create(
-                agent_spiffeID, workload_spiffeID, [workload_selector]
-            )
-
-            # Do not stop if entry already exists, stop for any other error
-            if result == None or not result.stderr.decode().find(
-                "similar entry already exists"
-            ):
-                return {
-                    "success": False,
-                    "message": "token created, it expires in 60 seconds. An error occured while registering workloads.",
-                    "client_id": client_id,
-                    "token": agent_token,
-                }
+        write_client_role(vault_client, f"client_{client_id}", workload_spiffeID)
+
+        # Register workload entry using unix:uid selector (simpler and more reliable than docker)
+        # All client containers run as root (uid:0)
+        workload_selector = "unix:uid:0"
+        result = entry_create(
+            agent_spiffeID, workload_spiffeID, [workload_selector]
+        )
+
+        # Do not stop if entry already exists, stop for any other error
+        if result == None or not result.stderr.decode().find(
+            "similar entry already exists"
+        ):
+            return {
+                "success": False,
+                "message": "token created, it expires in 60 seconds. An error occured while registering workloads.",
+                "client_id": client_id,
+                "token": agent_token,
+            }
 
         # Spire-Agent binary
         result = entry_create(
@@ -289,11 +303,14 @@ async def handle_workload_creation():
             compute_nodes_added[compute_node]["users"] = users_added
             compute_nodes_added[compute_node]["groups"] = groups_added
 
+        # Get fresh Vault client to avoid token expiration issues
+        vault_client = get_vault_client()
+
         # Generate and create a policy that gives read-only access to the application's secret
-        write_user_policy(hvac_client, f"client_{client_id}", data["secret"])
+        write_user_policy(vault_client, f"client_{client_id}", data["secret"])
 
         # Generate and create a role bound to the policy and to the spiffeID
-        write_user_role(hvac_client, f"client_{client_id}", data["secret"], spiffeID)
+        write_user_role(vault_client, f"client_{client_id}", data["secret"], spiffeID)
 
         # Success
         return {

From b6765eeb9967c93a20a115b97fb571cd9dc3d3c8 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Mon, 12 Jan 2026 15:34:06 +0200
Subject: [PATCH 02/28] Update Dockerfile pinned requirements to existing
 packages

---
 client/container_preparation/Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/container_preparation/Dockerfile b/client/container_preparation/Dockerfile
index 5429bfa..4172d01 100644
--- a/client/container_preparation/Dockerfile
+++ b/client/container_preparation/Dockerfile
@@ -16,11 +16,11 @@ WORKDIR /container_preparation
 
 # Install necessary packages
 RUN apk add --no-cache \
-    git=2.47.2-r0 \
-    curl=8.12.1-r1 \
-    jq=1.7.1-r0 \
+    git=2.49.1-r0 \
+    curl=8.14.1-r2 \
+    jq=1.8.1-r0 \
     build-base=0.5-r3 \
-    libffi-dev=3.4.7-r0
+    libffi-dev=3.4.8-r0
 # Install spire-agent
 RUN curl -LsSf -o spire-1.9.0-linux-amd64-musl.tar.gz https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz && \
     tar xvf spire-1.9.0-linux-amd64-musl.tar.gz && \

From ad66c7d996296d6f1deb153690197b34ac320446 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Mon, 12 Jan 2026 15:36:17 +0200
Subject: [PATCH 03/28] Add --force and --sif-host-path

Container building can somewhat confusingly fail when the sif image
already exists, or when the host /tmp ends up getting mounted to the
build container's /output. These are intended to defaultly avoid those
problems, and provide some configurability.
---
 client/container_preparation/entrypoint.sh    | 17 ++++++++--
 .../container_preparation/lib/image_build.py  | 34 +++++++++++++------
 .../prepare_container.py                      |  4 +++
 client/container_preparation/tools/cli/cli.py | 15 +++++++-
 4 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh
index e547655..2ed84ea 100755
--- a/client/container_preparation/entrypoint.sh
+++ b/client/container_preparation/entrypoint.sh
@@ -6,6 +6,7 @@
 # Default values for arguments
 docker_path="/var/run/docker.sock"
 docker_host_path=${docker_path}
+sif_host_path=""
 
 # Argument parser, arguments for both container preparation and key shipping should be handled here.
 parse_args() {
@@ -59,6 +60,10 @@ parse_args() {
 			docker_host_path="$2"
 			shift 2
 			;;
+		--sif-host-path)
+			sif_host_path="$2"
+			shift 2
+			;;
 		-h | --help)
 			python3 ./prepare_container.py --help
 			python3 ./utils/ship_a_key.py --help
@@ -148,9 +153,17 @@ printf "%b\n" "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Run co
 #
 
 if [ -z "$encrypted" ]; then
-	python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	if [ -n "$sif_host_path" ]; then
+		python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" --sif-host-path "${sif_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	else
+		python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	fi
 else
-	python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	if [ -n "$sif_host_path" ]; then
+		python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" --sif-host-path "${sif_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	else
+		python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	fi
 fi
 
 #
diff --git a/client/container_preparation/lib/image_build.py b/client/container_preparation/lib/image_build.py
index 97f3210..90f14b1 100644
--- a/client/container_preparation/lib/image_build.py
+++ b/client/container_preparation/lib/image_build.py
@@ -100,18 +100,22 @@ def build_prepared_image(base_image_tag, docker_client: DockerClient):
 def create_sif_image(
     prepared_image_tag,
     destination_path,
+    destination_host_path,
     docker_client: DockerClient,
     docker_socket_path: str,
     docker_host_socket_path: str,
     encrypted=False,
+    force=True,
 ):
     """Creates an Apptainer SIF image (encrypted or not) using the sd-container/build_env
     out of a prepared OCI image.
 
     Args:
         prepared_image_tag (string): The OCI Image tag (i.e sd-container/prepared_cowsay)
-        destination_path (string): The path where the .sif image will be written (i.e /tmp => /tmp/prepared_cowsay.sif)
+        destination_path (string): The path where the .sif image will be written from this container's perspective (i.e /tmp => /tmp/prepared_cowsay.sif)
+        destination_host_path (string): The path on the host machine where the .sif image will be written (for nested container volume mounts)
         encrypted (bool, optional): Wether or not to encrypt the container, keys are generated relatively to the current path, and are called "keys, keys.pub". Defaults to False.
+        force (bool, optional): Force overwrite of existing SIF image. Defaults to True.
     """
     # Check that the build environment exists
     build_env_exists = check_build_env_exists(docker_client=docker_client)
@@ -127,59 +131,67 @@ def create_sif_image(
     prepared_image_name = prepared_image_tag.split("/")[-1].split(":")[0]
 
     # Composing the final command to run (the build env consists in apptainer build + the command var)
-    command = f"/output/{prepared_image_name}.sif docker-daemon://{prepared_image_tag}"
+    force_flag = "--force " if force else ""
+    command = f"{force_flag}/output/{prepared_image_name}.sif docker-daemon://{prepared_image_tag}"
 
     # Composing the dictionary of volumes (docker socket for image reading, output directory for result image)
+    # Use destination_host_path for the volume mount (host perspective), not destination_path (container perspective)
     volumes = [
         f"{docker_host_socket_path}:{docker_socket_path}",
-        f"{destination_path}:/output",
+        f"{destination_host_path}:/output",
         "/etc/passwd:/etc/passwd",
         "/etc/group:/etc/group",
     ]
 
     # Run the secured container build
     if not encrypted:
-        docker_client.containers.run(
+        output = docker_client.containers.run(
             image=build_env_image_tag,
             command=command,
             volumes=volumes,
             remove=True,
             user=os.getuid(),
             group_add=[f"{os.stat(docker_socket_path).st_gid}"],
+            stderr=True,
         )
+        if output:
+            print(f"Apptainer build output: {output.decode('utf-8')}")
 
     # Keeping the if/else for easy code rollback if encrypted containers can be used (that's why we don't have only one "docker_client.containers.run()")
     else:
-        docker_client.containers.run(
+        output = docker_client.containers.run(
             image=build_env_image_tag,
             command=command,
             volumes=volumes,
             remove=True,
             user=os.getuid(),
             group_add=[f"{os.stat(docker_socket_path).st_gid}"],
+            stderr=True,
         )
+        if output:
+            print(f"Apptainer build output: {output.decode('utf-8')}")
 
         # Generate necessary keys to encrypt the SIF file
         sif_decryption_key = x25519.Identity.generate()
         sif_enryption_key = sif_decryption_key.to_public()
 
         # Encrypt the SIF image
-        with open(f"/tmp/{prepared_image_name}.sif", "rb") as inputfile:
+        with open(f"{destination_path}/{prepared_image_name}.sif", "rb") as inputfile:
             encrypted = encrypt(inputfile.read(), [sif_enryption_key])
 
         # Write the encrypted SIF image to the encrypted SIF file
-        open(f"/tmp/encrypted_{prepared_image_name}.sif", "wb+").write(encrypted)
+        open(f"{destination_path}/encrypted_{prepared_image_name}.sif", "wb+").write(encrypted)
 
         # Remove the unencrypted image
-        os.remove(f"/tmp/{prepared_image_name}.sif")
+        os.remove(f"{destination_path}/{prepared_image_name}.sif")
 
         print(
-            f"SIF image encrypted, written to /tmp/encrypted_{prepared_image_name}.sif"
+            f"SIF image encrypted, written to {destination_path}/encrypted_{prepared_image_name}.sif"
         )
 
-        open(f"/tmp/keys", "w+").write(str(sif_decryption_key))
+        open(f"{destination_path}/keys", "w+").write(str(sif_decryption_key))
 
-        print(f"Encryption : Keys written to /tmp/keys")
+        print(f"Encryption : Keys written to {destination_path}/keys")
 
         # Commenting out the encrypted container runtime part since we can't run them atm
         # # Define pem path
diff --git a/client/container_preparation/prepare_container.py b/client/container_preparation/prepare_container.py
index 6eae713..5892904 100644
--- a/client/container_preparation/prepare_container.py
+++ b/client/container_preparation/prepare_container.py
@@ -11,9 +11,11 @@
     # For future usage
     base_image = arguments.base_oci_image
     sif_path = arguments.sif_path
+    sif_host_path = arguments.sif_host_path if arguments.sif_host_path else arguments.sif_path
     encrypted = arguments.encrypted
     docker_socket_path = arguments.docker_path
     docker_host_socket_path = arguments.docker_host_path
+    force = arguments.force
 
     # Checking the access to the docker socket - avoid waiting for an error to kill the program
     print(f"Checking the access to docker socket at {docker_socket_path}")
@@ -42,10 +44,12 @@
     create_sif_image(
         prepared_image_tag,
         destination_path=sif_path,
+        destination_host_path=sif_host_path,
         docker_client=docker_client,
         docker_socket_path=docker_socket_path,
         docker_host_socket_path=docker_host_socket_path,
         encrypted=encrypted,
+        force=force,
     )
 
     print(f"{sif_path}/{prepared_image_tag.split(':')[0]}.sif successfully created")
diff --git a/client/container_preparation/tools/cli/cli.py b/client/container_preparation/tools/cli/cli.py
index 470819c..da47cf1 100644
--- a/client/container_preparation/tools/cli/cli.py
+++ b/client/container_preparation/tools/cli/cli.py
@@ -37,8 +37,21 @@ def parse_arguments() -> argparse.ArgumentParser:
     parser.add_argument(
         "--docker-host-path",
         type=str,
-        help="Path to the docker socket (default : /var/run/docker.sock)",
+        help="Path to the docker socket on the host (default : /var/run/docker.sock)",
         default="/var/run/docker.sock",
     )
+    parser.add_argument(
+        "--sif-host-path",
+        type=str,
+        help="Path to the SIF directory on the host (default : same as --sif-path). Use this when running in a container with volume mounts",
+        default=None,
+    )
+    parser.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        help="Force overwrite existing SIF image (default : True)",
+        default=True,
+    )
 
     return parser.parse_args()

From 8584020e64d386bd011d8ad84bc14d93840ab757 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Mon, 12 Jan 2026 15:53:50 +0200
Subject: [PATCH 04/28] Only print apptainer build logs if verbose

---
 client/container_preparation/lib/image_build.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/client/container_preparation/lib/image_build.py b/client/container_preparation/lib/image_build.py
index 90f14b1..c0d0351 100644
--- a/client/container_preparation/lib/image_build.py
+++ b/client/container_preparation/lib/image_build.py
@@ -106,6 +106,7 @@ def create_sif_image(
     docker_host_socket_path: str,
     encrypted=False,
     force=True,
+    verbose=False,
 ):
     """Creates an Apptainer SIF image (encrypted or not) using the sd-container/build_env
     out of a prepared OCI image.
@@ -154,7 +155,7 @@ def create_sif_image(
             group_add=[f"{os.stat(docker_socket_path).st_gid}"],
             stderr=True,
         )
-        if output:
+        if verbose:
             print(f"Apptainer build output: {output.decode('utf-8')}")
 
     # Keeping the if/else for easy code rollback if encrypted containers can be used (that's why we don't have only one "docker_client.containers.run()")
@@ -168,7 +169,7 @@ def create_sif_image(
             group_add=[f"{os.stat(docker_socket_path).st_gid}"],
             stderr=True,
         )
-        if output:
+        if verbose:
             print(f"Apptainer build output: {output.decode('utf-8')}")
 
         # Generate necessary keys to encrypt the SIF file

From 330d03f78a58688a3df09535bd719f70f36f0808 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Mon, 19 Jan 2026 14:25:04 +0200
Subject: [PATCH 05/28] Reinstate docker selectors for attestation

---
 server/app.py | 44 ++++++++++++++++++++------------------------
 1 file changed, 20 insertions(+), 24 deletions(-)

diff --git a/server/app.py b/server/app.py
index 0105659..2933387 100644
--- a/server/app.py
+++ b/server/app.py
@@ -63,13 +63,6 @@
 # Defining the trust domain (SPIRE Trust Domain)
 trust_domain = configuration["spire-server"]["trust-domain"]
 
-# Perform vault login, to be able to run later operations against vault
-hvac_client = vault_login(
-    configuration["vault"]["url"],
-    get_server_identity_JWT(),
-    configuration["vault"]["server-role"],
-)
-
 
 def get_vault_client():
     """Get a fresh Vault client with a new JWT token.
@@ -154,23 +147,26 @@ async def handle_client_registration():
         # Write the role bound to the workload's spiffeID
         write_client_role(vault_client, f"client_{client_id}", workload_spiffeID)
 
-        # Register workload entry using unix:uid selector (simpler and more reliable than docker)
-        # All client containers run as root (uid:0)
-        workload_selector = "unix:uid:0"
-        result = entry_create(
-            agent_spiffeID, workload_spiffeID, [workload_selector]
-        )
-
-        # Do not stop if entry already exists, stop for any other error
-        if result == None or not result.stderr.decode().find(
-            "similar entry already exists"
-        ):
-            return {
-                "success": False,
-                "message": "token created, it expires in 60 seconds. An error occured while registering workloads.",
-                "client_id": client_id,
-                "token": agent_token,
-            }
+        # For each authorized container preparation process (Here, a list of docker container_preparation image names)
+        for digest in get_build_env_image_digests():
+            digest = digest.replace("\n", "")
+            workload_selector = f"docker:image_id:{digest}"
+
+            # Register a workload bound to the agent, and the workload (Here, a container image)
+            result = entry_create(
+                agent_spiffeID, workload_spiffeID, [workload_selector]
+            )
+
+            # Do not stop if entry already exists, stop for any other error
+            if result == None or not result.stderr.decode().find(
+                "similar entry already exists"
+            ):
+                return {
+                    "success": False,
+                    "message": "token created, it expires in 60 seconds. An error occured while registering workloads.",
+                    "client_id": client_id,
+                    "token": agent_token,
+                }
 
         # Spire-Agent binary
         result = entry_create(

From e81f7218a2df777f22f2088d79a77ee255a1bb41 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 22 Jan 2026 17:12:10 +0200
Subject: [PATCH 06/28] Prompt for SSH password if encrypted

---
 utils/ssh_utils.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/utils/ssh_utils.py b/utils/ssh_utils.py
index 5b0f35e..845f961 100644
--- a/utils/ssh_utils.py
+++ b/utils/ssh_utils.py
@@ -1,5 +1,5 @@
 from paramiko.client import SSHClient
-from paramiko import SSHException, AutoAddPolicy, RSAKey
+from paramiko import SSHException, AutoAddPolicy, RSAKey, PasswordRequiredException
 from scp import SCPClient
 
 # Hostname and port configuration
@@ -30,7 +30,15 @@ def ssh_connect(username: str) -> SSHClient:
 
     # Probably running in a container
     except SSHException:
-        pkey = RSAKey.from_private_key_file("/tmp/.ssh/id_rsa")
+        try:
+            pkey = RSAKey.from_private_key_file("/tmp/.ssh/id_rsa")
+        except PasswordRequiredException:
+            import getpass
+
+            p = getpass.getpass(
+                prompt="Please enter your ssh key decryption password\n"
+            )
+            pkey = RSAKey.from_private_key_file("/tmp/.ssh/id_rsa", p)
         client.connect(
             host,
             port,

From d156831dea69db156d80bd3e531d2639a84a607d Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Wed, 18 Mar 2026 11:28:41 +0200
Subject: [PATCH 07/28] Add real (current) digests

---
 server/tools/digests | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/server/tools/digests b/server/tools/digests
index b32e66d..6982ca8 100644
--- a/server/tools/digests
+++ b/server/tools/digests
@@ -1,3 +1,4 @@
-ghcr.io/cscfi/hpcs/data-prep:dev-20251022
-ghcr.io/cscfi/hpcs/container-prep:dev-20251022
-ghcr.io/cscfi/hpcs/job-prep:dev-20251022
+sha256:98bf31bf00d59897e489ef24c5cead47e0a28ddf884a7568e9def07b78df7076
+sha256:74b6e0bf8024d5e35876b4199a004a32e25e4cf327ee3f76b35a6ec5885fe989
+sha256:1b2c346699b177ac40231d73a5631e1821ce32ec3d859bb2896cfae6f7f0c360
+

From 9cabdd545d64dbe36a3527d42947b855493e8124 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Tue, 24 Mar 2026 10:55:41 +0200
Subject: [PATCH 08/28] Add UID attestation to server

---
 server/app.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/server/app.py b/server/app.py
index 2933387..043c1a7 100644
--- a/server/app.py
+++ b/server/app.py
@@ -243,8 +243,25 @@ async def handle_workload_creation():
             # Compute node's agent spiffeID
             parentID = SpiffeId(f"spiffe://{trust_domain}/h/{compute_node}")
 
-            # For each user
-            if data["users"] != None:
+            # For each user — use uid selectors if provided (more reliable on HPC systems
+            # where static binaries can't do LDAP lookups for unix:user resolution)
+            uids = data.get("uids")
+            if uids is not None:
+                for uid in uids:
+                    selectors = [f"unix:uid:{uid}"]
+                    result = entry_create(parentID, spiffeID, selectors)
+                    if result == None or not result.stderr.decode().find(
+                        "similar entry already exists"
+                    ):
+                        return {
+                            "success": False,
+                            "message": f"error occured while registering a workload : {str(spiffeID)}. Registered selectors available in compute_nodes_added subobject.",
+                            "clientID": client_id,
+                            "spiffeID": str(spiffeID),
+                            "compute_nodes_added": compute_nodes_added,
+                        }
+                    users_added.append(uid)
+            elif data["users"] != None:
                 for user in data["users"]:
 
                     # Create proper selector

From f54a967a1ae789036954089eebc78c240642a689 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Tue, 24 Mar 2026 11:15:21 +0200
Subject: [PATCH 09/28] Add evicting tool

---
 server/tools/evict.sh | 126 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100755 server/tools/evict.sh

diff --git a/server/tools/evict.sh b/server/tools/evict.sh
new file mode 100755
index 0000000..f054c02
--- /dev/null
+++ b/server/tools/evict.sh
@@ -0,0 +1,126 @@
+#!/bin/sh
+#
+# evict.sh — Evict SPIRE agents or delete SPIRE entries from the hpcs-stack deployment.
+#
+# Usage:
+#   evict.sh agents  [--all | --jq-selector <jq>] [--verbose]
+#   evict.sh entries [--all | --jq-selector <jq>] [--verbose]
+#
+# --all              Select all agents/entries
+# --jq-selector      A jq expression producing one ID per line from the JSON list output.
+#                    For agents:  input is `spire-server agent list -output json`
+#                                 expression should yield agent path, e.g. '.agents[].id.path'
+#                    For entries: input is `spire-server entry show -output json`
+#                                 expression should yield entry ID, e.g. '.entries[].id'
+# --verbose          Print each successful eviction/deletion, and any failures with detail
+#
+# Examples:
+#   evict.sh agents --all --verbose
+#   evict.sh entries --all
+#   evict.sh entries --jq-selector '.entries[] | select(.selectors[].value | startswith("user:")) | .id' --verbose
+
+SPIRE_POD="spire-server-0"
+SPIRE_CONTAINER="spire-server"
+SPIRE_NAMESPACE="hpcs"
+SPIRE_SOCKET="/tmp/spire-server/private/api.sock"
+SPIRE_BIN="/opt/spire/bin/spire-server"
+TRUST_DOMAIN="hpcs"
+
+spire_exec() {
+    kubectl exec -i "$SPIRE_POD" -c "$SPIRE_CONTAINER" -n "$SPIRE_NAMESPACE" -- \
+        "$SPIRE_BIN" "$@" -socketPath "$SPIRE_SOCKET"
+}
+
+usage() {
+    echo "Usage: $0 <agents|entries> [--all | --jq-selector <jq>] [--verbose]"
+    exit 1
+}
+
+verb=""
+all=false
+jq_selector=""
+verbose=false
+
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        agents|entries)
+            verb="$1"
+            shift
+            ;;
+        --all)
+            all=true
+            shift
+            ;;
+        --jq-selector)
+            jq_selector="$2"
+            shift 2
+            ;;
+        --verbose)
+            verbose=true
+            shift
+            ;;
+        -h|--help)
+            usage
+            ;;
+        *)
+            echo "Unknown option: $1"
+            usage
+            ;;
+    esac
+done
+
+if [ -z "$verb" ]; then
+    echo "Error: verb required (agents or entries)"
+    usage
+fi
+
+if [ "$all" = false ] && [ -z "$jq_selector" ]; then
+    echo "Error: specify --all or --jq-selector"
+    usage
+fi
+
+if [ "$verb" = "agents" ]; then
+    if [ -z "$jq_selector" ]; then
+        jq_selector='.agents[].id.path'
+    fi
+
+    items=$(spire_exec agent list -output json | jq -r "$jq_selector" | tr -d ' \t')
+
+    if [ -z "$items" ]; then
+        echo "No agents matched."
+        exit 0
+    fi
+
+    for agent_path in $items; do
+        spiffe_id="spiffe://${TRUST_DOMAIN}${agent_path}"
+        result=$(spire_exec agent evict -spiffeID "$spiffe_id" 2>&1)
+        if [ $? -eq 0 ]; then
+            [ "$verbose" = true ] && echo "Evicted agent: $spiffe_id"
+        else
+            echo "Failed to evict agent: $spiffe_id"
+            [ "$verbose" = true ] && echo "$result"
+        fi
+    done
+
+elif [ "$verb" = "entries" ]; then
+    if [ -z "$jq_selector" ]; then
+        jq_selector='.entries[].id'
+    fi
+
+    items=$(spire_exec entry show -output json | jq -r "$jq_selector" | tr -d ' \t')
+
+    if [ -z "$items" ]; then
+        echo "No entries matched."
+        exit 0
+    fi
+
+    for entry_id in $items; do
+        result=$(spire_exec entry delete -entryID "$entry_id" 2>&1)
+        if [ $? -eq 0 ]; then
+            [ "$verbose" = true ] && echo "Deleted entry: $entry_id"
+        else
+            echo "Failed to delete entry: $entry_id"
+            [ "$verbose" = true ] && echo "$result"
+        fi
+    done
+fi

From 395a3cbde112576d043a59b8340f8176405318cd Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Tue, 24 Mar 2026 11:24:44 +0200
Subject: [PATCH 10/28] New digests

---
 server/tools/digests | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/tools/digests b/server/tools/digests
index 6982ca8..e759319 100644
--- a/server/tools/digests
+++ b/server/tools/digests
@@ -1,4 +1,6 @@
 sha256:98bf31bf00d59897e489ef24c5cead47e0a28ddf884a7568e9def07b78df7076
 sha256:74b6e0bf8024d5e35876b4199a004a32e25e4cf327ee3f76b35a6ec5885fe989
 sha256:1b2c346699b177ac40231d73a5631e1821ce32ec3d859bb2896cfae6f7f0c360
-
+sha256:4bf6770b8833b56bc18c19235c6b1cff1ca9907ba0b163467ecefdbc5ec2a9b7
+sha256:f7caf92037d7ec44f12d9a9c8c1e1f119290f797d72aa25c0dcaa33763f2cbd0
+sha256:81caef82d0d14c49473d2fefebafe1ba2da43d70e405bb65dacd4e16ab0e616f

From 4c98308ab23386f1625bed9d0dba3d6112de20d6 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Wed, 25 Mar 2026 10:53:40 +0200
Subject: [PATCH 11/28] Add digest

---
 server/tools/digests | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server/tools/digests b/server/tools/digests
index e759319..74122ff 100644
--- a/server/tools/digests
+++ b/server/tools/digests
@@ -4,3 +4,6 @@ sha256:1b2c346699b177ac40231d73a5631e1821ce32ec3d859bb2896cfae6f7f0c360
 sha256:4bf6770b8833b56bc18c19235c6b1cff1ca9907ba0b163467ecefdbc5ec2a9b7
 sha256:f7caf92037d7ec44f12d9a9c8c1e1f119290f797d72aa25c0dcaa33763f2cbd0
 sha256:81caef82d0d14c49473d2fefebafe1ba2da43d70e405bb65dacd4e16ab0e616f
+sha256:1a0bb841735829bae5974afec9538e3db6180b9aff27acda04924c3e5cd7d267
+sha256:caee83226b7853960fec665a24ded5c8559bbced6de359ced8db6837c4f079a8
+sha256:b0274cd2632456d192bbd617d06320aacffcfbb56ae43baec84418bca6692860

From 885fa3348cf448f9fd514066fcd9efd435c10e46 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Wed, 25 Mar 2026 15:15:21 +0200
Subject: [PATCH 12/28] Add configmap for digest additions / changes

---
 .../charts/hpcs/templates/hpcs-server-configmap.yaml          | 2 ++
 .../charts/hpcs/templates/hpcs-server-statefulset.yaml        | 4 ++++
 k8s/hpcs-stack/charts/hpcs/values.yaml                        | 1 +
 3 files changed, 7 insertions(+)

diff --git a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml
index 4a59f55..dd3a94c 100644
--- a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml
+++ b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-configmap.yaml
@@ -3,6 +3,8 @@ kind: ConfigMap
 metadata:
   name: hpcs-server
 data:
+  digests: |
+{{ .Values.server.digests | indent 4 }}
   hpcs-server.conf: |
     [spire-server]
     address = {{ .Values.server.spire.address }}
diff --git a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml
index 8bc8a24..d0c7a33 100644
--- a/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml
+++ b/k8s/hpcs-stack/charts/hpcs/templates/hpcs-server-statefulset.yaml
@@ -28,6 +28,10 @@ spec:
             - name: hpcs-server-configs
               mountPath: /tmp/
               readOnly: false
+            - name: hpcs-server-configs
+              mountPath: /server/tools/digests
+              subPath: digests
+              readOnly: true
             - name: hpcs-spire-sockets
               mountPath: /var/run/sockets
               readOnly: false
diff --git a/k8s/hpcs-stack/charts/hpcs/values.yaml b/k8s/hpcs-stack/charts/hpcs/values.yaml
index 12eb15f..10d6ff7 100644
--- a/k8s/hpcs-stack/charts/hpcs/values.yaml
+++ b/k8s/hpcs-stack/charts/hpcs/values.yaml
@@ -4,6 +4,7 @@ image:
   # tag:
   pullPolicy: IfNotPresent
 server:
+  digests: ""
   port: 10080
   spire:
     address: localhost

From 2cb7e362c389bcb69816d093738646c0fb92fe46 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 15:45:22 +0200
Subject: [PATCH 13/28] Changes to --follow and --verbose in job_prep

Previously, "follow" meant to stream job log output, and otherwise we
persisted with the job, printing changes in its status. Now, "follow"
means the old default state, --verbose means to stream the job log,
and the default is to exit after the job has been submitted.
---
 client/job_preparation/prepare_job.py   | 49 +++++++++++--------------
 client/job_preparation/utils/cli/cli.py |  9 ++++-
 2 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/client/job_preparation/prepare_job.py b/client/job_preparation/prepare_job.py
index 11a0daf..69ca24d 100644
--- a/client/job_preparation/prepare_job.py
+++ b/client/job_preparation/prepare_job.py
@@ -88,48 +88,43 @@
     jobid = stdout.read().decode().split(" ")[-1].replace("\n", "")
     print(f"Job successfully created with id {jobid}")
 
-    # If user asked to follow the logs of the job
-    if options.follow:
-        # Follow the job's output
-        command = f"cd {options.workdir} ; touch {options.job_name}.out ; tail -f -n 0 {options.job_name}.out"
-        stdin, stdout, stderr = ssh_run_command(ssh_client, command, True)
-        stdin.close()
+    if options.follow or options.verbose:
+        # If verbose, stream job log output in a background thread with indentation
+        if options.verbose:
+            verbose_client = ssh_connect(options.username)
+            command = f"cd {options.workdir} ; touch {options.job_name}.out ; tail -f -n 0 {options.job_name}.out"
+            _, log_stdout, _ = ssh_run_command(verbose_client, command, True)
 
-        # Keep read alive
-        for line in iter(stdout.readline, ""):
-            print(line, end="")
+            def stream_log():
+                for line in iter(log_stdout.readline, ""):
+                    print(f"    {line}", end="")
 
-    # If user doesn't follow job's logs
-    else:
-        # Still, follow the job's status at fixed interval
-        print(
-            f"Waiting for the job to run. You can now exit this script if needed, outputs will be available in {options.workdir}/output when finished"
-        )
+            Thread(target=stream_log, daemon=True).start()
 
-        # Specific output format squeue command to parse informations about submitted job
+        # Poll squeue until the job completes
         command = f"squeue -o '%A;%u;%T' | grep {options.username} | grep {jobid}"
-
-        # Run first status update
         stdin, stdout, stderr = ssh_run_command(ssh_client, command, True)
         job_status = ""
         stdout = stdout.read().decode().replace("\n", "")
         stdin.close()
 
-        # While job runs (while it has an entry in squeue)
         while stdout != "":
-            # If status has changed
             if job_status != stdout.split(";")[-1]:
                 job_status = stdout.split(";")[-1]
-                print(f"Job's state changed, job is now in state : {job_status}")
-
-            # Wait
+                print(f"Job status: {job_status}")
             sleep(10)
-
-            # Run status update
             stdin, stdout, stderr = ssh_run_command(ssh_client, command, True)
             stdout = stdout.read().decode().replace("\n", "")
             stdin.close()
 
-    # Job is over
-    print(f"Job finished, please find outputs in {options.workdir}/output")
+        print(f"Job finished, outputs available in {options.workdir}/output")
+
+        if options.verbose:
+            verbose_client.close()
+
+    else:
+        print(
+            f"Outputs will be available in {options.workdir}/output when the job finishes"
+        )
+
     ssh_client.close()
diff --git a/client/job_preparation/utils/cli/cli.py b/client/job_preparation/utils/cli/cli.py
index 2ad71b9..6594eca 100644
--- a/client/job_preparation/utils/cli/cli.py
+++ b/client/job_preparation/utils/cli/cli.py
@@ -109,7 +109,14 @@ def parse_arguments() -> argparse.Namespace:
         "--follow",
         "-f",
         action="store_true",
-        help="Follow job's output (default : False)",
+        help="Follow job status, polling squeue until the job completes (default: False)",
+        default=False,
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Stream job log output in addition to status updates, implies --follow (default: False)",
         default=False,
     )
 

From 82c32060f6e36cb1796c1cbfd30baf403c0704d3 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 15:56:58 +0200
Subject: [PATCH 14/28] Don't needlessly print if some temp files don't exist

---
 client/container_preparation/entrypoint.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh
index 2ed84ea..198e426 100755
--- a/client/container_preparation/entrypoint.sh
+++ b/client/container_preparation/entrypoint.sh
@@ -100,10 +100,10 @@ end_entrypoint() {
 			mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true
 		fi
 		for suffix in log conf sock ; do
-			rm "/tmp/agent.${suffix}"
+			rm -f "/tmp/agent.${suffix}"
 		done
 		rm -f /tmp/keys
-		rm /tmp/dataset_info.yaml
+		rm -f /tmp/dataset_info.yaml
 		kill "$1"
 	fi
 	exit "$2"

From 64f85bc1c0b6b9d65965f2909c99f8a2a32e6ac6 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:10:32 +0200
Subject: [PATCH 15/28] Unify and fix log preservation on failure

---
 client/container_preparation/entrypoint.sh |  3 ++-
 client/data_preparation/entrypoint.sh      | 12 ++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh
index 198e426..1cbd107 100755
--- a/client/container_preparation/entrypoint.sh
+++ b/client/container_preparation/entrypoint.sh
@@ -94,7 +94,8 @@ end_entrypoint() {
 	else
 		echo "Cleaning everything before leaving ..."
 		rm -rf /tmp/data
-		if [ "${1}" -ne 0 ] ; then
+		# On failure, preserve agent logs and config for debugging before removing them
+		if [ "${2}" -ne 0 ] ; then
 			timestamp=$(date +%s)
 			mv /tmp/agent.log "/tmp/agent.log-${timestamp}" || true
 			mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true
diff --git a/client/data_preparation/entrypoint.sh b/client/data_preparation/entrypoint.sh
index 15bd42c..e9e6c2d 100755
--- a/client/data_preparation/entrypoint.sh
+++ b/client/data_preparation/entrypoint.sh
@@ -70,9 +70,17 @@ parse_args() {
 end_entrypoint() {
 	echo "Cleaning everything before leaving ..."
 	rm -rf /tmp/data
-	rm /tmp/agent*
+	# On failure, preserve agent logs and config for debugging before removing them
+	if [ "${2}" -ne 0 ]; then
+		timestamp=$(date +%s)
+		mv /tmp/agent.log "/tmp/agent.log-${timestamp}" || true
+		mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true
+	fi
+	for suffix in log conf sock; do
+		rm -f "/tmp/agent.${suffix}"
+	done
 	rm -f /tmp/keys
-	rm /tmp/dataset_info.yaml
+	rm -f /tmp/dataset_info.yaml
 	kill "$1"
 	exit "$2"
 }

From 3f88ed46f697702334cda18729fef2ee71511a59 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:18:25 +0200
Subject: [PATCH 16/28] pgrep -> ps

---
 client/data_preparation/entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/data_preparation/entrypoint.sh b/client/data_preparation/entrypoint.sh
index e9e6c2d..601e6e5 100755
--- a/client/data_preparation/entrypoint.sh
+++ b/client/data_preparation/entrypoint.sh
@@ -108,7 +108,7 @@ spire_agent_pid=$!
 until [ -e /tmp/agent.sock ]; do
 	printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 10 seconds ${NC}"
 	sleep 10
-	if ! pgrep -f "$spire_agent_pid" > /dev/null; then
+	if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
 		echo "spire agent died, aborting"
 		end_entrypoint "$spire_agent_pid" 1
 	fi

From 57632b82ece8a88d54c03ef50c0b4f94e4a80fd8 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:28:50 +0200
Subject: [PATCH 17/28] Unify waiting for Spire agent socket

First wait, then check if process died, then print that we're waiting
some more. This way there's usually no concerning needless printing.
---
 client/container_preparation/entrypoint.sh | 12 +++++++-----
 client/data_preparation/entrypoint.sh      | 14 ++++++++------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/client/container_preparation/entrypoint.sh b/client/container_preparation/entrypoint.sh
index 1cbd107..4f6308b 100755
--- a/client/container_preparation/entrypoint.sh
+++ b/client/container_preparation/entrypoint.sh
@@ -131,12 +131,14 @@ if [ -n "$encrypted" ]; then
 	spire_agent_pid=$!
 
 	# Wait for socket to be created
+	sleep 2
 	until [ -e /tmp/agent.sock ]; do
-		sleep 2
-		if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
-			echo "spire agent died, aborting"
-			end_entrypoint "$spire_agent_pid" 1
-		fi
+	    if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
+		echo "spire agent died, aborting"
+		end_entrypoint "$spire_agent_pid" 1
+	    fi
+	    printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 5 seconds ${NC}"
+	    sleep 5
 	done
 
 	# Make socket accessible to all processes
diff --git a/client/data_preparation/entrypoint.sh b/client/data_preparation/entrypoint.sh
index 601e6e5..080fd45 100755
--- a/client/data_preparation/entrypoint.sh
+++ b/client/data_preparation/entrypoint.sh
@@ -105,13 +105,15 @@ printf "%b\n" "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Registering
 python3 ./utils/spawn_agent.py --config "$config" >/dev/null 2>/dev/null &
 spire_agent_pid=$!
 
+# Wait for socket to be created
+sleep 2
 until [ -e /tmp/agent.sock ]; do
-	printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 10 seconds ${NC}"
-	sleep 10
-	if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
-		echo "spire agent died, aborting"
-		end_entrypoint "$spire_agent_pid" 1
-	fi
+    if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
+	echo "spire agent died, aborting"
+	end_entrypoint "$spire_agent_pid" 1
+    fi
+    printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 5 seconds ${NC}"
+    sleep 5
 done
 
 # Make socket accessible to all processes

From 8f105bc503d5a06dfea591ade0dc7b0ebdfaf037 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:35:04 +0200
Subject: [PATCH 18/28] On failure, print agent log to job output

---
 client/job_preparation/utils/sbatch.template | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template
index 107bf55..52911a7 100644
--- a/client/job_preparation/utils/sbatch.template
+++ b/client/job_preparation/utils/sbatch.template
@@ -28,6 +28,8 @@
 
 # Cleanup
 cleanup() {
+    echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} === spire-agent log ==="
+    cat /tmp/agent.log 2>/dev/null || echo "(no agent log found)"
     rm -r /tmp/data
     rm /tmp/agent*
     kill "$1"

From 5f731b879239cbecc2afe06d686012288dd16c1f Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:38:10 +0200
Subject: [PATCH 19/28] Fetch spire-agent if not available

(Because of the need to use our own binary which may not be able to
access system's LDAP, this is why we also need to use UID attestation
instead of username)
---
 client/job_preparation/utils/sbatch.template | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template
index 52911a7..cea02b9 100644
--- a/client/job_preparation/utils/sbatch.template
+++ b/client/job_preparation/utils/sbatch.template
@@ -82,6 +82,13 @@ if ! which gocryptfs ; then
     rm -r gocryptfs*
 fi
 
+if ! which spire-agent ; then
+    mkdir -p bin
+    curl -O -L https://github.com/spiffe/spire/releases/download/v1.9.6/spire-1.9.6-linux-amd64-musl.tar.gz || exit 1
+    tar xvf spire-1.9.6-linux-amd64-musl.tar.gz
+    mv spire-1.9.6/bin/spire-agent ./bin/
+    rm -r spire-1.9.6*
+fi
 
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Running agent registration"
 

From cfff588b5aa11c8c2044f968cc5a72b1d650e867 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:43:53 +0200
Subject: [PATCH 20/28] Fix buffering and logging of spawn_agent.py

---
 client/job_preparation/utils/sbatch.template | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template
index cea02b9..feacf1a 100644
--- a/client/job_preparation/utils/sbatch.template
+++ b/client/job_preparation/utils/sbatch.template
@@ -93,7 +93,7 @@ fi
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Running agent registration"
 
 # Spawn spire-agent
-python3 ./utils/spawn_agent.py --config ~/.config/hpcs-client.conf -cn > $WORKING_DIRECTORY/agent.log 2> $WORKING_DIRECTORY/agent.log &
+python3 -u ./utils/spawn_agent.py --config ~/.config/hpcs-client.conf -cn > $WORKING_DIRECTORY/agent.log 2>&1 &
 spire_agent_pid=$!
 
 # Wait until agent runs properly

From dbd33433e20a9c94b4e1e4fdc70822d5e6acda3f Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:47:09 +0200
Subject: [PATCH 21/28] Import Thread for the verbose logging

---
 client/job_preparation/prepare_job.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/job_preparation/prepare_job.py b/client/job_preparation/prepare_job.py
index 69ca24d..c7856cb 100644
--- a/client/job_preparation/prepare_job.py
+++ b/client/job_preparation/prepare_job.py
@@ -8,6 +8,7 @@
 from utils.ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command
 from utils.conf.client.conf import parse_configuration
 from time import sleep
+from threading import Thread
 from pyrage import x25519
 
 if __name__ == "__main__":

From f0f5d6f79c407f288079c6a85caeea3f5dd134ea Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:47:39 +0200
Subject: [PATCH 22/28] Wait a while for SVID fetch

---
 client/job_preparation/utils/sbatch.template | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template
index feacf1a..3fa232d 100644
--- a/client/job_preparation/utils/sbatch.template
+++ b/client/job_preparation/utils/sbatch.template
@@ -112,8 +112,14 @@ echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Decrypting container image"
 
 echo "Getting SVID from workload API ..."
 
-# Get SVID from workload
-svid=$(spire-agent api fetch jwt -audience TESTING -socketPath /tmp/agent.sock -spiffeID APPLICATION_SPIFFEID | head -n2 | tail -n 1 | tr -d ' ' | tr -d '\t' )
+# Retry SVID fetch — entries may take a few seconds to sync after attestation
+svid=""
+for i in 1 2 3 4 5; do
+    svid=$(spire-agent api fetch jwt -audience TESTING -socketPath /tmp/agent.sock -spiffeID APPLICATION_SPIFFEID 2>/dev/null | head -n2 | tail -n 1 | tr -d ' ' | tr -d '\t')
+    [ -n "$svid" ] && break
+    echo "SVID not yet available, retrying in 5 seconds (attempt $i/5) ..."
+    sleep 5
+done
 
 echo "Logging in to the vault ..."
 

From 7b2d3f44c3c256e6a8236ef7856fae7607e6bc4c Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:48:31 +0200
Subject: [PATCH 23/28] Log on this failure too

---
 client/job_preparation/utils/sbatch.template | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/job_preparation/utils/sbatch.template b/client/job_preparation/utils/sbatch.template
index 3fa232d..d6632c6 100644
--- a/client/job_preparation/utils/sbatch.template
+++ b/client/job_preparation/utils/sbatch.template
@@ -136,7 +136,7 @@ echo "$data_key" > /tmp/container_key
 echo "Decrypting container image ..."
 
 # Decrypt the container image
-age --decrypt -i  /tmp/container_key -o $WORKING_DIRECTORY/app.sif APPLICATION_PATH || exit 1
+age --decrypt -i  /tmp/container_key -o $WORKING_DIRECTORY/app.sif APPLICATION_PATH || cleanup $spire_agent_pid 1
 
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Creating encrypted volumes"
 

From 7e945874fd4c2780e876aa8af171cbd83a551e17 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:48:58 +0200
Subject: [PATCH 24/28] Add function for expanding nodelists

---
 utils/ship_a_key.py | 48 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py
index 36967dd..b31e41a 100644
--- a/utils/ship_a_key.py
+++ b/utils/ship_a_key.py
@@ -1,6 +1,54 @@
 import argparse
 from re import search
 from os import path
+
+
+def expand_nodelist(nodelist_expr: str) -> list:
+    """Expand a Slurm nodelist expression into individual node names.
+
+    Supports formats like:
+        nid003044
+        nid[003044-003046]
+        nid[005124,005126,005128-005133,005136-005153]
+        nid[003044-003046],other005
+    """
+    nodes = []
+
+    # Split on top-level commas (i.e. not inside brackets)
+    parts = []
+    depth = 0
+    current = []
+    for char in nodelist_expr:
+        if char == "[":
+            depth += 1
+            current.append(char)
+        elif char == "]":
+            depth -= 1
+            current.append(char)
+        elif char == "," and depth == 0:
+            parts.append("".join(current))
+            current = []
+        else:
+            current.append(char)
+    if current:
+        parts.append("".join(current))
+
+    for part in parts:
+        if "[" in part:
+            prefix, rest = part.split("[", 1)
+            rest = rest.rstrip("]")
+            for item in rest.split(","):
+                if "-" in item:
+                    start, end = item.split("-", 1)
+                    width = len(start)  # preserve zero-padding
+                    for i in range(int(start), int(end) + 1):
+                        nodes.append(f"{prefix}{str(i).zfill(width)}")
+                else:
+                    nodes.append(f"{prefix}{item}")
+        else:
+            nodes.append(part)
+
+    return nodes
 from pyspiffe.workloadapi import default_jwt_source, default_workload_api_client
 from pyspiffe.spiffe_id import spiffe_id
 from pyspiffe.exceptions import SpiffeIdError

From c9ea119ed18a0e6893488df97a3ca89db62daf21 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:49:17 +0200
Subject: [PATCH 25/28] Fix typo

---
 utils/ship_a_key.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py
index b31e41a..1f14023 100644
--- a/utils/ship_a_key.py
+++ b/utils/ship_a_key.py
@@ -379,7 +379,7 @@ def create_authorized_workloads(
     )
 
     print(
-        f"Data and info file were shipped to te supercomputer. Info about the dataset are available at {options.data_path_at_rest}/{secret_name}.info.yaml"
+        f"Data and info file were shipped to the supercomputer. Info about the dataset are available at {options.data_path_at_rest.rstrip('/')}/{secret_name}.info.yaml"
     )
 
     ssh_client.close()

From cb355217fec04da072ffd654e4df2ff9e59c1d01 Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:50:02 +0200
Subject: [PATCH 26/28] Enable UID attestation

---
 utils/ship_a_key.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py
index 1f14023..0347cd6 100644
--- a/utils/ship_a_key.py
+++ b/utils/ship_a_key.py
@@ -236,7 +236,7 @@ def validate_options(options: argparse.ArgumentParser):
 
 
 def create_authorized_workloads(
-    SVID: JwtSvid, secret, url, users, groups, compute_nodes
+    SVID: JwtSvid, secret, url, users, groups, compute_nodes, uids=None
 ):
     """Create workloads that are authorized to access to a secret
 
@@ -248,6 +248,7 @@ def create_authorized_workloads(
         users ([str]): List of users in UNIX format, comma separated
         groups ([str]): List of groups in UNIX format, comma separated
         compute_nodes ([str]): List of compute nodes UNIX format, comma separated
+        uids ([str]): List of numeric UIDs resolved from the HPC system, used instead of usernames for workload attestation
 
     Returns:
         Data returned by the server (spiffeID to access the secret, this client's id, the path to the secret created workloads have access to)
@@ -261,6 +262,7 @@ def create_authorized_workloads(
         "users": users,
         "groups": groups,
         "compute_nodes": compute_nodes,
+        "uids": uids,
     }
 
     # POST request
@@ -308,6 +310,22 @@ def create_authorized_workloads(
         username,
     ) = validate_options(options)
 
+    # Create SSH connection early so we can resolve UIDs from the HPC system
+    ssh_client = ssh_connect(username=username)
+
+    # Resolve UIDs for each user via SSH — the unix workload attestor on compute nodes
+    # uses static binaries that can't do LDAP lookups, so only produces unix:uid selectors
+    uids = None
+    if users is not None:
+        uids = []
+        for user in users:
+            _, stdout, _ = ssh_run_command(ssh_client, f"id -u {user}")
+            uid = stdout.read().decode().strip()
+            if not uid.isdigit():
+                print(f"Error: could not resolve UID for user '{user}' on the supercomputer")
+                exit(1)
+            uids.append(uid)
+
     # Create the workloadAPI access
     jwt_workload_api = default_jwt_source.DefaultJwtSource(
         spiffe_socket_path=f"unix://{socketpath}",
@@ -326,6 +344,7 @@ def create_authorized_workloads(
         users,
         groups,
         compute_nodes,
+        uids,
     )
 
     # Login to the vault using client's certificate

From 7da7a0a14d91600aa6573ef2740b0c64950804ec Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:50:48 +0200
Subject: [PATCH 27/28] Fix data path and dir creation

---
 utils/ship_a_key.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py
index 0347cd6..99cf7b8 100644
--- a/utils/ship_a_key.py
+++ b/utils/ship_a_key.py
@@ -57,7 +57,7 @@ def expand_nodelist(nodelist_expr: str) -> list:
 from vault.vault_utils import vault_login, write_secret
 import yaml
 from hashlib import sha512
-from ssh_utils import ssh_connect, ssh_copy_file
+from ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command
 from conf.client.conf import parse_configuration
 
 # Provide client_id from cli$
@@ -367,11 +367,9 @@ def create_authorized_workloads(
     # Compute file's checksum
     checksum = sha512(open(options.data_path, "rb").read()).hexdigest()
 
-    # Create SSH connection
-    ssh_client = ssh_connect(username=username)
-
-    # Ship data via SSH
-    ssh_copy_file(ssh_client, data_path, options.data_path_at_rest)
+    # Ensure destination directory exists, then ship data via SSH
+    ssh_run_command(ssh_client, f"mkdir -p {options.data_path_at_rest}")
+    ssh_copy_file(ssh_client, data_path, f"{options.data_path_at_rest}/{data_path.split('/')[-1]}".replace("//", "/"))
 
     # Write datasets info object
     dataset_info_object = {

From 0d00493dd50905230e3721a22b28f3e1fa85fe4e Mon Sep 17 00:00:00 2001
From: Sam Hardwick <sam.hardwick@csc.fi>
Date: Thu, 26 Mar 2026 16:52:09 +0200
Subject: [PATCH 28/28] Use nodelist expansion in ship_a_key

---
 utils/ship_a_key.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/ship_a_key.py b/utils/ship_a_key.py
index 99cf7b8..e946ab9 100644
--- a/utils/ship_a_key.py
+++ b/utils/ship_a_key.py
@@ -187,7 +187,7 @@ def validate_options(options: argparse.ArgumentParser):
     # Parse compute, check that their names are correctly formed (using UNIX regex's)
     compute_nodes = options.compute_nodes
     if compute_nodes != None:
-        compute_nodes = options.compute_nodes.split(",")
+        compute_nodes = expand_nodelist(options.compute_nodes)
         for compute_node in compute_nodes:
             if not search(
                 "^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$", compute_node