Dev fixes

client/container_preparation/Dockerfile

@@ -16,11 +16,11 @@ WORKDIR /container_preparation
 
 # Install necessary packages
 RUN apk add --no-cache \
-    git=2.47.2-r0 \
-    curl=8.12.1-r1 \
-    jq=1.7.1-r0 \
+    git=2.49.1-r0 \
+    curl=8.14.1-r2 \
+    jq=1.8.1-r0 \
     build-base=0.5-r3 \
-    libffi-dev=3.4.7-r0
+    libffi-dev=3.4.8-r0
 # Install spire-agent
 RUN curl -LsSf -o spire-1.9.0-linux-amd64-musl.tar.gz https://github.com/spiffe/spire/releases/download/v1.9.0/spire-1.9.0-linux-amd64-musl.tar.gz && \
     tar xvf spire-1.9.0-linux-amd64-musl.tar.gz && \

client/container_preparation/entrypoint.sh

@@ -6,6 +6,7 @@
 # Default values for arguments
 docker_path="/var/run/docker.sock"
 docker_host_path=${docker_path}
+sif_host_path=""
 
 # Argument parser, arguments for both container preparation and key shipping should be handled here.
 parse_args() {
@@ -59,6 +60,10 @@ parse_args() {
 			docker_host_path="$2"
 			shift 2
 			;;
+		--sif-host-path)
+			sif_host_path="$2"
+			shift 2
+			;;
 		-h | --help)
 			python3 ./prepare_container.py --help
 			python3 ./utils/ship_a_key.py --help
@@ -89,16 +94,17 @@ end_entrypoint() {
 	else
 		echo "Cleaning everything before leaving ..."
 		rm -rf /tmp/data
-		if [ "${1}" -ne 0 ] ; then
+		# On failure, preserve agent logs and config for debugging before removing them
+		if [ "${2}" -ne 0 ] ; then
 			timestamp=$(date +%s)
 			mv /tmp/agent.log "/tmp/agent.log-${timestamp}" || true
 			mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true
 		fi
 		for suffix in log conf sock ; do
-			rm "/tmp/agent.${suffix}"
+			rm -f "/tmp/agent.${suffix}"
 		done
 		rm -f /tmp/keys
-		rm /tmp/dataset_info.yaml
+		rm -f /tmp/dataset_info.yaml
 		kill "$1"
 	fi
 	exit "$2"
@@ -124,12 +130,20 @@ if [ -n "$encrypted" ]; then
 	python3 ./utils/spawn_agent.py --config "$config" >/dev/null 2>/dev/null &
 	spire_agent_pid=$!
 
-fi
+	# Wait for socket to be created
+	sleep 2
+	until [ -e /tmp/agent.sock ]; do
+	    if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
+		echo "spire agent died, aborting"
+		end_entrypoint "$spire_agent_pid" 1
+	    fi
+	    printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 5 seconds ${NC}"
+	    sleep 5
+	done
 
-ps "$spire_agent_pid" >/dev/null || (
-	echo "spire agent died, aborting"
-	end_entrypoint "$spire_agent_pid" 1
-)
+	# Make socket accessible to all processes
+	chmod 777 /tmp/agent.sock
+fi
 
 #
 ## [END] Perform node attestation
@@ -142,9 +156,17 @@ printf "%b\n" "${YELLOW}[LUMI-SD]${NC}${BLUE}[Container preparation]${NC} Run co
 #
 
 if [ -z "$encrypted" ]; then
-	python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	if [ -n "$sif_host_path" ]; then
+		python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" --sif-host-path "${sif_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	else
+		python3 ./prepare_container.py -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	fi
 else
-	python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	if [ -n "$sif_host_path" ]; then
+		python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" --sif-host-path "${sif_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	else
+		python3 ./prepare_container.py -e -b "$base_oci_image" -s "$sif_path" -d "$docker_path" --docker-host-path "${docker_host_path}" || end_entrypoint "$spire_agent_pid" 1
+	fi
 fi
 
 #

client/container_preparation/lib/image_build.py

@@ -100,18 +100,23 @@ def build_prepared_image(base_image_tag, docker_client: DockerClient):
 def create_sif_image(
     prepared_image_tag,
     destination_path,
+    destination_host_path,
     docker_client: DockerClient,
     docker_socket_path: str,
     docker_host_socket_path: str,
     encrypted=False,
+    force=True,
+    verbose=False,
 ):
     """Creates an Apptainer SIF image (encrypted or not) using the sd-container/build_env
     out of a prepared OCI image.
 
     Args:
         prepared_image_tag (string): The OCI Image tag (i.e sd-container/prepared_cowsay)
-        destination_path (string): The path where the .sif image will be written (i.e /tmp => /tmp/prepared_cowsay.sif)
+        destination_path (string): The path where the .sif image will be written from this container's perspective (i.e /tmp => /tmp/prepared_cowsay.sif)
+        destination_host_path (string): The path on the host machine where the .sif image will be written (for nested container volume mounts)
         encrypted (bool, optional): Wether or not to encrypt the container, keys are generated relatively to the current path, and are called "keys, keys.pub". Defaults to False.
+        force (bool, optional): Force overwrite of existing SIF image. Defaults to True.
     """
     # Check that the build environment exists
     build_env_exists = check_build_env_exists(docker_client=docker_client)
@@ -127,59 +132,67 @@ def create_sif_image(
     prepared_image_name = prepared_image_tag.split("/")[-1].split(":")[0]
 
     # Composing the final command to run (the build env consists in apptainer build + the command var)
-    command = f"/output/{prepared_image_name}.sif docker-daemon://{prepared_image_tag}"
+    force_flag = "--force " if force else ""
+    command = f"{force_flag}/output/{prepared_image_name}.sif docker-daemon://{prepared_image_tag}"
 
     # Composing the dictionary of volumes (docker socket for image reading, output directory for result image)
+    # Use destination_host_path for the volume mount (host perspective), not destination_path (container perspective)
     volumes = [
         f"{docker_host_socket_path}:{docker_socket_path}",
-        f"{destination_path}:/output",
+        f"{destination_host_path}:/output",
         "/etc/passwd:/etc/passwd",
         "/etc/group:/etc/group",
     ]
 
     # Run the secured container build
     if not encrypted:
-        docker_client.containers.run(
+        output = docker_client.containers.run(
             image=build_env_image_tag,
             command=command,
             volumes=volumes,
             remove=True,
             user=os.getuid(),
             group_add=[f"{os.stat(docker_socket_path).st_gid}"],
+            stderr=True,
         )
+        if verbose:
+            print(f"Apptainer build output: {output.decode('utf-8')}")
 
     # Keeping the if/else for easy code rollback if encrypted containers can be used (that's why we don't have only one "docker_client.containers.run()")
     else:
-        docker_client.containers.run(
+        output = docker_client.containers.run(
             image=build_env_image_tag,
             command=command,
             volumes=volumes,
             remove=True,
             user=os.getuid(),
             group_add=[f"{os.stat(docker_socket_path).st_gid}"],
+            stderr=True,
         )
+        if verbose:
+            print(f"Apptainer build output: {output.decode('utf-8')}")
 
         # Generate necessary keys to encrypt the SIF file
         sif_decryption_key = x25519.Identity.generate()
         sif_enryption_key = sif_decryption_key.to_public()
 
         # Encrypt the SIF image
-        with open(f"/tmp/{prepared_image_name}.sif", "rb") as inputfile:
+        with open(f"{destination_path}/{prepared_image_name}.sif", "rb") as inputfile:
             encrypted = encrypt(inputfile.read(), [sif_enryption_key])
 
         # Write the encrypted SIF image to the encrypted SIF file
-        open(f"/tmp/encrypted_{prepared_image_name}.sif", "wb+").write(encrypted)
+        open(f"{destination_path}/encrypted_{prepared_image_name}.sif", "wb+").write(encrypted)
 
         # Remove the unencrypted image
-        os.remove(f"/tmp/{prepared_image_name}.sif")
+        os.remove(f"{destination_path}/{prepared_image_name}.sif")
 
         print(
-            f"SIF image encrypted, written to /tmp/encrypted_{prepared_image_name}.sif"
+            f"SIF image encrypted, written to {destination_path}/encrypted_{prepared_image_name}.sif"
         )
 
-        open(f"/tmp/keys", "w+").write(str(sif_decryption_key))
+        open(f"{destination_path}/keys", "w+").write(str(sif_decryption_key))
 
-        print(f"Encryption : Keys written to /tmp/keys")
+        print(f"Encryption : Keys written to {destination_path}/keys")
 
         # Commenting out the encrypted container runtime part since we can't run them atm
         # # Define pem path

client/container_preparation/prepare_container.py

@@ -11,9 +11,11 @@
     # For future usage
     base_image = arguments.base_oci_image
     sif_path = arguments.sif_path
+    sif_host_path = arguments.sif_host_path if arguments.sif_host_path else arguments.sif_path
     encrypted = arguments.encrypted
     docker_socket_path = arguments.docker_path
     docker_host_socket_path = arguments.docker_host_path
+    force = arguments.force
 
     # Checking the access to the docker socket - avoid waiting for an error to kill the program
     print(f"Checking the access to docker socket at {docker_socket_path}")
@@ -42,10 +44,12 @@
     create_sif_image(
         prepared_image_tag,
         destination_path=sif_path,
+        destination_host_path=sif_host_path,
         docker_client=docker_client,
         docker_socket_path=docker_socket_path,
         docker_host_socket_path=docker_host_socket_path,
         encrypted=encrypted,
+        force=force,
     )
 
     print(f"{sif_path}/{prepared_image_tag.split(':')[0]}.sif successfully created")

client/container_preparation/tools/cli/cli.py

@@ -37,8 +37,21 @@ def parse_arguments() -> argparse.ArgumentParser:
     parser.add_argument(
         "--docker-host-path",
         type=str,
-        help="Path to the docker socket (default : /var/run/docker.sock)",
+        help="Path to the docker socket on the host (default : /var/run/docker.sock)",
         default="/var/run/docker.sock",
     )
+    parser.add_argument(
+        "--sif-host-path",
+        type=str,
+        help="Path to the SIF directory on the host (default : same as --sif-path). Use this when running in a container with volume mounts",
+        default=None,
+    )
+    parser.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        help="Force overwrite existing SIF image (default : True)",
+        default=True,
+    )
 
     return parser.parse_args()

client/data_preparation/entrypoint.sh

@@ -70,9 +70,17 @@ parse_args() {
 end_entrypoint() {
 	echo "Cleaning everything before leaving ..."
 	rm -rf /tmp/data
-	rm /tmp/agent*
+	# On failure, preserve agent logs and config for debugging before removing them
+	if [ "${2}" -ne 0 ]; then
+		timestamp=$(date +%s)
+		mv /tmp/agent.log "/tmp/agent.log-${timestamp}" || true
+		mv /tmp/agent.conf "/tmp/agent.conf-${timestamp}" || true
+	fi
+	for suffix in log conf sock; do
+		rm -f "/tmp/agent.${suffix}"
+	done
 	rm -f /tmp/keys
-	rm /tmp/dataset_info.yaml
+	rm -f /tmp/dataset_info.yaml
 	kill "$1"
 	exit "$2"
 }
@@ -97,15 +105,20 @@ printf "%b\n" "${YELLOW}[LUMI-SD]${NC}${BLUE}[Data preparation]${NC} Registering
 python3 ./utils/spawn_agent.py --config "$config" >/dev/null 2>/dev/null &
 spire_agent_pid=$!
 
+# Wait for socket to be created
+sleep 2
 until [ -e /tmp/agent.sock ]; do
-	printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 10 seconds ${NC}"
-	sleep 10
-	if ! pgrep -f "$spire_agent_pid" > /dev/null; then
-		echo "spire agent died, aborting"
-		end_entrypoint "$spire_agent_pid" 1
-	fi
+    if ! ps "$spire_agent_pid" >/dev/null 2>&1; then
+	echo "spire agent died, aborting"
+	end_entrypoint "$spire_agent_pid" 1
+    fi
+    printf "%b\n" "${RED}[LUMI-SD][Data preparation] Spire workload api socket doesn't exist, waiting 5 seconds ${NC}"
+    sleep 5
 done
 
+# Make socket accessible to all processes
+chmod 777 /tmp/agent.sock
+
 #
 ## [END] Perform node attestation
 #

client/job_preparation/prepare_job.py

@@ -8,6 +8,7 @@
 from utils.ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command
 from utils.conf.client.conf import parse_configuration
 from time import sleep
+from threading import Thread
 from pyrage import x25519
 
 if __name__ == "__main__":
@@ -88,48 +89,43 @@
     jobid = stdout.read().decode().split(" ")[-1].replace("\n", "")
     print(f"Job successfully created with id {jobid}")
 
-    # If user asked to follow the logs of the job
-    if options.follow:
-        # Follow the job's output
-        command = f"cd {options.workdir} ; touch {options.job_name}.out ; tail -f -n 0 {options.job_name}.out"
-        stdin, stdout, stderr = ssh_run_command(ssh_client, command, True)
-        stdin.close()
+    if options.follow or options.verbose:
+        # If verbose, stream job log output in a background thread with indentation
+        if options.verbose:
+            verbose_client = ssh_connect(options.username)
+            command = f"cd {options.workdir} ; touch {options.job_name}.out ; tail -f -n 0 {options.job_name}.out"
+            _, log_stdout, _ = ssh_run_command(verbose_client, command, True)
 
-        # Keep read alive
-        for line in iter(stdout.readline, ""):
-            print(line, end="")
+            def stream_log():
+                for line in iter(log_stdout.readline, ""):
+                    print(f"    {line}", end="")
 
-    # If user doesn't follow job's logs
-    else:
-        # Still, follow the job's status at fixed interval
-        print(
-            f"Waiting for the job to run. You can now exit this script if needed, outputs will be available in {options.workdir}/output when finished"
-        )
+            Thread(target=stream_log, daemon=True).start()
 
-        # Specific output format squeue command to parse informations about submitted job
+        # Poll squeue until the job completes
         command = f"squeue -o '%A;%u;%T' | grep {options.username} | grep {jobid}"
-
-        # Run first status update
         stdin, stdout, stderr = ssh_run_command(ssh_client, command, True)
         job_status = ""
         stdout = stdout.read().decode().replace("\n", "")
         stdin.close()
 
-        # While job runs (while it has an entry in squeue)
         while stdout != "":
-            # If status has changed
             if job_status != stdout.split(";")[-1]:
                 job_status = stdout.split(";")[-1]
-                print(f"Job's state changed, job is now in state : {job_status}")
-
-            # Wait
+                print(f"Job status: {job_status}")
             sleep(10)
-
-            # Run status update
             stdin, stdout, stderr = ssh_run_command(ssh_client, command, True)
             stdout = stdout.read().decode().replace("\n", "")
             stdin.close()
 
-    # Job is over
-    print(f"Job finished, please find outputs in {options.workdir}/output")
+        print(f"Job finished, outputs available in {options.workdir}/output")
+
+        if options.verbose:
+            verbose_client.close()
+
+    else:
+        print(
+            f"Outputs will be available in {options.workdir}/output when the job finishes"
+        )
+
     ssh_client.close()

client/job_preparation/utils/cli/cli.py

@@ -109,7 +109,14 @@ def parse_arguments() -> argparse.Namespace:
         "--follow",
         "-f",
         action="store_true",
-        help="Follow job's output (default : False)",
+        help="Follow job status, polling squeue until the job completes (default: False)",
+        default=False,
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Stream job log output in addition to status updates, implies --follow (default: False)",
         default=False,
     )