SRE-1220: Distrust compromised CA roots in Docker images

apps/brunch-agent/docker/Dockerfile

@@ -11,11 +11,15 @@ ENV MISE_DATA_DIR="/mise"
 ENV MISE_CACHE_DIR="/mise/cache"
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends ca-certificates curl && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \

apps/hash-ai-worker-ts/docker/Dockerfile

@@ -8,11 +8,15 @@ ENV MISE_DATA_DIR="/mise"
 ENV MISE_CACHE_DIR="/mise/cache"
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \

apps/hash-ai-worker-ts/package.json

@@ -36,7 +36,7 @@
     "run-flow-scenario": "NODE_ENV=development yarn exe ./scripts/flow-scenarios/run-flow-scenario.ts",
     "sanitize-html": "NODE_ENV=development yarn exe ./scripts/sanitize-html.ts",
     "sentry:sourcemaps": "sentry-cli sourcemaps inject --org hashintel --project hash-temporal-worker-ai ./dist && sentry-cli sourcemaps upload --org hashintel --project hash-temporal-worker-ai ./dist",
-    "start": "NODE_ENV=production NODE_OPTIONS=--max-old-space-size=2048 node ./dist/main.js",
+    "start": "NODE_ENV=production NODE_OPTIONS=\"${NODE_OPTIONS:-} --max-old-space-size=2048\" node ./dist/main.js",
     "start:healthcheck": "wait-on --timeout 1200000 http-get://localhost:4100/health",
     "start:test": "NODE_ENV=test NODE_OPTIONS=--max-old-space-size=2048 node ./dist/main.js",
     "start:test:healthcheck": "wait-on --timeout 600000 http-get://localhost:4100/health",

apps/hash-api/docker/Dockerfile

@@ -8,11 +8,15 @@ ENV MISE_DATA_DIR="/mise"
 ENV MISE_CACHE_DIR="/mise/cache"
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \

apps/hash-api/package.json

@@ -19,7 +19,7 @@
     "lint:eslint": "eslint --report-unused-disable-directives .",
     "lint:tsc": "tsc --noEmit",
     "setup:admin": "NODE_ENV=production OTEL_SERVICE_NAME=\"Setup Admin\" tsx --import ./src/instrument.mjs ./src/setup-admin.ts",
-    "start": "NODE_ENV=production NODE_OPTIONS=--max-old-space-size=2048 tsx --import ./src/instrument.mjs ./src/index.ts",
+    "start": "NODE_ENV=production NODE_OPTIONS=\"${NODE_OPTIONS:-} --max-old-space-size=2048\" tsx --import ./src/instrument.mjs ./src/index.ts",
     "start:healthcheck": "wait-on --timeout 1200000 http://localhost:5001",
     "start:migrate": "NODE_ENV=production OTEL_SERVICE_NAME=\"Node Migrations\" tsx --import ./src/instrument.mjs ./src/ensure-system-graph-is-initialized.ts",
     "start:test": "NODE_ENV=test NODE_OPTIONS=--max-old-space-size=2048 tsx ./src/index.ts",

apps/hash-frontend/docker/Dockerfile

@@ -6,11 +6,15 @@ ENV MISE_DATA_DIR="/mise"
 ENV MISE_CACHE_DIR="/mise/cache"
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \

apps/hash-graph/docker/Dockerfile

@@ -9,11 +9,16 @@ ENV MISE_CACHE_DIR="/mise/cache"
 ENV MISE_NODE_COREPACK=1
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
+# The runner copies this bundle.
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \

apps/hash-integration-worker/docker/Dockerfile

@@ -8,11 +8,15 @@ ENV MISE_DATA_DIR="/mise"
 ENV MISE_CACHE_DIR="/mise/cache"
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \

apps/hash-integration-worker/package.json

@@ -14,7 +14,7 @@
     "lint:eslint": "eslint --report-unused-disable-directives .",
     "lint:tsc": "tsc --noEmit",
     "sentry:sourcemaps": "sentry-cli sourcemaps inject --org hashintel --project hash-temporal-worker-integration ./dist && sentry-cli sourcemaps upload --org hashintel --project hash-temporal-worker-integration ./dist",
-    "start": "NODE_ENV=production NODE_OPTIONS=--max-old-space-size=2048 tsx ./src/main.ts",
+    "start": "NODE_ENV=production NODE_OPTIONS=\"${NODE_OPTIONS:-} --max-old-space-size=2048\" tsx ./src/main.ts",
     "start:healthcheck": "wait-on --timeout 1200000 http-get://localhost:4300/health",
     "start:test": "NODE_ENV=test NODE_OPTIONS=--max-old-space-size=2048 tsx ./dist/main.js",
     "start:test:healthcheck": "wait-on --timeout 600000 http-get://localhost:4300/health"

apps/petrinaut-opt/docker/Dockerfile

@@ -17,13 +17,17 @@ ENV MISE_CACHE_DIR="/mise/cache"
 ENV MISE_CARGO_HOME="/usr/local/cargo"
 ENV PATH="/mise/shims:/usr/local/cargo/bin:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 # turbo reads the Cargo workspace through `cargo metadata`, in the prune as well as in the build.
 COPY rust-toolchain.toml /tmp/rust-toolchain.toml
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419~deb12u2 && \
     /etc/mise/install.sh && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
@@ -113,6 +117,9 @@ ENV PATH="/opt/venv/bin:${PATH}" \
     PYTHONUNBUFFERED=1 \
     PETRINAUT_CHILD_NODE_OPTIONS="--permission --allow-fs-read=/opt/petrinaut-runtime --allow-fs-read=/usr/local/bin/petrinaut --disable-proto=throw --max-old-space-size=768 --no-addons --no-global-search-paths"
 
+# gRPC otherwise loads the roots bundled in grpcio.
+ENV GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt
+
 WORKDIR /app
 
 # Node is what the bindings' executable runs on today.
@@ -125,6 +132,12 @@ COPY apps/petrinaut-opt/src ./src
 # built workspace packages it resolves.
 COPY --from=bindings-runtime / /opt/petrinaut-runtime
 
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419~deb12u2 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 # `petrinaut` is the command the bindings spawn by default, and they give the
 # child a fixed PATH of /usr/local/bin:/usr/bin:/bin, so it is linked there.
 RUN chmod +x /opt/petrinaut-runtime/node_modules/.bin/petrinaut && \

infra/compose/hydra/Dockerfile

@@ -2,6 +2,10 @@ FROM oryd/hydra:v25.4
 
 USER root
 
+# TODO(SRE-1231): Remove this exclusion once the upstream image rejects the affected eMudhra chains.
+RUN sed -i -e 's|^mozilla/emSign_ECC_Root_CA_-_G3.crt|!&|' /etc/ca-certificates.conf && \
+    update-ca-certificates
+
 RUN mkdir -p /home/nonroot/.postgresql
 
 # Add AWS certificate bundle for SSL connection

infra/compose/kratos/Dockerfile

@@ -1,5 +1,15 @@
+# TODO(SRE-1231): Remove this stage and its bundle copy once upstream Kratos rejects the affected eMudhra chains.
+FROM debian:13.3-slim AS ca-certificates
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1
+
+
 FROM oryd/kratos:v26.2.0-distroless
 
+COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
 COPY ./templates /etc/config/kratos/templates
 COPY ./hooks /etc/config/kratos/hooks
 COPY ./identity.schema.json /etc/config/kratos/

libs/@local/graph/atlas/docker/Dockerfile

@@ -12,11 +12,15 @@ ENV MISE_CACHE_DIR="/mise/cache"
 ENV MISE_NODE_COREPACK=1
 ENV PATH="/mise/shims:$PATH"
 
+# TODO(SRE-1221): Remove this override after upgrading Node.js to a release without the affected eMudhra roots: https://github.com/nodejs/node/pull/66871
+ENV NODE_OPTIONS=--use-openssl-ca
+
 COPY .config/mise /etc/mise
 
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20250419+deb13u1 && \
     /etc/mise/install.sh && \
     mise --version && \
     apt-get clean && \
@@ -61,6 +65,7 @@ COPY rust-toolchain.toml .
 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
     apt-get update && \
     apt-get install -y --no-install-recommends curl ca-certificates build-essential && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20260816ubuntu0.24.04.1 && \
     /etc/mise/install.sh && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
@@ -107,7 +112,8 @@ RUN cuda="${CUDA_VERSION%.*}" && \
     cuda="${cuda/./-}" && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-    "cuda-nvrtc-${cuda}" "cuda-cudart-dev-${cuda}" "cuda-cccl-${cuda}" && \
+    "cuda-nvrtc-${cuda}" "cuda-cudart-dev-${cuda}" "cuda-cccl-${cuda}" ca-certificates && \
+    dpkg --compare-versions "$(dpkg-query -W -f='${Version}' ca-certificates)" ge 20260816ubuntu0.24.04.1 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
     ldconfig && \