test: run the end-to-end tests on the image

.github/workflows/ci.yml

@@ -3,12 +3,12 @@
 # coverage threshold (reported to Codecov), the type check and the build,
 # an audit of the production dependencies, the Docker image (a smoke test
 # with a read-only root file system) and its scan for known vulnerabilities
-# (Trivy), the end-to-end tests against mock services (`e2e/mock/`), the
-# documentation (strict MkDocs build), the release readiness, and, on
-# pushes, push the image by digest (with SBOM and provenance attestations),
-# scan exactly that digest, then attest, sign (Sigstore) and tag it with
-# `dev` and `sha-<commit>`
-# (never `latest`, which is for releases). The gate `passed` ("CI passed")
+# (Trivy), the end-to-end tests against mock services (`e2e/mock/`) on the
+# build of the checkout and, separately, on the image, the documentation
+# (strict MkDocs build), the release readiness, and, on pushes, push the
+# image by digest (with SBOM and provenance attestations), scan exactly
+# that digest, then attest, sign (Sigstore) and tag it with `dev` and
+# `sha-<commit>` (never `latest`, which is for releases). The gate `passed` ("CI passed")
 # is the only required check. No `paths:` filters, so that the gate always
 # reports; skip jobs with `if:` instead (and allow the skip in the gate).
 # Runs also when a pull request is edited, so that the title check follows
@@ -236,7 +236,7 @@ jobs:
   # read-only root file system and a writable `/tmp` (see "Container image"
   # in `docs/operator-guide/deployment.md`): for pull requests an image
   # built as `docker build` builds it, without settings, which is handed to
-  # `scan`;
+  # `scan` and `e2e-image`;
   # for pushes exactly the digest pushed by `push` (public, so pulling it
   # needs no login), in a job without credentials, so that the published
   # image is the checked one
@@ -329,6 +329,73 @@ jobs:
           path: playwright-report/
           retention-days: 7
 
+  # The journey of `e2e` in Chromium against the image instead of the build
+  # of the checkout (`E2E_IMAGE`, see `playwright.config.ts`): started as it
+  # is deployed (non-root, read-only root file system, writable `/tmp`, all
+  # settings at start), so that a difference between the two (e.g. a file
+  # missing from the standalone output, or a setting read at build time)
+  # fails here. For pull requests the image of `image`, for pushes exactly
+  # the digest pushed by `push` (public, so pulling it needs no login).
+  # Unlike in `e2e`, Playwright runs on the runner, not in its image, as it
+  # starts the app with the runner's Docker (`docker run` in
+  # `playwright.config.ts`); Chromium and its system packages come from
+  # `playwright install`, in the version of `@playwright/test`. On Ubuntu
+  # 24.04, as Playwright 1.56 doesn't support `ubuntu-latest` (26.04): move
+  # to `ubuntu-latest` with a version of `@playwright/test` that does
+  e2e-image:
+    name: End-to-end tests (image)
+    # For pull requests, `push` is skipped
+    needs: [image, push]
+    if: >-
+      ${{ !cancelled() && needs.image.result == 'success' && (
+        github.event_name == 'pull_request' || needs.push.result == 'success') }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Download the image
+        if: ${{ github.event_name != 'push' }}
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: image
+          path: ${{ runner.temp }}
+      - name: Load the image
+        if: ${{ github.event_name != 'push' }}
+        run: docker load -i "${RUNNER_TEMP}/image.tar"
+      - name: Pull the pushed image
+        if: ${{ github.event_name == 'push' }}
+        env:
+          DIGEST: ${{ needs.push.outputs.digest }}
+        run: |
+          if [[ ! "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "::error::Invalid image digest '${DIGEST}'"
+            exit 1
+          fi
+          docker pull "${IMAGE_NAME}@${DIGEST}"
+          docker tag "${IMAGE_NAME}@${DIGEST}" "${IMAGE_NAME}:scan"
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version-file: package.json
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+      - name: Run end-to-end tests on the image
+        run: npx playwright test --project chromium
+        env:
+          E2E_IMAGE: ${{ env.IMAGE_NAME }}:scan
+      - name: Upload the test report
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: playwright-report-image
+          path: playwright-report/
+          retention-days: 7
+
   # On pushes, builds the image once and pushes it by digest only (untagged,
   # with SBOM and provenance attestations, `mode=max`), for `image` to check,
   # `scan` to scan and `publish` to sign and tag. Runs alongside the
@@ -458,7 +525,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     if: ${{ github.event_name == 'push' }}
-    needs: [lint, test, build, audit, e2e, docs, readiness, image, push, scan]
+    needs: [lint, test, build, audit, e2e, e2e-image, docs, readiness, image, push, scan]
     # Docker Hub secrets; the environment only allows the branch `dev`
     environment: dev-images
     permissions:
@@ -655,7 +722,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     if: always()
-    needs: [title, lint, test, build, audit, e2e, docs, readiness, image, push, scan, publish]
+    needs: [title, lint, test, build, audit, e2e, e2e-image, docs, readiness, image, push, scan, publish]
     permissions: {}
     steps:
       - name: Check results of all jobs

.github/workflows/release.yml

@@ -5,7 +5,8 @@
 # (`prepare`); build and push the Docker image (with SBOM and provenance
 # attestations) by digest, untagged (`push`); scan exactly that digest for
 # vulnerabilities in a job of its own, without the release credentials and
-# signing identity (`scan`), and smoke-test it as CI does (`smoke`); and
+# signing identity (`scan`), smoke-test it and run the end-to-end tests on
+# it as CI does (`smoke`, `e2e`); and
 # only then attest its build provenance, sign it (Sigstore keyless signing)
 # and tag it (`release-image`; see "Verifying images" in
 # `docs/operator-guide/deployment.md`). Last, create the GitHub release
@@ -304,10 +305,57 @@ jobs:
           DIGEST: ${{ needs.push.outputs.digest }}
         run: scripts/check-image.sh "${IMAGE_NAME}@${DIGEST}"
 
+  # Runs the end-to-end tests of CI's `e2e-image` job (Chromium, mock
+  # services of `e2e/mock/`) on exactly the pushed digest (public, so
+  # pulling it needs no login), in a job without credentials, so that the
+  # released image is the tested one. On Ubuntu 24.04, as `e2e-image`
+  e2e:
+    name: End-to-end tests (release image)
+    needs: [prepare, push]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      # The tests of the commit checked by `prepare`
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          ref: ${{ needs.prepare.outputs.sha }}
+      - name: Pull the pushed image
+        env:
+          DIGEST: ${{ needs.push.outputs.digest }}
+        run: |
+          if [[ ! "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "::error::Invalid image digest '${DIGEST}'"
+            exit 1
+          fi
+          docker pull "${IMAGE_NAME}@${DIGEST}"
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version-file: package.json
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+      - name: Run end-to-end tests on the image
+        run: npx playwright test --project chromium
+        env:
+          E2E_IMAGE: ${{ env.IMAGE_NAME }}@${{ needs.push.outputs.digest }}
+      - name: Upload the test report
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: playwright-report-release
+          path: playwright-report/
+          retention-days: 7
+
   # Attests, signs and tags the scanned digest
   release-image:
     name: Release image
-    needs: [prepare, push, smoke, scan]
+    needs: [prepare, push, smoke, e2e, scan]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     environment: release

docs/development/index.md

@@ -103,6 +103,25 @@ calls real services.
     tests in the Playwright image, as CI does (see the `e2e` job in
     `.github/workflows/ci.yml`).
 
+    To run them against a Docker image instead of the build of the
+    checkout, set `E2E_IMAGE` to the image. Playwright then starts it as it
+    is deployed: as the image's non-root user, with a read-only root file
+    system, a writable `/tmp` and all settings at start. A difference
+    between the two, such as a file missing from the standalone output or a
+    setting read at build time, fails the tests then. This needs Docker with
+    the host network (Linux) and a free container name `krini-e2e`. CI does
+    this in Chromium for every image it builds (`e2e-image` job), and the
+    release workflow for every release image (`e2e` job):
+
+    ```bash
+    docker build -t krini:local .
+    E2E_IMAGE=krini:local bun run test:e2e --project chromium
+    ```
+
+    The container is removed when the tests end. If a run was killed and
+    left it behind, the next run fails at once, as the name is taken:
+    remove it with `docker rm -f krini-e2e`.
+
 ## Documentation
 
 The documentation is built with [MkDocs](https://www.mkdocs.org/) and the

docs/development/releasing.md

@@ -130,9 +130,11 @@ all releases are GitHub pre-releases; stable versions (without `-alphaN`,
     versions, the curated notes and the CI run (`Prepare release`); builds
     and pushes the image by digest (`Push release image`, needs approval);
     scans exactly that digest (`Scan release image`) and runs the smoke test
-    of CI on it (`Smoke test`, `scripts/check-image.sh`); attests, signs and
-    tags it `X.Y.Z` (and `latest` for the highest stable version)
-    (`Release image`, needs approval); and then, without another approval,
+    and the end-to-end tests of CI on it (`Smoke test`, with
+    `scripts/check-image.sh`, and `End-to-end tests (release image)`); only
+    when all three passed, attests, signs and tags it `X.Y.Z` (and `latest`
+    for the highest stable version) (`Release image`, needs approval); and
+    then, without another approval,
     creates the GitHub release (`Publish release`): the curated notes of the
     tagged commit, then GitHub's generated notes under `## Commits` (without
     their `## What's Changed` header, with "New Contributors"), and GitHub's

docs/operator-guide/configuration.md

@@ -46,7 +46,9 @@ PRIVACY_LOGIN_NOTICE_URL=https://login.example.org/privacy
   `openid profile email offline_access`.
 - `CLOUD_REGISTRY_URL`: Base URL of the Cloud Registry API.
 - `APP_ORIGIN`: Origin of your application (e.g., `http://localhost:3000`
-  for local development or your production URL).
+  for local development or your production URL), as users' browsers reach
+  it. The login sends users back to it, and the logout accepts requests
+  only from it.
 
 Make sure the OIDC client has the redirect URI of `OIDC_REDIRECT_URI`
 registered, e.g. for local development:

docs/operator-guide/deployment.md

@@ -46,7 +46,8 @@ reserved for releases. Deployments pull these images and update themselves
 
 Before CI tags an image, the commit passes lint, unit tests, type check,
 build, a dependency audit and the end-to-end tests, and exactly the
-pushed digest passes a smoke test (non-root, read-only root file system)
+pushed digest passes a smoke test (non-root, read-only root file system),
+the end-to-end tests (started as it is deployed, against mock services)
 and a scan for known vulnerabilities (Trivy, failing on fixable critical
 and high findings). CI then attests its build provenance and signs it
 with Sigstore keyless signing. The image also carries an SBOM and
@@ -126,8 +127,8 @@ To publish images from a fork or for another organization, the
 repository needs:
 
 - the variable `DOCKERHUB_ORG`, the Docker Hub namespace (by default
-  `elixircloud`), with a public repository `krini` in it, as the scan and
-  the smoke test pull the pushed digest without login;
+  `elixircloud`), with a public repository `krini` in it, as the scan, the
+  smoke test and the end-to-end tests pull the pushed digest without login;
 - the environment `dev-images`, limited to the branch `dev`, with the
   secrets `DOCKERHUB_LOGIN` and `DOCKERHUB_TOKEN` (a token
   that can push to that repository). The name is kept from when only `dev`

playwright.config.ts

@@ -1,3 +1,4 @@
+import { resolve } from "node:path";
 import { defineConfig, devices } from "@playwright/test";
 import {
     appPort,
@@ -12,11 +13,83 @@ import {
 // end-to-end tests (`e2e/`) against the mock services of
 // `e2e/mock/server.ts` (login provider, registry, WES service), which start
 // first; then the app is built and started, on its own port. Nothing calls
-// real public services. Ports are in `e2e/settings.ts`.
+// real public services. Ports are in `e2e/settings.ts`. With `E2E_IMAGE`
+// set to a Docker image (e.g. `krini:local` or `<name>@<digest>`), the app
+// is that image instead, started as it is deployed: as the image's
+// non-root user, with a read-only root file system, a writable `/tmp` and
+// all settings at start. That needs Docker with the host network (Linux).
 const port = appPort;
 const baseURL = appURL;
 const mockHttp = mockURL;
 const certDir = "e2e/.certs";
+const image = process.env.E2E_IMAGE;
+
+// The image and the certificate directory (where the mock services write
+// it: relative to this file, the web servers' working directory) are part
+// of a shell command: only the characters of image references and of plain
+// paths, without quotes, spaces or `$`
+const certPath = resolve(__dirname, certDir);
+if (image !== undefined && !/^\w[\w.:/@-]*$/.test(image)) {
+    throw new Error(
+        `E2E_IMAGE has characters of no image reference: '${image}'`,
+    );
+}
+if (image !== undefined && !/^[\w./-]+$/.test(certPath)) {
+    throw new Error(
+        `Run the tests with E2E_IMAGE from a plain path, not '${certPath}'`,
+    );
+}
+
+/** Settings of the app, the same for the build of the checkout and the
+ * image */
+const appEnv = {
+    LS_LOGIN_URL: mockHttp,
+    OIDC_CLIENT_ID: client.id,
+    OIDC_CLIENT_SECRET: client.secret,
+    OIDC_REDIRECT_URI: `${baseURL}/auth/api/callback`,
+    OIDC_SCOPE: "openid profile email",
+    CLOUD_REGISTRY_URL: `${mockHttp}/ga4gh/registry/v1`,
+    APP_ORIGIN: baseURL,
+    PRIVACY_CONTROLLER: privacy.controller,
+    PRIVACY_CONTACT: privacy.contact,
+    PRIVACY_NOTICE_URL: privacy.noticeUrl,
+    PRIVACY_LOGIN_NOTICE_URL: privacy.loginNoticeUrl,
+};
+
+/** The app: the image, or the build of the checkout */
+const app = image
+    ? {
+          // `--env <name>` passes the variable from `env` below; the
+          // certificate directory is mounted at `/certs`. On SIGTERM, Docker
+          // stops the container, and `--rm` removes it; a container left by
+          // a killed run keeps the name, so the next run fails at once
+          command: [
+              "docker run --rm --init --name krini-e2e --network host",
+              "--read-only --tmpfs /tmp",
+              `--volume ${certPath}:/certs:ro`,
+              "--env PORT --env NODE_EXTRA_CA_CERTS",
+              ...Object.keys(appEnv).map((name) => `--env ${name}`),
+              image,
+          ].join(" "),
+          gracefulShutdown: { signal: "SIGTERM" as const, timeout: 10_000 },
+          env: {
+              PORT: String(port),
+              // the app checks the mock WES service's certificate
+              NODE_EXTRA_CA_CERTS: "/certs/cert.pem",
+              ...appEnv,
+          },
+      }
+    : {
+          // the build doesn't get the privacy settings, so the test fails if
+          // the notice is rendered at build time
+          command: `env -u PRIVACY_CONTROLLER -u PRIVACY_CONTACT -u PRIVACY_NOTICE_URL -u PRIVACY_LOGIN_NOTICE_URL bun run build && bun run start --port ${port}`,
+          env: {
+              NEXT_TELEMETRY_DISABLED: "1",
+              // the app checks the mock WES service's certificate
+              NODE_EXTRA_CA_CERTS: `${certDir}/cert.pem`,
+              ...appEnv,
+          },
+      };
 
 export default defineConfig({
     testDir: "e2e",
@@ -48,28 +121,11 @@ export default defineConfig({
             },
         },
         {
-            // the build doesn't get the privacy settings, so the test fails if
-            // the notice is rendered at build time
-            command: `env -u PRIVACY_CONTROLLER -u PRIVACY_CONTACT -u PRIVACY_NOTICE_URL -u PRIVACY_LOGIN_NOTICE_URL bun run build && bun run start --port ${port}`,
+            ...app,
             url: `${baseURL}/auth/login`,
-            reuseExistingServer: !process.env.CI,
+            // with an image, a running app may be another one: never reuse it
+            reuseExistingServer: !process.env.CI && !image,
             timeout: 300_000,
-            env: {
-                NEXT_TELEMETRY_DISABLED: "1",
-                // the app checks the mock WES service's certificate
-                NODE_EXTRA_CA_CERTS: `${certDir}/cert.pem`,
-                LS_LOGIN_URL: mockHttp,
-                OIDC_CLIENT_ID: client.id,
-                OIDC_CLIENT_SECRET: client.secret,
-                OIDC_REDIRECT_URI: `${baseURL}/auth/api/callback`,
-                OIDC_SCOPE: "openid profile email",
-                CLOUD_REGISTRY_URL: `${mockHttp}/ga4gh/registry/v1`,
-                APP_ORIGIN: baseURL,
-                PRIVACY_CONTROLLER: privacy.controller,
-                PRIVACY_CONTACT: privacy.contact,
-                PRIVACY_NOTICE_URL: privacy.noticeUrl,
-                PRIVACY_LOGIN_NOTICE_URL: privacy.loginNoticeUrl,
-            },
         },
     ],
 });