wchilders-nvidia · Oct 15, 2026 · Oct 14, 2026
diff --git a/.github/workflows/acknowledg-deploy.yml b/.github/workflows/acknowledg-deploy.yml
@@ -0,0 +1,166 @@
+# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
+# Exceptions.
+# See https://edgcpp.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# Build and deploy AcknowlEDG over SSH. Two independent jobs:
+#
+#   client  builds the Vue client and rsyncs the static files into the web
+#           root. Nothing is restarted; browsers pick up the new files once
+#           their caches expire.
+#   server  builds the server wheel, uploads it, and runs
+#           dev_tools/services/acknowledg/server/deploy-remote.sh on the host,
+#           which installs it and restarts the systemd unit only when the
+#           wheel's contents differ from what is already deployed.
+#
+# Host, paths, and credentials come from secrets in the `acknowledg`
+# environment (see dev_tools/services/acknowledg/README.md). The SSH setup
+# writes them into a `Host acknowledg` alias so they never appear on a command
+# line.
+
+name: AcknowlEDG Deploy
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - dev_tools/services/acknowledg/**
+      - dev_tools/pylibs/edgacknowledg/**
+      - .github/workflows/acknowledg-deploy.yml
+  workflow_dispatch:
+
+concurrency:
+  group: acknowledg-deploy
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  ACKNOWLEDG_DIR: dev_tools/services/acknowledg
+
+jobs:
+  client:
+    name: Deploy client
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: acknowledg
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Set up Node
+        uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: ${{ env.ACKNOWLEDG_DIR }}/client/package-lock.json
+
+      - name: Build client
+        working-directory: ${{ env.ACKNOWLEDG_DIR }}/client
+        env:
+          VITE_EDG_ACKNOWLEDG_API_HOST: ${{ secrets.ACKNOWLEDG_API_HOST }}
+        run: |
+          # The client opens `${VITE_EDG_ACKNOWLEDG_API_HOST}/review`; without
+          # a scheme the browser resolves that relative to the page instead.
+          case "${VITE_EDG_ACKNOWLEDG_API_HOST}" in
+            ws://?*|wss://?*) ;;
+            *)
+              echo '::error::ACKNOWLEDG_API_HOST must be a ws:// or wss:// URL'
+              exit 1
+              ;;
+          esac
+          npm ci
+          npm run build
+
+      - name: Configure SSH
+        env:
+          SSH_HOST: ${{ secrets.ACKNOWLEDG_SSH_HOST }}
+          SSH_USER: ${{ secrets.ACKNOWLEDG_SSH_USER }}
+          SSH_PORT: ${{ secrets.ACKNOWLEDG_SSH_PORT || '22' }}
+          SSH_PRIVATE_KEY: ${{ secrets.ACKNOWLEDG_SSH_PRIVATE_KEY }}
+          SSH_KNOWN_HOSTS: ${{ secrets.ACKNOWLEDG_SSH_KNOWN_HOSTS }}
+        run: &configure-ssh |
+          install -d -m 700 ~/.ssh
+          install -m 600 /dev/null ~/.ssh/deploy_key
+          printf '%s\n' "${SSH_PRIVATE_KEY}" > ~/.ssh/deploy_key
+          printf '%s\n' "${SSH_KNOWN_HOSTS}" > ~/.ssh/known_hosts
+          cat > ~/.ssh/config <<EOF
+          Host acknowledg
+            HostName ${SSH_HOST}
+            User ${SSH_USER}
+            Port ${SSH_PORT}
+            IdentityFile ~/.ssh/deploy_key
+            IdentitiesOnly yes
+            StrictHostKeyChecking yes
+            BatchMode yes
+          EOF
+          chmod 600 ~/.ssh/config ~/.ssh/known_hosts
+
+      - name: Upload client files
+        working-directory: ${{ env.ACKNOWLEDG_DIR }}/client
+        env:
+          CLIENT_PATH: ${{ secrets.ACKNOWLEDG_CLIENT_PATH }}
+        run: |
+          # --delay-updates swaps the new files in at the end of the transfer
+          # and --delete-after removes stale hashed assets only once the new
+          # index.html no longer references them.
+          rsync --recursive --links --compress --checksum --itemize-changes \
+            --delay-updates --delete-after \
+            dist/ "acknowledg:${CLIENT_PATH}/"
+
+  server:
+    name: Deploy server
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: acknowledg
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v10.3.0
+
+      - name: Build server wheel
+        id: build
+        working-directory: ${{ env.ACKNOWLEDG_DIR }}/server
+        run: |
+          uv build --wheel --out-dir dist
+          wheel=$(ls dist/*.whl)
+          # Fingerprint the wheel by its RECORD, which lists a hash of every
+          # file in it; the zip itself isn't reproducible (timestamps). WHEEL
+          # only names the build backend's version, so leave it out to avoid
+          # restarts when nothing we ship has changed.
+          fingerprint=$(unzip -p "${wheel}" '*.dist-info/RECORD' \
+            | grep -v -E '\.dist-info/(WHEEL|RECORD),' \
+            | sort | sha256sum | cut -d' ' -f1)
+          echo "Built $(basename "${wheel}") with fingerprint ${fingerprint}"
+          echo "wheel=${wheel}" >> "${GITHUB_OUTPUT}"
+          echo "fingerprint=${fingerprint}" >> "${GITHUB_OUTPUT}"
+
+      - name: Configure SSH
+        env:
+          SSH_HOST: ${{ secrets.ACKNOWLEDG_SSH_HOST }}
+          SSH_USER: ${{ secrets.ACKNOWLEDG_SSH_USER }}
+          SSH_PORT: ${{ secrets.ACKNOWLEDG_SSH_PORT || '22' }}
+          SSH_PRIVATE_KEY: ${{ secrets.ACKNOWLEDG_SSH_PRIVATE_KEY }}
+          SSH_KNOWN_HOSTS: ${{ secrets.ACKNOWLEDG_SSH_KNOWN_HOSTS }}
+        run: *configure-ssh
+
+      - name: Install server
+        working-directory: ${{ env.ACKNOWLEDG_DIR }}/server
+        env:
+          SERVER_PATH: ${{ secrets.ACKNOWLEDG_SERVER_PATH }}
+          SERVICE_NAME: ${{ secrets.ACKNOWLEDG_SERVICE_NAME }}
+          WHEEL: ${{ steps.build.outputs.wheel }}
+          FINGERPRINT: ${{ steps.build.outputs.fingerprint }}
+        run: |
+          wheel_name=$(basename "${WHEEL}")
+          ssh acknowledg "mkdir -p $(printf '%q' "${SERVER_PATH}/incoming")"
+          rsync --checksum "${WHEEL}" "acknowledg:${SERVER_PATH}/incoming/"
+          ssh acknowledg "bash -s -- $(printf '%q ' \
+            "${SERVER_PATH}" "${SERVICE_NAME}" "${wheel_name}" \
+            "${FINGERPRINT}")" < deploy-remote.sh
diff --git a/dev_tools/services/acknowledg/PROTOCOL.md b/dev_tools/services/acknowledg/PROTOCOL.md
@@ -42,9 +42,10 @@ server's current `r`).
 
 `<uuid>` is the shared review id. `<slug>` selects one section of that kind.
 
-When `EDG_ACKNOWLEDG_API_KEY` is set, `/source`, `/edit`, and `/batch` require
+When the server has an API key configured (a systemd `api-key` credential, or
+else `EDG_ACKNOWLEDG_API_KEY`), `/source`, `/edit`, and `/batch` require
 header `X-AcknowlEDG-API-Key`. The server accepts the WebSocket, then closes
-with code **3000** if the key is missing or wrong. When the env var is unset,
+with code **3000** if the key is missing or wrong. When no key is configured,
 authentication is skipped (local development).
 
 ## Roles and occupancy

diff --git a/dev_tools/services/acknowledg/README.md b/dev_tools/services/acknowledg/README.md
@@ -23,7 +23,8 @@ when the CLI starts, so reconnects keep the original time) shown in the
 viewer's local timezone.
 
 Write-capable `/source`, `/edit`, and `/batch` routes can be protected with a
-shared API key via the `EDG_ACKNOWLEDG_API_KEY` environment variable.
+shared API key via the `EDG_ACKNOWLEDG_API_KEY` environment variable or, for
+deployed servers, a systemd credential (see [Server](#server)).
 `edg-acknowledg-cli` multiplexes many sections over `/batch` from a JSON
 specification; `edgy-review` and `edg-bench-review` remain available as
 task-focused CLIs that use the direct `/source` and `/edit` endpoints.
@@ -88,4 +89,106 @@ a reverse proxy.  For Apache, this looks something like the following (with
 </VirtualHost>
 ```
 
+Hypercorn itself is best run as a systemd service. A unit along these lines
+(saved as e.g. `/etc/systemd/system/acknowledg-api.service`, with `<USER>` and
+`<SERVER PATH>` replaced, where `<SERVER PATH>/venv` has the server wheel
+installed) keeps it running and restarts it on failure:
+
+```
+[Unit]
+Description=AcknowlEDG API Server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=exec
+User=<USER>
+WorkingDirectory=<SERVER PATH>/wdir
+# Optional API key protecting the write routes (see below).
+LoadCredentialEncrypted=api-key:/etc/credstore.encrypted/acknowledg-api.api-key
+ExecStart=<SERVER PATH>/venv/bin/hypercorn --bind 127.0.0.1:2727 acknowledg_server:app
+Restart=on-failure
+RestartSec=5
+
+# Review state is held in memory, so the server needs no writable paths.
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=read-only
+
+[Install]
+WantedBy=multi-user.target
+```
+
+The API key is optional: without one, the write routes accept any client, which
+is only appropriate when the API host is reachable solely by trusted users.
+When a key is wanted, supply it as a [systemd credential][credentials] rather
+than an environment variable: systemd decrypts it at service start into a
+private directory readable only by the service, and the server reads it from
+`$CREDENTIALS_DIRECTORY/api-key` (falling back to `EDG_ACKNOWLEDG_API_KEY` when
+no credential is present). On disk the key is encrypted with the host's key
+(and TPM2, when available), so the file is useless if copied elsewhere, and
+neither the deploy user nor the service user can read it. Create it once as
+root (requires systemd 250 or later):
+
+```
+mkdir -p /etc/credstore.encrypted
+systemd-ask-password -n 'AcknowlEDG API key:' \
+  | systemd-creds encrypt --name=api-key - \
+      /etc/credstore.encrypted/acknowledg-api.api-key
+```
+
+As written, `LoadCredentialEncrypted=` makes systemd refuse to start the
+service if the credential file is missing, so a host meant to require a key
+fails loudly rather than quietly running without authentication. To run without
+a key, omit the `LoadCredentialEncrypted=` line. To rotate the key, re-run the
+command above and restart the service. On hosts with systemd 247–249, use
+`LoadCredential=` with a plain file instead (`root:root`, mode `0600`), which
+keeps the same isolation from the service and deploy users but stores the key
+unencrypted.
+
+[credentials]: https://systemd.io/CREDENTIALS/
+
+Enable the service with `systemctl daemon-reload` and `systemctl enable --now
+acknowledg-api`. Review sessions live only in the server's memory, so
+restarting the service drops any open reviews.
+
 Locally, `server/run-server.sh` starts Hypercorn on `127.0.0.1:2727`.
+
+### Automated Deployment
+
+`.github/workflows/acknowledg-deploy.yml` builds and deploys both halves over
+SSH on pushes to `main` that touch AcknowlEDG. The client is rsynced into the
+web root (no restart needed). The server wheel is uploaded and handed to
+`server/deploy-remote.sh`, which installs it and restarts the systemd unit
+**only** when the wheel's contents differ from the last successful deploy
+(tracked in `<server path>/deployed.fingerprint`). The last few deployed
+wheels are kept under `<server path>/wheels/` for manual rollback.
+
+The workflow reads these secrets from the `acknowledg` GitHub environment:
+
+| Secret | Purpose |
+| --- | --- |
+| `ACKNOWLEDG_SSH_HOST` | Host name or IP address |
+| `ACKNOWLEDG_SSH_USER` | Deploy user |
+| `ACKNOWLEDG_SSH_PORT` | SSH port (optional; defaults to 22) |
+| `ACKNOWLEDG_SSH_PRIVATE_KEY` | Private key for the deploy user |
+| `ACKNOWLEDG_SSH_KNOWN_HOSTS` | `ssh-keyscan` output for the host |
+| `ACKNOWLEDG_CLIENT_PATH` | Web root for the client files |
+| `ACKNOWLEDG_SERVER_PATH` | Server directory containing `venv/` |
+| `ACKNOWLEDG_SERVICE_NAME` | systemd unit running the server (e.g. `acknowledg-api`) |
+| `ACKNOWLEDG_API_HOST` | API URL baked into the client (`VITE_EDG_ACKNOWLEDG_API_HOST`), including the scheme, e.g. `wss://acknowledg-api.example.com` |
+
+The host must already be set up; the workflow does not manage the unit file:
+
+- The deploy user's `authorized_keys` holds the public key, `rsync` is
+  installed, and the user owns the client and server paths.
+- `<server path>/venv` is a Python 3.11+ venv, and the unit's `ExecStart`
+  runs Hypercorn from it (see the example unit under [Server](#server)).
+- The deploy user may restart the unit without a password, e.g. in
+  `/etc/sudoers.d/acknowledg`. sudo matches arguments exactly, so the unit
+  name here must be spelled as in `ACKNOWLEDG_SERVICE_NAME`:
+
+  ```
+  <user> ALL=(root) NOPASSWD: /usr/bin/systemctl restart acknowledg-api
+  ```
diff --git a/dev_tools/services/acknowledg/server/acknowledg_server/__init__.py b/dev_tools/services/acknowledg/server/acknowledg_server/__init__.py
@@ -4,10 +4,11 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 import asyncio
+import hmac
 import os
 import uuid
 
-from functools import wraps
+from functools import lru_cache, wraps
 from http import HTTPStatus
 from typing import (
   Any, AsyncGenerator, Awaitable, Callable, Dict, Optional, TypeVar
@@ -145,7 +146,23 @@ class NoTestEditSocket(Exception):
 class NoBenchEditSocket(Exception):
   pass
 
+API_KEY_CREDENTIAL = 'api-key'
+
+@lru_cache(maxsize = None)
 def _configured_api_key() -> Optional[str]:
+  '''Return the API key protecting write-capable routes, if any.
+
+  A systemd credential named ``api-key`` (``LoadCredential=`` or
+  ``LoadCredentialEncrypted=``) takes precedence, so deployments can keep the
+  key out of the process environment. Otherwise ``EDG_ACKNOWLEDG_API_KEY`` is
+  used. The key is read once and cached.
+  '''
+  credentials_dir = os.environ.get('CREDENTIALS_DIRECTORY')
+  if credentials_dir is not None:
+    path = os.path.join(credentials_dir, API_KEY_CREDENTIAL)
+    if os.path.isfile(path):
+      with open(path, encoding = 'utf-8') as credential:
+        return credential.read().rstrip('\r\n')
   return os.environ.get('EDG_ACKNOWLEDG_API_KEY')
 
 def _extract_request_api_key() -> Optional[str]:
@@ -154,9 +171,9 @@ def _extract_request_api_key() -> Optional[str]:
 def require_api_key(func: F) -> F:
   '''Authenticate write-capable websocket routes with a shared API key.
 
-  When ``EDG_ACKNOWLEDG_API_KEY`` is unset, authentication is skipped (local
-  development). Otherwise the key must be supplied via the
-  ``X-AcknowlEDG-API-Key`` header.
+  When no key is configured (see ``_configured_api_key``), authentication is
+  skipped (local development). Otherwise the key must be supplied via the
+  ``X-AcknowlEDG-API-Key`` header; it is compared in constant time.
 
   The websocket is accepted before authentication so it can be closed with an
   application close code on failure.
@@ -167,7 +184,8 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any:
     expected = _configured_api_key()
     if expected is not None:
       provided = _extract_request_api_key()
-      if provided != expected:
+      if provided is None or not hmac.compare_digest(
+          provided.encode('utf-8'), expected.encode('utf-8')):
         await websocket.close(WS_CLOSE_UNAUTHORIZED)
         abort(HTTPStatus.UNAUTHORIZED)
     return await func(*args, **kwargs)