test(storage): replay the capture over TLS in the parity oracle

.changeset/tls-parity-leg.md

@@ -0,0 +1,11 @@
+---
+---
+
+test(storage): replay the capture over TLS in the parity oracle
+
+`scripts/storage-parity.sh` now starts a second Postgres with `ssl=on` and a self-signed
+pair generated per run, and asserts the same 118-op replay over `sslmode=require` matches
+both the TypeScript capture and the plaintext run. `pg_stat_ssl` confirms the session the
+server sees, so a silently plaintext connection cannot pass, and the step fails if the
+test is skipped rather than run. With `tls-none` restored the leg fails to connect instead
+of falling back.

crates/nostream-storage/tests/parity.rs

@@ -1012,27 +1012,27 @@ async fn a_tls_connection_is_attempted_rather_than_refused_by_this_build() {
     }
 }
 
-#[tokio::test]
-#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"]
-async fn the_typescript_script_replays_with_an_empty_diff() {
+/// Run the capture's script against a fresh database created through `admin_url`, and return
+/// each op's result in order. The one helper both the plaintext and the TLS leg call, so the
+/// two runs differ only in how the connection is made.
+async fn replay(admin_url: &str, database: &str) -> Vec<Value> {
     let fixture = fixture();
-    let admin_url = admin_url();
-    let admin = PgPool::connect(&admin_url)
+    let admin = PgPool::connect(admin_url)
         .await
         .expect("the administrative database is reachable");
     admin
         .execute(sqlx::AssertSqlSafe(format!(
-            "DROP DATABASE IF EXISTS {DATABASE} WITH (FORCE)"
+            "DROP DATABASE IF EXISTS {database} WITH (FORCE)"
         )))
         .await
         .expect("the parity database drops");
     admin
-        .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {DATABASE}")))
+        .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {database}")))
         .await
         .expect("the parity database is created");
     drop(admin);
 
-    let backend = PostgresBackend::connect(&with_database(&admin_url, DATABASE))
+    let backend = PostgresBackend::connect(&with_database(admin_url, database))
         .await
         .expect("the parity database is reachable")
         .with_settings(Arc::new(common::settings_from_fixture(&fixture)));
@@ -1049,6 +1049,7 @@ async fn the_typescript_script_replays_with_an_empty_diff() {
         .expect("the corpus inserts");
 
     let mut refs = HashMap::new();
+    let mut results = Vec::new();
     for (index, op) in fixture["script"]
         .as_array()
         .expect("script")
@@ -1063,10 +1064,92 @@ async fn the_typescript_script_replays_with_an_empty_diff() {
                 .await
                 .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}"))
         };
-        let expected = &op["result"]["compare"];
+        results.push(actual);
+    }
+    results
+}
+
+/// Compare each op's result with the TypeScript capture's.
+fn assert_matches_capture(actual: &[Value]) {
+    let fixture = fixture();
+    let script = fixture["script"].as_array().expect("script");
+    assert_eq!(actual.len(), script.len(), "the replay ran every op");
+    for (index, (op, got)) in script.iter().zip(actual).enumerate() {
+        let name = op["op"].as_str().expect("an op name");
         assert_eq!(
-            &actual, expected,
+            got, &op["result"]["compare"],
             "op {index} ({name}) differs from the TypeScript capture"
         );
     }
 }
+
+#[tokio::test]
+#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"]
+async fn the_typescript_script_replays_with_an_empty_diff() {
+    let actual = replay(&admin_url(), DATABASE).await;
+    assert_matches_capture(&actual);
+}
+
+/// The same script over TLS and over plaintext, compared op by op (#911).
+///
+/// `sslmode=require` cannot fall back: a build without the TLS feature fails to connect
+/// here rather than downgrading, which is the regression the leg exists to catch. The
+/// connection-string option is not taken on trust either: `pg_stat_ssl` reports what the
+/// server sees for this session, so a silently plaintext connection fails the leg too.
+#[tokio::test]
+#[ignore = "needs PARITY_TLS_ADMIN_URL; scripts/storage-parity.sh sets it"]
+async fn the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run() {
+    // CI runs every ignored test with only the plaintext URL, so the leg skips there.
+    // scripts/storage-parity.sh sets PARITY_TLS_REQUIRED, under which a skip is a failure:
+    // the oracle must not report a green run for a leg that never executed.
+    let tls_admin_url = match std::env::var("PARITY_TLS_ADMIN_URL") {
+        Ok(url) => url,
+        Err(_) if std::env::var_os("PARITY_TLS_REQUIRED").is_some() => {
+            panic!("PARITY_TLS_REQUIRED is set but PARITY_TLS_ADMIN_URL is not")
+        }
+        Err(_) => {
+            eprintln!(
+                "PARITY_TLS_ADMIN_URL is unset: the TLS leg runs from scripts/storage-parity.sh"
+            );
+            return;
+        }
+    };
+
+    let admin = PgPool::connect(&tls_admin_url)
+        .await
+        .expect("the TLS administrative database is reachable (is ssl=on?)");
+    let encrypted: bool =
+        sqlx::query_scalar("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()")
+            .fetch_one(&admin)
+            .await
+            .expect("pg_stat_ssl answers");
+    assert!(
+        encrypted,
+        "the server reports a plaintext session for a sslmode=require URL"
+    );
+    drop(admin);
+
+    let over_tls = replay(&tls_admin_url, "nostream_parity_tls").await;
+    assert_matches_capture(&over_tls);
+
+    let over_plain = replay(&admin_url(), "nostream_parity_plain").await;
+    if over_plain != over_tls {
+        let index = over_plain
+            .iter()
+            .zip(&over_tls)
+            .position(|(plain, tls)| plain != tls)
+            .unwrap_or(0);
+        let fixture = fixture();
+        let name = fixture["script"]
+            .get(index)
+            .and_then(|op| op["op"].as_str())
+            .unwrap_or("?");
+        panic!(
+            "op {index} ({name}) differs between plaintext and TLS: plaintext {:?}, TLS {:?}",
+            over_plain.get(index),
+            over_tls.get(index)
+        );
+    }
+    // Read by scripts/storage-parity.sh with --nocapture as proof the assertions ran.
+    println!("TLS_PARITY_COMPLETE");
+}

scripts/storage-parity.sh

@@ -11,6 +11,7 @@
 #                         host) set the address that machine reaches, e.g. the tailnet one.
 #   PARITY_HOST           host the harness dials. Default $PARITY_BIND.
 #   PARITY_PG_PORT        published Postgres port (default 18776)
+#   PARITY_PG_TLS_PORT    published port of the TLS Postgres server (default 18778)
 #   PARITY_REDIS_PORT     published Redis port (default 18777)
 #   PARITY_PG_PASSWORD    Postgres password; generated per run (openssl rand -hex 16) if unset
 #   PARITY_REDIS_PASSWORD Redis password; generated per run if unset
@@ -35,6 +36,7 @@ cd "$root"
 PARITY_BIND=${PARITY_BIND:-127.0.0.1}
 PARITY_HOST=${PARITY_HOST:-$PARITY_BIND}
 PARITY_PG_PORT=${PARITY_PG_PORT:-18776}
+PARITY_PG_TLS_PORT=${PARITY_PG_TLS_PORT:-18778}
 PARITY_REDIS_PORT=${PARITY_REDIS_PORT:-18777}
 PARITY_PG_PASSWORD=${PARITY_PG_PASSWORD:-$(openssl rand -hex 16)}
 PARITY_REDIS_PASSWORD=${PARITY_REDIS_PASSWORD:-$(openssl rand -hex 16)}
@@ -48,10 +50,31 @@ if [ -z "${CUCUMBER_BIN:-}" ]; then
 fi
 
 pg_container="nostream-parity-pg-$$"
+tls_container="nostream-parity-pg-tls-$$"
 redis_container="nostream-parity-redis-$$"
 
+# The TLS server keeps its certificate pair and its PGDATA inside the worktree. Ports,
+# labels and the per-run credential match the plaintext server; only the bind sources are
+# new, and they need the path the daemon sees (the sandbox mounts this tree at /workspace,
+# the daemon host at /home/cameri/Workspace). NOSTREAM_HOST_DIR overrides both, as it does
+# for scripts/rust.sh.
+# The directory is per invocation: two runs of this script (different ports, or the same)
+# must not delete each other's live PGDATA, so the suffix carries the pid and a random
+# token, and the exit trap removes it.
+tls_dir="$root/.test-reports/parity-tls-$$-$(openssl rand -hex 4)"
+if [ -n "${NOSTREAM_HOST_DIR:-}" ]; then
+    host_root=$NOSTREAM_HOST_DIR
+else
+    case "$root" in
+        /workspace/*) host_root="/home/cameri/Workspace/${root#/workspace/}" ;;
+        *) host_root=$root ;;
+    esac
+fi
+tls_dir_host="$host_root${tls_dir#"$root"}"
+
 cleanup() {
-    docker rm -f "$pg_container" "$redis_container" >/dev/null 2>&1 || true
+    docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true
+    rm -rf "$tls_dir"
 }
 trap cleanup EXIT
 trap 'exit 130' INT HUP TERM
@@ -171,6 +194,9 @@ fi
 pg_password_url=$(url_escape "$PARITY_PG_PASSWORD")
 redis_password_url=$(url_escape "$PARITY_REDIS_PASSWORD")
 admin_url="postgres://postgres:$pg_password_url@$host_url:$PARITY_PG_PORT/postgres"
+# sslmode=require: the leg is about a server that insists on TLS, and `prefer` would quietly
+# pass over plaintext, which is the failure #911 is about.
+tls_admin_url="postgres://postgres:$pg_password_url@$host_url:$PARITY_PG_TLS_PORT/postgres?sslmode=require"
 docker version >/dev/null 2>&1 \
     || fail "the Docker daemon at ${DOCKER_HOST:-/var/run/docker.sock} is not reachable"
 
@@ -191,7 +217,7 @@ printf '%s\n' "$redaction_output" | grep -q 'test result: ok. 1 passed' || {
 # variables instead of surfacing as a sqlx timeout later.
 probe_dial() {
     # The probe wants the address itself; URLs keep the bracketed IPv6 spelling.
-    node -e 'const net = require("net"); const s = net.connect({ host: process.argv[1], port: Number(process.argv[2]) }); s.setTimeout(3000); s.on("connect", () => { s.destroy(); process.exit(0) }); s.on("timeout", () => { s.destroy(); process.exit(1) }); s.on("error", () => process.exit(1))' "$host_plain" "$PARITY_PG_PORT"
+    node -e 'const net = require("net"); const s = net.connect({ host: process.argv[1], port: Number(process.argv[2]) }); s.setTimeout(3000); s.on("connect", () => { s.destroy(); process.exit(0) }); s.on("timeout", () => { s.destroy(); process.exit(1) }); s.on("error", () => process.exit(1))' "$host_plain" "$1"
 }
 
 wait_for_log() {
@@ -200,29 +226,54 @@ wait_for_log() {
     attempts=0
     until docker logs "$container" 2>&1 | grep -q "$pattern"; do
         attempts=$((attempts + 1))
-        [ "$attempts" -ge 90 ] && fail "$container did not report: $pattern"
+        if [ "$attempts" -ge 90 ]; then
+            docker logs "$container" 2>&1 | tail -20
+            fail "$container did not report: $pattern"
+        fi
         sleep 1
     done
 }
 
 echo "== containers (bound to $PARITY_BIND, per-run credentials)"
-docker rm -f "$pg_container" "$redis_container" >/dev/null 2>&1 || true
+
+# A fresh self-signed pair per run. The key stays 0600, and the server runs as the invoking
+# uid rather than the image's postgres user (uid 70): a 0600 file written here is owned by
+# this user and unreadable to uid 70, so the stock user cannot start with it. The issue's
+# alternative, chmod 644 on a copy, is not one PostgreSQL accepts either — it refuses a key
+# with group or world access and the server exits. Running as this uid with its own PGDATA
+# on a bind keeps the key strict and the image unchanged.
+mkdir -p "$tls_dir/certs" "$tls_dir/pgdata"
+chmod 755 "$tls_dir" "$tls_dir/certs"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=localhost" \
+    -keyout "$tls_dir/certs/server.key" -out "$tls_dir/certs/server.crt" >/dev/null 2>&1 \
+    || fail "the TLS certificate pair could not be generated (openssl is required)"
+chmod 600 "$tls_dir/certs/server.key"
+chmod 644 "$tls_dir/certs/server.crt"
+
+docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true
 docker create --name "$pg_container" --label org.testcontainers=true \
     -e POSTGRES_PASSWORD="$PARITY_PG_PASSWORD" -e POSTGRES_DB=postgres \
     -p "$PARITY_BIND:$PARITY_PG_PORT:5432" postgres:16-alpine >/dev/null
+docker create --name "$tls_container" --label org.testcontainers=true \
+    --user "$(id -u):$(id -g)" \
+    -e POSTGRES_PASSWORD="$PARITY_PG_PASSWORD" -e POSTGRES_DB=postgres -e PGDATA=/pgdata \
+    -v "$tls_dir_host/certs:/certs:ro" -v "$tls_dir_host/pgdata:/pgdata" \
+    -p "$PARITY_BIND:$PARITY_PG_TLS_PORT:5432" postgres:16-alpine \
+    -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key >/dev/null
 docker create --name "$redis_container" --label org.testcontainers=true \
     -p "$PARITY_BIND:$PARITY_REDIS_PORT:6379" redis:7-alpine \
     redis-server --requirepass "$PARITY_REDIS_PASSWORD" >/dev/null
-docker start "$pg_container" "$redis_container" >/dev/null
+docker start "$pg_container" "$tls_container" "$redis_container" >/dev/null
 wait_for_log "$pg_container" "database system is ready to accept connections"
+wait_for_log "$tls_container" "database system is ready to accept connections"
 wait_for_log "$redis_container" "Ready to accept connections"
 attempts=0
-until probe_dial; do
+until probe_dial "$PARITY_PG_PORT" && probe_dial "$PARITY_PG_TLS_PORT"; do
     attempts=$((attempts + 1))
-    [ "$attempts" -ge 5 ] && fail "the published Postgres port is not reachable at $PARITY_HOST:$PARITY_PG_PORT: set PARITY_BIND and PARITY_HOST to an address this machine can dial (127.0.0.1 on the daemon host; the daemon host's reachable address from elsewhere)"
+    [ "$attempts" -ge 5 ] && fail "a published Postgres port is not reachable at $PARITY_HOST ($PARITY_PG_PORT, $PARITY_PG_TLS_PORT): set PARITY_BIND and PARITY_HOST to an address this machine can dial (127.0.0.1 on the daemon host; the daemon host's reachable address from elsewhere)"
     sleep 1
 done
-echo "   postgres and redis are accepting connections on $PARITY_HOST"
+echo "   postgres (plaintext and TLS) and redis are accepting connections on $PARITY_HOST"
 
 echo "== migrations and schema parity"
 ./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" \
@@ -234,6 +285,23 @@ echo "== behavioural parity"
     cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \
     || fail "the parity target failed"
 
+echo "== tls parity (the same replay over sslmode=require)"
+# The test skips itself when the variable is absent, so the run has to prove it executed:
+# a renamed or quietly skipped TLS leg is the failure mode this check closes.
+# PARITY_TLS_REQUIRED makes the test panic instead of skipping when it has no TLS URL, and
+# the completion marker is printed only after the assertions, so a skip cannot pass.
+tls_output=$(./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" PARITY_TLS_ADMIN_URL="$tls_admin_url" PARITY_TLS_REQUIRED=1 \
+    cargo test -p nostream-storage --test parity the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run \
+    -- --ignored --exact --test-threads=1 --nocapture 2>&1) || {
+    printf '%s\n' "$tls_output"
+    fail "the TLS leg failed: the replay over sslmode=require did not match the plaintext run"
+}
+printf '%s\n' "$tls_output" | grep -q 'TLS_PARITY_COMPLETE' || {
+    printf '%s\n' "$tls_output"
+    fail "the TLS parity test did not complete (renamed, skipped, or it returned early?)"
+}
+echo "   the replay over sslmode=require matches the plaintext run ($(printf '%s\n' "$tls_output" | grep -o 'finished in [0-9.]*s'))"
+
 echo "== cucumber on the sqlx-migrated schema"
 [ -x "$CUCUMBER_BIN" ] || fail "cucumber-js not found at $CUCUMBER_BIN"
 mkdir -p .test-reports/integration
@@ -245,4 +313,4 @@ NODE_OPTIONS=--no-experimental-strip-types \
     "$CUCUMBER_BIN" --format summary \
     || fail "the cucumber suite failed"
 
-echo "== storage-parity: all three steps passed"
+echo "== storage-parity: all four steps passed"