From 0bb9e725fa67b9401b5212b063840e58700d4b35 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 19 Oct 2026 22:43:43 -0400 Subject: [PATCH 1/2] test(storage): replay the capture over TLS in the parity oracle The oracle started Postgres without TLS, so nothing exercised a server that insists on it. It now starts a second server with ssl=on and a per-run self-signed pair, and the same replay runs over sslmode=require, compared op by op against the TypeScript capture and against the plaintext run; pg_stat_ssl confirms the session the server sees. The postgres image runs as uid 70 and PostgreSQL refuses a 0600 key it cannot read, so that server runs as the invoking uid with its own PGDATA on a bind, keeping the key 0600. With tls-none restored the leg fails to connect instead of falling back. Fixes #911. --- .changeset/tls-parity-leg.md | 11 +++ crates/nostream-storage/tests/parity.rs | 90 ++++++++++++++++++++++--- scripts/storage-parity.sh | 81 +++++++++++++++++++--- 3 files changed, 163 insertions(+), 19 deletions(-) create mode 100644 .changeset/tls-parity-leg.md diff --git a/.changeset/tls-parity-leg.md b/.changeset/tls-parity-leg.md new file mode 100644 index 00000000..a0b63693 --- /dev/null +++ b/.changeset/tls-parity-leg.md @@ -0,0 +1,11 @@ +--- +--- + +test(storage): replay the capture over TLS in the parity oracle + +`scripts/storage-parity.sh` now starts a second Postgres with `ssl=on` and a self-signed +pair generated per run, and asserts the same 118-op replay over `sslmode=require` matches +both the TypeScript capture and the plaintext run. `pg_stat_ssl` confirms the session the +server sees, so a silently plaintext connection cannot pass, and the step fails if the +test is skipped rather than run. With `tls-none` restored the leg fails to connect instead +of falling back. diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index f63bae17..11eea332 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -1012,27 +1012,27 @@ async fn a_tls_connection_is_attempted_rather_than_refused_by_this_build() { } } -#[tokio::test] -#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"] -async fn the_typescript_script_replays_with_an_empty_diff() { +/// Run the capture's script against a fresh database created through `admin_url`, and return +/// each op's result in order. The one helper both the plaintext and the TLS leg call, so the +/// two runs differ only in how the connection is made. +async fn replay(admin_url: &str, database: &str) -> Vec { let fixture = fixture(); - let admin_url = admin_url(); - let admin = PgPool::connect(&admin_url) + let admin = PgPool::connect(admin_url) .await .expect("the administrative database is reachable"); admin .execute(sqlx::AssertSqlSafe(format!( - "DROP DATABASE IF EXISTS {DATABASE} WITH (FORCE)" + "DROP DATABASE IF EXISTS {database} WITH (FORCE)" ))) .await .expect("the parity database drops"); admin - .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {DATABASE}"))) + .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {database}"))) .await .expect("the parity database is created"); drop(admin); - let backend = PostgresBackend::connect(&with_database(&admin_url, DATABASE)) + let backend = PostgresBackend::connect(&with_database(admin_url, database)) .await .expect("the parity database is reachable") .with_settings(Arc::new(common::settings_from_fixture(&fixture))); @@ -1049,6 +1049,7 @@ async fn the_typescript_script_replays_with_an_empty_diff() { .expect("the corpus inserts"); let mut refs = HashMap::new(); + let mut results = Vec::new(); for (index, op) in fixture["script"] .as_array() .expect("script") @@ -1063,10 +1064,79 @@ async fn the_typescript_script_replays_with_an_empty_diff() { .await .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}")) }; - let expected = &op["result"]["compare"]; + results.push(actual); + } + results +} + +/// Compare each op's result with the TypeScript capture's. +fn assert_matches_capture(actual: &[Value]) { + let fixture = fixture(); + let script = fixture["script"].as_array().expect("script"); + assert_eq!(actual.len(), script.len(), "the replay ran every op"); + for (index, (op, got)) in script.iter().zip(actual).enumerate() { + let name = op["op"].as_str().expect("an op name"); assert_eq!( - &actual, expected, + got, &op["result"]["compare"], "op {index} ({name}) differs from the TypeScript capture" ); } } + +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"] +async fn the_typescript_script_replays_with_an_empty_diff() { + let actual = replay(&admin_url(), DATABASE).await; + assert_matches_capture(&actual); +} + +/// The same script over TLS and over plaintext, compared op by op (#911). +/// +/// `sslmode=require` cannot fall back: a build without the TLS feature fails to connect +/// here rather than downgrading, which is the regression the leg exists to catch. The +/// connection-string option is not taken on trust either: `pg_stat_ssl` reports what the +/// server sees for this session, so a silently plaintext connection fails the leg too. +#[tokio::test] +#[ignore = "needs PARITY_TLS_ADMIN_URL; scripts/storage-parity.sh sets it"] +async fn the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run() { + let Ok(tls_admin_url) = std::env::var("PARITY_TLS_ADMIN_URL") else { + eprintln!("PARITY_TLS_ADMIN_URL is unset: the TLS leg runs from scripts/storage-parity.sh"); + return; + }; + + let admin = PgPool::connect(&tls_admin_url) + .await + .expect("the TLS administrative database is reachable (is ssl=on?)"); + let encrypted: bool = + sqlx::query_scalar("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()") + .fetch_one(&admin) + .await + .expect("pg_stat_ssl answers"); + assert!( + encrypted, + "the server reports a plaintext session for a sslmode=require URL" + ); + drop(admin); + + let over_tls = replay(&tls_admin_url, "nostream_parity_tls").await; + assert_matches_capture(&over_tls); + + let over_plain = replay(&admin_url(), "nostream_parity_plain").await; + if over_plain != over_tls { + let index = over_plain + .iter() + .zip(&over_tls) + .position(|(plain, tls)| plain != tls) + .unwrap_or(0); + let fixture = fixture(); + let name = fixture["script"] + .get(index) + .and_then(|op| op["op"].as_str()) + .unwrap_or("?"); + panic!( + "op {index} ({name}) differs between plaintext and TLS: plaintext {:?}, TLS {:?}", + over_plain.get(index), + over_tls.get(index) + ); + } +} diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index 9d7f128d..bd0736e5 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -11,6 +11,7 @@ # host) set the address that machine reaches, e.g. the tailnet one. # PARITY_HOST host the harness dials. Default $PARITY_BIND. # PARITY_PG_PORT published Postgres port (default 18776) +# PARITY_PG_TLS_PORT published port of the TLS Postgres server (default 18778) # PARITY_REDIS_PORT published Redis port (default 18777) # PARITY_PG_PASSWORD Postgres password; generated per run (openssl rand -hex 16) if unset # PARITY_REDIS_PASSWORD Redis password; generated per run if unset @@ -35,6 +36,7 @@ cd "$root" PARITY_BIND=${PARITY_BIND:-127.0.0.1} PARITY_HOST=${PARITY_HOST:-$PARITY_BIND} PARITY_PG_PORT=${PARITY_PG_PORT:-18776} +PARITY_PG_TLS_PORT=${PARITY_PG_TLS_PORT:-18778} PARITY_REDIS_PORT=${PARITY_REDIS_PORT:-18777} PARITY_PG_PASSWORD=${PARITY_PG_PASSWORD:-$(openssl rand -hex 16)} PARITY_REDIS_PASSWORD=${PARITY_REDIS_PASSWORD:-$(openssl rand -hex 16)} @@ -48,10 +50,27 @@ if [ -z "${CUCUMBER_BIN:-}" ]; then fi pg_container="nostream-parity-pg-$$" +tls_container="nostream-parity-pg-tls-$$" redis_container="nostream-parity-redis-$$" +# The TLS server keeps its certificate pair and its PGDATA inside the worktree. Ports, +# labels and the per-run credential match the plaintext server; only the bind sources are +# new, and they need the path the daemon sees (the sandbox mounts this tree at /workspace, +# the daemon host at /home/cameri/Workspace). NOSTREAM_HOST_DIR overrides both, as it does +# for scripts/rust.sh. +tls_dir="$root/.test-reports/parity-tls" +if [ -n "${NOSTREAM_HOST_DIR:-}" ]; then + host_root=$NOSTREAM_HOST_DIR +else + case "$root" in + /workspace/*) host_root="/home/cameri/Workspace/${root#/workspace/}" ;; + *) host_root=$root ;; + esac +fi +tls_dir_host="$host_root/.test-reports/parity-tls" + cleanup() { - docker rm -f "$pg_container" "$redis_container" >/dev/null 2>&1 || true + docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true } trap cleanup EXIT trap 'exit 130' INT HUP TERM @@ -171,6 +190,9 @@ fi pg_password_url=$(url_escape "$PARITY_PG_PASSWORD") redis_password_url=$(url_escape "$PARITY_REDIS_PASSWORD") admin_url="postgres://postgres:$pg_password_url@$host_url:$PARITY_PG_PORT/postgres" +# sslmode=require: the leg is about a server that insists on TLS, and `prefer` would quietly +# pass over plaintext, which is the failure #911 is about. +tls_admin_url="postgres://postgres:$pg_password_url@$host_url:$PARITY_PG_TLS_PORT/postgres?sslmode=require" docker version >/dev/null 2>&1 \ || fail "the Docker daemon at ${DOCKER_HOST:-/var/run/docker.sock} is not reachable" @@ -191,7 +213,7 @@ printf '%s\n' "$redaction_output" | grep -q 'test result: ok. 1 passed' || { # variables instead of surfacing as a sqlx timeout later. probe_dial() { # The probe wants the address itself; URLs keep the bracketed IPv6 spelling. - node -e 'const net = require("net"); const s = net.connect({ host: process.argv[1], port: Number(process.argv[2]) }); s.setTimeout(3000); s.on("connect", () => { s.destroy(); process.exit(0) }); s.on("timeout", () => { s.destroy(); process.exit(1) }); s.on("error", () => process.exit(1))' "$host_plain" "$PARITY_PG_PORT" + node -e 'const net = require("net"); const s = net.connect({ host: process.argv[1], port: Number(process.argv[2]) }); s.setTimeout(3000); s.on("connect", () => { s.destroy(); process.exit(0) }); s.on("timeout", () => { s.destroy(); process.exit(1) }); s.on("error", () => process.exit(1))' "$host_plain" "$1" } wait_for_log() { @@ -200,29 +222,55 @@ wait_for_log() { attempts=0 until docker logs "$container" 2>&1 | grep -q "$pattern"; do attempts=$((attempts + 1)) - [ "$attempts" -ge 90 ] && fail "$container did not report: $pattern" + if [ "$attempts" -ge 90 ]; then + docker logs "$container" 2>&1 | tail -20 + fail "$container did not report: $pattern" + fi sleep 1 done } echo "== containers (bound to $PARITY_BIND, per-run credentials)" -docker rm -f "$pg_container" "$redis_container" >/dev/null 2>&1 || true + +# A fresh self-signed pair per run. The key stays 0600, and the server runs as the invoking +# uid rather than the image's postgres user (uid 70): a 0600 file written here is owned by +# this user and unreadable to uid 70, so the stock user cannot start with it. The issue's +# alternative, chmod 644 on a copy, is not one PostgreSQL accepts either — it refuses a key +# with group or world access and the server exits. Running as this uid with its own PGDATA +# on a bind keeps the key strict and the image unchanged. +rm -rf "$tls_dir" +mkdir -p "$tls_dir/certs" "$tls_dir/pgdata" +chmod 755 "$tls_dir" "$tls_dir/certs" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=localhost" \ + -keyout "$tls_dir/certs/server.key" -out "$tls_dir/certs/server.crt" >/dev/null 2>&1 \ + || fail "the TLS certificate pair could not be generated (openssl is required)" +chmod 600 "$tls_dir/certs/server.key" +chmod 644 "$tls_dir/certs/server.crt" + +docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true docker create --name "$pg_container" --label org.testcontainers=true \ -e POSTGRES_PASSWORD="$PARITY_PG_PASSWORD" -e POSTGRES_DB=postgres \ -p "$PARITY_BIND:$PARITY_PG_PORT:5432" postgres:16-alpine >/dev/null +docker create --name "$tls_container" --label org.testcontainers=true \ + --user "$(id -u):$(id -g)" \ + -e POSTGRES_PASSWORD="$PARITY_PG_PASSWORD" -e POSTGRES_DB=postgres -e PGDATA=/pgdata \ + -v "$tls_dir_host/certs:/certs:ro" -v "$tls_dir_host/pgdata:/pgdata" \ + -p "$PARITY_BIND:$PARITY_PG_TLS_PORT:5432" postgres:16-alpine \ + -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key >/dev/null docker create --name "$redis_container" --label org.testcontainers=true \ -p "$PARITY_BIND:$PARITY_REDIS_PORT:6379" redis:7-alpine \ redis-server --requirepass "$PARITY_REDIS_PASSWORD" >/dev/null -docker start "$pg_container" "$redis_container" >/dev/null +docker start "$pg_container" "$tls_container" "$redis_container" >/dev/null wait_for_log "$pg_container" "database system is ready to accept connections" +wait_for_log "$tls_container" "database system is ready to accept connections" wait_for_log "$redis_container" "Ready to accept connections" attempts=0 -until probe_dial; do +until probe_dial "$PARITY_PG_PORT" && probe_dial "$PARITY_PG_TLS_PORT"; do attempts=$((attempts + 1)) - [ "$attempts" -ge 5 ] && fail "the published Postgres port is not reachable at $PARITY_HOST:$PARITY_PG_PORT: set PARITY_BIND and PARITY_HOST to an address this machine can dial (127.0.0.1 on the daemon host; the daemon host's reachable address from elsewhere)" + [ "$attempts" -ge 5 ] && fail "a published Postgres port is not reachable at $PARITY_HOST ($PARITY_PG_PORT, $PARITY_PG_TLS_PORT): set PARITY_BIND and PARITY_HOST to an address this machine can dial (127.0.0.1 on the daemon host; the daemon host's reachable address from elsewhere)" sleep 1 done -echo " postgres and redis are accepting connections on $PARITY_HOST" +echo " postgres (plaintext and TLS) and redis are accepting connections on $PARITY_HOST" echo "== migrations and schema parity" ./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" \ @@ -234,6 +282,21 @@ echo "== behavioural parity" cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \ || fail "the parity target failed" +echo "== tls parity (the same replay over sslmode=require)" +# The test skips itself when the variable is absent, so the run has to prove it executed: +# a renamed or quietly skipped TLS leg is the failure mode this check closes. +tls_output=$(./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" PARITY_TLS_ADMIN_URL="$tls_admin_url" \ + cargo test -p nostream-storage --test parity the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run \ + -- --ignored --exact --test-threads=1 2>&1) || { + printf '%s\n' "$tls_output" + fail "the TLS leg failed: the replay over sslmode=require did not match the plaintext run" +} +printf '%s\n' "$tls_output" | grep -q 'test result: ok. 1 passed' || { + printf '%s\n' "$tls_output" + fail "the TLS parity test did not run (renamed or skipped?)" +} +echo " the replay over sslmode=require matches the plaintext run ($(printf '%s\n' "$tls_output" | grep -o 'finished in [0-9.]*s'))" + echo "== cucumber on the sqlx-migrated schema" [ -x "$CUCUMBER_BIN" ] || fail "cucumber-js not found at $CUCUMBER_BIN" mkdir -p .test-reports/integration @@ -245,4 +308,4 @@ NODE_OPTIONS=--no-experimental-strip-types \ "$CUCUMBER_BIN" --format summary \ || fail "the cucumber suite failed" -echo "== storage-parity: all three steps passed" +echo "== storage-parity: all four steps passed" From 0ad608e6a26dfce9643a991dcb84c61da457bb61 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 19 Oct 2026 22:49:40 -0400 Subject: [PATCH 2/2] fix(tooling): isolate the oracle's TLS run and prove the leg ran Two runs of the oracle shared .test-reports/parity-tls, so a second run deleted the first one's live PGDATA. The directory now carries the pid and a random token and the exit trap removes it, so concurrent runs (on different ports) cannot touch each other. The TLS test returned early without a TLS URL, which also printed '1 passed', so the script's pass-count check could not tell a skip from a run. PARITY_TLS_REQUIRED makes the test panic instead of skipping, and a completion marker printed after the assertions is what the script checks. --- crates/nostream-storage/tests/parity.rs | 19 ++++++++++++++++--- scripts/storage-parity.sh | 19 ++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index 11eea332..4a1caae0 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -1099,9 +1099,20 @@ async fn the_typescript_script_replays_with_an_empty_diff() { #[tokio::test] #[ignore = "needs PARITY_TLS_ADMIN_URL; scripts/storage-parity.sh sets it"] async fn the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run() { - let Ok(tls_admin_url) = std::env::var("PARITY_TLS_ADMIN_URL") else { - eprintln!("PARITY_TLS_ADMIN_URL is unset: the TLS leg runs from scripts/storage-parity.sh"); - return; + // CI runs every ignored test with only the plaintext URL, so the leg skips there. + // scripts/storage-parity.sh sets PARITY_TLS_REQUIRED, under which a skip is a failure: + // the oracle must not report a green run for a leg that never executed. + let tls_admin_url = match std::env::var("PARITY_TLS_ADMIN_URL") { + Ok(url) => url, + Err(_) if std::env::var_os("PARITY_TLS_REQUIRED").is_some() => { + panic!("PARITY_TLS_REQUIRED is set but PARITY_TLS_ADMIN_URL is not") + } + Err(_) => { + eprintln!( + "PARITY_TLS_ADMIN_URL is unset: the TLS leg runs from scripts/storage-parity.sh" + ); + return; + } }; let admin = PgPool::connect(&tls_admin_url) @@ -1139,4 +1150,6 @@ async fn the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run() { over_tls.get(index) ); } + // Read by scripts/storage-parity.sh with --nocapture as proof the assertions ran. + println!("TLS_PARITY_COMPLETE"); } diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index bd0736e5..f8021cbb 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -58,7 +58,10 @@ redis_container="nostream-parity-redis-$$" # new, and they need the path the daemon sees (the sandbox mounts this tree at /workspace, # the daemon host at /home/cameri/Workspace). NOSTREAM_HOST_DIR overrides both, as it does # for scripts/rust.sh. -tls_dir="$root/.test-reports/parity-tls" +# The directory is per invocation: two runs of this script (different ports, or the same) +# must not delete each other's live PGDATA, so the suffix carries the pid and a random +# token, and the exit trap removes it. +tls_dir="$root/.test-reports/parity-tls-$$-$(openssl rand -hex 4)" if [ -n "${NOSTREAM_HOST_DIR:-}" ]; then host_root=$NOSTREAM_HOST_DIR else @@ -67,10 +70,11 @@ else *) host_root=$root ;; esac fi -tls_dir_host="$host_root/.test-reports/parity-tls" +tls_dir_host="$host_root${tls_dir#"$root"}" cleanup() { docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true + rm -rf "$tls_dir" } trap cleanup EXIT trap 'exit 130' INT HUP TERM @@ -238,7 +242,6 @@ echo "== containers (bound to $PARITY_BIND, per-run credentials)" # alternative, chmod 644 on a copy, is not one PostgreSQL accepts either — it refuses a key # with group or world access and the server exits. Running as this uid with its own PGDATA # on a bind keeps the key strict and the image unchanged. -rm -rf "$tls_dir" mkdir -p "$tls_dir/certs" "$tls_dir/pgdata" chmod 755 "$tls_dir" "$tls_dir/certs" openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=localhost" \ @@ -285,15 +288,17 @@ echo "== behavioural parity" echo "== tls parity (the same replay over sslmode=require)" # The test skips itself when the variable is absent, so the run has to prove it executed: # a renamed or quietly skipped TLS leg is the failure mode this check closes. -tls_output=$(./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" PARITY_TLS_ADMIN_URL="$tls_admin_url" \ +# PARITY_TLS_REQUIRED makes the test panic instead of skipping when it has no TLS URL, and +# the completion marker is printed only after the assertions, so a skip cannot pass. +tls_output=$(./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" PARITY_TLS_ADMIN_URL="$tls_admin_url" PARITY_TLS_REQUIRED=1 \ cargo test -p nostream-storage --test parity the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run \ - -- --ignored --exact --test-threads=1 2>&1) || { + -- --ignored --exact --test-threads=1 --nocapture 2>&1) || { printf '%s\n' "$tls_output" fail "the TLS leg failed: the replay over sslmode=require did not match the plaintext run" } -printf '%s\n' "$tls_output" | grep -q 'test result: ok. 1 passed' || { +printf '%s\n' "$tls_output" | grep -q 'TLS_PARITY_COMPLETE' || { printf '%s\n' "$tls_output" - fail "the TLS parity test did not run (renamed or skipped?)" + fail "the TLS parity test did not complete (renamed, skipped, or it returned early?)" } echo " the replay over sslmode=require matches the plaintext run ($(printf '%s\n' "$tls_output" | grep -o 'finished in [0-9.]*s'))"