diff --git a/.changeset/tls-parity-leg.md b/.changeset/tls-parity-leg.md new file mode 100644 index 00000000..a0b63693 --- /dev/null +++ b/.changeset/tls-parity-leg.md @@ -0,0 +1,11 @@ +--- +--- + +test(storage): replay the capture over TLS in the parity oracle + +`scripts/storage-parity.sh` now starts a second Postgres with `ssl=on` and a self-signed +pair generated per run, and asserts the same 118-op replay over `sslmode=require` matches +both the TypeScript capture and the plaintext run. `pg_stat_ssl` confirms the session the +server sees, so a silently plaintext connection cannot pass, and the step fails if the +test is skipped rather than run. With `tls-none` restored the leg fails to connect instead +of falling back. diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index f63bae17..4a1caae0 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -1012,27 +1012,27 @@ async fn a_tls_connection_is_attempted_rather_than_refused_by_this_build() { } } -#[tokio::test] -#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"] -async fn the_typescript_script_replays_with_an_empty_diff() { +/// Run the capture's script against a fresh database created through `admin_url`, and return +/// each op's result in order. The one helper both the plaintext and the TLS leg call, so the +/// two runs differ only in how the connection is made. +async fn replay(admin_url: &str, database: &str) -> Vec { let fixture = fixture(); - let admin_url = admin_url(); - let admin = PgPool::connect(&admin_url) + let admin = PgPool::connect(admin_url) .await .expect("the administrative database is reachable"); admin .execute(sqlx::AssertSqlSafe(format!( - "DROP DATABASE IF EXISTS {DATABASE} WITH (FORCE)" + "DROP DATABASE IF EXISTS {database} WITH (FORCE)" ))) .await .expect("the parity database drops"); admin - .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {DATABASE}"))) + .execute(sqlx::AssertSqlSafe(format!("CREATE DATABASE {database}"))) .await .expect("the parity database is created"); drop(admin); - let backend = PostgresBackend::connect(&with_database(&admin_url, DATABASE)) + let backend = PostgresBackend::connect(&with_database(admin_url, database)) .await .expect("the parity database is reachable") .with_settings(Arc::new(common::settings_from_fixture(&fixture))); @@ -1049,6 +1049,7 @@ async fn the_typescript_script_replays_with_an_empty_diff() { .expect("the corpus inserts"); let mut refs = HashMap::new(); + let mut results = Vec::new(); for (index, op) in fixture["script"] .as_array() .expect("script") @@ -1063,10 +1064,92 @@ async fn the_typescript_script_replays_with_an_empty_diff() { .await .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}")) }; - let expected = &op["result"]["compare"]; + results.push(actual); + } + results +} + +/// Compare each op's result with the TypeScript capture's. +fn assert_matches_capture(actual: &[Value]) { + let fixture = fixture(); + let script = fixture["script"].as_array().expect("script"); + assert_eq!(actual.len(), script.len(), "the replay ran every op"); + for (index, (op, got)) in script.iter().zip(actual).enumerate() { + let name = op["op"].as_str().expect("an op name"); assert_eq!( - &actual, expected, + got, &op["result"]["compare"], "op {index} ({name}) differs from the TypeScript capture" ); } } + +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"] +async fn the_typescript_script_replays_with_an_empty_diff() { + let actual = replay(&admin_url(), DATABASE).await; + assert_matches_capture(&actual); +} + +/// The same script over TLS and over plaintext, compared op by op (#911). +/// +/// `sslmode=require` cannot fall back: a build without the TLS feature fails to connect +/// here rather than downgrading, which is the regression the leg exists to catch. The +/// connection-string option is not taken on trust either: `pg_stat_ssl` reports what the +/// server sees for this session, so a silently plaintext connection fails the leg too. +#[tokio::test] +#[ignore = "needs PARITY_TLS_ADMIN_URL; scripts/storage-parity.sh sets it"] +async fn the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run() { + // CI runs every ignored test with only the plaintext URL, so the leg skips there. + // scripts/storage-parity.sh sets PARITY_TLS_REQUIRED, under which a skip is a failure: + // the oracle must not report a green run for a leg that never executed. + let tls_admin_url = match std::env::var("PARITY_TLS_ADMIN_URL") { + Ok(url) => url, + Err(_) if std::env::var_os("PARITY_TLS_REQUIRED").is_some() => { + panic!("PARITY_TLS_REQUIRED is set but PARITY_TLS_ADMIN_URL is not") + } + Err(_) => { + eprintln!( + "PARITY_TLS_ADMIN_URL is unset: the TLS leg runs from scripts/storage-parity.sh" + ); + return; + } + }; + + let admin = PgPool::connect(&tls_admin_url) + .await + .expect("the TLS administrative database is reachable (is ssl=on?)"); + let encrypted: bool = + sqlx::query_scalar("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()") + .fetch_one(&admin) + .await + .expect("pg_stat_ssl answers"); + assert!( + encrypted, + "the server reports a plaintext session for a sslmode=require URL" + ); + drop(admin); + + let over_tls = replay(&tls_admin_url, "nostream_parity_tls").await; + assert_matches_capture(&over_tls); + + let over_plain = replay(&admin_url(), "nostream_parity_plain").await; + if over_plain != over_tls { + let index = over_plain + .iter() + .zip(&over_tls) + .position(|(plain, tls)| plain != tls) + .unwrap_or(0); + let fixture = fixture(); + let name = fixture["script"] + .get(index) + .and_then(|op| op["op"].as_str()) + .unwrap_or("?"); + panic!( + "op {index} ({name}) differs between plaintext and TLS: plaintext {:?}, TLS {:?}", + over_plain.get(index), + over_tls.get(index) + ); + } + // Read by scripts/storage-parity.sh with --nocapture as proof the assertions ran. + println!("TLS_PARITY_COMPLETE"); +} diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index 9d7f128d..f8021cbb 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -11,6 +11,7 @@ # host) set the address that machine reaches, e.g. the tailnet one. # PARITY_HOST host the harness dials. Default $PARITY_BIND. # PARITY_PG_PORT published Postgres port (default 18776) +# PARITY_PG_TLS_PORT published port of the TLS Postgres server (default 18778) # PARITY_REDIS_PORT published Redis port (default 18777) # PARITY_PG_PASSWORD Postgres password; generated per run (openssl rand -hex 16) if unset # PARITY_REDIS_PASSWORD Redis password; generated per run if unset @@ -35,6 +36,7 @@ cd "$root" PARITY_BIND=${PARITY_BIND:-127.0.0.1} PARITY_HOST=${PARITY_HOST:-$PARITY_BIND} PARITY_PG_PORT=${PARITY_PG_PORT:-18776} +PARITY_PG_TLS_PORT=${PARITY_PG_TLS_PORT:-18778} PARITY_REDIS_PORT=${PARITY_REDIS_PORT:-18777} PARITY_PG_PASSWORD=${PARITY_PG_PASSWORD:-$(openssl rand -hex 16)} PARITY_REDIS_PASSWORD=${PARITY_REDIS_PASSWORD:-$(openssl rand -hex 16)} @@ -48,10 +50,31 @@ if [ -z "${CUCUMBER_BIN:-}" ]; then fi pg_container="nostream-parity-pg-$$" +tls_container="nostream-parity-pg-tls-$$" redis_container="nostream-parity-redis-$$" +# The TLS server keeps its certificate pair and its PGDATA inside the worktree. Ports, +# labels and the per-run credential match the plaintext server; only the bind sources are +# new, and they need the path the daemon sees (the sandbox mounts this tree at /workspace, +# the daemon host at /home/cameri/Workspace). NOSTREAM_HOST_DIR overrides both, as it does +# for scripts/rust.sh. +# The directory is per invocation: two runs of this script (different ports, or the same) +# must not delete each other's live PGDATA, so the suffix carries the pid and a random +# token, and the exit trap removes it. +tls_dir="$root/.test-reports/parity-tls-$$-$(openssl rand -hex 4)" +if [ -n "${NOSTREAM_HOST_DIR:-}" ]; then + host_root=$NOSTREAM_HOST_DIR +else + case "$root" in + /workspace/*) host_root="/home/cameri/Workspace/${root#/workspace/}" ;; + *) host_root=$root ;; + esac +fi +tls_dir_host="$host_root${tls_dir#"$root"}" + cleanup() { - docker rm -f "$pg_container" "$redis_container" >/dev/null 2>&1 || true + docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true + rm -rf "$tls_dir" } trap cleanup EXIT trap 'exit 130' INT HUP TERM @@ -171,6 +194,9 @@ fi pg_password_url=$(url_escape "$PARITY_PG_PASSWORD") redis_password_url=$(url_escape "$PARITY_REDIS_PASSWORD") admin_url="postgres://postgres:$pg_password_url@$host_url:$PARITY_PG_PORT/postgres" +# sslmode=require: the leg is about a server that insists on TLS, and `prefer` would quietly +# pass over plaintext, which is the failure #911 is about. +tls_admin_url="postgres://postgres:$pg_password_url@$host_url:$PARITY_PG_TLS_PORT/postgres?sslmode=require" docker version >/dev/null 2>&1 \ || fail "the Docker daemon at ${DOCKER_HOST:-/var/run/docker.sock} is not reachable" @@ -191,7 +217,7 @@ printf '%s\n' "$redaction_output" | grep -q 'test result: ok. 1 passed' || { # variables instead of surfacing as a sqlx timeout later. probe_dial() { # The probe wants the address itself; URLs keep the bracketed IPv6 spelling. - node -e 'const net = require("net"); const s = net.connect({ host: process.argv[1], port: Number(process.argv[2]) }); s.setTimeout(3000); s.on("connect", () => { s.destroy(); process.exit(0) }); s.on("timeout", () => { s.destroy(); process.exit(1) }); s.on("error", () => process.exit(1))' "$host_plain" "$PARITY_PG_PORT" + node -e 'const net = require("net"); const s = net.connect({ host: process.argv[1], port: Number(process.argv[2]) }); s.setTimeout(3000); s.on("connect", () => { s.destroy(); process.exit(0) }); s.on("timeout", () => { s.destroy(); process.exit(1) }); s.on("error", () => process.exit(1))' "$host_plain" "$1" } wait_for_log() { @@ -200,29 +226,54 @@ wait_for_log() { attempts=0 until docker logs "$container" 2>&1 | grep -q "$pattern"; do attempts=$((attempts + 1)) - [ "$attempts" -ge 90 ] && fail "$container did not report: $pattern" + if [ "$attempts" -ge 90 ]; then + docker logs "$container" 2>&1 | tail -20 + fail "$container did not report: $pattern" + fi sleep 1 done } echo "== containers (bound to $PARITY_BIND, per-run credentials)" -docker rm -f "$pg_container" "$redis_container" >/dev/null 2>&1 || true + +# A fresh self-signed pair per run. The key stays 0600, and the server runs as the invoking +# uid rather than the image's postgres user (uid 70): a 0600 file written here is owned by +# this user and unreadable to uid 70, so the stock user cannot start with it. The issue's +# alternative, chmod 644 on a copy, is not one PostgreSQL accepts either — it refuses a key +# with group or world access and the server exits. Running as this uid with its own PGDATA +# on a bind keeps the key strict and the image unchanged. +mkdir -p "$tls_dir/certs" "$tls_dir/pgdata" +chmod 755 "$tls_dir" "$tls_dir/certs" +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=localhost" \ + -keyout "$tls_dir/certs/server.key" -out "$tls_dir/certs/server.crt" >/dev/null 2>&1 \ + || fail "the TLS certificate pair could not be generated (openssl is required)" +chmod 600 "$tls_dir/certs/server.key" +chmod 644 "$tls_dir/certs/server.crt" + +docker rm -f "$pg_container" "$tls_container" "$redis_container" >/dev/null 2>&1 || true docker create --name "$pg_container" --label org.testcontainers=true \ -e POSTGRES_PASSWORD="$PARITY_PG_PASSWORD" -e POSTGRES_DB=postgres \ -p "$PARITY_BIND:$PARITY_PG_PORT:5432" postgres:16-alpine >/dev/null +docker create --name "$tls_container" --label org.testcontainers=true \ + --user "$(id -u):$(id -g)" \ + -e POSTGRES_PASSWORD="$PARITY_PG_PASSWORD" -e POSTGRES_DB=postgres -e PGDATA=/pgdata \ + -v "$tls_dir_host/certs:/certs:ro" -v "$tls_dir_host/pgdata:/pgdata" \ + -p "$PARITY_BIND:$PARITY_PG_TLS_PORT:5432" postgres:16-alpine \ + -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key >/dev/null docker create --name "$redis_container" --label org.testcontainers=true \ -p "$PARITY_BIND:$PARITY_REDIS_PORT:6379" redis:7-alpine \ redis-server --requirepass "$PARITY_REDIS_PASSWORD" >/dev/null -docker start "$pg_container" "$redis_container" >/dev/null +docker start "$pg_container" "$tls_container" "$redis_container" >/dev/null wait_for_log "$pg_container" "database system is ready to accept connections" +wait_for_log "$tls_container" "database system is ready to accept connections" wait_for_log "$redis_container" "Ready to accept connections" attempts=0 -until probe_dial; do +until probe_dial "$PARITY_PG_PORT" && probe_dial "$PARITY_PG_TLS_PORT"; do attempts=$((attempts + 1)) - [ "$attempts" -ge 5 ] && fail "the published Postgres port is not reachable at $PARITY_HOST:$PARITY_PG_PORT: set PARITY_BIND and PARITY_HOST to an address this machine can dial (127.0.0.1 on the daemon host; the daemon host's reachable address from elsewhere)" + [ "$attempts" -ge 5 ] && fail "a published Postgres port is not reachable at $PARITY_HOST ($PARITY_PG_PORT, $PARITY_PG_TLS_PORT): set PARITY_BIND and PARITY_HOST to an address this machine can dial (127.0.0.1 on the daemon host; the daemon host's reachable address from elsewhere)" sleep 1 done -echo " postgres and redis are accepting connections on $PARITY_HOST" +echo " postgres (plaintext and TLS) and redis are accepting connections on $PARITY_HOST" echo "== migrations and schema parity" ./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" \ @@ -234,6 +285,23 @@ echo "== behavioural parity" cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \ || fail "the parity target failed" +echo "== tls parity (the same replay over sslmode=require)" +# The test skips itself when the variable is absent, so the run has to prove it executed: +# a renamed or quietly skipped TLS leg is the failure mode this check closes. +# PARITY_TLS_REQUIRED makes the test panic instead of skipping when it has no TLS URL, and +# the completion marker is printed only after the assertions, so a skip cannot pass. +tls_output=$(./scripts/rust.sh env PARITY_ADMIN_URL="$admin_url" PARITY_TLS_ADMIN_URL="$tls_admin_url" PARITY_TLS_REQUIRED=1 \ + cargo test -p nostream-storage --test parity the_same_replay_over_tls_matches_the_capture_and_the_plaintext_run \ + -- --ignored --exact --test-threads=1 --nocapture 2>&1) || { + printf '%s\n' "$tls_output" + fail "the TLS leg failed: the replay over sslmode=require did not match the plaintext run" +} +printf '%s\n' "$tls_output" | grep -q 'TLS_PARITY_COMPLETE' || { + printf '%s\n' "$tls_output" + fail "the TLS parity test did not complete (renamed, skipped, or it returned early?)" +} +echo " the replay over sslmode=require matches the plaintext run ($(printf '%s\n' "$tls_output" | grep -o 'finished in [0-9.]*s'))" + echo "== cucumber on the sqlx-migrated schema" [ -x "$CUCUMBER_BIN" ] || fail "cucumber-js not found at $CUCUMBER_BIN" mkdir -p .test-reports/integration @@ -245,4 +313,4 @@ NODE_OPTIONS=--no-experimental-strip-types \ "$CUCUMBER_BIN" --format summary \ || fail "the cucumber suite failed" -echo "== storage-parity: all three steps passed" +echo "== storage-parity: all four steps passed"