From 5cf157cbd000720694e09e7665670eb8b946f926 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 19 Oct 2026 21:34:29 -0400 Subject: [PATCH 1/2] fix(storage): connect over TLS and keep transaction refs addressable --- .changeset/tls-and-transaction-refs.md | 16 ++ Cargo.lock | 181 +++++++++++++++++- Cargo.toml | 2 +- .../tests/capture/gen-storage-parity.mjs | 17 +- .../tests/fixtures/repository-parity.json | 49 ++++- crates/nostream-storage/tests/parity.rs | 21 +- 6 files changed, 259 insertions(+), 27 deletions(-) create mode 100644 .changeset/tls-and-transaction-refs.md diff --git a/.changeset/tls-and-transaction-refs.md b/.changeset/tls-and-transaction-refs.md new file mode 100644 index 00000000..5c93de3a --- /dev/null +++ b/.changeset/tls-and-transaction-refs.md @@ -0,0 +1,16 @@ +--- +--- + +fix(storage): negotiate TLS, and keep an enqueued id addressable inside its transaction + +`sqlx` was selected with `tls-none`, so a database that requires `sslmode=require` +could not be reached at all (#911). The workspace dependency now enables +`tls-rustls-ring-webpki` — ring with webpki roots, no native-tls — and the TLS +path is proven by a probe against a Postgres with `ssl=on`. + +The parity harness's `run_transaction` handed every nested op a fresh refs map, +so an `outbox.enqueue` followed by `outbox.mark_delivered` in one group lost the +id and came back as a failed transaction (#912). The caller's map is threaded +through the nested calls, the capture generator does the same, and a new +transaction group in the fixture enqueues and then marks the row delivered, so +the TypeScript capture and the Rust replay agree on the id. diff --git a/Cargo.lock b/Cargo.lock index a214cc74..e134f127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,7 +62,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -73,7 +73,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -435,7 +435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -445,7 +445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -595,6 +595,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -1107,7 +1118,7 @@ checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1235,7 +1246,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1611,6 +1622,20 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.5" @@ -1621,7 +1646,41 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1803,7 +1862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1852,6 +1911,7 @@ dependencies = [ "log", "memchr", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -1861,6 +1921,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots", ] [[package]] @@ -2013,6 +2074,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -2071,7 +2138,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2132,7 +2199,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2373,6 +2440,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -2494,6 +2567,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "2.1.3" @@ -2581,6 +2663,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -2590,6 +2681,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -2666,6 +2821,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 7ca802f9..5a7d1cf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" sha2 = "0.11.1" -sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "postgres", "migrate", "macros", "chrono", "json", "tls-none"] } +sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "postgres", "migrate", "macros", "chrono", "json", "tls-rustls-ring-webpki"] } tempfile = "3" thiserror = "2" tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } diff --git a/crates/nostream-storage/tests/capture/gen-storage-parity.mjs b/crates/nostream-storage/tests/capture/gen-storage-parity.mjs index 4d6884f6..7e0f0be8 100644 --- a/crates/nostream-storage/tests/capture/gen-storage-parity.mjs +++ b/crates/nostream-storage/tests/capture/gen-storage-parity.mjs @@ -850,6 +850,17 @@ const script = [ }, }, { op: 'outbox.claim_batch', request: { limit: 10 } }, + // One group enqueues and then marks the same row delivered: the enqueue's id has to be + // addressable by the next op, inside the transaction. + { + op: 'transaction', + request: { + ops: [ + { op: 'outbox.enqueue', request: { event_type: 'transaction.enqueue.mark', payload: { probe: 'marked' } } }, + { op: 'outbox.mark_delivered', request: { enqueue_event_type: 'transaction.enqueue.mark' } }, + ], + }, + }, ] // --------------------------------------------------------------------------------------- @@ -1278,8 +1289,8 @@ const resolveId = (request) => { return value } -const runOp = async (op, client) => { - const result = await run(op, client) +const runOp = async (op, client, inTransaction = false) => { + const result = await run(op, client, inTransaction) if (op.op === 'outbox.enqueue') { const message = result.compare.message // The id is not part of the comparison; it is needed to address the row later. @@ -1300,7 +1311,7 @@ const runTransaction = async (op) => { await db.transaction(async (trx) => { for (const [index, nested] of op.request.ops.entries()) { try { - executed.push(await run(nested, trx, true)) + executed.push(await runOp(nested, trx, true)) } catch (error) { failedAt = index throw error diff --git a/crates/nostream-storage/tests/fixtures/repository-parity.json b/crates/nostream-storage/tests/fixtures/repository-parity.json index ff33cfe3..87358a7c 100644 --- a/crates/nostream-storage/tests/fixtures/repository-parity.json +++ b/crates/nostream-storage/tests/fixtures/repository-parity.json @@ -3,7 +3,7 @@ "repo": "cameri/nostream", "capture_commit": "242a3001d77149e4d70151ec03060c4ef44a1547", "generator": "crates/nostream-storage/tests/capture/gen-storage-parity.mjs", - "generated_on": "2026-10-19", + "generated_on": "2026-10-20", "server_image": "postgres:16-alpine (PostgreSQL 16.15)", "timezone": "UTC", "migration_command": "knex migrate:latest", @@ -11317,7 +11317,7 @@ } }, "ref": { - "id": "d05eb04a-9e3b-4fee-9226-57efc47138c2" + "id": "d6726eeb-3fc8-4d38-baa1-444875bfb324" } } }, @@ -11346,7 +11346,7 @@ } }, "ref": { - "id": "78642867-7c9f-4628-8dab-89a15adf2fe1" + "id": "ac74823d-fc2f-4963-a148-0266b982b07a" } } }, @@ -11684,6 +11684,49 @@ ] } } + }, + { + "op": "transaction", + "request": { + "ops": [ + { + "op": "outbox.enqueue", + "request": { + "event_type": "transaction.enqueue.mark", + "payload": { + "probe": "marked" + } + } + }, + { + "op": "outbox.mark_delivered", + "request": { + "enqueue_event_type": "transaction.enqueue.mark" + } + } + ] + }, + "result": { + "compare": { + "failed": false, + "results": [ + { + "message": { + "event_type": "transaction.enqueue.mark", + "payload": { + "probe": "marked" + }, + "status": "pending", + "attempt_count": 0, + "last_error": null + } + }, + { + "ok": true + } + ] + } + } } ], "projection": { diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index 2defdf19..6470e3d2 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -946,21 +946,22 @@ fn new_report(value: &Value) -> NewReport { } } -async fn run_transaction(backend: &PostgresBackend, op: &Value) -> Value { +async fn run_transaction( + backend: &PostgresBackend, + op: &Value, + refs: &mut HashMap, +) -> Value { let ops = op["request"]["ops"].as_array().expect("transaction ops"); let transaction = backend.begin().await.expect("a serializable transaction"); let mut results = Vec::new(); let mut failed_at = None; + // One map for the whole group: an enqueue's id has to be addressable by the + // next op in the same transaction, exactly as the top-level script does. + // A fresh map per nested op loses it, and `mark_delivered` then has no row + // to address. for (index, nested) in ops.iter().enumerate() { - match run_op( - backend, - Client::Transaction(&transaction), - nested, - &mut HashMap::new(), - ) - .await - { + match run_op(backend, Client::Transaction(&transaction), nested, refs).await { Ok(result) => results.push(result), Err(_) => { failed_at = Some(index); @@ -1026,7 +1027,7 @@ async fn the_typescript_script_replays_with_an_empty_diff() { { let name = op["op"].as_str().expect("an op name"); let actual = if name == "transaction" { - run_transaction(&backend, op).await + run_transaction(&backend, op, &mut refs).await } else { run_op(&backend, Client::Backend(&backend), op, &mut refs) .await From 28868833dcfc580f905cbdf7a4a93f0788d317c8 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 19 Oct 2026 21:42:16 -0400 Subject: [PATCH 2/2] test(storage): guard the TLS feature in the parity target --- .changeset/tls-and-transaction-refs.md | 4 +++- crates/nostream-storage/tests/parity.rs | 30 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.changeset/tls-and-transaction-refs.md b/.changeset/tls-and-transaction-refs.md index 5c93de3a..d9410020 100644 --- a/.changeset/tls-and-transaction-refs.md +++ b/.changeset/tls-and-transaction-refs.md @@ -6,7 +6,9 @@ fix(storage): negotiate TLS, and keep an enqueued id addressable inside its tran `sqlx` was selected with `tls-none`, so a database that requires `sslmode=require` could not be reached at all (#911). The workspace dependency now enables `tls-rustls-ring-webpki` — ring with webpki roots, no native-tls — and the TLS -path is proven by a probe against a Postgres with `ssl=on`. +path now has a committed regression test: the parity target requires a +`sslmode=require` connection to be attempted rather than refused by the build, +and completes `SELECT 1` when the server offers TLS. The parity harness's `run_transaction` handed every nested op a fresh refs map, so an `outbox.enqueue` followed by `outbox.mark_delivered` in one group lost the diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index 6470e3d2..f63bae17 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -982,6 +982,36 @@ async fn run_transaction( } } +/// The TLS feature is compiled in (#911). A server that does not offer TLS has +/// to refuse the upgrade; this build must not refuse it first. With `tls-none` +/// sqlx fails before any handshake with "built without TLS support", which is +/// the regression the feature change closes. A server that does offer TLS is the +/// stronger case: the connection succeeds and `SELECT 1` answers. +#[tokio::test] +#[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"] +async fn a_tls_connection_is_attempted_rather_than_refused_by_this_build() { + let admin = admin_url(); + let base = admin.split('?').next().unwrap_or(admin.as_str()); + let url = format!("{base}?sslmode=require"); + match PgPool::connect(&url).await { + Ok(pool) => { + let one: i32 = sqlx::query_scalar("SELECT 1") + .fetch_one(&pool) + .await + .expect("SELECT 1 over TLS"); + assert_eq!(one, 1); + pool.close().await; + } + Err(error) => { + let message = error.to_string(); + assert!( + !message.contains("built without TLS support"), + "sqlx refused to negotiate TLS locally: {message}" + ); + } + } +} + #[tokio::test] #[ignore = "needs PARITY_ADMIN_URL; scripts/storage-parity.sh sets it"] async fn the_typescript_script_replays_with_an_empty_diff() {