From a7a1189d477a2402dc57a87ef20428c45993f346 Mon Sep 17 00:00:00 2001 From: "Christopher Krawczyk (SQL)" Date: Tue, 6 Oct 2026 14:33:08 -0700 Subject: [PATCH 1/4] Adopt fallible startup and retained bounded shutdown in Node Preserve the prepared core and startup failures across repeated lifecycle calls, validate shutdown deadlines before mutation, and unref callbacks only after proven core quiescence without changing Promise results. Add supervised lifecycle regressions, test-hook wiring, and migration docs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 73748d4f-2eb6-47d2-9dbc-af8681e80e83 --- .../duroxide-node-orchestrations/SKILL.md | 2 +- .github/copilot-instructions.md | 2 +- CHANGELOG.md | 24 + CONTRIBUTING.md | 23 +- Cargo.toml | 3 + README.md | 26 + __tests__/fixtures/lifecycle-probe.cjs | 113 +++++ __tests__/lifecycle.test.js | 291 ++++++++++++ ci/smoke/smoke.mjs | 23 +- docs/architecture.md | 30 +- docs/user-guide.md | 60 +++ lib/duroxide.d.ts | 7 + lib/duroxide.js | 16 +- package.json | 3 +- src/lib.rs | 4 + src/runtime.rs | 444 ++++++++++++------ src/test_hooks.rs | 91 ++++ 17 files changed, 1009 insertions(+), 153 deletions(-) create mode 100644 __tests__/fixtures/lifecycle-probe.cjs create mode 100644 __tests__/lifecycle.test.js create mode 100644 src/test_hooks.rs diff --git a/.agents/skills/duroxide-node-orchestrations/SKILL.md b/.agents/skills/duroxide-node-orchestrations/SKILL.md index b6d85d7..21db55a 100644 --- a/.agents/skills/duroxide-node-orchestrations/SKILL.md +++ b/.agents/skills/duroxide-node-orchestrations/SKILL.md @@ -327,7 +327,7 @@ npm run test:all # Everything (52 tests) - Use `SqliteProvider.inMemory()` for fast isolated tests (SQLite smoketest only) - All PG tests need `DATABASE_URL` in `.env` (loaded by `dotenv`) - Each test file uses a separate PG schema for isolation -- Use short `runtime.shutdown(100)` timeout — it waits the full duration +- `runtime.shutdown(100)` permits 100 ms of grace plus five seconds of cleanup headroom. It can finish early; timeout rejects while cleanup remains owned and requires application/supervisor termination. - Set `RUST_LOG=info` to see traces in test output - Use `workerLockTimeoutMs: 2000` in tests needing fast activity cancellation detection diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index a14b8f5..334b67e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -203,6 +203,6 @@ validation so CI and published packages use registry crates. - All JS callbacks run on the Node.js main thread (single-threaded) - No `async function*` generators — breaks replay model - SQLite may hit "database is locked" under concurrency (retried automatically) -- `Runtime.shutdown(timeoutMs)` waits the full timeout (no short-circuit) +- `Runtime.shutdown(timeoutMs)` uses grace plus five seconds of bounded waiting; idle work can finish early, and failures reject while retaining cleanup. - Platform-specific `.node` binary — cross-platform needs per-platform builds - Activity cancellation detection latency depends on `workerLockTimeoutMs` (default 30s → ~15s detection) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1db583..38828dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Use fallible core startup and bounded shutdown while preserving `Promise`. + Shutdown uses the supplied grace plus five seconds, with a one-second omitted + grace, and idle workers can finish early. Invalid durations reject before + changing ownership, even for no-work/repeated calls. Repeated calls retain + the original operation and errors instead of succeeding on an empty runtime; + no call renews the first deadlines. Pre-core registry/preparation failures + become ordinary retained startup errors without changing registration policy. +- Pre-start shutdown is terminal; start/registration after activation or stop is + rejected. Post-stop metrics remain unavailable. Core completion remains retained + without keeping Node's event loop alive after proven retirement. +- Timeouts explicitly require application/supervisor process termination while + cleanup remains owned, even if a later observer proves retirement. Finite grace + does not guarantee bounded forced cleanup. This narrow adapter does not add + .NET's foreign-continuation lifetime barrier or own detached application work. + The matching published core dependency is a separate release prerequisite; + local development overrides are not distributable. + +### Tests + +- SQLite-only real-wrapper lifecycle coverage, feature-gated provider/fault controls, + ordered race repetitions, isolated held-provider probes, and production-export checks. + ## [0.1.29] - 2026-09-03 ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf68613..1f57a9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,4 +32,25 @@ npm run test:sessions cargo clippy --all-targets ``` -After Rust source changes (`src/*.rs`), re-run `npx napi build --platform` before running JavaScript tests. \ No newline at end of file +After Rust source changes (`src/*.rs`), re-run `npx napi build --platform` before running JavaScript tests. + +### Lifecycle checks without PostgreSQL + +The lifecycle suite requires a freshly built local native module, not an optional +prebuilt platform package. Its private controls are absent from production builds. +For example, in PowerShell: + +```powershell +npm run build:debug -- --features test-hooks --js false --dts target\lifecycle-native.d.ts +$env:DUROXIDE_LIFECYCLE_TEST_HOOKS = '1' +npm run test:lifecycle +Remove-Item Env:DUROXIDE_LIFECYCLE_TEST_HOOKS +npm run build:debug -- --js false --dts target\lifecycle-native.d.ts +npm run test:lifecycle +``` + +Instrumented cases exercise real provider waits and contained owned-task faults. +The suite checks native import provenance, 100 repetitions per ordered race, +bounded child-process cleanup, and production export absence. Production mode +skips only hook-dependent cases. Do not publish instrumented assets or temporary +local core overrides. A matching published core minimum is required before release. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index d8272d9..fa2fb87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,9 @@ edition = "2021" [lib] crate-type = ["cdylib"] +[features] +test-hooks = ["duroxide/test-hooks"] + [dependencies] duroxide = { version = "0.1.30", features = ["sqlite"] } duroxide-pg = "0.1.34" diff --git a/README.md b/README.md index 8dec748..cc0d3fd 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,32 @@ Node.js/TypeScript SDK for the [Duroxide](https://github.com/microsoft/duroxide) - **SQLite & PostgreSQL** — pluggable storage backends - **KV store** — durable per-instance key-value state with snapshots and pruning via `ctx.getKvAllValues()` / `client.getKvAllValues()` +## Runtime lifecycle (unreleased) + +`await runtime.start()` returns after startup and propagates fallible core errors. +`await runtime.shutdown(timeoutMs)` still resolves to `undefined` on success. +The argument is a nonnegative safe-integer grace period in milliseconds, within +the native monotonic deadline range. Omission means 1000 ms; total waiting is +grace plus 5000 ms. Idle work can finish early. + +The first stop fixes both deadlines. Repeated calls observe the same operation, +including timeout/operational errors and genuine late completion. Shutdown before +start is now terminal; later start or registration is rejected. Metrics remain +unavailable after stop. Serialize lifecycle and registration calls on each runtime. + +A `lifecycle_shutdown_timed_out` rejection means cleanup is still owned and +incomplete. Do not restart the worker: arrange process termination through the +application or supervisor, even if a later observation reports completion. The SDK +does not terminate the process. This narrow adapter change does not add tracking +of arbitrary JavaScript continuations or the .NET SDK's additional managed +foreign-retirement barrier. Independently owned provider/Client resources have +their own lifetime. See the [lifecycle guide](docs/user-guide.md#runtime-lifecycle) +for validation and an executable early-drain/repeated-stop example. + +These unreleased changes require the corresponding core lifecycle release before +distribution; a temporary local Cargo override is development evidence, not a +release dependency pin. + ## Quick Start ```bash diff --git a/__tests__/fixtures/lifecycle-probe.cjs b/__tests__/fixtures/lifecycle-probe.cjs new file mode 100644 index 0000000..058a9cb --- /dev/null +++ b/__tests__/fixtures/lifecycle-probe.cjs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { createInterface } = require('node:readline'); +const { performance } = require('node:perf_hooks'); +const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); +const native = require(Object.keys(require.cache).find(file => file.endsWith('.node'))); +const emit = (phase, detail = {}) => console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase, ...detail })); + +async function main() { + const scenario = process.argv[2]; + let runtime = new Runtime(await SqliteProvider.inMemory(), { + dispatcherPollIntervalMs: 5, serviceName: 'lifecycle-private-sentinel', + }); + if (scenario === 'registry-invalid' || scenario === 'registry-descending') { + const versions = scenario === 'registry-invalid' ? ['not-semver'] : ['2.0.0', '1.0.0']; + for (const version of versions) { + runtime.registerOrchestrationVersioned('registry-private-sentinel', version, function* () { return 'ok'; }); + } + let startMessage; + await assert.rejects(runtime.start(), error => { + assert.ok(error instanceof Error); + assert.match(error.message, /lifecycle_start_failed/); + assert.doesNotMatch(error.message, /sentinel|Panic in async function/); + startMessage = error.message; + return true; + }); + for (const timeout of [0, 30_000]) { + await assert.rejects(runtime.shutdown(timeout), error => { + assert.equal(error.message, startMessage); + return true; + }); + } + assert.equal(runtime.metricsSnapshot(), null); + await assert.rejects(runtime.start(), /lifecycle_terminal/); + assert.throws(() => runtime.registerActivity('late', async () => 'late'), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestration('late', function* () {}), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestrationVersioned('late', '3.0.0', function* () {}), /lifecycle_terminal/); + globalThis.retainedRuntime = runtime; + emit('REGISTRY_FAILURE_RETAINED', { scenario }); + return; + } + if (scenario === 'exit') { + runtime.registerActivity('ExitActivity', async (_, value) => value); + runtime.registerOrchestration('ExitFlow', function* () { return 'ok'; }); + await runtime.start(); + await runtime.shutdown(100); + globalThis.retainedRuntime = runtime; + emit('EXIT_READY'); + return; + } + const hooks = native._lifecycleTestHooks(runtime._native); + hooks.hold('provider'); + await runtime.start(); + await hooks.waitEntered('provider'); + const input = createInterface({ input: process.stdin }); + const command = new Promise(resolve => input.once('line', resolve)); + emit('STOPPING'); + const start = performance.now(); + await assert.rejects(runtime.shutdown(scenario === 'late' ? undefined : 0), error => { + assert.match(error.message, /lifecycle_shutdown_timed_out/); + assert.match(error.message, /cleanup.*incomplete/); + assert.match(error.message, /terminate the process/); + assert.ok(!error.message.includes('sentinel')); + return true; + }); + const elapsedMs = performance.now() - start; + const repeated = performance.now(); + await assert.rejects(runtime.shutdown(30_000), error => { + assert.match(error.message, /lifecycle_shutdown_timed_out/); + assert.ok(!error.message.includes('sentinel')); + return true; + }); + const repeatedMs = performance.now() - repeated; + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + assert.equal(runtime.metricsSnapshot(), null); + emit('TIMED_OUT', { elapsedMs, repeatedMs }); + if (await command === 'retain') { + runtime = null; + global.gc(); + const state = JSON.parse(hooks.snapshot()); + assert.ok(state.active > 0); + assert.equal(state.coordinators, 1); + emit('RETAINED'); + await new Promise(() => {}); + } else { + hooks.release('provider'); + const deadline = performance.now() + 2000; + while (true) { + try { await runtime.shutdown(30_000); break; } + catch (error) { + assert.match(error.message, /lifecycle_shutdown_timed_out/); + assert.ok(performance.now() < deadline); + await new Promise(resolve => setTimeout(resolve, 1)); + } + } + const state = JSON.parse(hooks.snapshot()); + assert.equal(state.active, 0); + assert.equal(state.cleanupActive, 0); + assert.equal(state.started, state.completed); + assert.equal(state.coordinators, 1); + assert.equal(state.forceRequests, 1); + await assert.rejects(runtime.start(), /lifecycle_terminal/); + emit('RECLAIMED'); + input.close(); + } +} + +main().catch(error => { + emit('FAILED', { error: error.stack }); + process.exitCode = 1; +}); diff --git a/__tests__/lifecycle.test.js b/__tests__/lifecycle.test.js new file mode 100644 index 0000000..f24a523 --- /dev/null +++ b/__tests__/lifecycle.test.js @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const { readFileSync } = require('node:fs'); +const path = require('node:path'); +const { spawn, spawnSync } = require('node:child_process'); +const { createInterface } = require('node:readline'); +const { performance } = require('node:perf_hooks'); +const { SqliteProvider, Client, Runtime } = require('../lib/duroxide.js'); + +const root = path.resolve(__dirname, '..'); +const nativePath = Object.keys(require.cache).find(file => file.endsWith('.node')); +assert.ok(nativePath?.startsWith(root + path.sep), 'tests require the freshly built local native module'); +const native = require(nativePath); +const instrumented = process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1'; +const SECRET = 'Password=lifecycle-secret-sentinel;Host=private-sentinel'; +const MAX_GRACE_MS = Number(((2n ** 64n) - 1n) / 1_000_000n) - 5000; + +function diagnostic(category, incomplete = false) { + return error => { + assert.ok(error instanceof Error); + assert.match(error.message, new RegExp(category)); + assert.ok(!error.message.includes(SECRET)); + assert.ok(!error.message.includes('sentinel')); + if (incomplete) { + assert.match(error.message, /cleanup.*incomplete/); + assert.match(error.message, /terminate the process/); + } + if (category === 'lifecycle_shutdown_failed') { + assert.match(error.message, /cleanup completed and is quiescent/); + assert.doesNotMatch(error.message, /terminate the process/); + } + return true; + }; +} + +async function fixture(options) { + const provider = await SqliteProvider.inMemory(); + const runtime = new Runtime(provider, { dispatcherPollIntervalMs: 5, ...options }); + return { provider, runtime, hooks: instrumented ? native._lifecycleTestHooks(runtime._native) : null }; +} + +function retired(hooks, forced) { + const state = JSON.parse(hooks.snapshot()); + assert.equal(state.active, 0); + assert.equal(state.cleanupActive, 0); + assert.equal(state.started, state.completed); + assert.equal(state.coordinators, 1); + if (forced !== undefined) assert.equal(state.forceRequests, forced ? 1 : 0); +} + +test('native provenance and production export boundary', () => { + assert.equal(typeof native._lifecycleTestHooks === 'function', instrumented); + assert.equal(typeof native.LifecycleTestHooks === 'function', instrumented); + console.log('LIFECYCLE_NATIVE ' + JSON.stringify({ + path: nativePath, + sha256: createHash('sha256').update(readFileSync(nativePath)).digest('hex'), + instrumented, + })); +}); + +test('invalid durations preserve registration, startup, and repeated-stop ownership', async () => { + const { runtime, hooks } = await fixture(); + for (const value of [-1, 0.5, NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1, '0', null, MAX_GRACE_MS + 1, Number.MAX_SAFE_INTEGER]) { + await assert.rejects(runtime.shutdown(value), diagnostic('lifecycle_invalid_timeout')); + } + runtime.registerOrchestration('LifecycleInvalidInput', function* () { return 'ok'; }); + await runtime.start(); + assert.notEqual(runtime.metricsSnapshot(), null); + await assert.rejects(runtime.shutdown(-1), diagnostic('lifecycle_invalid_timeout')); + assert.notEqual(runtime.metricsSnapshot(), null); + assert.equal(await runtime.shutdown(0), undefined); + assert.equal(runtime.metricsSnapshot(), null); + await assert.rejects(runtime.shutdown(Number.MAX_SAFE_INTEGER), diagnostic('lifecycle_invalid_timeout')); + assert.equal(await runtime.shutdown(), undefined); + if (hooks) retired(hooks); +}); + +test('maximum representable default-total boundary is checked on no-work and repeated stop', async () => { + const { runtime } = await fixture(); + assert.equal(await runtime.shutdown(MAX_GRACE_MS), undefined); + await assert.rejects(runtime.shutdown(MAX_GRACE_MS + 1), diagnostic('lifecycle_invalid_timeout')); + assert.equal(await runtime.shutdown(), undefined); +}); + +test('pre-start shutdown is a terminal no-work success', async () => { + const { runtime } = await fixture(); + assert.equal(await runtime.shutdown(0), undefined); + assert.equal(await runtime.shutdown(), undefined); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + assert.throws(() => runtime.registerActivity('late', async () => 'late'), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestration('late', function* () {}), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestrationVersioned('late', '1.0.0', function* () {}), /lifecycle_terminal/); + assert.equal(runtime.metricsSnapshot(), null); +}); + +test('fallible preparation rejects invalid options without an escaping panic', async () => { + const { runtime } = await fixture({ sessionIdleTimeoutMs: 0 }); + await assert.rejects(runtime.start(), diagnostic('lifecycle_start_failed')); + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_start_failed')); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + assert.equal(runtime.metricsSnapshot(), null); +}); + +for (const scenario of ['registry-invalid', 'registry-descending']) { + test(`pre-core startup failure is ordinary and retained: ${scenario}`, () => { + const child = spawnSync(process.execPath, [path.join(__dirname, 'fixtures', 'lifecycle-probe.cjs'), scenario], { + cwd: root, encoding: 'utf8', timeout: 5000, env: process.env, + }); + assert.equal(child.error, undefined, child.error?.message); + assert.equal(child.status, 0, child.stdout + child.stderr); + assert.match(child.stdout, /REGISTRY_FAILURE_RETAINED/); + }); +} + +test('ascending and duplicate version registration keep their existing successful startup policy', async () => { + const { runtime } = await fixture(); + for (const version of ['1.0.0', '1.0.0', '2.0.0']) { + runtime.registerOrchestrationVersioned('LifecycleVersionPolicy', version, function* () { return 'ok'; }); + } + assert.equal(await runtime.start(), undefined); + assert.equal(await runtime.shutdown(100), undefined); + assert.equal(await runtime.shutdown(0), undefined); +}); + +test('idle shutdown short-circuits long grace and preserves successful shapes', async () => { + const { runtime, hooks } = await fixture(); + await runtime.start(); + const start = performance.now(); + assert.equal(await runtime.shutdown(30_000), undefined); + assert.ok(performance.now() - start < 2000); + assert.equal(await runtime.shutdown(0), undefined); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + if (hooks) retired(hooks, false); +}); + +test('SQLite activity and orchestration run through the unchanged public API', async () => { + const { runtime, provider } = await fixture(); + const client = new Client(provider); + runtime.registerActivity('LifecycleEcho', async (_, input) => input); + runtime.registerOrchestration('LifecycleEchoFlow', function* (context, input) { + return yield context.scheduleActivity('LifecycleEcho', input); + }); + await runtime.start(); + try { + await client.startOrchestration('lifecycle-echo', 'LifecycleEchoFlow', 'hello'); + const result = await client.waitForOrchestration('lifecycle-echo', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, 'hello'); + } finally { + await runtime.shutdown(100); + } +}); + +test('partial-startup failure retains descendants until actual rollback', { skip: !instrumented }, async () => { + const { runtime, hooks } = await fixture(); + hooks.hold('partial-startup'); + hooks.hold('provider'); + hooks.fail('partial-startup', SECRET); + const startup = runtime.start(); + const failed = assert.rejects(startup, diagnostic('lifecycle_start_failed')); + try { + await hooks.waitEntered('partial-startup'); + await hooks.waitEntered('provider'); + hooks.release('partial-startup'); + await failed; + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + } finally { + hooks.release('partial-startup'); + hooks.release('provider'); + } + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_shutdown_failed')); + await assert.rejects(runtime.shutdown(30_000), diagnostic('lifecycle_shutdown_failed')); + retired(hooks); +}); + +test('contained operational failure is preserved by repeated real shutdown calls', { skip: !instrumented }, async () => { + const { runtime, hooks } = await fixture(); + hooks.hold('worker'); + hooks.fail('worker', SECRET); + await runtime.start(); + await hooks.waitEntered('worker'); + hooks.release('worker'); + await assert.rejects(runtime.shutdown(100), diagnostic('lifecycle_shutdown_failed')); + await assert.rejects(runtime.shutdown(30_000), diagnostic('lifecycle_shutdown_failed')); + retired(hooks); +}); + +for (const force of [false, true]) { + test(`ordered ${force ? 'force-before-retirement' : 'retirement-before-force'} 100 times`, { skip: !instrumented }, async () => { + for (let iteration = 0; iteration < 100; iteration++) { + const { runtime, hooks } = await fixture(); + hooks.hold('provider'); + hooks.hold(force ? 'force' : 'coordinator'); + await runtime.start(); + await hooks.waitEntered('provider'); + const stop = runtime.shutdown(force ? 0 : 30_000); + try { + await hooks.waitEntered(force ? 'force' : 'coordinator'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } finally { + hooks.release('provider'); + hooks.release(force ? 'force' : 'coordinator'); + } + assert.equal(await stop, undefined); + assert.equal(await runtime.shutdown(), undefined); + retired(hooks, force); + } + }); +} + +test('retained completed Runtime does not prevent natural process exit', async () => { + const child = spawn(process.execPath, [path.join(__dirname, 'fixtures', 'lifecycle-probe.cjs'), 'exit'], { + cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: process.env, + }); + let output = ''; + child.stdout.on('data', chunk => { output += chunk; }); + child.stderr.on('data', chunk => { output += chunk; }); + let watchdog; + const exited = new Promise(resolve => child.once('exit', resolve)); + try { + await new Promise((resolve, reject) => { + watchdog = setTimeout(() => reject(new Error('completed runtime kept the process alive')), 3000); + child.once('error', reject); + child.once('exit', code => { + try { + assert.equal(code, 0, output); + assert.match(output, /EXIT_READY/); + resolve(); + } catch (error) { reject(error); } + }); + }); + } finally { + clearTimeout(watchdog); + if (child.exitCode === null && child.signalCode === null) child.kill(); + await exited; + } +}); + +for (const scenario of ['late', 'retained']) { + test(`isolated provider timeout: ${scenario}`, { skip: !instrumented }, async () => { + const child = spawn(process.execPath, ['--expose-gc', path.join(__dirname, 'fixtures', 'lifecycle-probe.cjs'), scenario], { + cwd: root, stdio: ['pipe', 'pipe', 'pipe'], env: process.env, + }); + let stderr = ''; + child.stderr.on('data', chunk => { stderr += chunk; }); + let watchdog; + const lines = createInterface({ input: child.stdout }); + const exited = new Promise(resolve => child.once('exit', resolve)); + try { + await new Promise((resolve, reject) => { + watchdog = setTimeout(() => reject(new Error('probe did not start')), 15_000); + child.once('error', reject); + child.once('exit', code => reject(new Error(`probe exited ${code}: ${stderr}`))); + lines.on('line', line => { + if (!line.startsWith('LIFECYCLE_PROBE ')) return; + try { + const message = JSON.parse(line.slice('LIFECYCLE_PROBE '.length)); + if (message.phase === 'STOPPING') { + clearTimeout(watchdog); + watchdog = setTimeout(() => reject(new Error('default shutdown exceeded seven seconds')), 7000); + } else if (message.phase === 'TIMED_OUT') { + clearTimeout(watchdog); + watchdog = setTimeout(() => reject(new Error('late cleanup/retention evidence missing')), 3000); + assert.ok(message.elapsedMs >= (scenario === 'late' ? 6000 : 5000)); + assert.ok(message.elapsedMs < (scenario === 'late' ? 7000 : 6000)); + assert.ok(message.repeatedMs < 1000); + child.stdin.write(scenario === 'late' ? 'release\n' : 'retain\n'); + } else if (message.phase === 'RECLAIMED' || message.phase === 'RETAINED') { + assert.equal(message.phase, scenario === 'late' ? 'RECLAIMED' : 'RETAINED'); + resolve(); + } else if (message.phase === 'FAILED') { + reject(new Error(message.error)); + } + + } catch (error) { reject(error); } + }); + }); + } finally { + clearTimeout(watchdog); + lines.close(); + child.stdin.destroy(); + if (child.exitCode === null && child.signalCode === null) child.kill(); + await exited; + } + }); +} diff --git a/ci/smoke/smoke.mjs b/ci/smoke/smoke.mjs index c93510a..3acfa0b 100644 --- a/ci/smoke/smoke.mjs +++ b/ci/smoke/smoke.mjs @@ -16,9 +16,22 @@ // output === "Hello, World!". import { SqliteProvider, Client, Runtime } from 'duroxide'; -import { mkdtempSync } from 'node:fs'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, sep } from 'node:path'; + +const require = createRequire(import.meta.url); +const nativePath = Object.keys(require.cache).find(file => file.endsWith('.node') && file.includes('duroxide')); +assert.ok(nativePath?.includes(`${sep}node_modules${sep}`), 'smoke must load the installed platform package'); +const native = require(nativePath); +assert.equal('_lifecycleTestHooks' in native, false); +assert.equal('LifecycleTestHooks' in native, false); +console.log('[smoke] native=' + JSON.stringify({ + path: nativePath, sha256: createHash('sha256').update(readFileSync(nativePath)).digest('hex'), +})); const dir = mkdtempSync(join(tmpdir(), 'duroxide-smoke-')); const dbPath = join(dir, 'smoke.db'); @@ -51,3 +64,9 @@ try { } finally { await runtime.shutdown(200); } + +assert.equal(await runtime.shutdown(0), undefined); +assert.equal(runtime.metricsSnapshot(), null); +const unstarted = new Runtime(provider); +await unstarted.shutdown(0); +await assert.rejects(unstarted.start(), /lifecycle_terminal/); diff --git a/docs/architecture.md b/docs/architecture.md index 0278ed4..58969cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -377,7 +377,9 @@ runtime.registerOrchestration('Parent', function* (ctx, input) { }); ``` -This is the **only fundamental limitation** vs the Rust core — all other features (typed APIs, cancellation propagation, etc.) have full parity. See the `async_blocks.test.js` tests for 12 comprehensive examples of this pattern. +See `async_blocks.test.js` for examples of this pattern. Lifecycle ownership is +also a boundary: this adapter does not track arbitrary JavaScript continuations +or provide the .NET adapter's additional foreign-retirement barrier. ### select/race Supports 2 Tasks @@ -389,7 +391,31 @@ File-based SQLite can hit "database is locked" errors under concurrent orchestra ### Runtime Shutdown -`Runtime.shutdown(timeoutMs)` waits for the full timeout duration unconditionally (the Rust runtime doesn't short-circuit when dispatchers drain). Use a small timeout (e.g., 100ms) in tests. +In the unreleased lifecycle, `Runtime.shutdown(timeoutMs)` remains a +`Promise` but waits for actual core retirement only within a finite budget: +supplied grace plus 5000 ms, with 1000 ms omitted grace. Idle execution can finish +early. Zero requests immediate force; force is not proof of quiescence. +The first valid stop fixes both deadlines, including for repeated observations. + +The adapter stores the prepared core Arc before awaiting fallible startup and +retains it after startup failure, stop, or timeout. Core task/cleanup ownership +survives a dropped waiter. Repeated shutdown observes the same operation rather +than succeeding because an Arc was taken from the wrapper. Valid pre-start stop +is terminal; registration/start afterward reject, and metrics remain unavailable +after stop/startup failure. Same-instance lifecycle/registration calls must remain +serialized. + +On proven core retirement, the shutdown Promise resolver runs on the JavaScript +thread and unreferences retained ThreadsafeFunctions so an otherwise idle process +can exit. Unref does not abort/free callbacks or discard completion. Incomplete +cleanup retains ownership and may keep the process alive. Timeout explicitly +requires application/supervisor process termination, even after late retirement; +the SDK itself never kills the process. + +This is core-owned execution, not complete ownership of every JavaScript task. +Independently owned Clients, providers, and provider token-refresh work remain +separate. See the [lifecycle guide](user-guide.md#runtime-lifecycle) for argument +validation, errors, and release prerequisites. ### Platform-Specific Binary diff --git a/docs/user-guide.md b/docs/user-guide.md index 8359827..add0f5b 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -49,6 +49,66 @@ async function main() { main(); ``` +## Runtime Lifecycle + +This section requires the **unreleased** adapter and matching core lifecycle; +published packages and temporary local core overrides are not release evidence. + +Register handlers before the first `await runtime.start()`. Startup returns after +activation, not after shutdown, and failures reject the Promise. +Registry/preparation failures, including invalid or descending versions, become +ordinary `lifecycle_start_failed` errors retained by subsequent shutdown calls. +Version ordering and duplicate-registration policy are unchanged. Adapter error +messages do not include panic payloads; this does not suppress Rust's separate +process-wide panic-hook output. + +`await runtime.shutdown()` uses one second of grace plus five seconds of cleanup +headroom. `shutdown(0)` requests immediate force, still with a five-second total. +The first stop fixes the deadlines; repeating it never grants a new budget. +Success remains `undefined`; errors carry `lifecycle_start_failed`, +`lifecycle_shutdown_timed_out`, or `lifecycle_shutdown_failed` categories. +Invalid non-finite, fractional, negative, unsafe-integer, or unsupported native +durations reject without consuming registrations or accepting stop. +Strings and `null` also reject; `undefined` means the omitted grace. Native +validation uses signed millisecond extraction and checked unsigned-nanosecond +deadlines: the interval ceiling for grace is +`floor((2^64 - 1) / 1_000_000) - 5000` milliseconds, also subject to the platform's +monotonic clock range. Validation still runs on unused or repeated calls. + +Stopping before start is terminal. Do not overlap lifecycle or registration calls +on the same instance. Metrics return `null` after stop. Repeated shutdown can +observe real late completion, but a timeout permanently rules out reuse and still +requires application/supervisor process termination. Cleanup remains owned; the +SDK never exits the process itself. This contract covers core-owned execution, +not arbitrary unregistered JavaScript continuations or the .NET adapter's +additional foreign-continuation barrier. Independent provider-owned background +work has a separate lifetime. Dropping/ignoring the returned Promise does not +cancel the core's retained cleanup. + +This complete example has no admitted work and can finish before grace: + +```javascript +const { SqliteProvider, Runtime } = require('duroxide'); + +async function main() { + const provider = await SqliteProvider.inMemory(); + const runtime = new Runtime(provider); + await runtime.start(); + await runtime.shutdown(); // 1000 ms grace, 6000 ms total; can finish early + await runtime.shutdown(0); // Observes the same actual completion, not a new stop + console.log(runtime.metricsSnapshot()); // null: stopped worker, not active metrics +} +main().catch(error => { + console.error(error); + process.exitCode = 1; +}); +``` + +For incomplete stop, setting an exit code alone may not terminate retained +noncooperative work. Arrange termination in the application's supervisor policy; +do not treat a caught error or successful late observation as permission to +reuse the runtime. A finite grace cannot promise bounded forced cleanup. + ## Orchestration Patterns ### Sequential Steps diff --git a/lib/duroxide.d.ts b/lib/duroxide.d.ts index df82968..4bded5d 100644 --- a/lib/duroxide.d.ts +++ b/lib/duroxide.d.ts @@ -313,7 +313,14 @@ export declare class Runtime { registerOrchestrationTyped(name: string, fn: (ctx: OrchestrationContext, input: TIn) => Generator): void; registerOrchestrationVersioned(name: string, version: string, fn: (ctx: OrchestrationContext, input: any) => Generator): void; registerOrchestrationVersionedTyped(name: string, version: string, fn: (ctx: OrchestrationContext, input: TIn) => Generator): void; + /** Start once; returns after startup. Do not overlap lifecycle/registration calls. */ start(): Promise; + /** + * Stop permanently, including before start. Default grace is 1000 ms; total is grace + 5000 ms. + * Repeated calls retain the original deadlines and observe actual completion/errors. + * Timeout rejects while cleanup remains owned; application/supervisor termination is required. + * timeoutMs must be a nonnegative safe integer within the native monotonic deadline range. + */ shutdown(timeoutMs?: number): Promise; metricsSnapshot(): JsMetricsSnapshot | null; } diff --git a/lib/duroxide.js b/lib/duroxide.js index ebc9b4a..6561da6 100644 --- a/lib/duroxide.js +++ b/lib/duroxide.js @@ -1138,8 +1138,8 @@ class Runtime { * @param {GeneratorFunction} fn - function*(ctx, input) { ... } */ registerOrchestration(name, fn) { - orchestrationFunctions.set(name, fn); this._native.registerOrchestration(name); + orchestrationFunctions.set(name, fn); } /** @@ -1180,24 +1180,30 @@ class Runtime { */ registerOrchestrationVersioned(name, version, fn) { const key = `${name}@${version}`; - orchestrationFunctions.set(key, fn); // Don't overwrite the plain name key — it serves as the fallback // for versions registered via registerOrchestration() (unversioned). this._native.registerOrchestrationVersioned(name, version); + orchestrationFunctions.set(key, fn); } /** - * Start the runtime. Blocks until shutdown is called. + * Start once and return after startup. Do not overlap lifecycle/registration calls. */ async start() { await this._native.start(); } /** - * Shutdown the runtime gracefully. - * @param {number} [timeoutMs] + * Stop permanently. Omitted grace is 1000 ms; the total bound is grace plus 5000 ms. + * Repeated calls observe the original stop, including failures or actual late completion. + * Timeout leaves cleanup owned and requires application/supervisor process termination. + * Even shutdown before start is terminal. Do not overlap lifecycle/registration calls. + * @param {number} [timeoutMs] Nonnegative safe-integer milliseconds in the native deadline range. */ async shutdown(timeoutMs) { + if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0)) { + throw new RangeError('lifecycle_invalid_timeout: timeoutMs must be a nonnegative safe integer'); + } await this._native.shutdown(timeoutMs); } diff --git a/package.json b/package.json index cc7bed2..4edc175 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "artifacts": "napi artifacts", "build": "napi build --platform --release", "build:debug": "napi build --platform", - "test": "node --test __tests__/e2e.test.js __tests__/kv_store.test.js __tests__/races.test.js __tests__/admin_api.test.js __tests__/sessions.test.js __tests__/advanced_features.test.js __tests__/async_blocks.test.js __tests__/typed_apis.test.js __tests__/copilot_chat.test.js", + "test": "node --test __tests__/e2e.test.js __tests__/kv_store.test.js __tests__/races.test.js __tests__/admin_api.test.js __tests__/sessions.test.js __tests__/advanced_features.test.js __tests__/async_blocks.test.js __tests__/typed_apis.test.js __tests__/copilot_chat.test.js __tests__/lifecycle.test.js", + "test:lifecycle": "node --test __tests__/lifecycle.test.js", "test:races": "node --test __tests__/races.test.js", "test:admin": "node --test __tests__/admin_api.test.js", "test:scenarios": "node --test __tests__/scenarios/*.test.js", diff --git a/src/lib.rs b/src/lib.rs index 6730651..0885a2e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,12 +12,16 @@ mod handlers; mod pg_provider; mod provider; mod runtime; +#[cfg(feature = "test-hooks")] +mod test_hooks; mod types; pub use client::JsClient; pub use pg_provider::JsPostgresProvider; pub use provider::JsSqliteProvider; pub use runtime::{JsRuntime, JsRuntimeOptions}; +#[cfg(feature = "test-hooks")] +pub use test_hooks::lifecycle_test_hooks; pub use types::{JsOrchestrationStatus, JsQueueDepths, JsSystemMetrics}; /// Emit an activity trace through the current Rust ActivityContext. diff --git a/src/runtime.rs b/src/runtime.rs index 0a1d4f1..f99f33a 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -4,18 +4,19 @@ use napi::bindgen_prelude::*; use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction}; use napi_derive::napi; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; -use duroxide::runtime::{self, OrchestrationRegistry}; use duroxide::runtime::OrchestrationHandler; +use duroxide::runtime::{self, OrchestrationRegistry}; -use duroxide::runtime::LogFormat; use duroxide::providers::TagFilter; +use duroxide::runtime::LogFormat; use crate::handlers::{JsActivityHandler, JsOrchestrationHandler}; -use crate::provider::JsSqliteProvider; use crate::pg_provider::JsPostgresProvider; +use crate::provider::JsSqliteProvider; use crate::types::JsMetricsSnapshot; /// Runtime options configurable from JavaScript. @@ -63,6 +64,12 @@ pub struct JsRuntime { dispose_fn: Option>, options: Option, inner: Option>, + callback_refs: Vec>, + started: bool, + shutdown_requested: bool, + startup_error: Option, + #[cfg(feature = "test-hooks")] + pub(crate) test_hooks: runtime::test_hooks::LifecycleHooks, } #[napi] @@ -78,6 +85,12 @@ impl JsRuntime { dispose_fn: None, options, inner: None, + callback_refs: Vec::new(), + started: false, + shutdown_requested: false, + startup_error: None, + #[cfg(feature = "test-hooks")] + test_hooks: runtime::test_hooks::LifecycleHooks::default(), } } @@ -93,6 +106,12 @@ impl JsRuntime { dispose_fn: None, options, inner: None, + callback_refs: Vec::new(), + started: false, + shutdown_requested: false, + startup_error: None, + #[cfg(feature = "test-hooks")] + test_hooks: runtime::test_hooks::LifecycleHooks::default(), } } @@ -104,10 +123,12 @@ impl JsRuntime { create_fn: ThreadsafeFunction, next_fn: ThreadsafeFunction, dispose_fn: ThreadsafeFunction, - ) { + ) -> Result<()> { + self.ensure_configurable()?; self.create_fn = Some(create_fn); self.next_fn = Some(next_fn); self.dispose_fn = Some(dispose_fn); + Ok(()) } /// Register a JavaScript activity function. @@ -117,21 +138,31 @@ impl JsRuntime { &mut self, name: String, callback: ThreadsafeFunction, - ) { + ) -> Result<()> { + self.ensure_configurable()?; self.activity_builders.push((name, callback)); + Ok(()) } /// Register a JavaScript orchestration (generator function). /// The orchestration name is used for both registration and the generator function lookup. #[napi] - pub fn register_orchestration(&mut self, name: String) { + pub fn register_orchestration(&mut self, name: String) -> Result<()> { + self.ensure_configurable()?; self.orchestration_names.push((name, None)); + Ok(()) } /// Register a versioned JavaScript orchestration. #[napi] - pub fn register_orchestration_versioned(&mut self, name: String, version: String) { + pub fn register_orchestration_versioned( + &mut self, + name: String, + version: String, + ) -> Result<()> { + self.ensure_configurable()?; self.orchestration_names.push((name, Some(version))); + Ok(()) } /// Start the runtime. This processes orchestrations and activities until shutdown. @@ -140,157 +171,290 @@ impl JsRuntime { /// This is async and takes &mut self. napi-rs requires async &mut methods to be marked unsafe. #[napi] pub async unsafe fn start(&mut self) -> Result<()> { - // Build activity registry - let mut activity_builder = duroxide::runtime::registry::ActivityRegistry::builder(); - for (name, callback) in self.activity_builders.drain(..) { - let handler = Arc::new(JsActivityHandler::new(name.clone(), callback)); - activity_builder = activity_builder.register(&name, move |ctx, input| { - let h = handler.clone(); - async move { h.invoke(ctx, input).await } - }); + self.ensure_configurable()?; + if self.create_fn.is_none() || self.next_fn.is_none() || self.dispose_fn.is_none() { + return Err(Error::from_reason( + "lifecycle_start_failed: generator driver not set", + )); } - let activities = activity_builder.build(); - - // Build orchestration registry - let create_fn = self - .create_fn - .take() - .ok_or_else(|| Error::from_reason("Generator driver not set. Call setGeneratorDriver() before start()."))?; - let next_fn = self - .next_fn - .take() - .ok_or_else(|| Error::from_reason("Generator driver not set."))?; - let dispose_fn = self - .dispose_fn - .take() - .ok_or_else(|| Error::from_reason("Generator driver not set."))?; + self.started = true; + let prepared = catch_unwind(AssertUnwindSafe(|| { + let mut callback_refs: Vec<_> = self + .activity_builders + .iter() + .map(|(_, callback)| callback.clone()) + .collect(); - let mut orch_builder = OrchestrationRegistry::builder(); - for (name, version) in self.orchestration_names.drain(..) { - let handler = Arc::new(JsOrchestrationHandler::new( - create_fn.clone(), - next_fn.clone(), - dispose_fn.clone(), - )); - if let Some(ver) = version { - orch_builder = orch_builder.register_versioned(&name, &ver, move |ctx, input| { - let h = handler.clone(); - async move { h.invoke(ctx, input).await } - }); - } else { - orch_builder = orch_builder.register(&name, move |ctx, input| { + // Build activity registry + let mut activity_builder = duroxide::runtime::registry::ActivityRegistry::builder(); + for (name, callback) in self.activity_builders.drain(..) { + let handler = Arc::new(JsActivityHandler::new(name.clone(), callback)); + activity_builder = activity_builder.register(&name, move |ctx, input| { let h = handler.clone(); async move { h.invoke(ctx, input).await } }); } - } - let orchestrations = orch_builder.build(); + let activities = activity_builder.build(); - // Build runtime options - let mut rt_options = runtime::RuntimeOptions::default(); - if let Some(ref opts) = self.options { - if let Some(c) = opts.orchestration_concurrency { - rt_options.orchestration_concurrency = c as usize; - } - if let Some(c) = opts.worker_concurrency { - rt_options.worker_concurrency = c as usize; - } - if let Some(ms) = opts.dispatcher_poll_interval_ms { - rt_options.dispatcher_min_poll_interval = Duration::from_millis(ms as u64); - } - if let Some(ms) = opts.worker_lock_timeout_ms { - rt_options.worker_lock_timeout = Duration::from_millis(ms as u64); - } - if let Some(ref fmt) = opts.log_format { - rt_options.observability.log_format = match fmt.as_str() { - "json" => LogFormat::Json, - "pretty" => LogFormat::Pretty, - _ => LogFormat::Compact, - }; - } - if let Some(ref level) = opts.log_level { - rt_options.observability.log_level = level.clone(); - } - if let Some(ref name) = opts.service_name { - rt_options.observability.service_name = name.clone(); - } - if let Some(ref ver) = opts.service_version { - rt_options.observability.service_version = Some(ver.clone()); - } - if let Some(max) = opts.max_sessions_per_runtime { - rt_options.max_sessions_per_runtime = max as usize; - } - if let Some(ms) = opts.session_idle_timeout_ms { - rt_options.session_idle_timeout = Duration::from_millis(ms as u64); + // Build orchestration registry + let create_fn = self + .create_fn + .take() + .ok_or(runtime::RuntimeStartError::StartupFailed)?; + let next_fn = self + .next_fn + .take() + .ok_or(runtime::RuntimeStartError::StartupFailed)?; + let dispose_fn = self + .dispose_fn + .take() + .ok_or(runtime::RuntimeStartError::StartupFailed)?; + callback_refs.extend([create_fn.clone(), next_fn.clone(), dispose_fn.clone()]); + + let mut orch_builder = OrchestrationRegistry::builder(); + for (name, version) in self.orchestration_names.drain(..) { + let handler = Arc::new(JsOrchestrationHandler::new( + create_fn.clone(), + next_fn.clone(), + dispose_fn.clone(), + )); + if let Some(ver) = version { + orch_builder = + orch_builder.register_versioned(&name, &ver, move |ctx, input| { + let h = handler.clone(); + async move { h.invoke(ctx, input).await } + }); + } else { + orch_builder = orch_builder.register(&name, move |ctx, input| { + let h = handler.clone(); + async move { h.invoke(ctx, input).await } + }); + } } - if let Some(ref nid) = opts.worker_node_id { - rt_options.worker_node_id = Some(nid.clone()); + let orchestrations = orch_builder.build(); + + // Build runtime options + let mut rt_options = runtime::RuntimeOptions::default(); + if let Some(ref opts) = self.options { + if let Some(c) = opts.orchestration_concurrency { + rt_options.orchestration_concurrency = c as usize; + } + if let Some(c) = opts.worker_concurrency { + rt_options.worker_concurrency = c as usize; + } + if let Some(ms) = opts.dispatcher_poll_interval_ms { + rt_options.dispatcher_min_poll_interval = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.worker_lock_timeout_ms { + rt_options.worker_lock_timeout = Duration::from_millis(ms as u64); + } + if let Some(ref fmt) = opts.log_format { + rt_options.observability.log_format = match fmt.as_str() { + "json" => LogFormat::Json, + "pretty" => LogFormat::Pretty, + _ => LogFormat::Compact, + }; + } + if let Some(ref level) = opts.log_level { + rt_options.observability.log_level = level.clone(); + } + if let Some(ref name) = opts.service_name { + rt_options.observability.service_name = name.clone(); + } + if let Some(ref ver) = opts.service_version { + rt_options.observability.service_version = Some(ver.clone()); + } + if let Some(max) = opts.max_sessions_per_runtime { + rt_options.max_sessions_per_runtime = max as usize; + } + if let Some(ms) = opts.session_idle_timeout_ms { + rt_options.session_idle_timeout = Duration::from_millis(ms as u64); + } + if let Some(ref nid) = opts.worker_node_id { + rt_options.worker_node_id = Some(nid.clone()); + } + if let Some(ref filter) = opts.worker_tag_filter { + rt_options.worker_tag_filter = match filter.as_str() { + "defaultOnly" => TagFilter::DefaultOnly, + "any" => TagFilter::Any, + "none" => TagFilter::None, + "tags" => { + let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); + TagFilter::Tags(tags.into_iter().collect()) + } + "defaultAnd" => { + let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); + TagFilter::DefaultAnd(tags.into_iter().collect()) + } + _ => TagFilter::DefaultOnly, + }; + } } - if let Some(ref filter) = opts.worker_tag_filter { - rt_options.worker_tag_filter = match filter.as_str() { - "defaultOnly" => TagFilter::DefaultOnly, - "any" => TagFilter::Any, - "none" => TagFilter::None, - "tags" => { - let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); - TagFilter::Tags(tags.into_iter().collect()) - } - "defaultAnd" => { - let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); - TagFilter::DefaultAnd(tags.into_iter().collect()) - } - _ => TagFilter::DefaultOnly, - }; + + let provider = self.provider.clone(); + #[cfg(feature = "test-hooks")] + let provider = Arc::new(runtime::test_hooks::TestProvider::new( + provider, + self.test_hooks.clone(), + )); + let rt = runtime::Runtime::prepare(provider, activities, orchestrations, rt_options)?; + self.callback_refs = callback_refs; + Ok::<_, runtime::RuntimeStartError>(rt) + })); + let rt = match prepared { + Ok(result) => result, + Err(payload) => { + // Payload destruction must not reintroduce a panic across the language boundary. + if let Err(secondary) = catch_unwind(AssertUnwindSafe(|| drop(payload))) { + std::mem::forget(secondary); + } + Err(runtime::RuntimeStartError::StartupFailed) } } - - let provider = self.provider.clone(); - let rt = runtime::Runtime::start_with_options( - provider, - activities, - orchestrations, - rt_options, - ) - .await; - - self.inner = Some(rt); - Ok(()) + .map_err(|error| { + self.startup_error = Some(error); + start_error(error) + })?; + self.inner = Some(rt.clone()); + #[cfg(feature = "test-hooks")] + rt.set_lifecycle_hooks(self.test_hooks.clone()) + .map_err(|error| { + self.startup_error = Some(error); + start_error(error) + })?; + rt.start_execution().await.map_err(|error| { + self.startup_error = Some(error); + start_error(error) + }) } /// Get a snapshot of runtime metrics. #[napi] pub fn metrics_snapshot(&self) -> Option { - self.inner.as_ref()?.metrics_snapshot().map(|m| JsMetricsSnapshot { - orch_starts: m.orch_starts as i64, - orch_completions: m.orch_completions as i64, - orch_failures: m.orch_failures as i64, - orch_application_errors: m.orch_application_errors as i64, - orch_infrastructure_errors: m.orch_infrastructure_errors as i64, - orch_configuration_errors: m.orch_configuration_errors as i64, - orch_poison: m.orch_poison as i64, - activity_success: m.activity_success as i64, - activity_app_errors: m.activity_app_errors as i64, - activity_infra_errors: m.activity_infra_errors as i64, - activity_config_errors: m.activity_config_errors as i64, - activity_poison: m.activity_poison as i64, - orch_dispatcher_items_fetched: m.orch_dispatcher_items_fetched as i64, - worker_dispatcher_items_fetched: m.worker_dispatcher_items_fetched as i64, - orch_continue_as_new: m.orch_continue_as_new as i64, - suborchestration_calls: m.suborchestration_calls as i64, - provider_errors: m.provider_errors as i64, - }) + if self.shutdown_requested || self.startup_error.is_some() { + return None; + } + self.inner + .as_ref()? + .metrics_snapshot() + .map(|m| JsMetricsSnapshot { + orch_starts: m.orch_starts as i64, + orch_completions: m.orch_completions as i64, + orch_failures: m.orch_failures as i64, + orch_application_errors: m.orch_application_errors as i64, + orch_infrastructure_errors: m.orch_infrastructure_errors as i64, + orch_configuration_errors: m.orch_configuration_errors as i64, + orch_poison: m.orch_poison as i64, + activity_success: m.activity_success as i64, + activity_app_errors: m.activity_app_errors as i64, + activity_infra_errors: m.activity_infra_errors as i64, + activity_config_errors: m.activity_config_errors as i64, + activity_poison: m.activity_poison as i64, + orch_dispatcher_items_fetched: m.orch_dispatcher_items_fetched as i64, + worker_dispatcher_items_fetched: m.worker_dispatcher_items_fetched as i64, + orch_continue_as_new: m.orch_continue_as_new as i64, + suborchestration_calls: m.suborchestration_calls as i64, + provider_errors: m.provider_errors as i64, + }) } - /// Shutdown the runtime gracefully. - /// - /// # Safety - /// Must not be called concurrently from multiple threads. - #[napi] - pub async unsafe fn shutdown(&mut self, timeout_ms: Option) -> Result<()> { - if let Some(rt) = self.inner.take() { - let timeout = timeout_ms.map(|ms| ms as u64); - rt.shutdown(timeout).await; + /// Stop with bounded waiting. Do not overlap lifecycle/registration calls. + #[napi(ts_return_type = "Promise")] + pub fn shutdown(&mut self, env: Env, timeout_ms: Option) -> Result { + let (grace, grace_deadline, total_deadline) = shutdown_deadlines(timeout_ms)?; + if let Some(rt) = &self.inner { + rt.request_shutdown_until(grace_deadline, total_deadline) + .map_err(shutdown_error)?; + } + self.shutdown_requested = true; + if self.inner.is_none() { + self.activity_builders.clear(); + self.orchestration_names.clear(); + self.create_fn = None; + self.next_fn = None; + self.dispose_fn = None; + } + let runtime = self.inner.clone(); + let startup_error = self.startup_error; + let mut callbacks = self.callback_refs.clone(); + env.execute_tokio_future( + async move { + Ok(match runtime { + Some(runtime) => Some(runtime.shutdown_with_grace(grace).await), + None => None, + }) + }, + move |env, result| { + let quiescent = match result { + None | Some(Ok(_)) => true, + Some(Err(error)) => error.is_quiescent(), + }; + // Keep the core completion record, without keeping Node's event loop + // alive after retirement. N-API unref must run on the JS thread. + if quiescent { + for callback in &mut callbacks { + callback.unref(env).map_err(|_| { + Error::from_reason( + "lifecycle_shutdown_failed: callback event-loop reference release failed", + ) + })?; + } + } + match result { + Some(result) => result.map_err(shutdown_error).and_then(|_| { + startup_error.map_or(Ok(()), |error| Err(start_error(error))) + }), + None => startup_error.map_or(Ok(()), |error| Err(start_error(error))), + } + }, + ) + } +} + +impl JsRuntime { + fn ensure_configurable(&self) -> Result<()> { + if self.started || self.shutdown_requested { + return Err(Error::from_reason( + "lifecycle_terminal: runtime has already started or shutdown was requested", + )); } Ok(()) } } + +fn start_error(error: runtime::RuntimeStartError) -> Error { + Error::from_reason(format!("lifecycle_start_failed: {error}")) +} + +fn shutdown_error(error: runtime::RuntimeShutdownError) -> Error { + let category = match error { + runtime::RuntimeShutdownError::TimedOut => "lifecycle_shutdown_timed_out", + runtime::RuntimeShutdownError::InvalidTimeouts + | runtime::RuntimeShutdownError::DeadlineOverflow => "lifecycle_invalid_timeout", + _ => "lifecycle_shutdown_failed", + }; + Error::from_reason(format!("{category}: {error}")) +} + +fn shutdown_deadlines(timeout_ms: Option) -> Result<(Duration, Instant, Instant)> { + let invalid = || { + Error::from_reason( + "lifecycle_invalid_timeout: timeoutMs must be a nonnegative supported millisecond duration", + ) + }; + let millis = u64::try_from(timeout_ms.unwrap_or(1000)).map_err(|_| invalid())?; + let grace = Duration::from_millis(millis); + let total = grace + .checked_add(Duration::from_secs(5)) + .ok_or_else(invalid)?; + if total > Duration::from_nanos(u64::MAX) { + return Err(invalid()); + } + let now = Instant::now(); + let grace_deadline = now.checked_add(grace).ok_or_else(invalid)?; + let total_deadline = now.checked_add(total).ok_or_else(invalid)?; + total_deadline + .checked_add(Duration::from_millis(1)) + .ok_or_else(invalid)?; + Ok((grace, grace_deadline, total_deadline)) +} diff --git a/src/test_hooks.rs b/src/test_hooks.rs new file mode 100644 index 0000000..14c5876 --- /dev/null +++ b/src/test_hooks.rs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use duroxide::runtime::test_hooks::{ + LifecycleHooks, LifecyclePoint, LifecycleTask, ProviderOperation, +}; +use napi::{Error, Result}; +use napi_derive::napi; +use std::time::Duration; + +use crate::runtime::JsRuntime; + +#[napi] +pub struct LifecycleTestHooks { + hooks: LifecycleHooks, +} + +#[napi(js_name = "_lifecycleTestHooks")] +pub fn lifecycle_test_hooks(runtime: &JsRuntime) -> LifecycleTestHooks { + LifecycleTestHooks { + hooks: runtime.test_hooks.clone(), + } +} + +#[napi] +impl LifecycleTestHooks { + #[napi] + pub fn hold(&self, point: String) -> Result<()> { + self.hooks + .hold(parse_point(&point)?) + .map(|_| ()) + .map_err(|error| Error::from_reason(format!("test gate: {error:?}"))) + } + + #[napi] + pub fn release(&self, point: String) -> Result<()> { + self.hooks + .gate(parse_point(&point)?) + .ok_or_else(|| Error::from_reason("test gate not armed"))? + .release(); + Ok(()) + } + + #[napi] + pub async fn wait_entered(&self, point: String) -> Result<()> { + let gate = self + .hooks + .gate(parse_point(&point)?) + .ok_or_else(|| Error::from_reason("test gate not armed"))?; + tokio::time::timeout(Duration::from_secs(3), gate.entered()) + .await + .map_err(|_| Error::from_reason("test gate entry timed out")) + } + + #[napi] + pub fn fail(&self, point: String, message: String) -> Result<()> { + self.hooks + .fail_once_with_message(parse_point(&point)?, message) + .map_err(|error| Error::from_reason(format!("test fault: {error:?}"))) + } + + #[napi] + pub fn snapshot(&self) -> String { + let counts = self.hooks.counts(); + serde_json::json!({ + "started": counts.iter().map(|count| count.started).sum::(), + "completed": counts.iter().map(|count| count.completed).sum::(), + "active": counts.iter().map(|count| count.active).sum::(), + "cleanupActive": self.hooks.cleanup_counts().active, + "coordinators": self.hooks.hits(LifecyclePoint::ShutdownCoordinator), + "forceRequests": self.hooks.hits(LifecyclePoint::ForceRequested), + }) + .to_string() + } +} + +fn parse_point(point: &str) -> Result { + match point { + "startup" => Ok(LifecyclePoint::StartupBeforeIo), + "partial-startup" => Ok(LifecyclePoint::StartupAfterSpawn( + LifecycleTask::OrchestrationDispatcher, + )), + "provider" => Ok(LifecyclePoint::ProviderReturn( + ProviderOperation::FetchOrchestration, + )), + "worker" => Ok(LifecyclePoint::ParentWork(LifecycleTask::WorkDispatcher)), + "force" => Ok(LifecyclePoint::ForceRequested), + "coordinator" => Ok(LifecyclePoint::ShutdownCoordinator), + _ => Err(Error::from_reason("unknown lifecycle test point")), + } +} From 851ee84c269d7a772b594d3f3b8e983208003fc7 Mon Sep 17 00:00:00 2001 From: "Christopher Krawczyk (SQL)" Date: Wed, 21 Oct 2026 13:57:41 -0700 Subject: [PATCH 2/4] fix: release retired runtime ownership and enforce lifecycle coverage Release obsolete native callback holders only after proven cleanup. Add isolated recurring lifecycle instrumentation, installed-addon provenance and stronger timing, registration and ownership regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 73748d4f-2eb6-47d2-9dbc-af8681e80e83 --- .github/workflows/build.yml | 55 +++ CONTRIBUTING.md | 58 +++- README.md | 12 +- .../fixtures/lifecycle-collection-probe.cjs | 149 +++++++++ __tests__/fixtures/lifecycle-probe.cjs | 36 +- __tests__/helpers/lifecycle-oracles.cjs | 108 ++++++ __tests__/helpers/lifecycle-probe.cjs | 118 +++++++ __tests__/lifecycle-oracles.test.js | 140 ++++++++ __tests__/lifecycle.test.js | 312 ++++++++++-------- ci/lifecycle-report.cjs | 41 +++ ci/run-lifecycle.cjs | 58 ++++ ci/smoke/README.md | 6 + ci/smoke/native-provenance.cjs | 59 ++++ ci/smoke/run-local.ps1 | 1 + ci/smoke/run-local.sh | 1 + ci/smoke/run-registry.ps1 | 1 + ci/smoke/run-registry.sh | 1 + ci/smoke/smoke.mjs | 16 +- ci/stage-lifecycle.cjs | 50 +++ docs/user-guide.md | 40 +++ package.json | 2 +- src/runtime.rs | 120 ++++--- 22 files changed, 1166 insertions(+), 218 deletions(-) create mode 100644 __tests__/fixtures/lifecycle-collection-probe.cjs create mode 100644 __tests__/helpers/lifecycle-oracles.cjs create mode 100644 __tests__/helpers/lifecycle-probe.cjs create mode 100644 __tests__/lifecycle-oracles.test.js create mode 100644 ci/lifecycle-report.cjs create mode 100644 ci/run-lifecycle.cjs create mode 100644 ci/smoke/native-provenance.cjs create mode 100644 ci/stage-lifecycle.cjs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3919933..d55b749 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,6 +83,61 @@ jobs: name: binary-linux-x64-gnu - name: Run tests run: npm run test:all + - name: Stage isolated production lifecycle suite + run: node ci/stage-lifecycle.cjs . target/lifecycle-production/stage production + - name: Require production lifecycle and export boundary + run: node ci/run-lifecycle.cjs production target/lifecycle-production/stage + - name: Upload production lifecycle evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-production-evidence-linux-x64 + path: | + target/lifecycle-production/stage/lifecycle-native.json + target/lifecycle-production/stage/lifecycle-results.json + target/lifecycle-production/stage/lifecycle.tap + target/lifecycle-production/stage/lifecycle.stderr.log + if-no-files-found: error + + lifecycle-instrumented: + name: Instrumented lifecycle (isolated) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 20 + - name: Set up Rust from repository toolchain + run: | + test -f rust-toolchain.toml + rustup show active-toolchain + rustc --version --verbose + - name: Install dependencies + run: npm install --omit=optional --ignore-scripts --no-audit --no-fund --package-lock=false + # No fallback override: compatible published core APIs are a separate gate. + - name: Build isolated instrumented addon + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target/lifecycle-instrumented/cargo + run: >- + npx napi build --platform --features test-hooks --js false --dts bindings.d.ts + --cargo-flags="--locked" target/lifecycle-instrumented/native + - name: Stage matching source and native artifact + run: node ci/stage-lifecycle.cjs target/lifecycle-instrumented/native target/lifecycle-instrumented/stage instrumented + - name: Require every instrumented lifecycle case + run: node ci/run-lifecycle.cjs instrumented target/lifecycle-instrumented/stage + # Upload evidence only, never an instrumented binary-* packaging input. + - name: Upload instrumented lifecycle evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-instrumented-evidence-linux-x64 + path: | + target/lifecycle-instrumented/stage/lifecycle-native.json + target/lifecycle-instrumented/stage/lifecycle-results.json + target/lifecycle-instrumented/stage/lifecycle.tap + target/lifecycle-instrumented/stage/lifecycle.stderr.log + if-no-files-found: error # ── Pack tarballs for smoke (local install simulation) ───── pack: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ad9cb4..f201108 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,19 +42,55 @@ After Rust source changes (`src/*.rs`), re-run `npx napi build --platform` befor The lifecycle suite requires a freshly built local native module, not an optional prebuilt platform package. Its private controls are absent from production builds. -For example, in PowerShell: +Keep native outputs, Cargo targets and staged sources separate in both modes. +For example, from the repository root in PowerShell: ```powershell -npm run build:debug -- --features test-hooks --js false --dts target\lifecycle-native.d.ts -$env:DUROXIDE_LIFECYCLE_TEST_HOOKS = '1' -npm run test:lifecycle -Remove-Item Env:DUROXIDE_LIFECYCLE_TEST_HOOKS -npm run build:debug -- --js false --dts target\lifecycle-native.d.ts -npm run test:lifecycle +$env:CARGO_TARGET_DIR = Join-Path $PWD 'target\lifecycle-production\cargo' +npx napi build --platform --js false --dts bindings.d.ts --cargo-flags="--locked" target\lifecycle-production\native +node ci\stage-lifecycle.cjs target\lifecycle-production\native target\lifecycle-production\stage production +node ci\run-lifecycle.cjs production target\lifecycle-production\stage + +$env:CARGO_TARGET_DIR = Join-Path $PWD 'target\lifecycle-instrumented\cargo' +npx napi build --platform --features test-hooks --js false --dts bindings.d.ts --cargo-flags="--locked" target\lifecycle-instrumented\native +node ci\stage-lifecycle.cjs target\lifecycle-instrumented\native target\lifecycle-instrumented\stage instrumented +node ci\run-lifecycle.cjs instrumented target\lifecycle-instrumented\stage +Remove-Item Env:CARGO_TARGET_DIR ``` Instrumented cases exercise real provider waits and contained owned-task faults. -The suite checks native import provenance, 100 repetitions per ordered race, -bounded child-process cleanup, and production export absence. Production mode -skips only hook-dependent cases. Do not publish instrumented assets or temporary -local core overrides. A matching published core minimum is required before release. \ No newline at end of file +The recurring `lifecycle-instrumented` CI job runs on pull requests and main, +alongside production tests. It requires every hook-dependent case to execute: +missing hooks, mode/hash mismatches, skips, TODOs, aborted tests or empty selection +fail validation. Production mode skips only explicitly marked instrumented cases +and continues to assert that private exports are absent. + +The stage manifest identifies the exact local addon and SHA-256, selected feature +mode and source hashes. Tests match the Duroxide loader's native class identities, +not an arbitrary cached `.node`. Sources live under a stage-local `node_modules` +directory so root-level test discovery cannot run duplicate staged suites. The +runner sets mode/provenance only in its child environment and retains TAP, stderr +and JSON totals beside the manifest. Instrumented evidence artifacts are not +named `binary-*`, contain no native binaries, and never feed packing/release jobs. +`CARGO_TARGET_DIR` also tells napi where to copy the compiled library from; +passing only Cargo's `--target-dir` would not set napi's source path. +The `--dts` filename is relative to napi's destination directory. + +Both ordered races still run **100 iterations**. Held-provider probes still +require `6000 <= elapsedMs < 7000` for omitted grace, `5000 <= elapsedMs < 6000` +for zero grace, and `repeatedMs < 1000`. Their separate message-delivery watchdog +allows the first call, repeated call and bounded transport headroom; it cannot +relax those measured SDK limits. The suite also checks incomplete-cleanup +event-loop liveness without stdin/referenced timers, and activity-only +self-capturing callback collection in a live process after creating frames retire. +Collection uses bounded WeakRef/FinalizationRegistry settling with a noncapturing +control, not natural process exit or zero task counts. Successful, quiescent-error +and late cleanup are covered, including late release with no further stop caller. + +Native-independent harness/oracle checks can run before a native build: +`node --test __tests__/lifecycle-oracles.test.js`. They include deliberate child +failure, malformed protocol, zero-start counter and CI-skip negative controls. + +Do not publish instrumented assets or temporary local core overrides. A matching +published core minimum remains a separate prerequisite: CI must fail missing core +APIs rather than silently skip coverage or substitute an unofficial dependency. \ No newline at end of file diff --git a/README.md b/README.md index 2a36539..c081556 100644 --- a/README.md +++ b/README.md @@ -32,16 +32,26 @@ Node.js/TypeScript SDK for the [Duroxide](https://github.com/microsoft/duroxide) The argument is a nonnegative safe-integer grace period in milliseconds, within the native monotonic deadline range. Omission means 1000 ms; total waiting is grace plus 5000 ms. Idle work can finish early. +At the first stop call, the supervisor's **remaining** time must exceed that total, +with room for other shutdown work, result delivery and process exit. Passing the +entire outer budget as grace cannot fit; see the +[budget-sizing guidance](docs/user-guide.md#sizing-the-supervisor-budget). The first stop fixes both deadlines. Repeated calls observe the same operation, including timeout/operational errors and genuine late completion. Shutdown before start is now terminal; later start or registration is rejected. Metrics remain unavailable after stop. Serialize lifecycle and registration calls on each runtime. +Actual quiescence releases obsolete native callback/core holders, including after +a quiescent failure or late cleanup without another caller. Stop results and the +original deadlines remain; incomplete or unproven cleanup retains ownership. A `lifecycle_shutdown_timed_out` rejection means cleanup is still owned and incomplete. Do not restart the worker: arrange process termination through the application or supervisor, even if a later observation reports completion. The SDK -does not terminate the process. This narrow adapter change does not add tracking +does not terminate the process. `lifecycle_shutdown_failed` is an error category, +not proof of quiescence: it also covers an unproven/lost cleanup owner and a +callback-reference release failure after core cleanup. Do not infer release +permission from that prefix. This narrow adapter change does not add tracking of arbitrary JavaScript continuations or the .NET SDK's additional managed foreign-retirement barrier. Independently owned provider/Client resources have their own lifetime. See the [lifecycle guide](docs/user-guide.md#runtime-lifecycle) diff --git a/__tests__/fixtures/lifecycle-collection-probe.cjs b/__tests__/fixtures/lifecycle-collection-probe.cjs new file mode 100644 index 0000000..cfb7e38 --- /dev/null +++ b/__tests__/fixtures/lifecycle-collection-probe.cjs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { performance } = require('node:perf_hooks'); +const { setImmediate: nextTurn, setTimeout: delay } = require('node:timers/promises'); +const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); +const { nativeProvenance } = require('../../ci/smoke/native-provenance.cjs'); +const { incompleteTimeout, knownQuiescentCoreFailure, retiredState } = require('../helpers/lifecycle-oracles.cjs'); + +const { native, provenance } = nativeProvenance(path.resolve(__dirname, '..', '..'), { + instrumented: process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1', + manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, +}); +const finalized = new Set(); +const registry = new FinalizationRegistry(label => finalized.add(label)); +const emit = (phase, detail) => console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase, ...detail })); + +function capturingActivity(owner) { + return async (_, input) => `${owner.prefix}:${input}`; +} + +function noncapturingActivity() { + return async (_, input) => input; +} + +async function createOwner(label, scenario, captures) { + const runtime = new Runtime(await SqliteProvider.inMemory(), { dispatcherPollIntervalMs: 5 }); + const owner = { runtime, prefix: label }; + const callback = captures ? capturingActivity(owner) : noncapturingActivity(); + const records = {}; + for (const [name, value] of Object.entries({ owner, runtime, native: runtime._native, callback })) { + const key = `${label}:${name}`; + registry.register(value, key); + records[name] = { label: key, weak: new WeakRef(value) }; + } + // Activity-only registration excludes the pre-existing global orchestration map. + runtime.registerActivity(`Collection${label}`, callback); + const hooks = scenario === 'completed' ? undefined : native._lifecycleTestHooks(runtime._native); + if (scenario === 'quiescent-failure') { + hooks.hold('worker'); + hooks.fail('worker', 'collection-private-sentinel'); + } else if (scenario.startsWith('late')) { + hooks.hold('blocking-provider'); + } + await runtime.start(); + if (scenario === 'quiescent-failure') { + try { await hooks.waitEntered('worker'); } + finally { hooks.release('worker'); } + await assert.rejects(runtime.shutdown(100), knownQuiescentCoreFailure); + await assert.rejects(runtime.shutdown(30_000), knownQuiescentCoreFailure); + retiredState(JSON.parse(hooks.snapshot()), { rollback: true, executionWitness: true }); + } else if (scenario.startsWith('late')) { + await hooks.waitEntered('blocking-provider'); + await assert.rejects(runtime.shutdown(0), incompleteTimeout); + await assert.rejects(runtime.shutdown(30_000), incompleteTimeout); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } else { + await runtime.shutdown(100); + await runtime.shutdown(0); + } + assert.equal(runtime.metricsSnapshot(), null); + // Rejected async start calls can retain napi-rs receiver references. Terminal + // rejection has separate tests and must not contaminate this collection control. + return { records, hooks }; +} + +function observations(records) { + return Object.fromEntries(Object.entries(records).map(([name, record]) => [name, { + weakGone: record.weak.deref() === undefined, + finalized: finalized.has(record.label), + }])); +} + +function collected(state) { + return Object.values(state).every(value => value.weakGone && value.finalized); +} + +async function settle(control, captured, milliseconds = 3000) { + const deadline = performance.now() + milliseconds; + let state; + let cycles = 0; + do { + // A dereferenced WeakRef is kept alive until the end of that JS job. + await nextTurn(); + global.gc(); + await delay(20); + state = { control: observations(control), captured: observations(captured) }; + cycles++; + if (collected(state.control) && collected(state.captured)) break; + } while (performance.now() < deadline); + return { ...state, cycles }; +} + +async function observeLateCompletion(runtimeRef) { + const runtime = runtimeRef.deref(); + assert.ok(runtime, 'incomplete native ownership must keep its self-capturing Runtime alive'); + const deadline = performance.now() + 2000; + while (true) { + try { await runtime.shutdown(30_000); break; } + catch (error) { + incompleteTimeout(error); + assert.ok(performance.now() < deadline, 'late cleanup did not complete'); + await delay(1); + } + } + assert.equal(await runtime.shutdown(0), undefined); + assert.equal(runtime.metricsSnapshot(), null); +} + +async function main() { + assert.equal(typeof global.gc, 'function', 'collection probe requires --expose-gc'); + const requested = process.argv[2]; + const scenario = requested === 'baseline' ? 'completed' : requested; + assert.ok(['completed', 'quiescent-failure', 'late', 'late-unobserved'].includes(scenario)); + const [control, captured] = await Promise.all([ + createOwner('Control', 'completed', false), + createOwner('Captured', scenario, true), + ]); + let retainedWhileIncomplete; + if (scenario.startsWith('late')) { + const pending = await settle(control.records, captured.records, 300); + retainedWhileIncomplete = Object.values(pending.captured).every(value => !value.weakGone && !value.finalized); + assert.ok(retainedWhileIncomplete, JSON.stringify(pending)); + assert.ok(JSON.parse(captured.hooks.snapshot()).active > 0); + captured.hooks.release('blocking-provider'); + if (scenario === 'late') await observeLateCompletion(captured.records.runtime.weak); + } + // Only WeakRefs and non-owning test controls survive the creating/observing frames. + const state = await settle(control.records, captured.records); + if (scenario.startsWith('late')) { + retiredState(JSON.parse(captured.hooks.snapshot()), { forced: true, executionWitness: true }); + } + const result = { + scenario: requested, native: provenance, ...state, + controlCollected: collected(state.control), + capturedCollected: collected(state.captured), + retainedWhileIncomplete, + }; + assert.ok(result.controlCollected, `noncapturing control did not collect: ${JSON.stringify(result)}`); + if (requested !== 'baseline') assert.ok(result.capturedCollected, `self-capturing owner did not collect: ${JSON.stringify(result)}`); + emit('COLLECTION', result); +} + +main().catch(error => { + emit('FAILED', { error: error.stack }); + process.exitCode = 1; +}); diff --git a/__tests__/fixtures/lifecycle-probe.cjs b/__tests__/fixtures/lifecycle-probe.cjs index 3b939a9..261347e 100644 --- a/__tests__/fixtures/lifecycle-probe.cjs +++ b/__tests__/fixtures/lifecycle-probe.cjs @@ -4,9 +4,16 @@ const assert = require('node:assert/strict'); const { createInterface } = require('node:readline'); const { performance } = require('node:perf_hooks'); +const path = require('node:path'); const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); -const native = require(Object.keys(require.cache).find(file => file.endsWith('.node'))); +const { nativeProvenance } = require('../../ci/smoke/native-provenance.cjs'); +const { incompleteTimeout, retiredState } = require('../helpers/lifecycle-oracles.cjs'); +const { native, provenance } = nativeProvenance(path.resolve(__dirname, '..', '..'), { + instrumented: process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1', + manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, +}); const emit = (phase, detail = {}) => console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase, ...detail })); +console.log('LIFECYCLE_NATIVE ' + JSON.stringify(provenance)); async function main() { const scenario = process.argv[2]; @@ -52,19 +59,15 @@ async function main() { } const hooks = native._lifecycleTestHooks(runtime._native); hooks.hold('blocking-provider'); + runtime.registerActivity('PendingCleanupActivity', async (_, value) => value); await runtime.start(); await hooks.waitEntered('blocking-provider'); - const input = createInterface({ input: process.stdin }); - const command = new Promise(resolve => input.once('line', resolve)); + const standalone = scenario === 'incomplete-liveness'; + const input = standalone ? undefined : createInterface({ input: process.stdin }); + const command = input && new Promise(resolve => input.once('line', resolve)); emit('STOPPING'); const start = performance.now(); - await assert.rejects(runtime.shutdown(scenario === 'late' ? undefined : 0), error => { - assert.match(error.message, /lifecycle_shutdown_timed_out/); - assert.match(error.message, /cleanup.*incomplete/); - assert.match(error.message, /terminate the process/); - assert.ok(!error.message.includes('sentinel')); - return true; - }); + await assert.rejects(runtime.shutdown(scenario === 'late' ? undefined : 0), incompleteTimeout); const elapsedMs = performance.now() - start; const repeated = performance.now(); await assert.rejects(runtime.shutdown(30_000), error => { @@ -76,6 +79,13 @@ async function main() { assert.ok(JSON.parse(hooks.snapshot()).active > 0); assert.equal(runtime.metricsSnapshot(), null); emit('TIMED_OUT', { elapsedMs, repeatedMs }); + if (standalone) { + globalThis.retainedRuntime = runtime; + process.once('beforeExit', () => emit('PREMATURE_BEFORE_EXIT', { state: JSON.parse(hooks.snapshot()) })); + let pulse = 0; + setInterval(() => emit('ALIVE', { pulse: ++pulse, state: JSON.parse(hooks.snapshot()) }), 100).unref(); + return; + } if (await command === 'retain') { runtime = null; global.gc(); @@ -96,11 +106,7 @@ async function main() { } } const state = JSON.parse(hooks.snapshot()); - assert.equal(state.active, 0); - assert.equal(state.cleanupActive, 0); - assert.equal(state.started, state.completed); - assert.equal(state.coordinators, 1); - assert.equal(state.forceRequests, 1); + retiredState(state, { forced: true, executionWitness: true }); await assert.rejects(runtime.start(), /lifecycle_terminal/); emit('RECLAIMED'); input.close(); diff --git a/__tests__/helpers/lifecycle-oracles.cjs b/__tests__/helpers/lifecycle-oracles.cjs new file mode 100644 index 0000000..c2a4b6b --- /dev/null +++ b/__tests__/helpers/lifecycle-oracles.cjs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); + +function diagnostic(category) { + return error => { + assert.ok(error instanceof Error); + assert.match(error.message, new RegExp(`^${category}:`)); + assert.doesNotMatch(error.message, /sentinel|Host=private/); + return true; + }; +} + +function knownQuiescentCoreFailure(error) { + diagnostic('lifecycle_shutdown_failed')(error); + assert.match(error.message, /cleanup completed and is quiescent/); + assert.doesNotMatch(error.message, /terminate the process/); + return true; +} + +function incompleteTimeout(error) { + diagnostic('lifecycle_shutdown_timed_out')(error); + assert.match(error.message, /cleanup.*incomplete/); + assert.match(error.message, /terminate the process/); + return true; +} + +function retiredState(state, { forced, rollback = false, executionWitness = false } = {}) { + if (executionWitness) assert.ok(state.started > 0, 'witnessed execution must have a recorded task start'); + assert.equal(state.active, 0); + assert.equal(state.cleanupActive, 0); + assert.equal(state.started, state.completed); + if (rollback) assert.ok(state.coordinators >= 1 && state.coordinators <= 2); + else assert.equal(state.coordinators, 1); + if (forced !== undefined) assert.equal(state.forceRequests, forced ? 1 : 0); +} + +function shutdownTiming(scenario) { + const minimum = scenario === 'late' ? 6000 : 5000; + return { minimum, maximum: minimum + 1000, repeatedMaximum: 1000, delivery: minimum + 3000 }; +} + +function assertShutdownTiming(scenario, message) { + const budget = shutdownTiming(scenario); + assert.ok(message.elapsedMs >= budget.minimum, `first stop ${message.elapsedMs}ms < ${budget.minimum}ms`); + assert.ok(message.elapsedMs < budget.maximum, `first stop ${message.elapsedMs}ms >= ${budget.maximum}ms`); + assert.ok(message.repeatedMs >= 0 && message.repeatedMs < budget.repeatedMaximum, + `repeated stop ${message.repeatedMs}ms must be < ${budget.repeatedMaximum}ms`); +} + +async function orderedRaceAttempt(runtime, hooks, force, iteration) { + const order = force ? 'force-before-retirement' : 'retirement-before-force'; + const stopGate = force ? 'force' : 'coordinator'; + const armed = []; + let phase = 'arming'; + let snapshot; + let stop; + let failure; + const context = () => `${order} iteration ${iteration + 1}/100 phase=${phase} lastSnapshot=${JSON.stringify(snapshot ?? null)}`; + try { + for (const gate of ['blocking-provider', stopGate]) { + phase = `arm:${gate}`; + hooks.hold(gate); + armed.push(gate); + } + phase = 'start'; + await runtime.start(); + phase = 'entered:blocking-provider'; + await hooks.waitEntered('blocking-provider'); + snapshot = JSON.parse(hooks.snapshot()); + assert.ok(snapshot.started > 0, 'entered provider execution was not counted'); + phase = `entered:${stopGate}`; + stop = runtime.shutdown(force ? 0 : 30_000); + stop.catch(() => {}); + await hooks.waitEntered(stopGate); + snapshot = JSON.parse(hooks.snapshot()); + assert.ok(snapshot.active > 0); + } catch (error) { + failure = new Error(context(), { cause: error }); + } finally { + for (const gate of armed) { + try { hooks.release(gate); } + catch (error) { failure ??= new Error(`${context()} releasing ${gate}`, { cause: error }); } + } + try { + if (!stop) stop = runtime.shutdown(0); + assert.equal(await stop, undefined); + } catch (error) { + failure ??= new Error(`${context()} cleanup`, { cause: error }); + } + } + if (failure) throw failure; + try { + phase = 'repeated-stop'; + assert.equal(await runtime.shutdown(), undefined); + phase = 'retired'; + snapshot = JSON.parse(hooks.snapshot()); + retiredState(snapshot, { forced: force, executionWitness: true }); + } catch (error) { + throw new Error(context(), { cause: error }); + } +} + +module.exports = { + diagnostic, knownQuiescentCoreFailure, incompleteTimeout, retiredState, + shutdownTiming, assertShutdownTiming, orderedRaceAttempt, +}; diff --git a/__tests__/helpers/lifecycle-probe.cjs b/__tests__/helpers/lifecycle-probe.cjs new file mode 100644 index 0000000..7ae83fe --- /dev/null +++ b/__tests__/helpers/lifecycle-probe.cjs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const { createInterface } = require('node:readline'); +const path = require('node:path'); + +const OUTPUT_LIMIT = 8192; + +function startProbe(scenario, { input = false, fixture = 'lifecycle-probe.cjs', script, env = process.env } = {}) { + const args = script === undefined + ? ['--expose-gc', path.join(__dirname, '..', 'fixtures', fixture), scenario] + : ['-e', script]; + const child = spawn(process.execPath, args, { + cwd: path.resolve(__dirname, '..', '..'), stdio: [input ? 'pipe' : 'ignore', 'pipe', 'pipe'], env, + }); + let stdout = ''; + let stderr = ''; + let lastPhase = 'spawn'; + let expectedPhase = 'none'; + let failure; + let waiter; + const messages = []; + const queue = []; + const describe = () => + `scenario=${scenario} expected=${expectedPhase} last=${lastPhase} exit=${child.exitCode} signal=${child.signalCode}` + + `\nstdout (tail):\n${stdout}\nstderr (tail):\n${stderr}`; + const reject = error => { + failure ??= error; + if (waiter) { + const pending = waiter; + waiter = undefined; + clearTimeout(pending.timer); + pending.reject(new Error(describe(), { cause: failure })); + } + }; + const deliver = () => { + if (!waiter || queue.length === 0) return; + const pending = waiter; + waiter = undefined; + clearTimeout(pending.timer); + const message = queue.shift(); + try { + assert.equal(message.phase, expectedPhase, describe()); + pending.resolve(message); + } catch (error) { + failure ??= error; + pending.reject(new Error(describe(), { cause: error })); + } + }; + child.stdout.on('data', chunk => { stdout = (stdout + chunk).slice(-OUTPUT_LIMIT); }); + child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(-OUTPUT_LIMIT); }); + const lines = createInterface({ input: child.stdout }); + lines.on('line', line => { + if (!line.startsWith('LIFECYCLE_PROBE ')) return; + try { + const message = JSON.parse(line.slice('LIFECYCLE_PROBE '.length)); + assert.equal(typeof message.phase, 'string'); + lastPhase = message.phase; + messages.push(message); + if (message.phase === 'FAILED' || message.phase === 'PREMATURE_BEFORE_EXIT') { + reject(new Error(message.error ?? `native cleanup pending at ${message.phase}`)); + return; + } + queue.push(message); + deliver(); + } catch (error) { reject(error); } + }); + child.on('error', reject); + const exited = new Promise(resolve => child.once('close', (code, signal) => { + if (waiter) reject(new Error(`child closed before ${expectedPhase}: ${code ?? signal}`)); + resolve({ code, signal }); + })); + + return { + child, messages, describe, + async next(phase, milliseconds) { + expectedPhase = phase; + if (failure) throw new Error(describe(), { cause: failure }); + if (child.exitCode !== null || child.signalCode !== null) { + if (queue.length === 0) throw new Error(describe(), { cause: new Error('child already exited') }); + } + assert.equal(waiter, undefined, 'only one protocol observation may be pending'); + return new Promise((resolve, rejectPromise) => { + const timer = setTimeout(() => reject(new Error(`message-delivery watchdog expired after ${milliseconds}ms`)), milliseconds); + waiter = { resolve, reject: rejectPromise, timer }; + deliver(); + }); + }, + async successfulExit(milliseconds) { + expectedPhase = 'exit(0)'; + let timer; + try { + const status = await Promise.race([ + exited, + new Promise((_, rejectPromise) => { + timer = setTimeout(() => rejectPromise(new Error(describe(), { + cause: new Error(`natural-exit watchdog expired after ${milliseconds}ms`), + })), milliseconds); + }), + ]); + if (failure) throw new Error(describe(), { cause: failure }); + assert.equal(status.code, 0, describe()); + assert.equal(status.signal, null, describe()); + } finally { clearTimeout(timer); } + }, + async dispose() { + if (waiter) reject(new Error('probe disposed')); + lines.close(); + child.stdin?.destroy(); + if (child.exitCode === null && child.signalCode === null) child.kill(); + await exited; + }, + }; +} + +module.exports = { startProbe, OUTPUT_LIMIT }; diff --git a/__tests__/lifecycle-oracles.test.js b/__tests__/lifecycle-oracles.test.js new file mode 100644 index 0000000..24b89f3 --- /dev/null +++ b/__tests__/lifecycle-oracles.test.js @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { identifyNative } = require('../ci/smoke/native-provenance.cjs'); +const { lifecycleReport, encodedRecords } = require('../ci/lifecycle-report.cjs'); +const { startProbe, OUTPUT_LIMIT } = require('./helpers/lifecycle-probe.cjs'); +const { + diagnostic, knownQuiescentCoreFailure, incompleteTimeout, retiredState, + shutdownTiming, assertShutdownTiming, orderedRaceAttempt, +} = require('./helpers/lifecycle-oracles.cjs'); + +test('failure category does not claim quiescence for owner loss or callback release errors', () => { + for (const message of [ + 'native shutdown owner failed; quiescence is unproven; application or supervisor must terminate the process', + 'callback event-loop reference release failed', + ]) { + const error = new Error(`lifecycle_shutdown_failed: ${message}`); + assert.equal(diagnostic('lifecycle_shutdown_failed')(error), true); + assert.throws(() => knownQuiescentCoreFailure(error)); + } + assert.equal(knownQuiescentCoreFailure(new Error('lifecycle_shutdown_failed: cleanup completed and is quiescent')), true); + assert.equal(incompleteTimeout(new Error('lifecycle_shutdown_timed_out: cleanup remains owned and incomplete; application or supervisor must terminate the process')), true); + assert.throws(() => diagnostic('lifecycle_shutdown_failed')(new Error('lifecycle_shutdown_failed: private-sentinel'))); +}); + +test('retirement oracle rejects zero execution starts despite a working coordinator', () => { + const zeroExecution = { active: 0, cleanupActive: 0, started: 0, completed: 0, coordinators: 1, forceRequests: 0 }; + retiredState(zeroExecution, { forced: false }); + assert.throws(() => retiredState(zeroExecution, { executionWitness: true }), /recorded task start/); +}); + +test('delivery allowance cannot admit over-budget first or repeated shutdown measurements', () => { + for (const scenario of ['late', 'retained']) { + const budget = shutdownTiming(scenario); + assert.ok(budget.delivery > budget.maximum + budget.repeatedMaximum); + assertShutdownTiming(scenario, { elapsedMs: budget.maximum - 1, repeatedMs: 999 }); + assert.throws(() => assertShutdownTiming(scenario, { elapsedMs: budget.maximum, repeatedMs: 0 })); + assert.throws(() => assertShutdownTiming(scenario, { elapsedMs: budget.minimum, repeatedMs: 1000 })); + assert.throws(() => assertShutdownTiming(scenario, { elapsedMs: budget.minimum - 1, repeatedMs: 0 })); + } +}); + +test('native identity follows Duroxide exports, not the first cached addon', () => { + const loaderPath = path.resolve('index.js'); + const otherPath = path.resolve('unrelated.node'); + const nativePath = path.resolve('duroxide.test.node'); + const bindings = { JsRuntime: class {}, JsClient: class {}, JsSqliteProvider: class {} }; + const intended = { filename: nativePath, exports: bindings }; + const cache = { + [otherPath]: { filename: otherPath, exports: null }, + [loaderPath]: { exports: bindings }, + [nativePath]: intended, + }; + assert.equal(identifyNative(loaderPath, cache), intended); + delete cache[nativePath]; + assert.throws(() => identifyNative(loaderPath, cache), /exactly one native module/); +}); + +test('ordered-race failures preserve cause, iteration and early armed-gate cleanup', async () => { + const original = new Error('deliberate second gate failure'); + const released = []; + let stopped = 0; + const hooks = { + hold(gate) { if (gate === 'force') throw original; }, + release(gate) { released.push(gate); }, + snapshot() { throw new Error('diagnostics must not call snapshots after failure'); }, + }; + const runtime = { + async start() { throw new Error('must not reach startup'); }, + async shutdown() { stopped++; }, + }; + await assert.rejects(orderedRaceAttempt(runtime, hooks, true, 41), error => { + assert.equal(error.cause, original); + assert.match(error.message, /force-before-retirement iteration 42\/100 phase=arm:force lastSnapshot=null/); + return true; + }); + assert.deepEqual(released, ['blocking-provider']); + assert.equal(stopped, 1); +}); + +test('probe failures retain bounded stdout, stderr, protocol phase and original cause', async () => { + const probe = startProbe('deliberate-failure', { + script: `console.log('x'.repeat(${OUTPUT_LIMIT * 2})); console.error('diagnostic-stderr'); + console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase: 'FAILED', error: 'deliberate child stack' }));`, + }); + try { + await assert.rejects(probe.next('STOPPING', 3000), error => { + assert.match(error.message, /scenario=deliberate-failure expected=STOPPING last=FAILED exit=/); + assert.match(error.message, /diagnostic-stderr/); + assert.match(error.cause.message, /deliberate child stack/); + assert.ok(error.message.length < OUTPUT_LIMIT * 2 + 1000); + return true; + }); + } finally { await probe.dispose(); } +}); + +test('malformed child protocol retains its parse failure and reliable cleanup', async () => { + const probe = startProbe('malformed', { script: "console.log('LIFECYCLE_PROBE {broken');" }); + try { + await assert.rejects(probe.next('STOPPING', 3000), error => { + assert.ok(error.cause instanceof SyntaxError); + assert.match(error.message, /expected=STOPPING last=spawn/); + assert.match(error.message, /\{broken/); + return true; + }); + } finally { await probe.dispose(); } +}); + +test('instrumented CI rejects skips, absent required cases, empty selection and aborted totals', () => { + const tap = `# Subtest: [instrumented] required +ok 1 - [instrumented] required +# Subtest: production +ok 2 - production +# tests 2 +# pass 2 +# fail 0 +# skipped 0 +# cancelled 0 +# todo 0 +`; + assert.equal(lifecycleReport(tap, 'instrumented').requiredHookCases, 1); + const skipped = tap.replace('ok 1 - [instrumented] required', 'ok 1 - [instrumented] required # SKIP') + .replace('# pass 2', '# pass 1').replace('# skipped 0', '# skipped 1'); + assert.throws(() => lifecycleReport(skipped, 'instrumented'), /must execute, not skip/); + assert.equal(lifecycleReport(skipped, 'production').skipped, 1); + assert.throws(() => lifecycleReport(tap.replaceAll('[instrumented]', 'unmarked'), 'instrumented'), /not selected/); + assert.throws(() => lifecycleReport(tap.replace('# tests 2', '# tests 0'), 'instrumented'), /selection/); + assert.throws(() => lifecycleReport(tap.replace('# cancelled 0', '# cancelled 1'), 'instrumented'), /aborted/); +}); + +test('native provenance records survive TAP escaping of Windows paths', () => { + const record = { path: 'C:\\native\\duroxide.win32-x64-msvc.node', sha256: 'a'.repeat(64), instrumented: false }; + const encoded = Buffer.from(JSON.stringify(record)).toString('base64'); + assert.deepEqual(encodedRecords(`# LIFECYCLE_NATIVE_RECORD ${encoded}\r\n`, 'LIFECYCLE_NATIVE_RECORD'), [record]); + assert.deepEqual(encodedRecords('# unrelated\n', 'LIFECYCLE_NATIVE_RECORD'), []); + assert.throws(() => encodedRecords('# LIFECYCLE_NATIVE_RECORD e2Jyb2tlbg==\n', 'LIFECYCLE_NATIVE_RECORD')); +}); diff --git a/__tests__/lifecycle.test.js b/__tests__/lifecycle.test.js index e4d421a..e42ae04 100644 --- a/__tests__/lifecycle.test.js +++ b/__tests__/lifecycle.test.js @@ -3,39 +3,26 @@ const { test } = require('node:test'); const assert = require('node:assert/strict'); -const { createHash } = require('node:crypto'); -const { readFileSync } = require('node:fs'); const path = require('node:path'); -const { spawn, spawnSync } = require('node:child_process'); -const { createInterface } = require('node:readline'); const { performance } = require('node:perf_hooks'); const { SqliteProvider, Client, Runtime } = require('../lib/duroxide.js'); +const { nativeProvenance } = require('../ci/smoke/native-provenance.cjs'); +const { startProbe } = require('./helpers/lifecycle-probe.cjs'); +const { + diagnostic, knownQuiescentCoreFailure, retiredState, + shutdownTiming, assertShutdownTiming, orderedRaceAttempt, +} = require('./helpers/lifecycle-oracles.cjs'); const root = path.resolve(__dirname, '..'); -const nativePath = Object.keys(require.cache).find(file => file.endsWith('.node')); -assert.ok(nativePath?.startsWith(root + path.sep), 'tests require the freshly built local native module'); -const native = require(nativePath); const instrumented = process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1'; +const { native, provenance } = nativeProvenance(root, { + instrumented, manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, +}); const SECRET = 'Password=lifecycle-secret-sentinel;Host=private-sentinel'; const MAX_GRACE_MS = Number(((2n ** 64n) - 1n) / 1_000_000n) - 5000; -function diagnostic(category, incomplete = false) { - return error => { - assert.ok(error instanceof Error); - assert.match(error.message, new RegExp(category)); - assert.ok(!error.message.includes(SECRET)); - assert.ok(!error.message.includes('sentinel')); - if (incomplete) { - assert.match(error.message, /cleanup.*incomplete/); - assert.match(error.message, /terminate the process/); - } - if (category === 'lifecycle_shutdown_failed') { - assert.match(error.message, /cleanup completed and is quiescent/); - assert.doesNotMatch(error.message, /terminate the process/); - } - return true; - }; -} +const hookTest = (name, body) => test(`[instrumented] ${name}`, { skip: !instrumented }, body); +const record = (name, value) => console.log(`${name} ${Buffer.from(JSON.stringify(value)).toString('base64')}`); async function fixture(options) { const provider = await SqliteProvider.inMemory(); @@ -43,24 +30,13 @@ async function fixture(options) { return { provider, runtime, hooks: instrumented ? native._lifecycleTestHooks(runtime._native) : null }; } -function retired(hooks, forced, rollback = false) { - const state = JSON.parse(hooks.snapshot()); - assert.equal(state.active, 0); - assert.equal(state.cleanupActive, 0); - assert.equal(state.started, state.completed); - if (rollback) assert.ok(state.coordinators >= 1 && state.coordinators <= 2); - else assert.equal(state.coordinators, 1); - if (forced !== undefined) assert.equal(state.forceRequests, forced ? 1 : 0); +function retired(hooks, options) { + retiredState(JSON.parse(hooks.snapshot()), options); } test('native provenance and production export boundary', () => { - assert.equal(typeof native._lifecycleTestHooks === 'function', instrumented); - assert.equal(typeof native.LifecycleTestHooks === 'function', instrumented); - console.log('LIFECYCLE_NATIVE ' + JSON.stringify({ - path: nativePath, - sha256: createHash('sha256').update(readFileSync(nativePath)).digest('hex'), - instrumented, - })); + console.log('LIFECYCLE_NATIVE ' + JSON.stringify(provenance)); + record('LIFECYCLE_NATIVE_RECORD', provenance); }); test('invalid durations preserve registration, startup, and repeated-stop ownership', async () => { @@ -80,6 +56,31 @@ test('invalid durations preserve registration, startup, and repeated-stop owners if (hooks) retired(hooks); }); +test('direct-native negative integers do not consume registered work or stop ownership', async () => { + const { runtime, provider, hooks } = await fixture(); + const client = new Client(provider); + runtime.registerActivity('NativeNegativeActivity', async (_, input) => `original:${input}`); + runtime.registerOrchestration('NativeNegativeFlow', function* (context, input) { + return yield context.scheduleActivity('NativeNegativeActivity', input); + }); + const rejectsNative = value => assert.rejects( + async () => runtime._native.shutdown(value), diagnostic('lifecycle_invalid_timeout')); + for (const value of [-1, -1000, -Number.MAX_SAFE_INTEGER]) await rejectsNative(value); + await runtime.start(); + try { + await rejectsNative(-1); + assert.notEqual(runtime.metricsSnapshot(), null); + await client.startOrchestration('native-negative-first', 'NativeNegativeFlow', 'before-stop'); + const result = await client.waitForOrchestration('native-negative-first', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, 'original:before-stop'); + } finally { await runtime.shutdown(100); } + await rejectsNative(-1); + assert.equal(await runtime._native.shutdown(0), undefined); + assert.equal(await runtime.shutdown(), undefined); + if (hooks) retired(hooks, { executionWitness: true }); +}); + test('maximum representable default-total boundary is checked on no-work and repeated stop', async () => { const { runtime } = await fixture(); assert.equal(await runtime.shutdown(MAX_GRACE_MS), undefined); @@ -98,6 +99,46 @@ test('pre-start shutdown is a terminal no-work success', async () => { assert.equal(runtime.metricsSnapshot(), null); }); +for (const typed of [false, true]) { + test(`rejected ${typed ? 'typed' : 'raw'} registrations cannot replace live handlers or shadow plain fallback`, async () => { + const { runtime: live, provider } = await fixture(); + const terminal = new Runtime(provider); + const client = new Client(provider); + const suffix = typed ? 'Typed' : 'Raw'; + const plain = `RegistrationOriginal${suffix}`; + const versioned = `RegistrationVersioned${suffix}`; + const fallback = `RegistrationFallback${suffix}`; + const register = typed ? 'registerOrchestrationTyped' : 'registerOrchestration'; + const registerVersioned = typed ? 'registerOrchestrationVersionedTyped' : 'registerOrchestrationVersioned'; + live[register](plain, function* (_, input) { return `plain:${input}`; }); + live[registerVersioned](versioned, '1.0.0', function* (_, input) { return `versioned:${input}`; }); + live[register](fallback, function* (_, input) { return `fallback:${input}`; }); + await terminal.shutdown(0); + await live.start(); + try { + const replacement = function* () { return 'rejected-replacement'; }; + assert.throws(() => terminal[register](plain, replacement), /lifecycle_terminal/); + assert.throws(() => terminal[registerVersioned](versioned, '1.0.0', replacement), /lifecycle_terminal/); + assert.throws(() => terminal[registerVersioned](fallback, '1.0.0', replacement), /lifecycle_terminal/); + for (const [name, expected, version] of [ + [plain, 'plain:fresh', undefined], + [versioned, 'versioned:fresh', '1.0.0'], + [fallback, 'fallback:fresh', '1.0.0'], + ]) { + const id = `rejected-registration-${name}`; + if (version) await client.startOrchestrationVersioned(id, name, 'fresh', version); + else await client.startOrchestration(id, name, 'fresh'); + const result = await client.waitForOrchestration(id, 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, expected, `${name} must resolve its original handler after rejection`); + } + } finally { + await live.shutdown(100); + await terminal.shutdown(0); + } + }); +} + test('fallible preparation rejects invalid options without an escaping panic', async () => { const { runtime } = await fixture({ sessionIdleTimeoutMs: 0 }); await assert.rejects(runtime.start(), diagnostic('lifecycle_start_failed')); @@ -107,13 +148,12 @@ test('fallible preparation rejects invalid options without an escaping panic', a }); for (const scenario of ['registry-invalid', 'registry-descending']) { - test(`pre-core startup failure is ordinary and retained: ${scenario}`, () => { - const child = spawnSync(process.execPath, [path.join(__dirname, 'fixtures', 'lifecycle-probe.cjs'), scenario], { - cwd: root, encoding: 'utf8', timeout: 5000, env: process.env, - }); - assert.equal(child.error, undefined, child.error?.message); - assert.equal(child.status, 0, child.stdout + child.stderr); - assert.match(child.stdout, /REGISTRY_FAILURE_RETAINED/); + test(`pre-core startup failure is ordinary and retained: ${scenario}`, async () => { + const probe = startProbe(scenario); + try { + await probe.successfulExit(5000); + assert.ok(probe.messages.some(message => message.phase === 'REGISTRY_FAILURE_RETAINED'), probe.describe()); + } finally { await probe.dispose(); } }); } @@ -135,11 +175,11 @@ test('idle shutdown short-circuits long grace and preserves successful shapes', assert.ok(performance.now() - start < 2000); assert.equal(await runtime.shutdown(0), undefined); await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); - if (hooks) retired(hooks, false); + if (hooks) retired(hooks, { forced: false }); }); -test('SQLite activity and orchestration run through the unchanged public API', async () => { - const { runtime, provider } = await fixture(); +test('SQLite public work and independent provider/client lifetimes survive worker shutdown', async () => { + const { runtime, provider, hooks } = await fixture(); const client = new Client(provider); runtime.registerActivity('LifecycleEcho', async (_, input) => input); runtime.registerOrchestration('LifecycleEchoFlow', function* (context, input) { @@ -154,6 +194,17 @@ test('SQLite activity and orchestration run through the unchanged public API', a } finally { await runtime.shutdown(100); } + if (hooks) retired(hooks, { executionWitness: true }); + assert.equal((await client.waitForOrchestration('lifecycle-echo', 5000)).output, 'hello'); + const replacement = new Runtime(provider); + replacement.registerOrchestration('LifecycleIndependentProvider', function* (_, input) { return input; }); + await replacement.start(); + try { + await client.startOrchestration('lifecycle-replacement', 'LifecycleIndependentProvider', 'still-open'); + const result = await client.waitForOrchestration('lifecycle-replacement', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, 'still-open'); + } finally { await replacement.shutdown(100); } }); for (const typed of [false, true]) { @@ -193,11 +244,11 @@ for (const typed of [false, true]) { await runtime.shutdown(100); } assert.equal(await runtime.shutdown(0), undefined); - if (hooks) retired(hooks); + if (hooks) retired(hooks, { executionWitness: true }); }); } -test('partial-startup failure retains descendants until actual rollback', { skip: !instrumented }, async () => { +hookTest('partial-startup failure retains descendants until actual rollback', async () => { const { runtime, hooks } = await fixture(); hooks.hold('partial-startup'); hooks.hold('blocking-provider'); @@ -215,132 +266,109 @@ test('partial-startup failure retains descendants until actual rollback', { skip hooks.release('partial-startup'); hooks.release('blocking-provider'); } - await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_shutdown_failed')); - await assert.rejects(runtime.shutdown(30_000), diagnostic('lifecycle_shutdown_failed')); - retired(hooks, undefined, true); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + await assert.rejects(runtime.shutdown(30_000), knownQuiescentCoreFailure); + retired(hooks, { rollback: true, executionWitness: true }); }); -test('contained operational failure is preserved by repeated real shutdown calls', { skip: !instrumented }, async () => { +hookTest('contained operational failure is preserved by repeated real shutdown calls', async () => { const { runtime, hooks } = await fixture(); hooks.hold('worker'); hooks.fail('worker', SECRET); await runtime.start(); await hooks.waitEntered('worker'); hooks.release('worker'); - await assert.rejects(runtime.shutdown(100), diagnostic('lifecycle_shutdown_failed')); - await assert.rejects(runtime.shutdown(30_000), diagnostic('lifecycle_shutdown_failed')); - retired(hooks, undefined, true); + await assert.rejects(runtime.shutdown(100), knownQuiescentCoreFailure); + await assert.rejects(runtime.shutdown(30_000), knownQuiescentCoreFailure); + retired(hooks, { rollback: true, executionWitness: true }); }); -test('cancellation-safe provider fetch retires without waiting for its gate', { skip: !instrumented }, async () => { +hookTest('cancellation-safe provider fetch retires without waiting for its gate', async () => { const { runtime, hooks } = await fixture(); hooks.hold('provider'); await runtime.start(); await hooks.waitEntered('provider'); try { await runtime.shutdown(1000); - retired(hooks, false); + retired(hooks, { forced: false, executionWitness: true }); } finally { hooks.release('provider'); } }); for (const force of [false, true]) { - test(`ordered ${force ? 'force-before-retirement' : 'retirement-before-force'} 100 times`, { skip: !instrumented }, async () => { + hookTest(`ordered ${force ? 'force-before-retirement' : 'retirement-before-force'} 100 times`, async () => { for (let iteration = 0; iteration < 100; iteration++) { const { runtime, hooks } = await fixture(); - hooks.hold('blocking-provider'); - hooks.hold(force ? 'force' : 'coordinator'); - await runtime.start(); - await hooks.waitEntered('blocking-provider'); - const stop = runtime.shutdown(force ? 0 : 30_000); - try { - await hooks.waitEntered(force ? 'force' : 'coordinator'); - assert.ok(JSON.parse(hooks.snapshot()).active > 0); - } finally { - hooks.release('blocking-provider'); - hooks.release(force ? 'force' : 'coordinator'); - } - assert.equal(await stop, undefined); - assert.equal(await runtime.shutdown(), undefined); - retired(hooks, force); + await orderedRaceAttempt(runtime, hooks, force, iteration); } }); } test('retained completed Runtime does not prevent natural process exit', async () => { - const child = spawn(process.execPath, [path.join(__dirname, 'fixtures', 'lifecycle-probe.cjs'), 'exit'], { - cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: process.env, - }); - let output = ''; - child.stdout.on('data', chunk => { output += chunk; }); - child.stderr.on('data', chunk => { output += chunk; }); - let watchdog; - const exited = new Promise(resolve => child.once('exit', resolve)); + const probe = startProbe('exit'); try { - await new Promise((resolve, reject) => { - watchdog = setTimeout(() => reject(new Error('completed runtime kept the process alive')), 3000); - child.once('error', reject); - child.once('exit', code => { - try { - assert.equal(code, 0, output); - assert.match(output, /EXIT_READY/); - resolve(); - } catch (error) { reject(error); } - }); - }); - } finally { - clearTimeout(watchdog); - if (child.exitCode === null && child.signalCode === null) child.kill(); - await exited; - } + await probe.successfulExit(3000); + assert.ok(probe.messages.some(message => message.phase === 'EXIT_READY'), probe.describe()); + } finally { await probe.dispose(); } }); for (const scenario of ['late', 'retained']) { - test(`isolated non-yielding provider timeout: ${scenario}`, { skip: !instrumented }, async () => { - const child = spawn(process.execPath, ['--expose-gc', path.join(__dirname, 'fixtures', 'lifecycle-probe.cjs'), scenario], { - cwd: root, stdio: ['pipe', 'pipe', 'pipe'], env: process.env, - }); - let stderr = ''; - child.stderr.on('data', chunk => { stderr += chunk; }); - let watchdog; - const lines = createInterface({ input: child.stdout }); - const exited = new Promise(resolve => child.once('exit', resolve)); + hookTest(`isolated non-yielding provider timeout: ${scenario}`, async () => { + const probe = startProbe(scenario, { input: true }); try { - await new Promise((resolve, reject) => { - watchdog = setTimeout(() => reject(new Error('probe did not start')), 15_000); - child.once('error', reject); - child.once('exit', code => reject(new Error(`probe exited ${code}: ${stderr}`))); - lines.on('line', line => { - if (!line.startsWith('LIFECYCLE_PROBE ')) return; - try { - const message = JSON.parse(line.slice('LIFECYCLE_PROBE '.length)); - if (message.phase === 'STOPPING') { - clearTimeout(watchdog); - watchdog = setTimeout(() => reject(new Error('default shutdown exceeded seven seconds')), 7000); - } else if (message.phase === 'TIMED_OUT') { - clearTimeout(watchdog); - watchdog = setTimeout(() => reject(new Error('late cleanup/retention evidence missing')), 3000); - assert.ok(message.elapsedMs >= (scenario === 'late' ? 6000 : 5000)); - assert.ok(message.elapsedMs < (scenario === 'late' ? 7000 : 6000)); - assert.ok(message.repeatedMs < 1000); - child.stdin.write(scenario === 'late' ? 'release\n' : 'retain\n'); - } else if (message.phase === 'RECLAIMED' || message.phase === 'RETAINED') { - assert.equal(message.phase, scenario === 'late' ? 'RECLAIMED' : 'RETAINED'); - resolve(); - } else if (message.phase === 'FAILED') { - reject(new Error(message.error)); - } + await probe.next('STOPPING', 15_000); + const message = await probe.next('TIMED_OUT', shutdownTiming(scenario).delivery); + assertShutdownTiming(scenario, message); + probe.child.stdin.write(scenario === 'late' ? 'release\n' : 'retain\n'); + const completed = await probe.next(scenario === 'late' ? 'RECLAIMED' : 'RETAINED', 3000); + record('LIFECYCLE_PROBE_RECORD', { scenario, ...message, finalPhase: completed.phase }); + } catch (error) { + throw new Error(probe.describe(), { cause: error }); + } finally { await probe.dispose(); } + }); +} - } catch (error) { reject(error); } - }); - }); - } finally { - clearTimeout(watchdog); - lines.close(); - child.stdin.destroy(); - if (child.exitCode === null && child.signalCode === null) child.kill(); - await exited; +hookTest('incomplete cleanup keeps the event loop live without stdin or referenced timers', async () => { + const probe = startProbe('incomplete-liveness'); + try { + await probe.next('STOPPING', 15_000); + const message = await probe.next('TIMED_OUT', shutdownTiming('retained').delivery); + assertShutdownTiming('retained', message); + for (let pulse = 1; pulse <= 3; pulse++) { + const alive = await probe.next('ALIVE', 2000); + assert.equal(alive.pulse, pulse); + assert.ok(alive.state.active > 0); + assert.ok(alive.state.started > 0); + assert.equal(alive.state.coordinators, 1); } + record('LIFECYCLE_PROBE_RECORD', { scenario: 'incomplete-liveness', ...message, livePulses: 3 }); + } catch (error) { + throw new Error(probe.describe(), { cause: error }); + } finally { await probe.dispose(); } +}); + +test('activity-only self-capturing callbacks collect after successful shutdown in a live process', async () => { + const probe = startProbe('completed', { fixture: 'lifecycle-collection-probe.cjs' }); + try { + const result = await probe.next('COLLECTION', 10_000); + assert.ok(result.controlCollected); + assert.ok(result.capturedCollected); + record('LIFECYCLE_PROBE_RECORD', result); + await probe.successfulExit(3000); + } finally { await probe.dispose(); } +}); + +for (const scenario of ['quiescent-failure', 'late', 'late-unobserved']) { + hookTest(`activity-only callbacks collect after ${scenario} cleanup`, async () => { + const probe = startProbe(scenario, { fixture: 'lifecycle-collection-probe.cjs' }); + try { + const result = await probe.next('COLLECTION', 15_000); + assert.ok(result.controlCollected); + assert.ok(result.capturedCollected); + if (scenario.startsWith('late')) assert.ok(result.retainedWhileIncomplete); + record('LIFECYCLE_PROBE_RECORD', result); + await probe.successfulExit(3000); + } finally { await probe.dispose(); } }); } diff --git a/ci/lifecycle-report.cjs b/ci/lifecycle-report.cjs new file mode 100644 index 0000000..bd8cce9 --- /dev/null +++ b/ci/lifecycle-report.cjs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); + +function lifecycleReport(stdout, mode) { + assert.ok(['production', 'instrumented'].includes(mode), 'explicit lifecycle mode is required'); + const count = label => { + const found = [...stdout.matchAll(new RegExp(`^# ${label} (\\d+)\\s*$`, 'gm'))]; + assert.equal(found.length, 1, `missing or ambiguous TAP total: ${label}`); + return Number(found[0][1]); + }; + const result = { + selected: count('tests'), passed: count('pass'), failed: count('fail'), + skipped: count('skipped'), aborted: count('cancelled'), todo: count('todo'), + requiredHookCases: [...stdout.matchAll(/^# Subtest: \[instrumented\] /gm)].length, + }; + assert.ok(result.selected > 0 && result.passed > 0, 'lifecycle selection must execute tests'); + assert.ok(result.requiredHookCases > 0, 'required instrumented cases were not selected'); + assert.equal(result.failed, 0, 'lifecycle tests failed'); + assert.equal(result.aborted, 0, 'lifecycle tests aborted'); + assert.equal(result.todo, 0, 'lifecycle tests may not be TODO'); + const skips = stdout.split(/\r?\n/).filter(line => /^ok \d+ - .* # SKIP(?:\s|$)/.test(line)); + assert.equal(skips.length, result.skipped, 'skip accounting mismatch'); + if (mode === 'instrumented') { + assert.equal(result.skipped, 0, 'required instrumented lifecycle cases must execute, not skip'); + } else { + assert.equal(result.skipped, result.requiredHookCases, 'production may skip only hook-dependent cases'); + assert.ok(skips.every(line => /^ok \d+ - \[instrumented\] /.test(line)), 'non-hook lifecycle test was skipped'); + } + assert.equal(result.selected, result.passed + result.skipped, 'incomplete lifecycle result totals'); + return result; +} + +function encodedRecords(stdout, name) { + assert.match(name, /^[A-Z_]+$/); + return [...stdout.matchAll(new RegExp(`^# ${name} ([A-Za-z0-9+/=]+)\\r?$`, 'gm'))] + .map(match => JSON.parse(Buffer.from(match[1], 'base64').toString('utf8'))); +} + +module.exports = { lifecycleReport, encodedRecords }; diff --git a/ci/run-lifecycle.cjs b/ci/run-lifecycle.cjs new file mode 100644 index 0000000..760c299 --- /dev/null +++ b/ci/run-lifecycle.cjs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const { readFileSync, writeFileSync } = require('node:fs'); +const path = require('node:path'); +const { lifecycleReport, encodedRecords } = require('./lifecycle-report.cjs'); + +const [mode, directory] = process.argv.slice(2); +assert.ok(['production', 'instrumented'].includes(mode) && directory, + 'usage: node ci/run-lifecycle.cjs '); +const stage = path.resolve(directory); +const manifestPath = path.join(stage, 'lifecycle-native.json'); +const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); +assert.equal(manifest.instrumented, mode === 'instrumented', 'stage mode differs from requested validation'); +assert.equal(manifest.sourceRoot, path.join(stage, 'node_modules', 'duroxide'), 'unexpected staged source root'); +const args = ['--test', '--test-reporter=tap', + path.join(manifest.sourceRoot, '__tests__', 'lifecycle-oracles.test.js'), + path.join(manifest.sourceRoot, '__tests__', 'lifecycle.test.js')]; +const child = spawnSync(process.execPath, args, { + cwd: manifest.sourceRoot, encoding: 'utf8', timeout: 180_000, maxBuffer: 16 * 1024 * 1024, + env: { + ...process.env, + DUROXIDE_LIFECYCLE_TEST_HOOKS: mode === 'instrumented' ? '1' : '0', + DUROXIDE_NATIVE_MANIFEST: manifestPath, + }, +}); +const stdout = child.stdout ?? ''; +const stderr = child.stderr ?? ''; +writeFileSync(path.join(stage, 'lifecycle.tap'), stdout); +writeFileSync(path.join(stage, 'lifecycle.stderr.log'), stderr); +process.stdout.write(stdout); +process.stderr.write(stderr); +const evidence = { + mode, native: manifest, command: [process.execPath, ...args], cwd: manifest.sourceRoot, + exit: child.status, signal: child.signal, error: child.error?.message, +}; +try { + assert.ifError(child.error); + assert.equal(child.status, 0, 'lifecycle child must exit successfully'); + assert.equal(child.signal, null, 'lifecycle child must not be terminated'); + evidence.counts = lifecycleReport(stdout, mode); + const records = encodedRecords(stdout, 'LIFECYCLE_NATIVE_RECORD'); + assert.equal(records.length, 1, 'lifecycle suite must report its native artifact'); + const actual = records[0]; + assert.equal(actual.path, manifest.path); + assert.equal(actual.sha256, manifest.sha256); + assert.equal(actual.instrumented, manifest.instrumented); + evidence.actualNative = actual; + evidence.probes = encodedRecords(stdout, 'LIFECYCLE_PROBE_RECORD'); +} catch (error) { + evidence.validationFailure = error.stack; + process.stderr.write(`${error.stack}\n`); + process.exitCode = 1; +} finally { + writeFileSync(path.join(stage, 'lifecycle-results.json'), JSON.stringify(evidence, null, 2) + '\n'); +} diff --git a/ci/smoke/README.md b/ci/smoke/README.md index ad12a05..af081a6 100644 --- a/ci/smoke/README.md +++ b/ci/smoke/README.md @@ -15,9 +15,15 @@ back to the local `.node` binary when run from the repo. | File | Purpose | |--------------------------|-------------------------------------------------| | `smoke.mjs` | Hello-world orchestration using `SqliteProvider`. | +| `native-provenance.cjs` | Verify the actual Duroxide binding, installed platform identity, production exports and SHA-256. | | `run-local.sh` / `.ps1` | Install from local tarballs (pre-publish). | | `run-registry.sh` / `.ps1` | Install from npm registry (post-publish). | +The launchers copy both the smoke script and its provenance helper. Verification +matches the loader's native class identities instead of selecting the first +cached addon. An installed smoke must resolve a declared Duroxide platform +subpackage at the SDK's version, with no lifecycle instrumentation exports. + ## Parity The smoke script MUST stay semantically in sync with the equivalent script in diff --git a/ci/smoke/native-provenance.cjs b/ci/smoke/native-provenance.cjs new file mode 100644 index 0000000..f37e3a0 --- /dev/null +++ b/ci/smoke/native-provenance.cjs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const { readFileSync, realpathSync } = require('node:fs'); +const path = require('node:path'); + +function identifyNative(loaderPath, cache = require.cache) { + const loader = cache[loaderPath]; + assert.ok(loader, `Duroxide loader was not loaded: ${loaderPath}`); + assert.equal(typeof loader.exports.JsRuntime, 'function', 'Duroxide loader must expose its native Runtime'); + const candidates = Object.values(cache).filter(module => + module.filename?.endsWith('.node') && + module.exports?.JsRuntime === loader.exports.JsRuntime && + module.exports?.JsClient === loader.exports.JsClient && + module.exports?.JsSqliteProvider === loader.exports.JsSqliteProvider); + assert.equal(candidates.length, 1, 'expected exactly one native module supplying the Duroxide bindings'); + const module = candidates[0]; + assert.match(path.basename(module.filename), /^duroxide\..+\.node$/); + return module; +} + +function nativeProvenance(root, { installed = false, instrumented = false, manifestPath } = {}) { + root = realpathSync(root); + const module = identifyNative(require.resolve(path.join(root, 'index.js'))); + const nativePath = realpathSync(module.filename); + const native = module.exports; + assert.equal('_lifecycleTestHooks' in native, instrumented, 'native hook export boundary mismatch'); + assert.equal('LifecycleTestHooks' in native, instrumented, 'native hook class export boundary mismatch'); + assert.equal(typeof native._lifecycleTestHooks === 'function', instrumented, 'native hook mode mismatch'); + assert.equal(typeof native.LifecycleTestHooks === 'function', instrumented, 'native hook class mode mismatch'); + if (installed) { + const sdkPackage = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')); + const platformPackage = JSON.parse(readFileSync(path.join(path.dirname(nativePath), 'package.json'), 'utf8')); + assert.equal(sdkPackage.name, 'duroxide'); + assert.ok(nativePath.includes(`${path.sep}node_modules${path.sep}`), 'smoke requires an installed platform package'); + assert.ok(Object.hasOwn(sdkPackage.optionalDependencies, platformPackage.name), 'native module is not a declared Duroxide platform package'); + assert.equal(platformPackage.version, sdkPackage.version, 'SDK/platform package version mismatch'); + assert.notEqual(path.dirname(nativePath), root, 'smoke must not use a repository-local addon'); + } else { + assert.equal(path.dirname(nativePath), root, 'lifecycle tests require the intended local addon, not an optional platform package'); + } + const provenance = { + path: nativePath, + sha256: createHash('sha256').update(readFileSync(nativePath)).digest('hex'), + instrumented, + }; + if (manifestPath) { + const expected = JSON.parse(readFileSync(manifestPath, 'utf8')); + assert.equal(realpathSync(expected.path), nativePath, 'native artifact path differs from staged build'); + assert.equal(expected.sha256, provenance.sha256, 'native artifact hash differs from staged build'); + assert.equal(expected.instrumented, instrumented, 'staged build mode mismatch'); + assert.deepEqual(expected.features, instrumented ? ['test-hooks'] : [], 'staged feature declaration mismatch'); + } + return { native, provenance }; +} + +module.exports = { identifyNative, nativeProvenance }; diff --git a/ci/smoke/run-local.ps1 b/ci/smoke/run-local.ps1 index 0290891..f7cbcac 100644 --- a/ci/smoke/run-local.ps1 +++ b/ci/smoke/run-local.ps1 @@ -21,6 +21,7 @@ try { if ($LASTEXITCODE -ne 0) { throw "npm install failed ($LASTEXITCODE)" } Copy-Item $env:SMOKE_SCRIPT ./smoke.mjs + Copy-Item (Join-Path (Split-Path $env:SMOKE_SCRIPT) 'native-provenance.cjs') ./native-provenance.cjs node ./smoke.mjs if ($LASTEXITCODE -ne 0) { throw "node smoke.mjs failed ($LASTEXITCODE)" } Write-Host "[smoke] local smoke OK" diff --git a/ci/smoke/run-local.sh b/ci/smoke/run-local.sh index aeb7fa3..362726e 100755 --- a/ci/smoke/run-local.sh +++ b/ci/smoke/run-local.sh @@ -29,5 +29,6 @@ npm init -y >/dev/null npm install --no-audit --no-fund --ignore-scripts "$TARBALL_PLATFORM" "$TARBALL_MAIN" cp "$SMOKE_SCRIPT" ./smoke.mjs +cp "$(dirname "$SMOKE_SCRIPT")/native-provenance.cjs" ./native-provenance.cjs node ./smoke.mjs echo "[smoke] local smoke OK" diff --git a/ci/smoke/run-registry.ps1 b/ci/smoke/run-registry.ps1 index ffc113c..3db1801 100644 --- a/ci/smoke/run-registry.ps1 +++ b/ci/smoke/run-registry.ps1 @@ -27,6 +27,7 @@ try { } Copy-Item $env:SMOKE_SCRIPT ./smoke.mjs + Copy-Item (Join-Path (Split-Path $env:SMOKE_SCRIPT) 'native-provenance.cjs') ./native-provenance.cjs node ./smoke.mjs if ($LASTEXITCODE -ne 0) { throw "node smoke.mjs failed ($LASTEXITCODE)" } Write-Host "[smoke] registry smoke OK" diff --git a/ci/smoke/run-registry.sh b/ci/smoke/run-registry.sh index cd23de9..d1832bc 100755 --- a/ci/smoke/run-registry.sh +++ b/ci/smoke/run-registry.sh @@ -36,5 +36,6 @@ for i in $(seq 1 "$attempts"); do done cp "$SMOKE_SCRIPT" ./smoke.mjs +cp "$(dirname "$SMOKE_SCRIPT")/native-provenance.cjs" ./native-provenance.cjs node ./smoke.mjs echo "[smoke] registry smoke OK" diff --git a/ci/smoke/smoke.mjs b/ci/smoke/smoke.mjs index 3acfa0b..4d4aaf4 100644 --- a/ci/smoke/smoke.mjs +++ b/ci/smoke/smoke.mjs @@ -16,22 +16,16 @@ // output === "Hello, World!". import { SqliteProvider, Client, Runtime } from 'duroxide'; -import { mkdtempSync, readFileSync } from 'node:fs'; +import { mkdtempSync } from 'node:fs'; import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { join, sep } from 'node:path'; +import { dirname, join } from 'node:path'; +import { nativeProvenance } from './native-provenance.cjs'; const require = createRequire(import.meta.url); -const nativePath = Object.keys(require.cache).find(file => file.endsWith('.node') && file.includes('duroxide')); -assert.ok(nativePath?.includes(`${sep}node_modules${sep}`), 'smoke must load the installed platform package'); -const native = require(nativePath); -assert.equal('_lifecycleTestHooks' in native, false); -assert.equal('LifecycleTestHooks' in native, false); -console.log('[smoke] native=' + JSON.stringify({ - path: nativePath, sha256: createHash('sha256').update(readFileSync(nativePath)).digest('hex'), -})); +const { provenance } = nativeProvenance(dirname(require.resolve('duroxide/package.json')), { installed: true }); +console.log('[smoke] native=' + JSON.stringify(provenance)); const dir = mkdtempSync(join(tmpdir(), 'duroxide-smoke-')); const dbPath = join(dir, 'smoke.db'); diff --git a/ci/stage-lifecycle.cjs b/ci/stage-lifecycle.cjs new file mode 100644 index 0000000..4cd6b67 --- /dev/null +++ b/ci/stage-lifecycle.cjs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const { cpSync, mkdirSync, readFileSync, readdirSync, realpathSync, writeFileSync } = require('node:fs'); +const path = require('node:path'); + +const [nativeDirectory, stageDirectory, mode] = process.argv.slice(2); +assert.ok(nativeDirectory && stageDirectory && ['production', 'instrumented'].includes(mode), + 'usage: node ci/stage-lifecycle.cjs '); +const root = path.resolve(__dirname, '..'); +const nativeRoot = realpathSync(nativeDirectory); +const stage = path.resolve(stageDirectory); +const target = path.join(root, 'target') + path.sep; +assert.ok(stage.startsWith(target), 'stage must be isolated under this repository target directory'); +assert.notEqual(stage, nativeRoot, 'native output and staged sources must use separate directories'); +const binaries = readdirSync(nativeRoot).filter(name => /^duroxide\..+\.node$/.test(name)); +assert.equal(binaries.length, 1, 'native output must contain exactly one intended platform addon'); +// Root-level `node --test` must not rediscover staged copies of the suite. +const sourceRoot = path.join(stage, 'node_modules', 'duroxide'); +mkdirSync(sourceRoot, { recursive: true }); +const existing = readdirSync(sourceRoot).filter(name => name.endsWith('.node')); +assert.ok(existing.every(name => name === binaries[0]), 'staging directory contains a different native addon'); +for (const name of ['index.js', 'index.d.ts', 'package.json', 'lib', '__tests__', 'ci']) { + cpSync(path.join(root, name), path.join(sourceRoot, name), { recursive: true }); +} +const nativePath = path.join(sourceRoot, binaries[0]); +cpSync(path.join(nativeRoot, binaries[0]), nativePath); +const hash = file => createHash('sha256').update(readFileSync(file)).digest('hex'); +const sources = {}; +function record(directory) { + for (const entry of readdirSync(path.join(root, directory), { withFileTypes: true })) { + const name = path.join(directory, entry.name); + if (entry.isDirectory()) record(name); + else sources[name] = hash(path.join(root, name)); + } +} +for (const name of ['src', 'lib', '__tests__', 'ci']) record(name); +for (const name of ['Cargo.toml', 'Cargo.lock', 'package.json', 'index.js']) sources[name] = hash(path.join(root, name)); +const manifest = { + sourceRoot, + path: realpathSync(nativePath), + sha256: hash(nativePath), + instrumented: mode === 'instrumented', + features: mode === 'instrumented' ? ['test-hooks'] : [], + sources, +}; +writeFileSync(path.join(stage, 'lifecycle-native.json'), JSON.stringify(manifest, null, 2) + '\n'); +console.log('LIFECYCLE_STAGE ' + JSON.stringify({ stage, path: manifest.path, sha256: manifest.sha256, mode })); diff --git a/docs/user-guide.md b/docs/user-guide.md index 3a76a15..b4347e7 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -67,6 +67,12 @@ headroom. `shutdown(0)` requests immediate force, still with a five-second total The first stop fixes the deadlines; repeating it never grants a new budget. Success remains `undefined`; errors carry `lifecycle_start_failed`, `lifecycle_shutdown_timed_out`, or `lifecycle_shutdown_failed` categories. +The `lifecycle_shutdown_failed` category alone is **not** cleanup proof. It covers +known-quiescent core failures, loss of the native shutdown owner with unproven +quiescence, and failure to release a Node callback's event-loop reference after +core quiescence. The last category's message does not itself assert quiescence. +Tests for known-quiescent core failures are specific to that outcome; neither +generic failure-category checks nor task-count telemetry authorize release. Invalid non-finite, fractional, negative, unsafe-integer, or unsupported native durations reject without consuming registrations or accepting stop. Strings and `null` also reject; `undefined` means the omitted grace. Native @@ -88,6 +94,17 @@ deadlines belong to this adapter; native Rust shutdown itself awaits cleanup without a total limit. Provider futures can be canceled under the provider's atomic acknowledgement and peek-lock recovery contract. +Callback/core holders are separate from persistent shutdown results and deadlines. +Only proven native quiescence permits their release on the JavaScript thread, +including known-quiescent failures and actual late completion without a further +shutdown call. This breaks obsolete activity callback/self-capture ownership +cycles; merely unref'ing an event-loop handle is not callback collection. +Unproven cleanup or a lost cleanup owner retains the graph and leaves the worker +unusable. Application-held references and the pre-existing global orchestration +handler map are outside this collection guarantee. Independent provider/client +ownership is not disposed by stopping a worker, and late release never retracts +the application's prior termination obligation. + This complete example has no admitted work and can finish before grace: ```javascript @@ -112,6 +129,29 @@ noncooperative work. Arrange termination in the application's supervisor policy; do not treat a caught error or successful late observation as permission to reuse the runtime. A finite grace cannot promise bounded forced cleanup. +### Sizing the supervisor budget + +Size against the supervisor time **remaining when the first shutdown call is +made**, not its original allocation: + +```text +remaining outer time > grace + 5000 ms + other shutdown work + delivery/exit margin +``` + +For example, with 30 seconds actually remaining, a 30-second grace asks the SDK +to observe cleanup for up to 35 seconds and cannot fit. An illustrative +20-second grace gives a 25-second SDK observation budget and leaves five seconds +for other work, scheduling, result delivery and process exit. Subtract time +already spent before calling `shutdown`. If less than the fixed five-second +allowance remains, `shutdown(0)` alone cannot make the SDK's timeout fit. + +Omission still means 1000 ms grace and 6000 ms total; zero grace still means +5000 ms total. The first call fixes those deadlines, and repeated calls neither +renew the budget nor retract a previous termination obligation. Native cleanup +itself has no total bound and may remain pending indefinitely. The application's +parent/supervisor termination policy must therefore fit outside these SDK +observation budgets; the SDK supplies no process-kill or supervisor mechanism. + ## Orchestration Patterns ### Sequential Steps diff --git a/package.json b/package.json index 156f140..964f7b4 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build": "napi build --platform --release", "build:debug": "napi build --platform", "test": "node --test __tests__/e2e.test.js __tests__/kv_store.test.js __tests__/races.test.js __tests__/admin_api.test.js __tests__/sessions.test.js __tests__/advanced_features.test.js __tests__/async_blocks.test.js __tests__/typed_apis.test.js __tests__/copilot_chat.test.js __tests__/lifecycle.test.js __tests__/replay_isolation.test.js __tests__/lock_options.test.js", - "test:lifecycle": "node --test __tests__/lifecycle.test.js", + "test:lifecycle": "node --test __tests__/lifecycle-oracles.test.js __tests__/lifecycle.test.js", "test:races": "node --test __tests__/races.test.js", "test:admin": "node --test __tests__/admin_api.test.js", "test:scenarios": "node --test __tests__/scenarios/*.test.js", diff --git a/src/runtime.rs b/src/runtime.rs index b7a6c24..376313c 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -7,7 +7,7 @@ use napi_derive::napi; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::watch; +use tokio::sync::{oneshot, watch}; use duroxide::runtime::OrchestrationHandler; use duroxide::runtime::{self, OrchestrationRegistry}; @@ -408,21 +408,41 @@ impl JsRuntime { #[napi(ts_return_type = "Promise")] pub fn shutdown(&mut self, env: Env, timeout_ms: Option) -> Result { let (grace_deadline, total_deadline) = shutdown_deadlines(timeout_ms)?; + self.shutdown_requested = true; if self.stop.is_none() { - if let Some(rt) = &self.inner { - rt.request_shutdown(); + if let Some(runtime) = self.inner.take() { + runtime.request_shutdown(); let (completed, completion) = watch::channel(None); - let runtime = rt.clone(); - napi::bindgen_prelude::spawn(async move { - completed.send_replace(Some(runtime.shutdown_until(grace_deadline).await)); - }); self.stop = Some(ShutdownOperation { completion, total_deadline, }); + let owned_runtime = runtime.clone(); + let resources = ShutdownResources(Some(ExecutionResources { + runtime, + callbacks: std::mem::take(&mut self.callback_refs), + })); + let (native_completed, native_completion) = oneshot::channel(); + napi::bindgen_prelude::spawn(async move { + let result = owned_runtime.shutdown_until(grace_deadline).await; + let _ = native_completed.send(result); + }); + // This owned observer outlives ordinary bounded waits. Its resolver + // runs on the JS thread, including when cleanup completes late. + env.execute_tokio_future( + async move { + Ok(match native_completion.await { + Ok(result) => result.map_err(ShutdownWaitError::Core), + Err(_) => Err(ShutdownWaitError::OwnerLost), + }) + }, + move |env, result| { + completed.send_replace(Some(resources.complete(env, result))); + Ok(()) + }, + )?; } } - self.shutdown_requested = true; if self.inner.is_none() { self.activity_builders.clear(); self.orchestration_names.clear(); @@ -432,7 +452,6 @@ impl JsRuntime { } let stop = self.stop.clone(); let startup_error = self.startup_error; - let mut callbacks = self.callback_refs.clone(); env.execute_tokio_future( async move { Ok(match stop { @@ -440,29 +459,11 @@ impl JsRuntime { None => None, }) }, - move |env, result| { - let quiescent = match result { - None | Some(Ok(_)) => true, - Some(Err(ShutdownWaitError::Core(error))) => error.is_quiescent(), - Some(Err(_)) => false, - }; - // Keep the core completion record, without keeping Node's event loop - // alive after retirement. N-API unref must run on the JS thread. - if quiescent { - for callback in &mut callbacks { - callback.unref(env).map_err(|_| { - Error::from_reason( - "lifecycle_shutdown_failed: callback event-loop reference release failed", - ) - })?; - } - } - match result { - Some(result) => result.map_err(shutdown_error).and_then(|_| { - startup_error.map_or(Ok(()), |error| Err(start_error(error))) - }), - None => startup_error.map_or(Ok(()), |error| Err(start_error(error))), - } + move |_, result| match result { + Some(result) => result + .map_err(shutdown_error) + .and_then(|_| startup_error.map_or(Ok(()), |error| Err(start_error(error)))), + None => startup_error.map_or(Ok(()), |error| Err(start_error(error))), }, ) } @@ -485,26 +486,68 @@ fn start_error(error: runtime::RuntimeStartError) -> Error { #[derive(Clone)] struct ShutdownOperation { - completion: watch::Receiver< - Option>, - >, + completion: watch::Receiver>, total_deadline: Instant, } +type ShutdownResult = std::result::Result; + +struct ExecutionResources { + runtime: Arc, + callbacks: Vec>, +} + +struct ShutdownResources(Option); + +impl ShutdownResources { + fn complete(mut self, env: &Env, result: ShutdownResult) -> ShutdownResult { + let quiescent = match result { + Ok(_) => true, + Err(ShutdownWaitError::Core(error)) => error.is_quiescent(), + Err(_) => false, + }; + if quiescent { + if let Some(resources) = self.0.as_mut() { + for callback in &mut resources.callbacks { + callback + .unref(env) + .map_err(|_| ShutdownWaitError::CallbackReleaseFailed)?; + } + } + if let Some(resources) = self.0.take() { + drop(resources.runtime); + drop(resources.callbacks); + } + } + result + } +} + +impl Drop for ShutdownResources { + fn drop(&mut self) { + if let Some(resources) = self.0.take() { + // Owner loss, failed JS dispatch or an unproven core result cannot + // authorize release. Retain the graph for supervisor termination. + std::mem::forget(resources); + } + } +} + #[derive(Clone, Copy)] enum ShutdownWaitError { TimedOut, OwnerLost, Core(runtime::RuntimeShutdownError), + CallbackReleaseFailed, } impl ShutdownOperation { - async fn wait(mut self) -> std::result::Result { + async fn wait(mut self) -> ShutdownResult { loop { { let completed = self.completion.borrow_and_update(); if let Some(result) = *completed { - return result.map_err(ShutdownWaitError::Core); + return result; } if Instant::now() >= self.total_deadline { return Err(ShutdownWaitError::TimedOut); @@ -534,6 +577,9 @@ fn shutdown_error(error: ShutdownWaitError) -> Error { ShutdownWaitError::Core(error) => { Error::from_reason(format!("lifecycle_shutdown_failed: {error}")) } + ShutdownWaitError::CallbackReleaseFailed => Error::from_reason( + "lifecycle_shutdown_failed: callback event-loop reference release failed", + ), } } From db590019fc4fb94654f5a3a377a3ca07e89d9ad4 Mon Sep 17 00:00:00 2001 From: "Christopher Krawczyk (SQL)" Date: Wed, 21 Oct 2026 16:03:20 -0700 Subject: [PATCH 3/4] docs: clarify Node shutdown waits and cleanup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 73748d4f-2eb6-47d2-9dbc-af8681e80e83 --- CHANGELOG.md | 40 ++++++------ CONTRIBUTING.md | 78 +++++++++++----------- README.md | 58 +++++++++-------- ci/smoke/README.md | 8 +-- docs/architecture.md | 69 +++++++++++++------- docs/user-guide.md | 150 ++++++++++++++++++++++--------------------- lib/duroxide.d.ts | 8 +-- lib/duroxide.js | 8 +-- 8 files changed, 230 insertions(+), 189 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0deb685..f57791b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,27 +9,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Use fallible core startup and bounded shutdown while preserving `Promise`. - Shutdown uses the supplied grace plus five seconds, with a one-second omitted - grace, and idle workers can finish early. Invalid durations reject before - changing ownership, even for no-work/repeated calls. Repeated calls retain - the original operation and errors instead of succeeding on an empty runtime; - no call renews the first deadlines. Pre-core registry/preparation failures - become ordinary retained startup errors without changing registration policy. -- Pre-start shutdown is terminal; start/registration after activation or stop is - rejected. Post-stop metrics remain unavailable. Core completion remains retained - without keeping Node's event loop alive after proven retirement. -- Timeouts explicitly require application/supervisor process termination while - cleanup remains owned, even if a later observer proves retirement. Finite grace - does not guarantee bounded forced cleanup. This narrow adapter does not add - .NET's foreign-continuation lifetime barrier or own detached application work. - The matching published core dependency is a separate release prerequisite; - local development overrides are not distributable. +- Use fallible core startup and bounded shutdown waits while preserving `Promise`. + Total waiting is the supplied grace plus five seconds. + Omitted grace is one second. Idle workers can finish early. + Invalid durations fail before ownership changes, including for no-work and repeated calls. + Repeated calls retain the original operation, errors, and deadlines. + They do not report success merely because the wrapper transferred its core. + Registry and preparation failures become ordinary retained startup errors. + Registration policy does not change. +- Pre-start shutdown is terminal. + Start and registration calls fail after activation or stop. + Post-stop metrics remain unavailable. + Proven cleanup permits release of obsolete callback and core holders on the JavaScript thread. + The terminal result and deadlines remain available. +- Timeouts require application- or supervisor-owned process termination while cleanup remains owned. + Later proven cleanup does not remove that requirement. + Finite grace does not guarantee bounded forced cleanup. + This adapter does not add .NET's foreign-continuation barrier or own detached application work. + Release requires a matching published core dependency. + Do not distribute local development overrides. ### Tests -- SQLite-only real-wrapper lifecycle coverage, feature-gated provider/fault controls, - ordered race repetitions, isolated held-provider probes, and production-export checks. +- Add SQLite-only lifecycle coverage through the real wrapper. + Cases include feature-gated provider and fault controls, repeated ordered races, + isolated held-provider probes, and production-export checks. ## [0.2.2] - 2026-10-21 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f201108..ff03c7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,9 +40,10 @@ After Rust source changes (`src/*.rs`), re-run `npx napi build --platform` befor ### Lifecycle checks without PostgreSQL -The lifecycle suite requires a freshly built local native module, not an optional -prebuilt platform package. Its private controls are absent from production builds. -Keep native outputs, Cargo targets and staged sources separate in both modes. +The lifecycle suite requires a freshly built local native module. +Do not use an optional prebuilt platform package. +Production builds omit private test controls. +Keep native outputs, Cargo targets, and staged sources separate for each mode. For example, from the repository root in PowerShell: ```powershell @@ -58,39 +59,42 @@ node ci\run-lifecycle.cjs instrumented target\lifecycle-instrumented\stage Remove-Item Env:CARGO_TARGET_DIR ``` -Instrumented cases exercise real provider waits and contained owned-task faults. -The recurring `lifecycle-instrumented` CI job runs on pull requests and main, -alongside production tests. It requires every hook-dependent case to execute: -missing hooks, mode/hash mismatches, skips, TODOs, aborted tests or empty selection -fail validation. Production mode skips only explicitly marked instrumented cases -and continues to assert that private exports are absent. - -The stage manifest identifies the exact local addon and SHA-256, selected feature -mode and source hashes. Tests match the Duroxide loader's native class identities, -not an arbitrary cached `.node`. Sources live under a stage-local `node_modules` -directory so root-level test discovery cannot run duplicate staged suites. The -runner sets mode/provenance only in its child environment and retains TAP, stderr -and JSON totals beside the manifest. Instrumented evidence artifacts are not -named `binary-*`, contain no native binaries, and never feed packing/release jobs. -`CARGO_TARGET_DIR` also tells napi where to copy the compiled library from; -passing only Cargo's `--target-dir` would not set napi's source path. +Instrumented cases exercise real provider waits and contained faults in owned tasks. +The `lifecycle-instrumented` CI job runs on pull requests and main, alongside production tests. +Every hook-dependent case must execute. +Missing hooks, mode or hash mismatches, skips, TODOs, aborted tests, and empty selections fail validation. +Production mode skips only explicitly marked instrumented cases. +It also checks that private exports are absent. + +The stage manifest identifies the exact local addon, its SHA-256, the selected feature mode, and source hashes. +Tests match the Duroxide loader's native class identities, not an arbitrary cached `.node`. +Sources use a stage-local `node_modules` directory. +This prevents root-level test discovery from running duplicate staged suites. +The runner sets mode and provenance only in its child environment. +It retains TAP, stderr, and JSON totals beside the manifest. +Instrumented evidence artifacts do not use `binary-*` names or contain native binaries. +They never feed packing or release jobs. +`CARGO_TARGET_DIR` also tells napi where to find the compiled library. +Cargo's `--target-dir` alone does not set napi's source path. The `--dts` filename is relative to napi's destination directory. -Both ordered races still run **100 iterations**. Held-provider probes still -require `6000 <= elapsedMs < 7000` for omitted grace, `5000 <= elapsedMs < 6000` -for zero grace, and `repeatedMs < 1000`. Their separate message-delivery watchdog -allows the first call, repeated call and bounded transport headroom; it cannot -relax those measured SDK limits. The suite also checks incomplete-cleanup -event-loop liveness without stdin/referenced timers, and activity-only -self-capturing callback collection in a live process after creating frames retire. -Collection uses bounded WeakRef/FinalizationRegistry settling with a noncapturing -control, not natural process exit or zero task counts. Successful, quiescent-error -and late cleanup are covered, including late release with no further stop caller. - -Native-independent harness/oracle checks can run before a native build: -`node --test __tests__/lifecycle-oracles.test.js`. They include deliberate child -failure, malformed protocol, zero-start counter and CI-skip negative controls. - -Do not publish instrumented assets or temporary local core overrides. A matching -published core minimum remains a separate prerequisite: CI must fail missing core -APIs rather than silently skip coverage or substitute an unofficial dependency. \ No newline at end of file +Both ordered races still run **100 iterations**. +Held-provider probes require `6000 <= elapsedMs < 7000` for omitted grace, +`5000 <= elapsedMs < 6000` for zero grace, and `repeatedMs < 1000`. +A separate message-delivery watchdog allows the first call, repeated call, and limited transport time. +It cannot relax those measured SDK limits. + +The suite checks event-loop liveness during incomplete cleanup, without stdin or referenced timers. +It also checks collection of activity-only self-capturing callbacks in a live process, after the creating frames return. +Collection uses bounded WeakRef/FinalizationRegistry settling and a noncapturing control. +It does not use natural process exit or zero task counts as proof. +Cases cover successful, quiescent-error, and late cleanup, including late release without another stop caller. + +Harness and oracle checks that do not need native code can run before a native build: +`node --test __tests__/lifecycle-oracles.test.js`. +They include negative controls for child failure, malformed protocol, a zero-start counter, and CI skips. + +Do not publish instrumented assets or temporary local core overrides. +A matching published core minimum remains a separate prerequisite. +CI must fail when core APIs are missing. +It must not silently skip coverage or substitute an unofficial dependency. \ No newline at end of file diff --git a/README.md b/README.md index c081556..053355b 100644 --- a/README.md +++ b/README.md @@ -27,39 +27,45 @@ Node.js/TypeScript SDK for the [Duroxide](https://github.com/microsoft/duroxide) ## Runtime lifecycle (unreleased) -`await runtime.start()` returns after startup and propagates fallible core errors. +`await runtime.start()` returns after startup. Core startup errors reject the Promise. `await runtime.shutdown(timeoutMs)` still resolves to `undefined` on success. -The argument is a nonnegative safe-integer grace period in milliseconds, within -the native monotonic deadline range. Omission means 1000 ms; total waiting is -grace plus 5000 ms. Idle work can finish early. -At the first stop call, the supervisor's **remaining** time must exceed that total, -with room for other shutdown work, result delivery and process exit. Passing the -entire outer budget as grace cannot fit; see the +The argument is a nonnegative safe-integer grace period in milliseconds. +It must fit the native monotonic deadline range. +Omitted grace is 1000 ms. Total waiting is grace plus 5000 ms. +Idle work can finish early. +At the first stop call, the supervisor's **remaining** time must exceed that total. +Leave time for other shutdown work, result delivery, and process exit. +Do not pass the entire outer budget as grace. See the [budget-sizing guidance](docs/user-guide.md#sizing-the-supervisor-budget). -The first stop fixes both deadlines. Repeated calls observe the same operation, -including timeout/operational errors and genuine late completion. Shutdown before -start is now terminal; later start or registration is rejected. Metrics remain -unavailable after stop. Serialize lifecycle and registration calls on each runtime. -Actual quiescence releases obsolete native callback/core holders, including after -a quiescent failure or late cleanup without another caller. Stop results and the -original deadlines remain; incomplete or unproven cleanup retains ownership. +The first stop fixes both deadlines. Repeated calls observe the same operation. +They can report timeout, operational errors, or actual late completion. +Shutdown before start is terminal. Later start or registration calls fail. +Metrics remain unavailable after stop. +Serialize lifecycle and registration calls on each runtime. + +**Quiescence** means that tracked core execution and cleanup have ended. +Only proven quiescence permits release of obsolete native callback and core holders. +This includes quiescent failures and late cleanup without another caller. +Stop results and the original deadlines remain available after release. +Incomplete cleanup, unproven cleanup, or a callback-release failure retains ownership. A `lifecycle_shutdown_timed_out` rejection means cleanup is still owned and -incomplete. Do not restart the worker: arrange process termination through the -application or supervisor, even if a later observation reports completion. The SDK -does not terminate the process. `lifecycle_shutdown_failed` is an error category, -not proof of quiescence: it also covers an unproven/lost cleanup owner and a -callback-reference release failure after core cleanup. Do not infer release -permission from that prefix. This narrow adapter change does not add tracking -of arbitrary JavaScript continuations or the .NET SDK's additional managed -foreign-retirement barrier. Independently owned provider/Client resources have -their own lifetime. See the [lifecycle guide](docs/user-guide.md#runtime-lifecycle) +incomplete. Do not restart the worker. +The application or supervisor must terminate the process, even if a later observation reports completion. +The SDK does not terminate the process. +`lifecycle_shutdown_failed` is an error category, not proof of quiescence. +It also covers a lost cleanup owner and failure to release callback references after core cleanup. +Do not infer release permission from that prefix. + +This adapter does not track arbitrary JavaScript continuations. +It does not add the .NET SDK's additional managed foreign-retirement barrier. +Independent provider and Client resources have their own lifetime. +See the [lifecycle guide](docs/user-guide.md#runtime-lifecycle) for validation and an executable early-drain/repeated-stop example. -These unreleased changes require the corresponding core lifecycle release before -distribution; a temporary local Cargo override is development evidence, not a -release dependency pin. +These unreleased changes require a compatible core lifecycle release before distribution. +A temporary local Cargo override provides development evidence, not a release dependency pin. ## Quick Start diff --git a/ci/smoke/README.md b/ci/smoke/README.md index af081a6..1279e52 100644 --- a/ci/smoke/README.md +++ b/ci/smoke/README.md @@ -19,10 +19,10 @@ back to the local `.node` binary when run from the repo. | `run-local.sh` / `.ps1` | Install from local tarballs (pre-publish). | | `run-registry.sh` / `.ps1` | Install from npm registry (post-publish). | -The launchers copy both the smoke script and its provenance helper. Verification -matches the loader's native class identities instead of selecting the first -cached addon. An installed smoke must resolve a declared Duroxide platform -subpackage at the SDK's version, with no lifecycle instrumentation exports. +The launchers copy both the smoke script and its provenance helper. +Verification matches the loader's native class identities, not the first cached addon. +The installed smoke must resolve a declared Duroxide platform subpackage at the SDK's version. +That package must not expose lifecycle instrumentation. ## Parity diff --git a/docs/architecture.md b/docs/architecture.md index 433e7a2..bf9e24e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -391,32 +391,53 @@ File-based SQLite can hit "database is locked" errors under concurrent orchestra ### Runtime Shutdown -In the unreleased lifecycle, `Runtime.shutdown(timeoutMs)` remains a -`Promise` but waits for actual core retirement only within a finite budget: -supplied grace plus 5000 ms, with 1000 ms omitted grace. Idle execution can finish -early. Zero requests immediate force; force is not proof of quiescence. +In the unreleased lifecycle, `Runtime.shutdown(timeoutMs)` remains a `Promise`. +It waits for actual core retirement within supplied grace plus 5000 ms. +Omitted grace is 1000 ms. Idle execution can finish early. +Zero requests immediate force. Force does not prove that cleanup ended. The first valid stop fixes both deadlines, including for repeated observations. -The adapter stores the prepared core Arc before awaiting fallible startup and -retains it after startup failure, stop, or timeout. The adapter retains one native -cleanup future independently of its bounded observers. Upstream core tracking -includes native task destruction; provider futures can be canceled under the -provider's atomic-acknowledgement and recovery contract. Repeated shutdown observes the same operation rather -than succeeding because an Arc was taken from the wrapper. Valid pre-start stop -is terminal; registration/start afterward reject, and metrics remain unavailable -after stop/startup failure. Same-instance lifecycle/registration calls must remain -serialized. - -On proven core retirement, the shutdown Promise resolver runs on the JavaScript -thread and unreferences retained ThreadsafeFunctions so an otherwise idle process -can exit. Unref does not abort/free callbacks or discard completion. Incomplete -cleanup retains ownership and may keep the process alive. Timeout explicitly -requires application/supervisor process termination, even after late retirement; -the SDK itself never kills the process. - -This is core-owned execution, not complete ownership of every JavaScript task. -Independently owned Clients, providers, and provider token-refresh work remain -separate. See the [lifecycle guide](user-guide.md#runtime-lifecycle) for argument +The adapter stores the prepared core Arc before it awaits fallible startup. +Startup failure, stop, and timeout do not discard cleanup ownership. +One native cleanup future remains owned independently of bounded caller waits. +Upstream core tracking includes native task destruction. +Provider futures can be canceled under the provider's atomic acknowledgment and recovery contract. +Repeated shutdown observes the same operation. +It does not report success merely because the wrapper transferred its Arc. + +```mermaid +flowchart TD + Stop["First valid stop: fix grace and total"] --> Cleanup["Keep one native cleanup operation"] + Stop --> Wait["Wait within the original total"] + Cleanup -->|Cleanup proven| Release["Release callback and core holders on the JavaScript thread"] + Cleanup -->|Cleanup unproven| Retain["Retain ownership"] + Release -->|Release fails| Retain + Wait -->|Completion published| Result["Resolve undefined or reject with an error"] + Wait -->|Total expires first| Timeout["Reject timeout; keep cleanup owned"] + Timeout --> Terminate["Application or supervisor must terminate the process"] +``` + +The caller's wait and cleanup have separate lifetimes. +An actual cleanup result can arrive after the caller's timeout. + +After proven core cleanup, the owned resolver runs on the JavaScript thread. +It removes event-loop references from retained ThreadsafeFunctions. +After successful removal, it releases obsolete callback and core holders. +The terminal result and deadlines remain available. +Removing an event-loop reference alone does not abort or collect a callback. +A callback-release failure or unproven cleanup retains ownership and can keep the process alive. + +A valid pre-start stop is terminal. +Later registration and start calls fail. +Metrics remain unavailable after stop or accepted startup failure. +Serialize lifecycle and registration calls on the same instance. +Timeout requires application- or supervisor-owned process termination, even after late cleanup. +The SDK never kills the process. + +This contract covers core-owned execution, not every JavaScript task. +Independent Clients, providers, and provider token-refresh work keep their separate lifetimes. +The global orchestration-handler map also remains unchanged. +See the [lifecycle guide](user-guide.md#runtime-lifecycle) for argument validation, errors, and release prerequisites. ### Platform-Specific Binary diff --git a/docs/user-guide.md b/docs/user-guide.md index b4347e7..4bff729 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -51,59 +51,65 @@ main(); ## Runtime Lifecycle -This section requires the **unreleased** adapter and matching core lifecycle; -published packages and temporary local core overrides are not release evidence. - -Register handlers before the first `await runtime.start()`. Startup returns after -activation, not after shutdown, and failures reject the Promise. -Registry/preparation failures, including invalid or descending versions, become -ordinary `lifecycle_start_failed` errors retained by subsequent shutdown calls. -Version ordering and duplicate-registration policy are unchanged. Adapter error -messages do not include panic payloads; this does not suppress Rust's separate -process-wide panic-hook output. - -`await runtime.shutdown()` uses one second of grace plus five seconds of cleanup -headroom. `shutdown(0)` requests immediate force, still with a five-second total. -The first stop fixes the deadlines; repeating it never grants a new budget. -Success remains `undefined`; errors carry `lifecycle_start_failed`, -`lifecycle_shutdown_timed_out`, or `lifecycle_shutdown_failed` categories. -The `lifecycle_shutdown_failed` category alone is **not** cleanup proof. It covers -known-quiescent core failures, loss of the native shutdown owner with unproven -quiescence, and failure to release a Node callback's event-loop reference after -core quiescence. The last category's message does not itself assert quiescence. -Tests for known-quiescent core failures are specific to that outcome; neither -generic failure-category checks nor task-count telemetry authorize release. -Invalid non-finite, fractional, negative, unsafe-integer, or unsupported native -durations reject without consuming registrations or accepting stop. -Strings and `null` also reject; `undefined` means the omitted grace. Native -validation uses signed millisecond extraction and checked unsigned-nanosecond -deadlines: the interval ceiling for grace is -`floor((2^64 - 1) / 1_000_000) - 5000` milliseconds, also subject to the platform's -monotonic clock range. Validation still runs on unused or repeated calls. - -Stopping before start is terminal. Do not overlap lifecycle or registration calls -on the same instance. Metrics return `null` after stop. Repeated shutdown can -observe real late completion, but a timeout permanently rules out reuse and still -requires application/supervisor process termination. Cleanup remains owned; the -SDK never exits the process itself. This contract covers core-owned execution, -not arbitrary unregistered JavaScript continuations or the .NET adapter's -additional foreign-continuation barrier. Independent provider-owned background -work has a separate lifetime. Dropping/ignoring the returned Promise does not -cancel the adapter-owned native cleanup future. Bounded waits and first-stop -deadlines belong to this adapter; native Rust shutdown itself awaits cleanup -without a total limit. Provider futures can be canceled under the provider's -atomic acknowledgement and peek-lock recovery contract. - -Callback/core holders are separate from persistent shutdown results and deadlines. -Only proven native quiescence permits their release on the JavaScript thread, -including known-quiescent failures and actual late completion without a further -shutdown call. This breaks obsolete activity callback/self-capture ownership -cycles; merely unref'ing an event-loop handle is not callback collection. -Unproven cleanup or a lost cleanup owner retains the graph and leaves the worker -unusable. Application-held references and the pre-existing global orchestration -handler map are outside this collection guarantee. Independent provider/client -ownership is not disposed by stopping a worker, and late release never retracts -the application's prior termination obligation. +This section requires the **unreleased** adapter and a matching core lifecycle build. +Current published packages and temporary local core overrides do not prove release readiness. + +Register handlers before the first `await runtime.start()`. +Startup returns after activation, not after shutdown. Failures reject the Promise. +Registry and preparation failures become ordinary `lifecycle_start_failed` errors. +These include invalid or descending versions. +Subsequent shutdown calls retain those startup errors. +Version ordering and duplicate-registration policy do not change. +Adapter error messages exclude panic payloads. +This does not suppress Rust's separate process-wide panic-hook output. + +`await runtime.shutdown()` uses one second of grace plus five seconds of total headroom. +`shutdown(0)` requests immediate force, with a five-second total. +The first stop fixes the deadlines. Repeated calls never receive a new budget. +Success remains `undefined`. +Errors use `lifecycle_start_failed`, `lifecycle_shutdown_timed_out`, or `lifecycle_shutdown_failed`. + +**Quiescence** means that tracked core execution and cleanup have ended. +The `lifecycle_shutdown_failed` category alone does **not** prove quiescence. +It covers core failures after proven cleanup and loss of the native shutdown owner without proof of cleanup. +It also covers failure to release a Node callback's event-loop reference after core cleanup. +That callback-release error message does not itself assert quiescence. +Tests for known-quiescent core failures apply only to that outcome. +Neither a generic failure-category check nor task-count telemetry permits release. + +Non-finite, fractional, negative, unsafe-integer, or unsupported native durations fail validation. +Validation failure does not consume registrations or accept stop. +Strings and `null` also fail. `undefined` selects the omitted grace. +Native validation extracts signed milliseconds and checks unsigned-nanosecond deadlines. +The maximum grace interval is `floor((2^64 - 1) / 1_000_000) - 5000` milliseconds. +It must also fit the platform's monotonic clock range. +Validation still runs on unused runtimes and repeated calls. + +Stopping before start is terminal. +Do not overlap lifecycle or registration calls on the same instance. +Metrics return `null` after stop. +Repeated shutdown can observe actual late completion, but a timeout permanently prevents reuse. +The application or supervisor must still terminate the process. +The SDK retains cleanup and never exits the process itself. + +This contract covers core-owned execution, not arbitrary unregistered JavaScript continuations. +It does not add the .NET adapter's additional foreign-continuation barrier. +Independent provider-owned background work has a separate lifetime. +Ignoring the returned Promise does not cancel the adapter-owned native cleanup future. +The adapter owns bounded waits and first-stop deadlines. +Native Rust shutdown waits for cleanup without a total limit. +Provider futures can be canceled under the provider's atomic acknowledgment and peek-lock recovery contract. + +Callback and core holders are separate from persistent shutdown results and deadlines. +Only proven native quiescence permits their release on the JavaScript thread. +This includes known-quiescent failures and actual late completion without another shutdown call. +Release breaks obsolete activity callbacks' self-capture cycles. +Removing an event-loop reference alone does not collect the callback. +Unproven cleanup, a lost cleanup owner, or a callback-release failure retains ownership and leaves the worker unusable. + +Application-held references and the global orchestration-handler map are outside this collection guarantee. +Stopping a worker does not dispose independent provider or Client ownership. +Late release never removes the application's prior termination requirement. This complete example has no admitted work and can finish before grace: @@ -124,33 +130,33 @@ main().catch(error => { }); ``` -For incomplete stop, setting an exit code alone may not terminate retained -noncooperative work. Arrange termination in the application's supervisor policy; -do not treat a caught error or successful late observation as permission to -reuse the runtime. A finite grace cannot promise bounded forced cleanup. +After an incomplete stop, setting an exit code alone might not terminate retained work. +Arrange process termination through the application's supervisor policy. +A caught error or successful late observation does not permit runtime reuse. +A finite grace does not guarantee bounded forced cleanup. ### Sizing the supervisor budget -Size against the supervisor time **remaining when the first shutdown call is -made**, not its original allocation: +Use the supervisor time **remaining at the first shutdown call**, not its original allocation: ```text remaining outer time > grace + 5000 ms + other shutdown work + delivery/exit margin ``` -For example, with 30 seconds actually remaining, a 30-second grace asks the SDK -to observe cleanup for up to 35 seconds and cannot fit. An illustrative -20-second grace gives a 25-second SDK observation budget and leaves five seconds -for other work, scheduling, result delivery and process exit. Subtract time -already spent before calling `shutdown`. If less than the fixed five-second -allowance remains, `shutdown(0)` alone cannot make the SDK's timeout fit. - -Omission still means 1000 ms grace and 6000 ms total; zero grace still means -5000 ms total. The first call fixes those deadlines, and repeated calls neither -renew the budget nor retract a previous termination obligation. Native cleanup -itself has no total bound and may remain pending indefinitely. The application's -parent/supervisor termination policy must therefore fit outside these SDK -observation budgets; the SDK supplies no process-kill or supervisor mechanism. +For example, assume that 30 seconds remain. +A 30-second grace requests up to 35 seconds of waiting and cannot fit. +A 20-second grace gives the SDK 25 seconds to wait. +That leaves five seconds for other work, scheduling, result delivery, and process exit. +Subtract time already spent before calling `shutdown`. +If less than five seconds remain, `shutdown(0)` alone cannot make the SDK's timeout fit. + +Omitted grace still means 1000 ms grace and 6000 ms total. +Zero grace still means 5000 ms total. +The first call fixes those deadlines. +Repeated calls cannot renew the budget or remove a previous termination requirement. +Native cleanup has no total bound and can remain pending indefinitely. +The application's parent or supervisor must allow time for process termination outside the SDK wait budget. +The SDK does not provide a process-kill or supervisor mechanism. ## Orchestration Patterns diff --git a/lib/duroxide.d.ts b/lib/duroxide.d.ts index 9149129..15eb251 100644 --- a/lib/duroxide.d.ts +++ b/lib/duroxide.d.ts @@ -321,12 +321,12 @@ export declare class Runtime { registerOrchestrationTyped(name: string, fn: (ctx: OrchestrationContext, input: TIn) => Generator): void; registerOrchestrationVersioned(name: string, version: string, fn: (ctx: OrchestrationContext, input: any) => Generator): void; registerOrchestrationVersionedTyped(name: string, version: string, fn: (ctx: OrchestrationContext, input: TIn) => Generator): void; - /** Start once; returns after startup. Do not overlap lifecycle/registration calls. */ + /** Start once and return after startup. Do not overlap lifecycle or registration calls. */ start(): Promise; /** - * Stop permanently, including before start. Default grace is 1000 ms; total is grace + 5000 ms. - * Repeated calls retain the original deadlines and observe actual completion/errors. - * Timeout rejects while cleanup remains owned; application/supervisor termination is required. + * Stop permanently, including before start. Default grace is 1000 ms. Total waiting is grace + 5000 ms. + * Repeated calls keep the original deadlines and observe actual completion or errors. + * Timeout rejects while cleanup remains owned. The application or supervisor must terminate the process. * timeoutMs must be a nonnegative safe integer within the native monotonic deadline range. */ shutdown(timeoutMs?: number): Promise; diff --git a/lib/duroxide.js b/lib/duroxide.js index 78c0331..2873554 100644 --- a/lib/duroxide.js +++ b/lib/duroxide.js @@ -1195,17 +1195,17 @@ class Runtime { } /** - * Start once and return after startup. Do not overlap lifecycle/registration calls. + * Start once and return after startup. Do not overlap lifecycle or registration calls. */ async start() { await this._native.start(); } /** - * Stop permanently. Omitted grace is 1000 ms; the total bound is grace plus 5000 ms. + * Stop permanently. Omitted grace is 1000 ms. Total waiting is grace plus 5000 ms. * Repeated calls observe the original stop, including failures or actual late completion. - * Timeout leaves cleanup owned and requires application/supervisor process termination. - * Even shutdown before start is terminal. Do not overlap lifecycle/registration calls. + * Timeout leaves cleanup owned. The application or supervisor must terminate the process. + * Even shutdown before start is terminal. Do not overlap lifecycle or registration calls. * @param {number} [timeoutMs] Nonnegative safe-integer milliseconds in the native deadline range. */ async shutdown(timeoutMs) { From 6bead773ce13dd5bc2325318e90ab3489544e70b Mon Sep 17 00:00:00 2001 From: "Christopher Krawczyk (SQL)" Date: Wed, 4 Nov 2026 17:25:26 -0800 Subject: [PATCH 4/4] Add metadata-only runtime stop observers Expose retained stop/fault state through independently owned observers with isolated AbortSignal waits and terminal local close. Keep observation metadata separate from worker ownership and event-loop keepalive. Cover real faults, source retirement, collection and notification races, and document the contracts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 73748d4f-2eb6-47d2-9dbc-af8681e80e83 --- CHANGELOG.md | 8 + README.md | 22 + .../fixtures/lifecycle-collection-probe.cjs | 104 ++++- __tests__/fixtures/lifecycle-probe.cjs | 34 +- __tests__/helpers/lifecycle-oracles.cjs | 144 +++++++ __tests__/lifecycle-oracles.test.js | 14 + __tests__/lifecycle.test.js | 404 +++++++++++++++++- docs/architecture.md | 26 ++ docs/user-guide.md | 87 ++++ index.d.ts | 19 +- index.js | 4 +- lib/duroxide.d.ts | 40 ++ lib/duroxide.js | 135 ++++++ src/lib.rs | 2 + src/runtime.rs | 25 +- src/stop_observation.rs | 252 +++++++++++ src/test_hooks.rs | 136 ++++++ 17 files changed, 1442 insertions(+), 14 deletions(-) create mode 100644 src/stop_observation.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d28d6..deafba5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add owned, metadata-only `RuntimeStopObserver` through `Runtime.stopObserver()`. + Retained stop/fault state supports independent AbortSignal waits, terminal + local close, and final-state delivery before source end. Observation does not + retain worker callbacks, keep Node alive by itself, or change shutdown + ownership, deadlines, and application policy. + ### Changed - Use fallible core startup and bounded shutdown waits while preserving `Promise`. diff --git a/README.md b/README.md index 053355b..39af3cb 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,28 @@ for validation and an executable early-drain/repeated-stop example. These unreleased changes require a compatible core lifecycle release before distribution. A temporary local Cargo override provides development evidence, not a release dependency pin. +### Passive stop observation + +`runtime.stopObserver()` returns an owned, metadata-only `RuntimeStopObserver`. +Read `observer.state` first, then call `waitForChange(lastSeen, { signal })`. +The states are `'NotRequested'`, `'Requested'`, and sticky `'Faulted'`. +They report recorded stop/fault state, not readiness or completed cleanup. +A late observer can already see a fault. + +Use one pending wait per observer. Independent consumers need separate observers. +`AbortSignal` cancels only its wait, rejects with `signal.reason`, and permits retry. +`close()` is terminal and idempotent; pending and later operations fail. +A normal source end returns an unseen final state before `null`. +Intermediate states can coalesce. Neither source end nor a settled Promise +proves worker cleanup. + +Observation does not start/stop the worker, change shutdown deadlines, or retain +its callbacks. Its native delivery does not keep Node alive by itself; the event +loop must still run to deliver results. The existing limit on unregistered +JavaScript continuations remains. See the +[observer example](docs/user-guide.md#passive-stop-observation) for initial state, +cancellation, local close, and normal source end. + ## Quick Start ```bash diff --git a/__tests__/fixtures/lifecycle-collection-probe.cjs b/__tests__/fixtures/lifecycle-collection-probe.cjs index cfb7e38..e7931a7 100644 --- a/__tests__/fixtures/lifecycle-collection-probe.cjs +++ b/__tests__/fixtures/lifecycle-collection-probe.cjs @@ -4,10 +4,11 @@ const assert = require('node:assert/strict'); const path = require('node:path'); const { performance } = require('node:perf_hooks'); +const { getEventListeners } = require('node:events'); const { setImmediate: nextTurn, setTimeout: delay } = require('node:timers/promises'); const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); const { nativeProvenance } = require('../../ci/smoke/native-provenance.cjs'); -const { incompleteTimeout, knownQuiescentCoreFailure, retiredState } = require('../helpers/lifecycle-oracles.cjs'); +const { incompleteTimeout, knownQuiescentCoreFailure, retiredState, bounded, until } = require('../helpers/lifecycle-oracles.cjs'); const { native, provenance } = nativeProvenance(path.resolve(__dirname, '..', '..'), { instrumented: process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1', @@ -109,9 +110,110 @@ async function observeLateCompletion(runtimeRef) { assert.equal(runtime.metricsSnapshot(), null); } +function observationOwner() { + return SqliteProvider.inMemory().then(provider => { + const runtime = new Runtime(provider); + const observer = runtime.stopObserver(); + const records = {}; + for (const [name, value] of Object.entries({ runtime, native: runtime._native })) { + const label = `SourceEnd:${name}`; + registry.register(value, label); + records[name] = { label, weak: new WeakRef(value) }; + } + const abort = new AbortController(); + const pending = observer.waitForChange('NotRequested', { signal: abort.signal }); + pending.catch(() => {}); + return { observer, records, pending, abort }; + }); +} + +async function sourceEndProbe() { + const { observer, records, pending, abort } = await observationOwner(); + try { + const state = await settle(records, records); + assert.ok(collected(state.control), JSON.stringify(state)); + assert.equal(await bounded(pending), null); + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + abort.abort(); + assert.equal(await pending, null); + assert.equal(observer.state, 'NotRequested'); + assert.equal(await observer.waitForChange('NotRequested'), null); + emit('OBSERVER_COLLECTION', { scenario: 'observer-source-end', collected: true, state, native: provenance }); + } finally { observer.close(); } +} + +async function retainedObservationOwner(label, captures) { + const runtime = new Runtime(await SqliteProvider.inMemory(), { dispatcherPollIntervalMs: 5 }); + const owner = { runtime, prefix: label }; + const callback = captures ? capturingActivity(owner) : noncapturingActivity(); + const records = {}; + for (const [name, value] of Object.entries({ owner, callback })) { + const key = `${label}:${name}`; + registry.register(value, key); + records[name] = { label: key, weak: new WeakRef(value) }; + } + runtime.registerActivity(`ObserverCollection${label}`, callback); + const observer = runtime.stopObserver(); + const hooks = provenance.instrumented ? native._lifecycleTestHooks(runtime._native) : undefined; + let first; + if (hooks) { + hooks.hold('observer-final-access'); + first = observer.waitForChange('NotRequested'); + first.catch(() => {}); + } + await runtime.start(); + await runtime.shutdown(100); + if (hooks) { + assert.equal(await bounded(first), 'Requested'); + await hooks.waitEntered('observer-final-access'); + assert.ok(JSON.parse(hooks.snapshot()).observerPending > 0); + } + const pendingState = { settled: false }; + const pending = observer.waitForChange('Requested'); + pending.then(() => { pendingState.settled = true; }, () => { pendingState.settled = true; }); + return { runtime, observer, pending, pendingState, hooks, records }; +} + +async function retainedObservationProbe() { + const control = await retainedObservationOwner('ObserverControl', false); + const captured = await retainedObservationOwner('ObserverCaptured', true); + try { + const state = await settle(control.records, captured.records); + assert.ok(collected(state.control), JSON.stringify(state)); + assert.ok(collected(state.captured), JSON.stringify(state)); + for (const value of [control, captured]) { + assert.equal(value.runtime.metricsSnapshot(), null); + assert.equal(value.observer.state, 'Requested'); + assert.equal(value.runtime._stopSource.snapshot().closed, false); + assert.equal(value.pendingState.settled, false); + if (value.hooks) { + const execution = JSON.parse(value.hooks.snapshot()); + retiredState(execution, { executionWitness: true }); + assert.ok(execution.observerPending >= 2, 'old delivery tail and new wait must both remain owned'); + } + } + emit('OBSERVER_COLLECTION', { + scenario: 'retained-observer', collected: true, state, native: provenance, + runtimeRetained: true, observerRetained: true, sourceOpen: true, + nativeTailHeld: provenance.instrumented, + }); + } finally { + for (const value of [control, captured]) { + value.observer.close(); + await assert.rejects(value.pending, /stop_observer_closed/); + if (value.hooks) { + value.hooks.release('observer-final-access'); + await until(() => JSON.parse(value.hooks.snapshot()).observerPending === 0); + } + } + } +} + async function main() { assert.equal(typeof global.gc, 'function', 'collection probe requires --expose-gc'); const requested = process.argv[2]; + if (requested === 'observer-source-end') return sourceEndProbe(); + if (requested === 'retained-observer') return retainedObservationProbe(); const scenario = requested === 'baseline' ? 'completed' : requested; assert.ok(['completed', 'quiescent-failure', 'late', 'late-unobserved'].includes(scenario)); const [control, captured] = await Promise.all([ diff --git a/__tests__/fixtures/lifecycle-probe.cjs b/__tests__/fixtures/lifecycle-probe.cjs index 261347e..395b9e5 100644 --- a/__tests__/fixtures/lifecycle-probe.cjs +++ b/__tests__/fixtures/lifecycle-probe.cjs @@ -7,7 +7,7 @@ const { performance } = require('node:perf_hooks'); const path = require('node:path'); const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); const { nativeProvenance } = require('../../ci/smoke/native-provenance.cjs'); -const { incompleteTimeout, retiredState } = require('../helpers/lifecycle-oracles.cjs'); +const { incompleteTimeout, retiredState, pendingObserverExit } = require('../helpers/lifecycle-oracles.cjs'); const { native, provenance } = nativeProvenance(path.resolve(__dirname, '..', '..'), { instrumented: process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1', manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, @@ -57,6 +57,38 @@ async function main() { emit('EXIT_READY'); return; } + if (scenario === 'observer-exit') { + const hooks = provenance.instrumented ? native._lifecycleTestHooks(runtime._native) : undefined; + const observer = runtime.stopObserver(); + runtime.registerActivity('ObserverExitActivity', async (_, value) => value); + let settled = false; + let pending; + if (hooks) { + hooks.hold('observer-delivery'); + pending = observer.waitForChange('NotRequested'); + pending.then(() => { settled = true; }, () => { settled = true; }); + } + await runtime.start(); + await runtime.shutdown(100); + if (hooks) { + await hooks.waitEntered('observer-delivery'); + } else { + pending = observer.waitForChange('Requested'); + pending.then(() => { settled = true; }, () => { settled = true; }); + } + const report = () => ({ + source: runtime._stopSource.snapshot(), settled, native: provenance, + execution: hooks ? JSON.parse(hooks.snapshot()) : undefined, + }); + globalThis.retainedObserver = { runtime, observer, pending, hooks }; + pendingObserverExit(report()); + emit('OBSERVER_EXIT_READY', report()); + process.once('beforeExit', () => { + pendingObserverExit(report()); + emit('OBSERVER_BEFORE_EXIT', report()); + }); + return; + } const hooks = native._lifecycleTestHooks(runtime._native); hooks.hold('blocking-provider'); runtime.registerActivity('PendingCleanupActivity', async (_, value) => value); diff --git a/__tests__/helpers/lifecycle-oracles.cjs b/__tests__/helpers/lifecycle-oracles.cjs index c2a4b6b..2e50082 100644 --- a/__tests__/helpers/lifecycle-oracles.cjs +++ b/__tests__/helpers/lifecycle-oracles.cjs @@ -2,6 +2,9 @@ // Licensed under the MIT License. const assert = require('node:assert/strict'); +const { getEventListeners } = require('node:events'); +const { performance } = require('node:perf_hooks'); +const { setTimeout: delay } = require('node:timers/promises'); function diagnostic(category) { return error => { @@ -102,7 +105,148 @@ async function orderedRaceAttempt(runtime, hooks, force, iteration) { } } +async function bounded(promise, milliseconds = 3000) { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('stop observation did not settle within its test bound')), milliseconds); + }), + ]); + } finally { clearTimeout(timer); } +} + +async function until(predicate, milliseconds = 3000) { + const deadline = performance.now() + milliseconds; + while (!predicate()) { + assert.ok(performance.now() < deadline, 'stop observation condition did not become true'); + await delay(1); + } +} + +function pendingObserverExit(value) { + assert.equal(value.source.closed, false, 'an ended source cannot prove pending observation liveness'); + assert.equal(value.settled, false, 'a settled promise cannot prove pending observation liveness'); + assert.equal(value.source.state, 'Requested'); + if (value.execution) { + retiredState(value.execution, { executionWitness: true }); + assert.ok(value.execution.observerPending > 0, 'notification context must still be owned'); + } +} + +async function observerRaceAttempt(runtime, hooks, changeFirst, iteration) { + const armed = []; + let delivered = 0; + const nativeWait = runtime._stopSource.waitForChange; + Object.defineProperty(runtime._stopSource, 'waitForChange', { + value(lastSeen, callback) { + return nativeWait.call(this, lastSeen, (...args) => { + delivered++; + callback(...args); + }); + }, + }); + const first = runtime.stopObserver(); + const closed = runtime.stopObserver(); + const initialAbort = new AbortController(); + const retryAbort = new AbortController(); + const reason = new Error('first-observation-aborted'); + const retryReason = new Error('retried-observation-aborted'); + let startup; + let failure; + let phase = 'arming'; + let late; + try { + for (const gate of ['startup', 'observer-delivery']) { + phase = `arm:${gate}`; + hooks.hold(gate); + armed.push(gate); + } + const firstWait = first.waitForChange('NotRequested', { signal: initialAbort.signal }); + const closeWait = closed.waitForChange('NotRequested'); + firstWait.catch(() => {}); + closeWait.catch(() => {}); + const staleAbort = assertSingleListener(initialAbort.signal); + startup = runtime.start(); + phase = 'held-startup'; + await hooks.waitEntered('startup'); + late = runtime.stopObserver(); + assert.equal(first.state, 'NotRequested'); + assert.equal(late.state, 'NotRequested'); + if (changeFirst) { + hooks.release('startup'); + await startup; + await runtime.shutdown(0); + await hooks.waitEntered('observer-delivery'); + } + phase = 'cancel-and-close'; + initialAbort.abort(reason); + closed.close(); + await bounded(Promise.all([ + assert.rejects(firstWait, error => error === reason), + assert.rejects(closeWait, /stop_observer_closed/), + ])); + assert.equal(getEventListeners(initialAbort.signal, 'abort').length, 0); + assert.equal(first.state, changeFirst ? 'Requested' : 'NotRequested'); + let retriedSettled = false; + const retry = first.waitForChange(changeFirst ? 'Requested' : 'NotRequested', { signal: retryAbort.signal }); + retry.then(() => { retriedSettled = true; }, () => { retriedSettled = true; }); + staleAbort(); + assertSingleListener(retryAbort.signal); + phase = 'retire-worker'; + if (!changeFirst) { + hooks.release('startup'); + await startup; + await runtime.shutdown(0); + await hooks.waitEntered('observer-delivery'); + } + const state = JSON.parse(hooks.snapshot()); + retiredState(state, { executionWitness: true }); + assert.ok(state.observerPending >= 2, 'old notification contexts must survive local settlement'); + assert.equal(late.state, 'Requested'); + assert.equal(runtime._stopSource.snapshot().closed, false); + phase = 'late-delivery'; + hooks.release('observer-delivery'); + if (changeFirst) { + await until(() => delivered >= 2); + assert.equal(retriedSettled, false, 'old callbacks must not settle the new pending wait'); + retryAbort.abort(retryReason); + await bounded(assert.rejects(retry, error => error === retryReason)); + } else { + assert.equal(await bounded(retry), 'Requested'); + retryAbort.abort(retryReason); + assert.equal(await retry, 'Requested', 'settled results are not rewritten by abort or close'); + } + assert.equal(getEventListeners(retryAbort.signal, 'abort').length, 0); + first.close(); + await until(() => JSON.parse(hooks.snapshot()).observerPending === 0); + } catch (error) { + failure = new Error(`stop observer ${changeFirst ? 'change-first' : 'cancel-first'} iteration ${iteration + 1}/100 phase=${phase}`, { cause: error }); + } finally { + first.close(); + closed.close(); + late?.close(); + for (const gate of armed) { + try { hooks.release(gate); } + catch (error) { failure ??= new Error(`stop observer cleanup gate ${gate}`, { cause: error }); } + } + try { + if (startup) await startup; + await runtime.shutdown(0); + } catch (error) { failure ??= error; } + } + if (failure) throw failure; +} + +function assertSingleListener(signal) { + const listeners = getEventListeners(signal, 'abort'); + assert.equal(listeners.length, 1); + return listeners[0]; +} + module.exports = { diagnostic, knownQuiescentCoreFailure, incompleteTimeout, retiredState, shutdownTiming, assertShutdownTiming, orderedRaceAttempt, + bounded, until, pendingObserverExit, observerRaceAttempt, }; diff --git a/__tests__/lifecycle-oracles.test.js b/__tests__/lifecycle-oracles.test.js index 24b89f3..1eca92e 100644 --- a/__tests__/lifecycle-oracles.test.js +++ b/__tests__/lifecycle-oracles.test.js @@ -10,6 +10,7 @@ const { startProbe, OUTPUT_LIMIT } = require('./helpers/lifecycle-probe.cjs'); const { diagnostic, knownQuiescentCoreFailure, incompleteTimeout, retiredState, shutdownTiming, assertShutdownTiming, orderedRaceAttempt, + pendingObserverExit, } = require('./helpers/lifecycle-oracles.cjs'); test('failure category does not claim quiescence for owner loss or callback release errors', () => { @@ -138,3 +139,16 @@ test('native provenance records survive TAP escaping of Windows paths', () => { assert.deepEqual(encodedRecords('# unrelated\n', 'LIFECYCLE_NATIVE_RECORD'), []); assert.throws(() => encodedRecords('# LIFECYCLE_NATIVE_RECORD e2Jyb2tlbg==\n', 'LIFECYCLE_NATIVE_RECORD')); }); + +test('stop observer natural-exit oracle rejects ended sources, settled promises and retired notifications', () => { + const pending = { + source: { state: 'Requested', closed: false }, + settled: false, + execution: { started: 1, completed: 1, active: 0, cleanupActive: 0, coordinators: 1, observerPending: 1 }, + }; + pendingObserverExit(pending); + assert.throws(() => pendingObserverExit({ ...pending, source: { ...pending.source, closed: true } }), /ended source/); + assert.throws(() => pendingObserverExit({ ...pending, settled: true }), /settled promise/); + assert.throws(() => pendingObserverExit({ ...pending, execution: { ...pending.execution, observerPending: 0 } }), /still be owned/); + assert.throws(() => pendingObserverExit({ ...pending, execution: { ...pending.execution, active: 1 } })); +}); diff --git a/__tests__/lifecycle.test.js b/__tests__/lifecycle.test.js index e42ae04..e1343d3 100644 --- a/__tests__/lifecycle.test.js +++ b/__tests__/lifecycle.test.js @@ -5,12 +5,15 @@ const { test } = require('node:test'); const assert = require('node:assert/strict'); const path = require('node:path'); const { performance } = require('node:perf_hooks'); -const { SqliteProvider, Client, Runtime } = require('../lib/duroxide.js'); +const { getEventListeners } = require('node:events'); +const { createHook } = require('node:async_hooks'); +const { SqliteProvider, Client, Runtime, RuntimeStopObserver } = require('../lib/duroxide.js'); const { nativeProvenance } = require('../ci/smoke/native-provenance.cjs'); const { startProbe } = require('./helpers/lifecycle-probe.cjs'); const { diagnostic, knownQuiescentCoreFailure, retiredState, shutdownTiming, assertShutdownTiming, orderedRaceAttempt, + bounded, until, pendingObserverExit, observerRaceAttempt, } = require('./helpers/lifecycle-oracles.cjs'); const root = path.resolve(__dirname, '..'); @@ -372,3 +375,402 @@ for (const scenario of ['quiescent-failure', 'late', 'late-unobserved']) { } finally { await probe.dispose(); } }); } + +test('stop observer arguments and already-aborted signals do not activate or stop the worker', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + try { + assert.ok(observer instanceof RuntimeStopObserver); + assert.throws(() => new RuntimeStopObserver(), TypeError); + for (const status of [undefined, null, 0, 'requested', {}, new String('Requested')]) { + await assert.rejects(observer.waitForChange(status), TypeError); + } + for (const options of [null, false, 1, 'options']) { + await assert.rejects(observer.waitForChange('NotRequested', options), TypeError); + } + for (const signal of [null, {}, false, new AbortController()]) { + await assert.rejects(observer.waitForChange('NotRequested', { signal }), TypeError); + } + for (const reason of [new Error('abort-reason'), null, false, 'abort-reason']) { + const abort = new AbortController(); + abort.abort(reason); + const observed = await observer.waitForChange('NotRequested', { signal: abort.signal }) + .then(() => assert.fail('an already-aborted wait must reject'), error => error); + assert.equal(observed, reason); + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + } + const ordinaryAbort = new AbortController(); + ordinaryAbort.abort(); + await assert.rejects(observer.waitForChange('NotRequested', { signal: ordinaryAbort.signal }), { name: 'AbortError' }); + assert.equal(observer.state, 'NotRequested'); + assert.equal(runtime.metricsSnapshot(), null); + if (hooks) { + const state = JSON.parse(hooks.snapshot()); + assert.equal(state.started, 0); + assert.equal(state.coordinators, 0); + assert.equal(state.observerPending, 0); + } + await runtime.start(); + assert.equal(observer.state, 'NotRequested'); + observer.close(); + assert.notEqual(runtime.metricsSnapshot(), null); + } finally { + observer.close(); + await runtime.shutdown(0); + } +}); + +test('stop observer waits isolate abort, overlap and terminal close and deliver unseen final state', async () => { + const { runtime } = await fixture(); + const first = runtime.stopObserver(); + const second = runtime.stopObserver(); + const closed = runtime.stopObserver(); + try { + const abort = new AbortController(); + const pending = first.waitForChange('NotRequested', { signal: abort.signal }); + pending.catch(() => {}); + const other = second.waitForChange('NotRequested'); + other.catch(() => {}); + const closingSignal = new AbortController().signal; + const closeWait = closed.waitForChange('NotRequested', { signal: closingSignal }); + closeWait.catch(() => {}); + await assert.rejects(first.waitForChange('NotRequested'), /stop_observer_busy/); + abort.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + assert.equal(first.state, 'NotRequested'); + closed.close(); + closed.close(); + await assert.rejects(closeWait, /stop_observer_closed/); + assert.equal(getEventListeners(closingSignal, 'abort').length, 0); + assert.throws(() => closed.state, /stop_observer_closed/); + await assert.rejects(closed.waitForChange('NotRequested'), /stop_observer_closed/); + const nextAbort = new AbortController(); + const retry = first.waitForChange('NotRequested', { signal: nextAbort.signal }); + retry.catch(() => {}); + await runtime.shutdown(0); + assert.deepEqual(await bounded(Promise.all([retry, other])), ['Requested', 'Requested']); + assert.equal(getEventListeners(nextAbort.signal, 'abort').length, 0); + nextAbort.abort(); + assert.equal(await retry, 'Requested'); + assert.equal(await first.waitForChange('NotRequested'), 'Requested'); + assert.equal(await first.waitForChange('Requested'), null); + assert.equal(first.state, 'Requested'); + const late = runtime.stopObserver(); + try { + assert.equal(await late.waitForChange('NotRequested'), 'Requested'); + assert.equal(await late.waitForChange('Requested'), null); + first.close(); + assert.equal(late.state, 'Requested'); + } finally { late.close(); } + } finally { + first.close(); + second.close(); + closed.close(); + await runtime.shutdown(0); + } +}); + +for (const action of ['abort', 'close', 'overlap']) { + test(`stop observer registration remains isolated under reentrant ${action}`, async () => { + for (const point of ['options', 'promise', 'native']) { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + const abort = new AbortController(); + const otherAbort = new AbortController(); + const reason = new Error(`registration-${action}-${point}`); + const resources = new Set(); + const destroyed = new Set(); + let entered = false; + let promises = 0; + let other; + const reenter = () => { + if (entered) return; + entered = true; + if (action === 'abort') abort.abort(reason); + else if (action === 'close') observer.close(); + else { + other = observer.waitForChange('NotRequested', { signal: otherAbort.signal }); + other.catch(() => {}); + } + }; + // Async hooks can run while the Promise or native notification is being constructed. + const hook = createHook({ + init(id, type) { + if (type === 'napi_rs_threadsafe_function') resources.add(id); + if (point === 'promise' && type === 'PROMISE' && ++promises === 2) reenter(); + if (point === 'native' && type === 'napi_rs_threadsafe_function') reenter(); + }, + destroy(id) { if (resources.has(id)) destroyed.add(id); }, + }); + try { + hook.enable(); + const options = point === 'options' + ? { get signal() { reenter(); return abort.signal; } } + : { signal: abort.signal }; + const pending = observer.waitForChange('NotRequested', options); + pending.catch(() => {}); + assert.ok(entered, `${action}: ${point}`); + if (action === 'abort') { + await bounded(assert.rejects(pending, error => error === reason)); + } else if (action === 'close') { + await bounded(assert.rejects(pending, /stop_observer_closed/)); + assert.throws(() => observer.state, /stop_observer_closed/); + } else { + const winner = point === 'native' ? pending : other; + const overlap = point === 'native' ? other : pending; + await bounded(assert.rejects(overlap, /stop_observer_busy/)); + (point === 'native' ? abort : otherAbort).abort(reason); + await bounded(assert.rejects(winner, error => error === reason)); + } + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + assert.equal(getEventListeners(otherAbort.signal, 'abort').length, 0); + await until(() => [...resources].every(id => destroyed.has(id))); + assert.equal(runtime.metricsSnapshot(), null); + if (action !== 'close') assert.equal(observer.state, 'NotRequested'); + if (hooks) { + const state = JSON.parse(hooks.snapshot()); + assert.equal(state.observerPending, 0); + assert.equal(state.started, 0); + assert.equal(state.coordinators, 0); + } + } finally { + hook.disable(); + observer.close(); + abort.abort(reason); + otherAbort.abort(reason); + await runtime.shutdown(0); + } + } + }); +} + +test('stop observer does not manufacture Faulted from a pre-core startup error', async () => { + const { runtime } = await fixture({ sessionIdleTimeoutMs: 0 }); + const observer = runtime.stopObserver(); + try { + await assert.rejects(runtime.start(), diagnostic('lifecycle_start_failed')); + assert.equal(observer.state, 'NotRequested'); + const pending = observer.waitForChange('NotRequested'); + pending.catch(() => {}); + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_start_failed')); + assert.equal(await bounded(pending), 'Requested'); + assert.equal(await observer.waitForChange('Requested'), null); + } finally { + observer.close(); + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_start_failed')); + } +}); + +hookTest('stop observer transport errors are explicit, sanitized and independent of stop state', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + const signal = new AbortController().signal; + try { + hooks.fail('observer-delivery', SECRET); + const wait = observer.waitForChange('NotRequested', { signal }); + const rejected = assert.rejects(wait, diagnostic('stop_observation_failed')); + await runtime.shutdown(0); + await bounded(rejected); + assert.equal(observer.state, 'Requested'); + assert.equal(getEventListeners(signal, 'abort').length, 0); + assert.equal(await observer.waitForChange('NotRequested'), 'Requested'); + assert.equal(await observer.waitForChange('Requested'), null); + } finally { + observer.close(); + await runtime.shutdown(0); + } +}); + +hookTest('stop observer sees a real retained fault before held cleanup without an SDK stop', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + hooks.hold('worker'); + hooks.hold('blocking-provider'); + hooks.fail('worker', SECRET); + try { + const fault = observer.waitForChange('NotRequested'); + fault.catch(() => {}); + await runtime.start(); + await hooks.waitEntered('worker'); + await hooks.waitEntered('blocking-provider'); + hooks.release('worker'); + assert.equal(await bounded(fault), 'Faulted'); + assert.equal(observer.state, 'Faulted'); + assert.notEqual(runtime.metricsSnapshot(), null, 'observation must not call SDK shutdown'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + const late = runtime.stopObserver(); + try { + assert.equal(late.state, 'Faulted'); + assert.equal(await late.waitForChange('NotRequested'), 'Faulted'); + const abort = new AbortController(); + const pending = late.waitForChange('Faulted', { signal: abort.signal }); + pending.catch(() => {}); + abort.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(late.state, 'Faulted'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } finally { late.close(); } + } finally { + observer.close(); + hooks.release('worker'); + hooks.release('blocking-provider'); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + } +}); + +hookTest('stop observer sees a real fault while a foreign callback remains pending without an SDK stop', async () => { + const { runtime, provider, hooks } = await fixture(); + const client = new Client(provider); + let enter, release; + const entered = new Promise(resolve => { enter = resolve; }); + const retirement = new Promise(resolve => { release = resolve; }); + let completed = false; + runtime.registerActivity('ObserverFaultActivity', async (_, input) => { + enter(); + await retirement; + completed = true; + return input; + }); + runtime.registerOrchestration('ObserverFaultFlow', function* (context, input) { + return yield context.scheduleActivity('ObserverFaultActivity', input); + }); + const observer = runtime.stopObserver(); + try { + const fault = observer.waitForChange('NotRequested'); + fault.catch(() => {}); + await runtime.start(); + await client.startOrchestration('observer-foreign-fault', 'ObserverFaultFlow', 'held'); + await bounded(entered); + hooks.fail('provider', SECRET); + await client.startOrchestration('observer-foreign-fault-trigger', 'ObserverFaultFlow', 'trigger'); + assert.equal(await bounded(fault), 'Faulted'); + assert.equal(observer.state, 'Faulted'); + assert.equal(completed, false, 'the real foreign callback must remain held at fault observation'); + assert.notEqual(runtime.metricsSnapshot(), null, 'observation must not call SDK shutdown'); + const pending = JSON.parse(hooks.snapshot()); + const late = runtime.stopObserver(); + try { + assert.equal(late.state, 'Faulted'); + assert.equal(await late.waitForChange('NotRequested'), 'Faulted'); + const abort = new AbortController(); + const pending = late.waitForChange('Faulted', { signal: abort.signal }); + pending.catch(() => {}); + abort.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(late.state, 'Faulted'); + assert.equal(completed, false); + } finally { late.close(); } + // Native retirement does not imply completion of unregistered JS continuations. + record('LIFECYCLE_PROBE_RECORD', { + scenario: 'observer-foreign-fault', native: provenance, + faultWithoutSdkStop: true, callbackHeldAtFault: !completed, + state: observer.state, execution: pending, + }); + } finally { + observer.close(); + release(); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + assert.equal(completed, true); + } +}); + +hookTest('stop observer publisher retirement freezes the final state without retiring execution', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + hooks.hold('worker'); + hooks.hold('blocking-provider'); + hooks.fail('worker', SECRET); + try { + await runtime.start(); + await hooks.waitEntered('worker'); + await hooks.waitEntered('blocking-provider'); + const ended = observer.waitForChange('NotRequested'); + ended.catch(() => {}); + hooks.retireObservationPublisher(runtime._native); + assert.equal(await bounded(ended), null); + assert.equal(observer.state, 'NotRequested'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + assert.equal(JSON.parse(hooks.snapshot()).coordinators, 0); + hooks.release('worker'); + await until(() => JSON.parse(hooks.snapshot()).coordinators === 1); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + assert.equal(observer.state, 'NotRequested', 'the closed source must not follow later core changes'); + assert.equal(await observer.waitForChange('NotRequested'), null); + } finally { + observer.close(); + hooks.release('worker'); + hooks.release('blocking-provider'); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + } +}); + +for (const changeFirst of [false, true]) { + hookTest(`stop observer ordered attachment/change/abort/close races 100 times: ${changeFirst ? 'change-first' : 'cancel-first'}`, async () => { + for (let iteration = 0; iteration < 100; iteration++) { + const { runtime, hooks } = await fixture(); + await observerRaceAttempt(runtime, hooks, changeFirst, iteration); + } + }); +} + +hookTest('stop observer Requested-to-Faulted escalation stays monotonic through 100 ordered shutdowns', async () => { + for (let iteration = 0; iteration < 100; iteration++) { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + hooks.hold('worker'); + hooks.hold('blocking-provider'); + hooks.fail('worker', SECRET); + let stop; + try { + await runtime.start(); + await hooks.waitEntered('worker'); + await hooks.waitEntered('blocking-provider'); + const requested = observer.waitForChange('NotRequested'); + requested.catch(() => {}); + stop = runtime.shutdown(30_000); + stop.catch(() => {}); + assert.equal(await bounded(requested), 'Requested'); + const fault = observer.waitForChange('Requested'); + fault.catch(() => {}); + hooks.release('worker'); + assert.equal(await bounded(fault), 'Faulted', `iteration ${iteration + 1}/100`); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } finally { + observer.close(); + hooks.release('worker'); + hooks.release('blocking-provider'); + await assert.rejects(stop ?? runtime.shutdown(0), knownQuiescentCoreFailure); + } + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + const late = runtime.stopObserver(); + assert.equal(late.state, 'Faulted'); + late.close(); + retired(hooks, { rollback: true, executionWitness: true }); + } +}); + +test('stop observer stays pending through natural exit after real worker cleanup', async () => { + const probe = startProbe('observer-exit'); + try { + const pending = await probe.next('OBSERVER_EXIT_READY', 5000); + pendingObserverExit(pending); + await probe.successfulExit(3000); + const exiting = probe.messages.find(message => message.phase === 'OBSERVER_BEFORE_EXIT'); + assert.ok(exiting, probe.describe()); + pendingObserverExit(exiting); + record('LIFECYCLE_PROBE_RECORD', { scenario: 'observer-exit', ...exiting }); + } finally { await probe.dispose(); } +}); + +for (const scenario of ['observer-source-end', 'retained-observer']) { + test(`stop observer metadata does not retain callback/owner roots: ${scenario}`, async () => { + const probe = startProbe(scenario, { fixture: 'lifecycle-collection-probe.cjs' }); + try { + const result = await probe.next('OBSERVER_COLLECTION', 10_000); + assert.ok(result.collected, JSON.stringify(result)); + record('LIFECYCLE_PROBE_RECORD', result); + await probe.successfulExit(3000); + } finally { await probe.dispose(); } + }); +} diff --git a/docs/architecture.md b/docs/architecture.md index bf9e24e..3ff1adb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -440,6 +440,32 @@ The global orchestration-handler map also remains unchanged. See the [lifecycle guide](user-guide.md#runtime-lifecycle) for argument validation, errors, and release prerequisites. +### Passive Stop Observation + +`Runtime` caches a receiver-only native source when it creates its wrapper. +`stopObserver()` uses that source rather than borrowing the mutable native +Runtime during startup or shutdown. Each observer owns metadata and one pending +wait slot, not the Runtime, provider, or callback registry. + +The SDK publisher freezes the final state and detaches its live core receiver +when the source ends. Reads use coherent status/closed metadata. +Core retirement alone is not SDK source end. A wait delivers an unseen final +state before `null`; local `close()` instead rejects pending and later use. + +Each wait owns cancellation metadata and an unreferenced observation +ThreadsafeFunction. Settlement removes its AbortSignal listener and clears +resolver/rejector fields. Registration rechecks reentrant state and closes any +native handle returned after settlement. An old callback cannot settle a later +wait on the same observer. Native delivery context remains valid through final +access without joining worker cleanup. + +Only observation delivery is unreferenced early. Existing worker/shutdown +callback release still requires its own cleanup proof. +Native queued-delivery counters are not JavaScript Promise-settlement counts. +Neither status, source end, nor these counters prove that an unregistered +JavaScript continuation ended. See the +[observer contract and example](user-guide.md#passive-stop-observation). + ### Platform-Specific Binary napi-rs compiles to a platform-specific `.node` binary (e.g., `duroxide.darwin-arm64.node`). Cross-platform distribution requires building on each target platform or using napi-rs's pre-built binary publishing flow. diff --git a/docs/user-guide.md b/docs/user-guide.md index 4bff729..b8affc3 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -158,6 +158,93 @@ Native cleanup has no total bound and can remain pending indefinitely. The application's parent or supervisor must allow time for process termination outside the SDK wait budget. The SDK does not provide a process-kill or supervisor mechanism. +### Passive stop observation + +`runtime.stopObserver()` creates an independent `RuntimeStopObserver`. +Do not construct the observer directly. Creating it, reading `state`, and +waiting do not start or stop execution. +Observation can run during native startup or shutdown; lifecycle and +registration calls themselves must still be serialized. + +| State | Meaning | +|-------|---------| +| `'NotRequested'` | No stop is recorded. The worker can be unconstructed, starting, or running; this is not readiness or health. | +| `'Requested'` | Stop was accepted, not necessarily received by native execution or completed. | +| `'Faulted'` | The core recorded a runtime-wide fault. It takes precedence and cannot regress. | + +These are string values; `RuntimeStopStatus` is a TypeScript type, not a +JavaScript enum. Read the initial snapshot before waiting. +A late observer can already be Faulted. Invalid input, pre-core validation +failure, cancellation, and observation errors do not create a runtime fault. + +This helper owns the observer passed to it. It reports the initial state and +each observed change, then ends its own observation after a fault. +Pass an observer rather than the Runtime so the pending function need not +retain the worker: + +```javascript +async function reportStopChanges(observer, signal) { + try { + let state = observer.state; + while (true) { + console.log('Runtime stop state:', state); + if (state === 'Faulted') return; + + const next = await observer.waitForChange(state, { signal }); + if (next === null) { + console.log('Stop observation source ended; cleanup is not proved.'); + return; + } + state = next; + } + } catch (error) { + if (signal?.aborted && Object.is(error, signal.reason)) { + console.log('Stop observation canceled; worker shutdown is unchanged.'); + } else if (error instanceof Error && error.message.startsWith('stop_observer_closed:')) { + console.log('Stop observation was closed.'); + } else { + throw error; + } + } finally { + observer.close(); + } +} +``` + +In an application that owns a Runtime, call +`reportStopChanges(runtime.stopObserver(), observationController.signal)`, +where `observationController` is an `AbortController`. +Observe the returned Promise and handle unexpected observation failures. +Cancel that controller to end only monitoring. +The application's lifecycle owner must still stop the Runtime normally. + +`waitForChange` compares against the supplied last-seen state; it is not an +event queue. Changes can coalesce. A final unseen state arrives before `null`. +The final snapshot stays readable until local close. +The source can remain open after worker cleanup. An unchanged `'Requested'` +or `'Faulted'` therefore still waits; cancel monitoring when no longer needed. + +Only one wait can be pending on an observer; overlap rejects with +`stop_observer_busy`. Separate observers do not consume each other's state. +An already-aborted or later-aborted signal rejects with its actual `reason`, +which is not necessarily an `AbortError`. Cancellation alone permits retry. +The helper above also closes its observer on exit. +`close()` is idempotent and terminal: pending and later operations fail with +`stop_observer_closed`. A settled Promise is not rewritten. + +The observer retains metadata, not Runtime/provider/registry ownership. +Native notification context stays valid through final access even after the +public wait settles. Closing observation does not join that tail or worker cleanup. +Observation delivery is unreferenced and cannot by itself keep Node alive. +A pending Promise alone is not a keepalive; leave the event loop free to deliver +changes while your application remains active. + +Neither a state nor source end proves quiescence or permits release. +A real core fault can be visible while an unregistered JavaScript callback +continues after native retirement. This API does not add ownership of such work. +It does not request shutdown, renew first-stop budgets, remove a termination +requirement, or select host/process policy. + ## Orchestration Patterns ### Sequential Steps diff --git a/index.d.ts b/index.d.ts index 9b8d35e..66f5fb3 100644 --- a/index.d.ts +++ b/index.d.ts @@ -102,6 +102,10 @@ export interface JsRuntimeOptions { /** Tag list for "tags" or "defaultAnd" filter modes. */ workerTagFilterTags?: Array } +export interface JsRuntimeStopSnapshot { + state: string + closed: boolean +} /** Orchestration status returned to JS. */ export interface JsOrchestrationStatus { status: string @@ -379,6 +383,7 @@ export declare class JsRuntime { * These three functions handle: creating generators, driving next steps, and disposing. */ setGeneratorDriver(createFn: (arg: string) => any, nextFn: (arg: string) => any, disposeFn: (arg: string) => any): void + stopSource(): JsRuntimeStopSource /** * Register a JavaScript activity function. * The JS function receives (contextInfoJson, input) and returns a Promise. @@ -400,11 +405,13 @@ export declare class JsRuntime { start(): Promise /** Get a snapshot of runtime metrics. */ metricsSnapshot(): JsMetricsSnapshot | null - /** - * Shutdown the runtime gracefully. - * - * # Safety - * Must not be called concurrently from multiple threads. - */ + /** Stop with bounded waiting. Do not overlap lifecycle/registration calls. */ shutdown(timeoutMs?: number | undefined | null): Promise } +export declare class JsRuntimeStopSource { + snapshot(): JsRuntimeStopSnapshot + waitForChange(lastSeen: string, callback: (err: Error | null, arg: string | undefined | null) => any): JsRuntimeStopWait +} +export declare class JsRuntimeStopWait { + close(): void +} diff --git a/index.js b/index.js index 9d65251..52583cd 100644 --- a/index.js +++ b/index.js @@ -313,12 +313,14 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { JsClient, JsPostgresProvider, JsSqliteProvider, JsRuntime, activityTraceLog, orchestrationTraceLog, activityIsCancelled, activityGetClient, activityTag, orchestrationSetCustomStatus, orchestrationResetCustomStatus, orchestrationGetCustomStatus, orchestrationSetValue, orchestrationGetValue, orchestrationClearValue, orchestrationClearAllValues, orchestrationGetKvAllValues, orchestrationGetKvAllKeys, orchestrationGetKvLength, orchestrationPruneKvValues, initTracing } = nativeBinding +const { JsClient, JsPostgresProvider, JsSqliteProvider, JsRuntime, JsRuntimeStopSource, JsRuntimeStopWait, activityTraceLog, orchestrationTraceLog, activityIsCancelled, activityGetClient, activityTag, orchestrationSetCustomStatus, orchestrationResetCustomStatus, orchestrationGetCustomStatus, orchestrationSetValue, orchestrationGetValue, orchestrationClearValue, orchestrationClearAllValues, orchestrationGetKvAllValues, orchestrationGetKvAllKeys, orchestrationGetKvLength, orchestrationPruneKvValues, initTracing } = nativeBinding module.exports.JsClient = JsClient module.exports.JsPostgresProvider = JsPostgresProvider module.exports.JsSqliteProvider = JsSqliteProvider module.exports.JsRuntime = JsRuntime +module.exports.JsRuntimeStopSource = JsRuntimeStopSource +module.exports.JsRuntimeStopWait = JsRuntimeStopWait module.exports.activityTraceLog = activityTraceLog module.exports.orchestrationTraceLog = orchestrationTraceLog module.exports.activityIsCancelled = activityIsCancelled diff --git a/lib/duroxide.d.ts b/lib/duroxide.d.ts index 15eb251..79543fd 100644 --- a/lib/duroxide.d.ts +++ b/lib/duroxide.d.ts @@ -311,6 +311,41 @@ export interface RuntimeOptions extends JsRuntimeOptions { workerTagFilter?: TagFilter; } +/** + * Retained stop state, not readiness or proof of cleanup. + * NotRequested includes unconstructed/starting/running; Requested records accepted stop. + * Faulted is a sticky recorded core-wide fault, not an observation error. + * This is a type-only union, not a runtime enum. + */ +export type RuntimeStopStatus = 'NotRequested' | 'Requested' | 'Faulted'; + +/** + * Independent, owned observation that does not retain worker/provider/registry resources. + * Obtain it through Runtime.stopObserver() and read state before waiting. + * Neither status nor source end proves cleanup or changes shutdown budgets/policy. + */ +export declare class RuntimeStopObserver { + private constructor(); + /** Retained snapshot, including after normal source end. Fails after local close. */ + readonly state: RuntimeStopStatus; + /** + * Compare against lastSeen; return a different state or null at normal source end. + * Return an unseen final state before null. Intermediate changes can coalesce. + * An unchanged Requested/Faulted still waits while the source is open. + * One pending wait per observer; overlap rejects with stop_observer_busy. + * Abort rejects with signal.reason, affects only this wait and permits retry. + * Local close instead rejects with stop_observer_closed and prevents reuse. + * Observation errors propagate without manufacturing Faulted. + * Delivery requires event-loop progress but does not keep Node alive by itself. + */ + waitForChange(lastSeen: RuntimeStopStatus, options?: { signal?: AbortSignal }): Promise; + /** + * End observation and reject a pending wait. Idempotent; later reads and waits fail. + * Does not rewrite settled results or join worker cleanup/native final access. + */ + close(): void; +} + /** Durable execution runtime. */ export declare class Runtime { constructor(provider: SqliteProvider | PostgresProvider, options?: RuntimeOptions); @@ -330,6 +365,11 @@ export declare class Runtime { * timeoutMs must be a nonnegative safe integer within the native monotonic deadline range. */ shutdown(timeoutMs?: number): Promise; + /** + * Create an independent observer, including during native startup/shutdown. + * Does not start/stop execution or retain this worker. The caller owns close(). + */ + stopObserver(): RuntimeStopObserver; metricsSnapshot(): JsMetricsSnapshot | null; } diff --git a/lib/duroxide.js b/lib/duroxide.js index 2873554..962aba7 100644 --- a/lib/duroxide.js +++ b/lib/duroxide.js @@ -1077,6 +1077,128 @@ class Client { } } +const stopObserverToken = Symbol('RuntimeStopObserver'); +const stopStatuses = new Set(['NotRequested', 'Requested', 'Faulted']); +const signalAborted = Object.getOwnPropertyDescriptor(AbortSignal.prototype, 'aborted').get; +const signalReason = Object.getOwnPropertyDescriptor(AbortSignal.prototype, 'reason').get; + +/** + * Owned stop metadata. Obtain an independent observer through Runtime.stopObserver(). + * Read state before waiting: a late reader can already see Faulted. + * NotRequested includes unconstructed, starting and running workers; it is not health. + * Requested records accepted stop. Faulted records a sticky core-wide fault. + * Neither state nor source end proves cleanup or permits resource release. + * Observation retains no worker/provider/registry ownership and applies no shutdown policy. + */ +class RuntimeStopObserver { + #source; + #closed = false; + #pending; + + constructor(source, token) { + if (token !== stopObserverToken) throw new TypeError('Use Runtime.stopObserver() to obtain an observer'); + this.#source = source; + } + + /** + * Read retained state without starting or stopping execution. + * The final snapshot remains readable after source end, until local close. + * @returns {'NotRequested'|'Requested'|'Faulted'} + */ + get state() { + if (this.#closed) throw new Error('stop_observer_closed: this observer is closed'); + return this.#source.snapshot().state; + } + + #assertCanWait() { + if (this.#closed) throw new Error('stop_observer_closed: this observer is closed'); + if (this.#pending) throw new Error('stop_observer_busy: this observer already has a pending wait'); + } + + /** + * Wait for a different retained state, or null at normal source end. + * An unseen final state arrives before null; intermediate states can coalesce. + * An unchanged Requested or Faulted still waits while the source is open. + * Only one wait may be pending on this observer; overlap rejects with stop_observer_busy. + * Abort rejects with signal.reason, consumes no state and permits retry. + * Local close instead rejects with stop_observer_closed and prevents reuse. + * Observation errors do not manufacture Faulted or stop the worker. + * Delivery needs event-loop progress but does not keep Node alive by itself. + * @param {'NotRequested'|'Requested'|'Faulted'} lastSeen State previously observed from this source. + * @param {{signal?: AbortSignal}} [options] Cancellation for this wait only. + * @returns {Promise<'NotRequested'|'Requested'|'Faulted'|null>} + */ + async waitForChange(lastSeen, options = {}) { + this.#assertCanWait(); + if (!stopStatuses.has(lastSeen)) throw new TypeError('lastSeen must be NotRequested, Requested or Faulted'); + if (options === null || typeof options !== 'object') throw new TypeError('options must be an object'); + let signal = options.signal; + if (signal !== undefined) { + let aborted; + try { aborted = signalAborted.call(signal); } + catch { throw new TypeError('signal must be an AbortSignal'); } + if (aborted) throw signalReason.call(signal); + } + this.#assertCanWait(); + const snapshot = this.#source.snapshot(); + if (snapshot.state !== lastSeen) return snapshot.state; + if (snapshot.closed) return null; + + return new Promise((resolve, reject) => { + this.#assertCanWait(); + const wait = { resolve, reject, native: undefined, finish: undefined }; + const abort = () => { + if (this.#pending === wait) wait.finish(true, signalReason.call(signal)); + }; + wait.finish = (failed, value) => { + if (this.#pending !== wait) return; + this.#pending = undefined; + if (signal !== undefined) { + EventTarget.prototype.removeEventListener.call(signal, 'abort', abort); + signal = undefined; + } + wait.native?.close(); + wait.native = undefined; + const settle = failed ? wait.reject : wait.resolve; + wait.resolve = wait.reject = undefined; + settle(value); + }; + this.#pending = wait; + try { + if (signal !== undefined) { + EventTarget.prototype.addEventListener.call(signal, 'abort', abort, { once: true }); + } + if (this.#pending !== wait) return; + if (signal !== undefined && signalAborted.call(signal)) { + abort(); + return; + } + const native = this.#source.waitForChange(lastSeen, (error, status) => { + if (error != null) wait.finish(true, error); + else if (status === null || stopStatuses.has(status)) wait.finish(false, status); + else wait.finish(true, new Error('stop_observation_failed: invalid native notification')); + }); + if (this.#pending === wait) wait.native = native; + else native.close(); + } catch (error) { + wait.finish(true, error); + } + }); + } + + /** + * End this observer and reject its pending wait. Idempotent and terminal. + * Later reads/waits fail; a settled Promise is not rewritten. + * Does not stop the worker or join cleanup or native notification final access. + */ + close() { + if (this.#closed) return; + this.#closed = true; + this.#pending?.finish(true, new Error('stop_observer_closed: this observer is closed')); + this.#source = undefined; + } +} + /** * Durable execution runtime. */ @@ -1108,6 +1230,8 @@ class Runtime { } else { this._native = new JsRuntime(provider._native, nativeOptions); } + // Keep observation separate from the native owner's asynchronous mutable startup borrow. + this._stopSource = this._native.stopSource(); // Wire up the generator driver functions this._native.setGeneratorDriver(createGenerator, nextStep, disposeGenerator); } @@ -1215,6 +1339,16 @@ class Runtime { await this._native.shutdown(timeoutMs); } + /** + * Obtain an independent, metadata-only observer without starting or stopping execution. + * Safe to obtain during native startup/shutdown. The caller owns close(). + * Read its initial state before waiting. It does not retain this worker. + * @returns {RuntimeStopObserver} + */ + stopObserver() { + return new RuntimeStopObserver(this._stopSource, stopObserverToken); + } + /** * Get a snapshot of runtime metrics. * @returns {object|null} @@ -1304,6 +1438,7 @@ module.exports = { PostgresProvider, Client, Runtime, + RuntimeStopObserver, OrchestrationContext, ActivityContext, initTracing, diff --git a/src/lib.rs b/src/lib.rs index 7797cc1..ef0a143 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ mod handlers; mod pg_provider; mod provider; mod runtime; +mod stop_observation; #[cfg(feature = "test-hooks")] mod test_hooks; mod types; @@ -20,6 +21,7 @@ pub use client::JsClient; pub use pg_provider::JsPostgresProvider; pub use provider::JsSqliteProvider; pub use runtime::{JsRuntime, JsRuntimeOptions}; +pub use stop_observation::{JsRuntimeStopSnapshot, JsRuntimeStopSource, JsRuntimeStopWait}; #[cfg(feature = "test-hooks")] pub use test_hooks::lifecycle_test_hooks; pub use types::{JsOrchestrationStatus, JsQueueDepths, JsSystemMetrics}; diff --git a/src/runtime.rs b/src/runtime.rs index 376313c..cfbfdd2 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -18,6 +18,7 @@ use duroxide::runtime::LogFormat; use crate::handlers::{JsActivityHandler, JsOrchestrationHandler}; use crate::pg_provider::JsPostgresProvider; use crate::provider::JsSqliteProvider; +use crate::stop_observation::{JsRuntimeStopSource, StopSource}; use crate::types::JsMetricsSnapshot; /// Runtime options configurable from JavaScript. @@ -94,6 +95,7 @@ pub struct JsRuntime { shutdown_requested: bool, stop: Option, startup_error: Option, + pub(crate) stop_source: StopSource, #[cfg(feature = "test-hooks")] pub(crate) test_hooks: runtime::test_hooks::LifecycleHooks, } @@ -116,6 +118,7 @@ impl JsRuntime { shutdown_requested: false, stop: None, startup_error: None, + stop_source: StopSource::new(), #[cfg(feature = "test-hooks")] test_hooks: runtime::test_hooks::LifecycleHooks::default(), } @@ -138,6 +141,7 @@ impl JsRuntime { shutdown_requested: false, stop: None, startup_error: None, + stop_source: StopSource::new(), #[cfg(feature = "test-hooks")] test_hooks: runtime::test_hooks::LifecycleHooks::default(), } @@ -159,6 +163,11 @@ impl JsRuntime { Ok(()) } + #[napi] + pub fn stop_source(&self) -> JsRuntimeStopSource { + self.stop_source.subscribe() + } + /// Register a JavaScript activity function. /// The JS function receives (contextInfoJson, input) and returns a Promise. #[napi] @@ -350,10 +359,7 @@ impl JsRuntime { let rt = match prepared { Ok(result) => result, Err(payload) => { - // Payload destruction must not reintroduce a panic across the language boundary. - if let Err(secondary) = catch_unwind(AssertUnwindSafe(|| drop(payload))) { - std::mem::forget(secondary); - } + discard_panic_payload(payload); Err(runtime::RuntimeStartError::StartupFailed) } } @@ -361,6 +367,7 @@ impl JsRuntime { self.startup_error = Some(error); start_error(error) })?; + self.stop_source.attach(rt.stop_observer()); self.inner = Some(rt.clone()); #[cfg(feature = "test-hooks")] rt.set_lifecycle_hooks(self.test_hooks.clone()) @@ -409,6 +416,7 @@ impl JsRuntime { pub fn shutdown(&mut self, env: Env, timeout_ms: Option) -> Result { let (grace_deadline, total_deadline) = shutdown_deadlines(timeout_ms)?; self.shutdown_requested = true; + self.stop_source.requested(); if self.stop.is_none() { if let Some(runtime) = self.inner.take() { runtime.request_shutdown(); @@ -444,6 +452,9 @@ impl JsRuntime { } } if self.inner.is_none() { + if self.stop.is_none() { + self.stop_source.close(); + } self.activity_builders.clear(); self.orchestration_names.clear(); self.create_fn = None; @@ -469,6 +480,12 @@ impl JsRuntime { } } +pub(crate) fn discard_panic_payload(payload: Box) { + if let Err(secondary) = catch_unwind(AssertUnwindSafe(|| drop(payload))) { + std::mem::forget(secondary); + } +} + impl JsRuntime { fn ensure_configurable(&self) -> Result<()> { if self.started || self.shutdown_requested { diff --git a/src/stop_observation.rs b/src/stop_observation.rs new file mode 100644 index 0000000..3ed990c --- /dev/null +++ b/src/stop_observation.rs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use duroxide::runtime::{RuntimeStopObserver, RuntimeStopStatus}; +use napi::bindgen_prelude::*; +use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}; +use napi_derive::napi; +use tokio::sync::watch; + +#[derive(Clone)] +struct SourceState { + status: RuntimeStopStatus, + core: Option, + closed: bool, +} + +fn merge(left: RuntimeStopStatus, right: RuntimeStopStatus) -> RuntimeStopStatus { + use RuntimeStopStatus::*; + match (left, right) { + (Faulted, _) | (_, Faulted) => Faulted, + (Requested, _) | (_, Requested) => Requested, + _ => NotRequested, + } +} + +impl SourceState { + fn status(&self) -> RuntimeStopStatus { + merge( + self.status, + self.core.as_ref().map_or(self.status, |core| core.state()), + ) + } +} + +pub(crate) struct StopSource { + state: watch::Sender, + #[cfg(feature = "test-hooks")] + pub(crate) hooks: crate::test_hooks::ObservationHooks, +} + +impl StopSource { + pub fn new() -> Self { + Self { + state: watch::channel(SourceState { + status: RuntimeStopStatus::NotRequested, + core: None, + closed: false, + }) + .0, + #[cfg(feature = "test-hooks")] + hooks: crate::test_hooks::ObservationHooks::default(), + } + } + + pub fn subscribe(&self) -> JsRuntimeStopSource { + JsRuntimeStopSource { + state: self.state.subscribe(), + #[cfg(feature = "test-hooks")] + hooks: self.hooks.clone(), + } + } + + pub fn attach(&self, core: RuntimeStopObserver) { + self.state.send_modify(|state| { + assert!(!state.closed && state.core.is_none()); + state.status = merge(state.status, core.state()); + state.core = Some(core); + }); + } + + pub fn requested(&self) { + self.state.send_if_modified(|state| { + let status = merge(state.status, RuntimeStopStatus::Requested); + if state.closed || state.status == status { + return false; + } + state.status = status; + true + }); + } + + pub fn close(&self) { + self.state.send_modify(|state| { + state.status = state.status(); + state.core = None; + state.closed = true; + }); + } +} + +impl Drop for StopSource { + fn drop(&mut self) { + // Dropping only the sender would leave a live core receiver behind. + self.close(); + } +} + +fn encode(status: RuntimeStopStatus) -> String { + match status { + RuntimeStopStatus::NotRequested => "NotRequested", + RuntimeStopStatus::Requested => "Requested", + RuntimeStopStatus::Faulted => "Faulted", + } + .to_owned() +} + +fn decode(status: &str) -> Result { + match status { + "NotRequested" => Ok(RuntimeStopStatus::NotRequested), + "Requested" => Ok(RuntimeStopStatus::Requested), + "Faulted" => Ok(RuntimeStopStatus::Faulted), + _ => Err(Error::new( + Status::InvalidArg, + "stop_observation_invalid_status: invalid last-seen status", + )), + } +} + +#[napi(object)] +pub struct JsRuntimeStopSnapshot { + pub state: String, + pub closed: bool, +} + +#[napi] +pub struct JsRuntimeStopSource { + state: watch::Receiver, + #[cfg(feature = "test-hooks")] + hooks: crate::test_hooks::ObservationHooks, +} + +#[napi] +impl JsRuntimeStopSource { + #[napi] + pub fn snapshot(&self) -> JsRuntimeStopSnapshot { + let state = self.state.borrow(); + JsRuntimeStopSnapshot { + state: encode(state.status()), + closed: state.closed, + } + } + + #[napi] + pub fn wait_for_change( + &self, + env: Env, + last_seen: String, + mut callback: ThreadsafeFunction>, + ) -> Result { + let last_seen = decode(&last_seen)?; + callback.unref(&env)?; + let mut source = self.state.clone(); + let (cancel, mut canceled) = watch::channel(()); + #[cfg(feature = "test-hooks")] + let hooks = self.hooks.clone(); + #[cfg(feature = "test-hooks")] + let pending = hooks.register(); + spawn(async move { + let result = match tokio::spawn(async move { + tokio::select! { + biased; + _ = canceled.changed() => Err(Error::from_reason( + "stop_observation_canceled: notification was canceled")), + status = changed(&mut source, last_seen) => Ok(status.map(encode)), + } + }) + .await + { + Ok(result) => result, + Err(error) => { + if error.is_panic() { + crate::runtime::discard_panic_payload(error.into_panic()); + } + Err(Error::from_reason( + "stop_observation_failed: native observation failed", + )) + } + }; + #[cfg(feature = "test-hooks")] + let result = if hooks.checkpoint("observer-delivery").await { + Err(Error::from_reason( + "stop_observation_failed: native observation failed", + )) + } else { + result + }; + let status = callback.call(result, ThreadsafeFunctionCallMode::NonBlocking); + if status != Status::Ok && status != Status::Closing { + eprintln!("stop_observation_failed: notification could not be delivered"); + } + #[cfg(feature = "test-hooks")] + { + hooks.delivered(status == Status::Ok); + hooks.checkpoint("observer-final-access").await; + } + drop(callback); + #[cfg(feature = "test-hooks")] + drop(pending); + }); + Ok(JsRuntimeStopWait { + cancel: Some(cancel), + }) + } +} + +async fn changed( + source: &mut watch::Receiver, + last_seen: RuntimeStopStatus, +) -> Option { + loop { + let (state, status) = { + let state = source.borrow_and_update(); + (state.clone(), state.status()) + }; + if status != last_seen { + return Some(status); + } + if state.closed { + return None; + } + if let Some(mut core) = state.core { + let native_status = core.state(); + if merge(state.status, native_status) != last_seen { + continue; + } + tokio::select! { + _ = source.changed() => {}, + result = core.changed(native_status) => { + if result.is_none() { + // The SDK publisher owns source lifetime, independently of core cleanup. + let _ = source.changed().await; + } + }, + } + } else { + let _ = source.changed().await; + } + } +} + +#[napi] +pub struct JsRuntimeStopWait { + cancel: Option>, +} + +#[napi] +impl JsRuntimeStopWait { + #[napi] + pub fn close(&mut self) { + self.cancel.take(); + } +} diff --git a/src/test_hooks.rs b/src/test_hooks.rs index f5ed951..da366da 100644 --- a/src/test_hooks.rs +++ b/src/test_hooks.rs @@ -6,19 +6,25 @@ use duroxide::runtime::test_hooks::{ }; use napi::{Error, Result}; use napi_derive::napi; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use std::time::Duration; +use tokio::sync::watch; use crate::runtime::JsRuntime; +use crate::stop_observation::StopSource; #[napi] pub struct LifecycleTestHooks { hooks: LifecycleHooks, + observation: ObservationHooks, } #[napi(js_name = "_lifecycleTestHooks")] pub fn lifecycle_test_hooks(runtime: &JsRuntime) -> LifecycleTestHooks { LifecycleTestHooks { hooks: runtime.test_hooks.clone(), + observation: runtime.stop_source.hooks.clone(), } } @@ -26,6 +32,9 @@ pub fn lifecycle_test_hooks(runtime: &JsRuntime) -> LifecycleTestHooks { impl LifecycleTestHooks { #[napi] pub fn hold(&self, point: String) -> Result<()> { + if point.starts_with("observer-") { + return self.observation.hold(&point); + } let parsed = parse_point(&point)?; let gate = if point == "blocking-provider" { self.hooks.hold_blocking(parsed) @@ -38,6 +47,10 @@ impl LifecycleTestHooks { #[napi] pub fn release(&self, point: String) -> Result<()> { + if point.starts_with("observer-") { + self.observation.gate(&point)?.released.send_replace(true); + return Ok(()); + } self.hooks .gate(parse_point(&point)?) .ok_or_else(|| Error::from_reason("test gate not armed"))? @@ -47,6 +60,15 @@ impl LifecycleTestHooks { #[napi] pub async fn wait_entered(&self, point: String) -> Result<()> { + if point.starts_with("observer-") { + let gate = self.observation.gate(&point)?; + return tokio::time::timeout( + Duration::from_secs(3), + signaled(gate.entered.subscribe()), + ) + .await + .map_err(|_| Error::from_reason("test gate entry timed out")); + } let gate = self .hooks .gate(parse_point(&point)?) @@ -58,14 +80,34 @@ impl LifecycleTestHooks { #[napi] pub fn fail(&self, point: String, message: String) -> Result<()> { + if point == "observer-delivery" { + self.observation.0.lock().failures.insert(point); + return Ok(()); + } self.hooks .fail_once_with_message(parse_point(&point)?, message) .map_err(|error| Error::from_reason(format!("test fault: {error:?}"))) } + #[napi] + pub fn retire_observation_publisher(&self, runtime: &mut JsRuntime) -> Result<()> { + if !Arc::ptr_eq(&self.observation.0, &runtime.stop_source.hooks.0) { + return Err(Error::from_reason( + "test observation publisher does not match", + )); + } + // Retain the worker so closure controls can test a later, real core fault. + drop(std::mem::replace( + &mut runtime.stop_source, + StopSource::new(), + )); + Ok(()) + } + #[napi] pub fn snapshot(&self) -> String { let counts = self.hooks.counts(); + let observation = self.observation.0.lock(); serde_json::json!({ "started": counts.iter().map(|count| count.started).sum::(), "completed": counts.iter().map(|count| count.completed).sum::(), @@ -73,11 +115,105 @@ impl LifecycleTestHooks { "cleanupActive": self.hooks.cleanup_counts().active, "coordinators": self.hooks.hits(LifecyclePoint::ShutdownCoordinator), "forceRequests": self.hooks.hits(LifecyclePoint::ForceRequested), + "observerPending": observation.pending, + "observerDeliveries": observation.deliveries, }) .to_string() } } +#[derive(Clone, Default)] +pub(crate) struct ObservationHooks(Arc>); + +#[derive(Default)] +struct ObservationState { + gates: HashMap>, + failures: HashSet, + pending: usize, + deliveries: usize, +} + +struct ObservationGate { + entered: watch::Sender, + released: watch::Sender, +} + +async fn signaled(mut signal: watch::Receiver) { + while !*signal.borrow_and_update() { + signal + .changed() + .await + .expect("test gate sender is retained"); + } +} + +impl ObservationHooks { + fn hold(&self, point: &str) -> Result<()> { + if !matches!(point, "observer-delivery" | "observer-final-access") { + return Err(Error::from_reason("unknown observation test point")); + } + let mut state = self.0.lock(); + if state + .gates + .get(point) + .is_some_and(|gate| !*gate.released.borrow()) + { + return Err(Error::from_reason("observation test gate already armed")); + } + state.gates.insert( + point.to_owned(), + Arc::new(ObservationGate { + entered: watch::channel(false).0, + released: watch::channel(false).0, + }), + ); + Ok(()) + } + + fn gate(&self, point: &str) -> Result> { + self.0 + .lock() + .gates + .get(point) + .cloned() + .ok_or_else(|| Error::from_reason("observation test gate not armed")) + } + + pub(crate) async fn checkpoint(&self, point: &str) -> bool { + let (gate, failed) = { + let mut state = self.0.lock(); + ( + state.gates.get(point).cloned(), + state.failures.remove(point), + ) + }; + if let Some(gate) = gate { + gate.entered.send_replace(true); + signaled(gate.released.subscribe()).await; + } + failed + } + + pub(crate) fn register(&self) -> ObservationTask { + self.0.lock().pending += 1; + ObservationTask(self.clone()) + } + + pub(crate) fn delivered(&self, queued: bool) { + if queued { + self.0.lock().deliveries += 1; + } + } +} + +pub(crate) struct ObservationTask(ObservationHooks); + +impl Drop for ObservationTask { + fn drop(&mut self) { + self.0 .0.lock().pending -= 1; + } +} + fn parse_point(point: &str) -> Result { match point { "startup" => Ok(LifecyclePoint::StartupBeforeIo),