diff --git a/.agents/skills/duroxide-node-orchestrations/SKILL.md b/.agents/skills/duroxide-node-orchestrations/SKILL.md index b6d85d7..21db55a 100644 --- a/.agents/skills/duroxide-node-orchestrations/SKILL.md +++ b/.agents/skills/duroxide-node-orchestrations/SKILL.md @@ -327,7 +327,7 @@ npm run test:all # Everything (52 tests) - Use `SqliteProvider.inMemory()` for fast isolated tests (SQLite smoketest only) - All PG tests need `DATABASE_URL` in `.env` (loaded by `dotenv`) - Each test file uses a separate PG schema for isolation -- Use short `runtime.shutdown(100)` timeout — it waits the full duration +- `runtime.shutdown(100)` permits 100 ms of grace plus five seconds of cleanup headroom. It can finish early; timeout rejects while cleanup remains owned and requires application/supervisor termination. - Set `RUST_LOG=info` to see traces in test output - Use `workerLockTimeoutMs: 2000` in tests needing fast activity cancellation detection diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bb6e8bf..aa0365c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -203,6 +203,6 @@ validation so CI and published packages use registry crates. - All JS callbacks run on the Node.js main thread (single-threaded) - No `async function*` generators — breaks replay model - SQLite may hit "database is locked" under concurrency (retried automatically) -- `Runtime.shutdown(timeoutMs)` waits the full timeout (no short-circuit) +- `Runtime.shutdown(timeoutMs)` uses grace plus five seconds of bounded waiting; idle work can finish early, and failures reject while retaining cleanup. - Platform-specific `.node` binary — cross-platform needs per-platform builds - Activity cancellation detection latency depends on `workerLockTimeoutMs` (default 30s → ~15s detection) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f7a640..72bfe92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,6 +83,61 @@ jobs: name: binary-linux-x64-gnu - name: Run tests run: npm run test:all + - name: Stage isolated production lifecycle suite + run: node ci/stage-lifecycle.cjs . target/lifecycle-production/stage production + - name: Require production lifecycle and export boundary + run: node ci/run-lifecycle.cjs production target/lifecycle-production/stage + - name: Upload production lifecycle evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-production-evidence-linux-x64 + path: | + target/lifecycle-production/stage/lifecycle-native.json + target/lifecycle-production/stage/lifecycle-results.json + target/lifecycle-production/stage/lifecycle.tap + target/lifecycle-production/stage/lifecycle.stderr.log + if-no-files-found: error + + lifecycle-instrumented: + name: Instrumented lifecycle (isolated) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24.22.1" + - name: Set up Rust from repository toolchain + run: | + test -f rust-toolchain.toml + rustup show active-toolchain + rustc --version --verbose + - name: Install dependencies + run: npm install --omit=optional --ignore-scripts --no-audit --no-fund --package-lock=false + # No fallback override: compatible published core APIs are a separate gate. + - name: Build isolated instrumented addon + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target/lifecycle-instrumented/cargo + run: >- + npx napi build --platform --features test-hooks --js false --dts bindings.d.ts + --cargo-flags="--locked" target/lifecycle-instrumented/native + - name: Stage matching source and native artifact + run: node ci/stage-lifecycle.cjs target/lifecycle-instrumented/native target/lifecycle-instrumented/stage instrumented + - name: Require every instrumented lifecycle case + run: node ci/run-lifecycle.cjs instrumented target/lifecycle-instrumented/stage + # Upload evidence only, never an instrumented binary-* packaging input. + - name: Upload instrumented lifecycle evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-instrumented-evidence-linux-x64 + path: | + target/lifecycle-instrumented/stage/lifecycle-native.json + target/lifecycle-instrumented/stage/lifecycle-results.json + target/lifecycle-instrumented/stage/lifecycle.tap + target/lifecycle-instrumented/stage/lifecycle.stderr.log + if-no-files-found: error # ── Pack tarballs for smoke (local install simulation) ───── pack: diff --git a/CHANGELOG.md b/CHANGELOG.md index 054a5e3..70801ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add owned, metadata-only `RuntimeStopObserver` through `Runtime.stopObserver()`. + Retained stop/fault state supports independent AbortSignal waits, terminal + local close, and final-state delivery before source end. Observation does not + retain worker callbacks, keep Node alive by itself, or change shutdown + ownership, deadlines, and application policy. + +### Changed + +- Use fallible core startup and bounded shutdown waits while preserving `Promise`. + Total waiting is the supplied grace plus five seconds. + Omitted grace is one second. Idle workers can finish early. + Invalid durations fail before ownership changes, including for no-work and repeated calls. + Repeated calls retain the original operation, errors, and deadlines. + They do not report success merely because the wrapper transferred its core. + Registry and preparation failures become ordinary retained startup errors. + Registration policy does not change. +- Pre-start shutdown is terminal. + Start and registration calls fail after activation or stop. + Post-stop metrics remain unavailable. + Proven cleanup permits release of obsolete callback and core holders on the JavaScript thread. + The terminal result and deadlines remain available. +- Timeouts require application- or supervisor-owned process termination while cleanup remains owned. + Later proven cleanup does not remove that requirement. + Finite grace does not guarantee bounded forced cleanup. + This adapter does not add .NET's foreign-continuation barrier or own detached application work. + Release requires a matching published core dependency. + Do not distribute local development overrides. + +### Tests + +- Add SQLite-only lifecycle coverage through the real wrapper. + Cases include feature-gated provider and fault controls, repeated ordered races, + isolated held-provider probes, and production-export checks. + ## [0.2.4] - 2026-10-27 ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79930d8..ff03c7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,4 +36,65 @@ npm run test:sessions cargo clippy --all-targets ``` -After Rust source changes (`src/*.rs`), re-run `npx napi build --platform` before running JavaScript tests. \ No newline at end of file +After Rust source changes (`src/*.rs`), re-run `npx napi build --platform` before running JavaScript tests. + +### Lifecycle checks without PostgreSQL + +The lifecycle suite requires a freshly built local native module. +Do not use an optional prebuilt platform package. +Production builds omit private test controls. +Keep native outputs, Cargo targets, and staged sources separate for each mode. +For example, from the repository root in PowerShell: + +```powershell +$env:CARGO_TARGET_DIR = Join-Path $PWD 'target\lifecycle-production\cargo' +npx napi build --platform --js false --dts bindings.d.ts --cargo-flags="--locked" target\lifecycle-production\native +node ci\stage-lifecycle.cjs target\lifecycle-production\native target\lifecycle-production\stage production +node ci\run-lifecycle.cjs production target\lifecycle-production\stage + +$env:CARGO_TARGET_DIR = Join-Path $PWD 'target\lifecycle-instrumented\cargo' +npx napi build --platform --features test-hooks --js false --dts bindings.d.ts --cargo-flags="--locked" target\lifecycle-instrumented\native +node ci\stage-lifecycle.cjs target\lifecycle-instrumented\native target\lifecycle-instrumented\stage instrumented +node ci\run-lifecycle.cjs instrumented target\lifecycle-instrumented\stage +Remove-Item Env:CARGO_TARGET_DIR +``` + +Instrumented cases exercise real provider waits and contained faults in owned tasks. +The `lifecycle-instrumented` CI job runs on pull requests and main, alongside production tests. +Every hook-dependent case must execute. +Missing hooks, mode or hash mismatches, skips, TODOs, aborted tests, and empty selections fail validation. +Production mode skips only explicitly marked instrumented cases. +It also checks that private exports are absent. + +The stage manifest identifies the exact local addon, its SHA-256, the selected feature mode, and source hashes. +Tests match the Duroxide loader's native class identities, not an arbitrary cached `.node`. +Sources use a stage-local `node_modules` directory. +This prevents root-level test discovery from running duplicate staged suites. +The runner sets mode and provenance only in its child environment. +It retains TAP, stderr, and JSON totals beside the manifest. +Instrumented evidence artifacts do not use `binary-*` names or contain native binaries. +They never feed packing or release jobs. +`CARGO_TARGET_DIR` also tells napi where to find the compiled library. +Cargo's `--target-dir` alone does not set napi's source path. +The `--dts` filename is relative to napi's destination directory. + +Both ordered races still run **100 iterations**. +Held-provider probes require `6000 <= elapsedMs < 7000` for omitted grace, +`5000 <= elapsedMs < 6000` for zero grace, and `repeatedMs < 1000`. +A separate message-delivery watchdog allows the first call, repeated call, and limited transport time. +It cannot relax those measured SDK limits. + +The suite checks event-loop liveness during incomplete cleanup, without stdin or referenced timers. +It also checks collection of activity-only self-capturing callbacks in a live process, after the creating frames return. +Collection uses bounded WeakRef/FinalizationRegistry settling and a noncapturing control. +It does not use natural process exit or zero task counts as proof. +Cases cover successful, quiescent-error, and late cleanup, including late release without another stop caller. + +Harness and oracle checks that do not need native code can run before a native build: +`node --test __tests__/lifecycle-oracles.test.js`. +They include negative controls for child failure, malformed protocol, a zero-start counter, and CI skips. + +Do not publish instrumented assets or temporary local core overrides. +A matching published core minimum remains a separate prerequisite. +CI must fail when core APIs are missing. +It must not silently skip coverage or substitute an unofficial dependency. \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 4bae888..671f8e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,9 @@ edition = "2021" [lib] crate-type = ["cdylib"] +[features] +test-hooks = ["duroxide/test-hooks"] + [dependencies] duroxide = { version = "0.1.32", features = ["sqlite"] } duroxide-pg = "0.1.38" diff --git a/README.md b/README.md index 5199401..1bacc85 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,70 @@ Node.js/TypeScript SDK for the [Duroxide](https://github.com/microsoft/duroxide) - **SQLite & PostgreSQL** — pluggable storage backends - **KV store** — durable per-instance key-value state with snapshots and pruning via `ctx.getKvAllValues()` / `client.getKvAllValues()` +## Runtime lifecycle (unreleased) + +`await runtime.start()` returns after startup. Core startup errors reject the Promise. +`await runtime.shutdown(timeoutMs)` still resolves to `undefined` on success. +The argument is a nonnegative safe-integer grace period in milliseconds. +It must fit the native monotonic deadline range. +Omitted grace is 1000 ms. Total waiting is grace plus 5000 ms. +Idle work can finish early. +At the first stop call, the supervisor's **remaining** time must exceed that total. +Leave time for other shutdown work, result delivery, and process exit. +Do not pass the entire outer budget as grace. See the +[budget-sizing guidance](docs/user-guide.md#sizing-the-supervisor-budget). + +The first stop fixes both deadlines. Repeated calls observe the same operation. +They can report timeout, operational errors, or actual late completion. +Shutdown before start is terminal. Later start or registration calls fail. +Metrics remain unavailable after stop. +Serialize lifecycle and registration calls on each runtime. + +**Quiescence** means that tracked core execution and cleanup have ended. +Only proven quiescence permits release of obsolete native callback and core holders. +This includes quiescent failures and late cleanup without another caller. +Stop results and the original deadlines remain available after release. +Incomplete cleanup, unproven cleanup, or a callback-release failure retains ownership. + +A `lifecycle_shutdown_timed_out` rejection means cleanup is still owned and +incomplete. Do not restart the worker. +The application or supervisor must terminate the process, even if a later observation reports completion. +The SDK does not terminate the process. +`lifecycle_shutdown_failed` is an error category, not proof of quiescence. +It also covers a lost cleanup owner and failure to release callback references after core cleanup. +Do not infer release permission from that prefix. + +This adapter does not track arbitrary JavaScript continuations. +It does not add the .NET SDK's additional managed foreign-retirement barrier. +Independent provider and Client resources have their own lifetime. +See the [lifecycle guide](docs/user-guide.md#runtime-lifecycle) +for validation and an executable early-drain/repeated-stop example. + +These unreleased changes require a compatible core lifecycle release before distribution. +A temporary local Cargo override provides development evidence, not a release dependency pin. + +### Passive stop observation + +`runtime.stopObserver()` returns an owned, metadata-only `RuntimeStopObserver`. +Read `observer.state` first, then call `waitForChange(lastSeen, { signal })`. +The states are `'NotRequested'`, `'Requested'`, and sticky `'Faulted'`. +They report recorded stop/fault state, not readiness or completed cleanup. +A late observer can already see a fault. + +Use one pending wait per observer. Independent consumers need separate observers. +`AbortSignal` cancels only its wait, rejects with `signal.reason`, and permits retry. +`close()` is terminal and idempotent; pending and later operations fail. +A normal source end returns an unseen final state before `null`. +Intermediate states can coalesce. Neither source end nor a settled Promise +proves worker cleanup. + +Observation does not start/stop the worker, change shutdown deadlines, or retain +its callbacks. Its native delivery does not keep Node alive by itself; the event +loop must still run to deliver results. The existing limit on unregistered +JavaScript continuations remains. See the +[observer example](docs/user-guide.md#passive-stop-observation) for initial state, +cancellation, local close, and normal source end. + ## Quick Start ```bash diff --git a/__tests__/fixtures/lifecycle-collection-probe.cjs b/__tests__/fixtures/lifecycle-collection-probe.cjs new file mode 100644 index 0000000..e7931a7 --- /dev/null +++ b/__tests__/fixtures/lifecycle-collection-probe.cjs @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { performance } = require('node:perf_hooks'); +const { getEventListeners } = require('node:events'); +const { setImmediate: nextTurn, setTimeout: delay } = require('node:timers/promises'); +const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); +const { nativeProvenance } = require('../../ci/smoke/native-provenance.cjs'); +const { incompleteTimeout, knownQuiescentCoreFailure, retiredState, bounded, until } = require('../helpers/lifecycle-oracles.cjs'); + +const { native, provenance } = nativeProvenance(path.resolve(__dirname, '..', '..'), { + instrumented: process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1', + manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, +}); +const finalized = new Set(); +const registry = new FinalizationRegistry(label => finalized.add(label)); +const emit = (phase, detail) => console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase, ...detail })); + +function capturingActivity(owner) { + return async (_, input) => `${owner.prefix}:${input}`; +} + +function noncapturingActivity() { + return async (_, input) => input; +} + +async function createOwner(label, scenario, captures) { + const runtime = new Runtime(await SqliteProvider.inMemory(), { dispatcherPollIntervalMs: 5 }); + const owner = { runtime, prefix: label }; + const callback = captures ? capturingActivity(owner) : noncapturingActivity(); + const records = {}; + for (const [name, value] of Object.entries({ owner, runtime, native: runtime._native, callback })) { + const key = `${label}:${name}`; + registry.register(value, key); + records[name] = { label: key, weak: new WeakRef(value) }; + } + // Activity-only registration excludes the pre-existing global orchestration map. + runtime.registerActivity(`Collection${label}`, callback); + const hooks = scenario === 'completed' ? undefined : native._lifecycleTestHooks(runtime._native); + if (scenario === 'quiescent-failure') { + hooks.hold('worker'); + hooks.fail('worker', 'collection-private-sentinel'); + } else if (scenario.startsWith('late')) { + hooks.hold('blocking-provider'); + } + await runtime.start(); + if (scenario === 'quiescent-failure') { + try { await hooks.waitEntered('worker'); } + finally { hooks.release('worker'); } + await assert.rejects(runtime.shutdown(100), knownQuiescentCoreFailure); + await assert.rejects(runtime.shutdown(30_000), knownQuiescentCoreFailure); + retiredState(JSON.parse(hooks.snapshot()), { rollback: true, executionWitness: true }); + } else if (scenario.startsWith('late')) { + await hooks.waitEntered('blocking-provider'); + await assert.rejects(runtime.shutdown(0), incompleteTimeout); + await assert.rejects(runtime.shutdown(30_000), incompleteTimeout); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } else { + await runtime.shutdown(100); + await runtime.shutdown(0); + } + assert.equal(runtime.metricsSnapshot(), null); + // Rejected async start calls can retain napi-rs receiver references. Terminal + // rejection has separate tests and must not contaminate this collection control. + return { records, hooks }; +} + +function observations(records) { + return Object.fromEntries(Object.entries(records).map(([name, record]) => [name, { + weakGone: record.weak.deref() === undefined, + finalized: finalized.has(record.label), + }])); +} + +function collected(state) { + return Object.values(state).every(value => value.weakGone && value.finalized); +} + +async function settle(control, captured, milliseconds = 3000) { + const deadline = performance.now() + milliseconds; + let state; + let cycles = 0; + do { + // A dereferenced WeakRef is kept alive until the end of that JS job. + await nextTurn(); + global.gc(); + await delay(20); + state = { control: observations(control), captured: observations(captured) }; + cycles++; + if (collected(state.control) && collected(state.captured)) break; + } while (performance.now() < deadline); + return { ...state, cycles }; +} + +async function observeLateCompletion(runtimeRef) { + const runtime = runtimeRef.deref(); + assert.ok(runtime, 'incomplete native ownership must keep its self-capturing Runtime alive'); + const deadline = performance.now() + 2000; + while (true) { + try { await runtime.shutdown(30_000); break; } + catch (error) { + incompleteTimeout(error); + assert.ok(performance.now() < deadline, 'late cleanup did not complete'); + await delay(1); + } + } + assert.equal(await runtime.shutdown(0), undefined); + assert.equal(runtime.metricsSnapshot(), null); +} + +function observationOwner() { + return SqliteProvider.inMemory().then(provider => { + const runtime = new Runtime(provider); + const observer = runtime.stopObserver(); + const records = {}; + for (const [name, value] of Object.entries({ runtime, native: runtime._native })) { + const label = `SourceEnd:${name}`; + registry.register(value, label); + records[name] = { label, weak: new WeakRef(value) }; + } + const abort = new AbortController(); + const pending = observer.waitForChange('NotRequested', { signal: abort.signal }); + pending.catch(() => {}); + return { observer, records, pending, abort }; + }); +} + +async function sourceEndProbe() { + const { observer, records, pending, abort } = await observationOwner(); + try { + const state = await settle(records, records); + assert.ok(collected(state.control), JSON.stringify(state)); + assert.equal(await bounded(pending), null); + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + abort.abort(); + assert.equal(await pending, null); + assert.equal(observer.state, 'NotRequested'); + assert.equal(await observer.waitForChange('NotRequested'), null); + emit('OBSERVER_COLLECTION', { scenario: 'observer-source-end', collected: true, state, native: provenance }); + } finally { observer.close(); } +} + +async function retainedObservationOwner(label, captures) { + const runtime = new Runtime(await SqliteProvider.inMemory(), { dispatcherPollIntervalMs: 5 }); + const owner = { runtime, prefix: label }; + const callback = captures ? capturingActivity(owner) : noncapturingActivity(); + const records = {}; + for (const [name, value] of Object.entries({ owner, callback })) { + const key = `${label}:${name}`; + registry.register(value, key); + records[name] = { label: key, weak: new WeakRef(value) }; + } + runtime.registerActivity(`ObserverCollection${label}`, callback); + const observer = runtime.stopObserver(); + const hooks = provenance.instrumented ? native._lifecycleTestHooks(runtime._native) : undefined; + let first; + if (hooks) { + hooks.hold('observer-final-access'); + first = observer.waitForChange('NotRequested'); + first.catch(() => {}); + } + await runtime.start(); + await runtime.shutdown(100); + if (hooks) { + assert.equal(await bounded(first), 'Requested'); + await hooks.waitEntered('observer-final-access'); + assert.ok(JSON.parse(hooks.snapshot()).observerPending > 0); + } + const pendingState = { settled: false }; + const pending = observer.waitForChange('Requested'); + pending.then(() => { pendingState.settled = true; }, () => { pendingState.settled = true; }); + return { runtime, observer, pending, pendingState, hooks, records }; +} + +async function retainedObservationProbe() { + const control = await retainedObservationOwner('ObserverControl', false); + const captured = await retainedObservationOwner('ObserverCaptured', true); + try { + const state = await settle(control.records, captured.records); + assert.ok(collected(state.control), JSON.stringify(state)); + assert.ok(collected(state.captured), JSON.stringify(state)); + for (const value of [control, captured]) { + assert.equal(value.runtime.metricsSnapshot(), null); + assert.equal(value.observer.state, 'Requested'); + assert.equal(value.runtime._stopSource.snapshot().closed, false); + assert.equal(value.pendingState.settled, false); + if (value.hooks) { + const execution = JSON.parse(value.hooks.snapshot()); + retiredState(execution, { executionWitness: true }); + assert.ok(execution.observerPending >= 2, 'old delivery tail and new wait must both remain owned'); + } + } + emit('OBSERVER_COLLECTION', { + scenario: 'retained-observer', collected: true, state, native: provenance, + runtimeRetained: true, observerRetained: true, sourceOpen: true, + nativeTailHeld: provenance.instrumented, + }); + } finally { + for (const value of [control, captured]) { + value.observer.close(); + await assert.rejects(value.pending, /stop_observer_closed/); + if (value.hooks) { + value.hooks.release('observer-final-access'); + await until(() => JSON.parse(value.hooks.snapshot()).observerPending === 0); + } + } + } +} + +async function main() { + assert.equal(typeof global.gc, 'function', 'collection probe requires --expose-gc'); + const requested = process.argv[2]; + if (requested === 'observer-source-end') return sourceEndProbe(); + if (requested === 'retained-observer') return retainedObservationProbe(); + const scenario = requested === 'baseline' ? 'completed' : requested; + assert.ok(['completed', 'quiescent-failure', 'late', 'late-unobserved'].includes(scenario)); + const [control, captured] = await Promise.all([ + createOwner('Control', 'completed', false), + createOwner('Captured', scenario, true), + ]); + let retainedWhileIncomplete; + if (scenario.startsWith('late')) { + const pending = await settle(control.records, captured.records, 300); + retainedWhileIncomplete = Object.values(pending.captured).every(value => !value.weakGone && !value.finalized); + assert.ok(retainedWhileIncomplete, JSON.stringify(pending)); + assert.ok(JSON.parse(captured.hooks.snapshot()).active > 0); + captured.hooks.release('blocking-provider'); + if (scenario === 'late') await observeLateCompletion(captured.records.runtime.weak); + } + // Only WeakRefs and non-owning test controls survive the creating/observing frames. + const state = await settle(control.records, captured.records); + if (scenario.startsWith('late')) { + retiredState(JSON.parse(captured.hooks.snapshot()), { forced: true, executionWitness: true }); + } + const result = { + scenario: requested, native: provenance, ...state, + controlCollected: collected(state.control), + capturedCollected: collected(state.captured), + retainedWhileIncomplete, + }; + assert.ok(result.controlCollected, `noncapturing control did not collect: ${JSON.stringify(result)}`); + if (requested !== 'baseline') assert.ok(result.capturedCollected, `self-capturing owner did not collect: ${JSON.stringify(result)}`); + emit('COLLECTION', result); +} + +main().catch(error => { + emit('FAILED', { error: error.stack }); + process.exitCode = 1; +}); diff --git a/__tests__/fixtures/lifecycle-probe.cjs b/__tests__/fixtures/lifecycle-probe.cjs new file mode 100644 index 0000000..395b9e5 --- /dev/null +++ b/__tests__/fixtures/lifecycle-probe.cjs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { createInterface } = require('node:readline'); +const { performance } = require('node:perf_hooks'); +const path = require('node:path'); +const { SqliteProvider, Runtime } = require('../../lib/duroxide.js'); +const { nativeProvenance } = require('../../ci/smoke/native-provenance.cjs'); +const { incompleteTimeout, retiredState, pendingObserverExit } = require('../helpers/lifecycle-oracles.cjs'); +const { native, provenance } = nativeProvenance(path.resolve(__dirname, '..', '..'), { + instrumented: process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1', + manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, +}); +const emit = (phase, detail = {}) => console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase, ...detail })); +console.log('LIFECYCLE_NATIVE ' + JSON.stringify(provenance)); + +async function main() { + const scenario = process.argv[2]; + let runtime = new Runtime(await SqliteProvider.inMemory(), { + dispatcherPollIntervalMs: 5, serviceName: 'lifecycle-private-sentinel', + }); + if (scenario === 'registry-invalid' || scenario === 'registry-descending') { + const versions = scenario === 'registry-invalid' ? ['not-semver'] : ['2.0.0', '1.0.0']; + for (const version of versions) { + runtime.registerOrchestrationVersioned('registry-private-sentinel', version, function* () { return 'ok'; }); + } + let startMessage; + await assert.rejects(runtime.start(), error => { + assert.ok(error instanceof Error); + assert.match(error.message, /lifecycle_start_failed/); + assert.doesNotMatch(error.message, /sentinel|Panic in async function/); + startMessage = error.message; + return true; + }); + for (const timeout of [0, 30_000]) { + await assert.rejects(runtime.shutdown(timeout), error => { + assert.equal(error.message, startMessage); + return true; + }); + } + assert.equal(runtime.metricsSnapshot(), null); + await assert.rejects(runtime.start(), /lifecycle_terminal/); + assert.throws(() => runtime.registerActivity('late', async () => 'late'), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestration('late', function* () {}), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestrationVersioned('late', '3.0.0', function* () {}), /lifecycle_terminal/); + globalThis.retainedRuntime = runtime; + emit('REGISTRY_FAILURE_RETAINED', { scenario }); + return; + } + if (scenario === 'exit') { + runtime.registerActivity('ExitActivity', async (_, value) => value); + runtime.registerOrchestration('ExitFlow', function* () { return 'ok'; }); + await runtime.start(); + await runtime.shutdown(100); + globalThis.retainedRuntime = runtime; + emit('EXIT_READY'); + return; + } + if (scenario === 'observer-exit') { + const hooks = provenance.instrumented ? native._lifecycleTestHooks(runtime._native) : undefined; + const observer = runtime.stopObserver(); + runtime.registerActivity('ObserverExitActivity', async (_, value) => value); + let settled = false; + let pending; + if (hooks) { + hooks.hold('observer-delivery'); + pending = observer.waitForChange('NotRequested'); + pending.then(() => { settled = true; }, () => { settled = true; }); + } + await runtime.start(); + await runtime.shutdown(100); + if (hooks) { + await hooks.waitEntered('observer-delivery'); + } else { + pending = observer.waitForChange('Requested'); + pending.then(() => { settled = true; }, () => { settled = true; }); + } + const report = () => ({ + source: runtime._stopSource.snapshot(), settled, native: provenance, + execution: hooks ? JSON.parse(hooks.snapshot()) : undefined, + }); + globalThis.retainedObserver = { runtime, observer, pending, hooks }; + pendingObserverExit(report()); + emit('OBSERVER_EXIT_READY', report()); + process.once('beforeExit', () => { + pendingObserverExit(report()); + emit('OBSERVER_BEFORE_EXIT', report()); + }); + return; + } + const hooks = native._lifecycleTestHooks(runtime._native); + hooks.hold('blocking-provider'); + runtime.registerActivity('PendingCleanupActivity', async (_, value) => value); + await runtime.start(); + await hooks.waitEntered('blocking-provider'); + const standalone = scenario === 'incomplete-liveness'; + const input = standalone ? undefined : createInterface({ input: process.stdin }); + const command = input && new Promise(resolve => input.once('line', resolve)); + emit('STOPPING'); + const start = performance.now(); + await assert.rejects(runtime.shutdown(scenario === 'late' ? undefined : 0), incompleteTimeout); + const elapsedMs = performance.now() - start; + const repeated = performance.now(); + await assert.rejects(runtime.shutdown(30_000), error => { + assert.match(error.message, /lifecycle_shutdown_timed_out/); + assert.ok(!error.message.includes('sentinel')); + return true; + }); + const repeatedMs = performance.now() - repeated; + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + assert.equal(runtime.metricsSnapshot(), null); + emit('TIMED_OUT', { elapsedMs, repeatedMs }); + if (standalone) { + globalThis.retainedRuntime = runtime; + process.once('beforeExit', () => emit('PREMATURE_BEFORE_EXIT', { state: JSON.parse(hooks.snapshot()) })); + let pulse = 0; + setInterval(() => emit('ALIVE', { pulse: ++pulse, state: JSON.parse(hooks.snapshot()) }), 100).unref(); + return; + } + if (await command === 'retain') { + runtime = null; + global.gc(); + const state = JSON.parse(hooks.snapshot()); + assert.ok(state.active > 0); + assert.equal(state.coordinators, 1); + emit('RETAINED'); + await new Promise(() => {}); + } else { + hooks.release('blocking-provider'); + const deadline = performance.now() + 2000; + while (true) { + try { await runtime.shutdown(30_000); break; } + catch (error) { + assert.match(error.message, /lifecycle_shutdown_timed_out/); + assert.ok(performance.now() < deadline); + await new Promise(resolve => setTimeout(resolve, 1)); + } + } + const state = JSON.parse(hooks.snapshot()); + retiredState(state, { forced: true, executionWitness: true }); + await assert.rejects(runtime.start(), /lifecycle_terminal/); + emit('RECLAIMED'); + input.close(); + } +} + +main().catch(error => { + emit('FAILED', { error: error.stack }); + process.exitCode = 1; +}); diff --git a/__tests__/helpers/lifecycle-oracles.cjs b/__tests__/helpers/lifecycle-oracles.cjs new file mode 100644 index 0000000..2e50082 --- /dev/null +++ b/__tests__/helpers/lifecycle-oracles.cjs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { getEventListeners } = require('node:events'); +const { performance } = require('node:perf_hooks'); +const { setTimeout: delay } = require('node:timers/promises'); + +function diagnostic(category) { + return error => { + assert.ok(error instanceof Error); + assert.match(error.message, new RegExp(`^${category}:`)); + assert.doesNotMatch(error.message, /sentinel|Host=private/); + return true; + }; +} + +function knownQuiescentCoreFailure(error) { + diagnostic('lifecycle_shutdown_failed')(error); + assert.match(error.message, /cleanup completed and is quiescent/); + assert.doesNotMatch(error.message, /terminate the process/); + return true; +} + +function incompleteTimeout(error) { + diagnostic('lifecycle_shutdown_timed_out')(error); + assert.match(error.message, /cleanup.*incomplete/); + assert.match(error.message, /terminate the process/); + return true; +} + +function retiredState(state, { forced, rollback = false, executionWitness = false } = {}) { + if (executionWitness) assert.ok(state.started > 0, 'witnessed execution must have a recorded task start'); + assert.equal(state.active, 0); + assert.equal(state.cleanupActive, 0); + assert.equal(state.started, state.completed); + if (rollback) assert.ok(state.coordinators >= 1 && state.coordinators <= 2); + else assert.equal(state.coordinators, 1); + if (forced !== undefined) assert.equal(state.forceRequests, forced ? 1 : 0); +} + +function shutdownTiming(scenario) { + const minimum = scenario === 'late' ? 6000 : 5000; + return { minimum, maximum: minimum + 1000, repeatedMaximum: 1000, delivery: minimum + 3000 }; +} + +function assertShutdownTiming(scenario, message) { + const budget = shutdownTiming(scenario); + assert.ok(message.elapsedMs >= budget.minimum, `first stop ${message.elapsedMs}ms < ${budget.minimum}ms`); + assert.ok(message.elapsedMs < budget.maximum, `first stop ${message.elapsedMs}ms >= ${budget.maximum}ms`); + assert.ok(message.repeatedMs >= 0 && message.repeatedMs < budget.repeatedMaximum, + `repeated stop ${message.repeatedMs}ms must be < ${budget.repeatedMaximum}ms`); +} + +async function orderedRaceAttempt(runtime, hooks, force, iteration) { + const order = force ? 'force-before-retirement' : 'retirement-before-force'; + const stopGate = force ? 'force' : 'coordinator'; + const armed = []; + let phase = 'arming'; + let snapshot; + let stop; + let failure; + const context = () => `${order} iteration ${iteration + 1}/100 phase=${phase} lastSnapshot=${JSON.stringify(snapshot ?? null)}`; + try { + for (const gate of ['blocking-provider', stopGate]) { + phase = `arm:${gate}`; + hooks.hold(gate); + armed.push(gate); + } + phase = 'start'; + await runtime.start(); + phase = 'entered:blocking-provider'; + await hooks.waitEntered('blocking-provider'); + snapshot = JSON.parse(hooks.snapshot()); + assert.ok(snapshot.started > 0, 'entered provider execution was not counted'); + phase = `entered:${stopGate}`; + stop = runtime.shutdown(force ? 0 : 30_000); + stop.catch(() => {}); + await hooks.waitEntered(stopGate); + snapshot = JSON.parse(hooks.snapshot()); + assert.ok(snapshot.active > 0); + } catch (error) { + failure = new Error(context(), { cause: error }); + } finally { + for (const gate of armed) { + try { hooks.release(gate); } + catch (error) { failure ??= new Error(`${context()} releasing ${gate}`, { cause: error }); } + } + try { + if (!stop) stop = runtime.shutdown(0); + assert.equal(await stop, undefined); + } catch (error) { + failure ??= new Error(`${context()} cleanup`, { cause: error }); + } + } + if (failure) throw failure; + try { + phase = 'repeated-stop'; + assert.equal(await runtime.shutdown(), undefined); + phase = 'retired'; + snapshot = JSON.parse(hooks.snapshot()); + retiredState(snapshot, { forced: force, executionWitness: true }); + } catch (error) { + throw new Error(context(), { cause: error }); + } +} + +async function bounded(promise, milliseconds = 3000) { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('stop observation did not settle within its test bound')), milliseconds); + }), + ]); + } finally { clearTimeout(timer); } +} + +async function until(predicate, milliseconds = 3000) { + const deadline = performance.now() + milliseconds; + while (!predicate()) { + assert.ok(performance.now() < deadline, 'stop observation condition did not become true'); + await delay(1); + } +} + +function pendingObserverExit(value) { + assert.equal(value.source.closed, false, 'an ended source cannot prove pending observation liveness'); + assert.equal(value.settled, false, 'a settled promise cannot prove pending observation liveness'); + assert.equal(value.source.state, 'Requested'); + if (value.execution) { + retiredState(value.execution, { executionWitness: true }); + assert.ok(value.execution.observerPending > 0, 'notification context must still be owned'); + } +} + +async function observerRaceAttempt(runtime, hooks, changeFirst, iteration) { + const armed = []; + let delivered = 0; + const nativeWait = runtime._stopSource.waitForChange; + Object.defineProperty(runtime._stopSource, 'waitForChange', { + value(lastSeen, callback) { + return nativeWait.call(this, lastSeen, (...args) => { + delivered++; + callback(...args); + }); + }, + }); + const first = runtime.stopObserver(); + const closed = runtime.stopObserver(); + const initialAbort = new AbortController(); + const retryAbort = new AbortController(); + const reason = new Error('first-observation-aborted'); + const retryReason = new Error('retried-observation-aborted'); + let startup; + let failure; + let phase = 'arming'; + let late; + try { + for (const gate of ['startup', 'observer-delivery']) { + phase = `arm:${gate}`; + hooks.hold(gate); + armed.push(gate); + } + const firstWait = first.waitForChange('NotRequested', { signal: initialAbort.signal }); + const closeWait = closed.waitForChange('NotRequested'); + firstWait.catch(() => {}); + closeWait.catch(() => {}); + const staleAbort = assertSingleListener(initialAbort.signal); + startup = runtime.start(); + phase = 'held-startup'; + await hooks.waitEntered('startup'); + late = runtime.stopObserver(); + assert.equal(first.state, 'NotRequested'); + assert.equal(late.state, 'NotRequested'); + if (changeFirst) { + hooks.release('startup'); + await startup; + await runtime.shutdown(0); + await hooks.waitEntered('observer-delivery'); + } + phase = 'cancel-and-close'; + initialAbort.abort(reason); + closed.close(); + await bounded(Promise.all([ + assert.rejects(firstWait, error => error === reason), + assert.rejects(closeWait, /stop_observer_closed/), + ])); + assert.equal(getEventListeners(initialAbort.signal, 'abort').length, 0); + assert.equal(first.state, changeFirst ? 'Requested' : 'NotRequested'); + let retriedSettled = false; + const retry = first.waitForChange(changeFirst ? 'Requested' : 'NotRequested', { signal: retryAbort.signal }); + retry.then(() => { retriedSettled = true; }, () => { retriedSettled = true; }); + staleAbort(); + assertSingleListener(retryAbort.signal); + phase = 'retire-worker'; + if (!changeFirst) { + hooks.release('startup'); + await startup; + await runtime.shutdown(0); + await hooks.waitEntered('observer-delivery'); + } + const state = JSON.parse(hooks.snapshot()); + retiredState(state, { executionWitness: true }); + assert.ok(state.observerPending >= 2, 'old notification contexts must survive local settlement'); + assert.equal(late.state, 'Requested'); + assert.equal(runtime._stopSource.snapshot().closed, false); + phase = 'late-delivery'; + hooks.release('observer-delivery'); + if (changeFirst) { + await until(() => delivered >= 2); + assert.equal(retriedSettled, false, 'old callbacks must not settle the new pending wait'); + retryAbort.abort(retryReason); + await bounded(assert.rejects(retry, error => error === retryReason)); + } else { + assert.equal(await bounded(retry), 'Requested'); + retryAbort.abort(retryReason); + assert.equal(await retry, 'Requested', 'settled results are not rewritten by abort or close'); + } + assert.equal(getEventListeners(retryAbort.signal, 'abort').length, 0); + first.close(); + await until(() => JSON.parse(hooks.snapshot()).observerPending === 0); + } catch (error) { + failure = new Error(`stop observer ${changeFirst ? 'change-first' : 'cancel-first'} iteration ${iteration + 1}/100 phase=${phase}`, { cause: error }); + } finally { + first.close(); + closed.close(); + late?.close(); + for (const gate of armed) { + try { hooks.release(gate); } + catch (error) { failure ??= new Error(`stop observer cleanup gate ${gate}`, { cause: error }); } + } + try { + if (startup) await startup; + await runtime.shutdown(0); + } catch (error) { failure ??= error; } + } + if (failure) throw failure; +} + +function assertSingleListener(signal) { + const listeners = getEventListeners(signal, 'abort'); + assert.equal(listeners.length, 1); + return listeners[0]; +} + +module.exports = { + diagnostic, knownQuiescentCoreFailure, incompleteTimeout, retiredState, + shutdownTiming, assertShutdownTiming, orderedRaceAttempt, + bounded, until, pendingObserverExit, observerRaceAttempt, +}; diff --git a/__tests__/helpers/lifecycle-probe.cjs b/__tests__/helpers/lifecycle-probe.cjs new file mode 100644 index 0000000..7ae83fe --- /dev/null +++ b/__tests__/helpers/lifecycle-probe.cjs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const { createInterface } = require('node:readline'); +const path = require('node:path'); + +const OUTPUT_LIMIT = 8192; + +function startProbe(scenario, { input = false, fixture = 'lifecycle-probe.cjs', script, env = process.env } = {}) { + const args = script === undefined + ? ['--expose-gc', path.join(__dirname, '..', 'fixtures', fixture), scenario] + : ['-e', script]; + const child = spawn(process.execPath, args, { + cwd: path.resolve(__dirname, '..', '..'), stdio: [input ? 'pipe' : 'ignore', 'pipe', 'pipe'], env, + }); + let stdout = ''; + let stderr = ''; + let lastPhase = 'spawn'; + let expectedPhase = 'none'; + let failure; + let waiter; + const messages = []; + const queue = []; + const describe = () => + `scenario=${scenario} expected=${expectedPhase} last=${lastPhase} exit=${child.exitCode} signal=${child.signalCode}` + + `\nstdout (tail):\n${stdout}\nstderr (tail):\n${stderr}`; + const reject = error => { + failure ??= error; + if (waiter) { + const pending = waiter; + waiter = undefined; + clearTimeout(pending.timer); + pending.reject(new Error(describe(), { cause: failure })); + } + }; + const deliver = () => { + if (!waiter || queue.length === 0) return; + const pending = waiter; + waiter = undefined; + clearTimeout(pending.timer); + const message = queue.shift(); + try { + assert.equal(message.phase, expectedPhase, describe()); + pending.resolve(message); + } catch (error) { + failure ??= error; + pending.reject(new Error(describe(), { cause: error })); + } + }; + child.stdout.on('data', chunk => { stdout = (stdout + chunk).slice(-OUTPUT_LIMIT); }); + child.stderr.on('data', chunk => { stderr = (stderr + chunk).slice(-OUTPUT_LIMIT); }); + const lines = createInterface({ input: child.stdout }); + lines.on('line', line => { + if (!line.startsWith('LIFECYCLE_PROBE ')) return; + try { + const message = JSON.parse(line.slice('LIFECYCLE_PROBE '.length)); + assert.equal(typeof message.phase, 'string'); + lastPhase = message.phase; + messages.push(message); + if (message.phase === 'FAILED' || message.phase === 'PREMATURE_BEFORE_EXIT') { + reject(new Error(message.error ?? `native cleanup pending at ${message.phase}`)); + return; + } + queue.push(message); + deliver(); + } catch (error) { reject(error); } + }); + child.on('error', reject); + const exited = new Promise(resolve => child.once('close', (code, signal) => { + if (waiter) reject(new Error(`child closed before ${expectedPhase}: ${code ?? signal}`)); + resolve({ code, signal }); + })); + + return { + child, messages, describe, + async next(phase, milliseconds) { + expectedPhase = phase; + if (failure) throw new Error(describe(), { cause: failure }); + if (child.exitCode !== null || child.signalCode !== null) { + if (queue.length === 0) throw new Error(describe(), { cause: new Error('child already exited') }); + } + assert.equal(waiter, undefined, 'only one protocol observation may be pending'); + return new Promise((resolve, rejectPromise) => { + const timer = setTimeout(() => reject(new Error(`message-delivery watchdog expired after ${milliseconds}ms`)), milliseconds); + waiter = { resolve, reject: rejectPromise, timer }; + deliver(); + }); + }, + async successfulExit(milliseconds) { + expectedPhase = 'exit(0)'; + let timer; + try { + const status = await Promise.race([ + exited, + new Promise((_, rejectPromise) => { + timer = setTimeout(() => rejectPromise(new Error(describe(), { + cause: new Error(`natural-exit watchdog expired after ${milliseconds}ms`), + })), milliseconds); + }), + ]); + if (failure) throw new Error(describe(), { cause: failure }); + assert.equal(status.code, 0, describe()); + assert.equal(status.signal, null, describe()); + } finally { clearTimeout(timer); } + }, + async dispose() { + if (waiter) reject(new Error('probe disposed')); + lines.close(); + child.stdin?.destroy(); + if (child.exitCode === null && child.signalCode === null) child.kill(); + await exited; + }, + }; +} + +module.exports = { startProbe, OUTPUT_LIMIT }; diff --git a/__tests__/lifecycle-oracles.test.js b/__tests__/lifecycle-oracles.test.js new file mode 100644 index 0000000..1eca92e --- /dev/null +++ b/__tests__/lifecycle-oracles.test.js @@ -0,0 +1,154 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { identifyNative } = require('../ci/smoke/native-provenance.cjs'); +const { lifecycleReport, encodedRecords } = require('../ci/lifecycle-report.cjs'); +const { startProbe, OUTPUT_LIMIT } = require('./helpers/lifecycle-probe.cjs'); +const { + diagnostic, knownQuiescentCoreFailure, incompleteTimeout, retiredState, + shutdownTiming, assertShutdownTiming, orderedRaceAttempt, + pendingObserverExit, +} = require('./helpers/lifecycle-oracles.cjs'); + +test('failure category does not claim quiescence for owner loss or callback release errors', () => { + for (const message of [ + 'native shutdown owner failed; quiescence is unproven; application or supervisor must terminate the process', + 'callback event-loop reference release failed', + ]) { + const error = new Error(`lifecycle_shutdown_failed: ${message}`); + assert.equal(diagnostic('lifecycle_shutdown_failed')(error), true); + assert.throws(() => knownQuiescentCoreFailure(error)); + } + assert.equal(knownQuiescentCoreFailure(new Error('lifecycle_shutdown_failed: cleanup completed and is quiescent')), true); + assert.equal(incompleteTimeout(new Error('lifecycle_shutdown_timed_out: cleanup remains owned and incomplete; application or supervisor must terminate the process')), true); + assert.throws(() => diagnostic('lifecycle_shutdown_failed')(new Error('lifecycle_shutdown_failed: private-sentinel'))); +}); + +test('retirement oracle rejects zero execution starts despite a working coordinator', () => { + const zeroExecution = { active: 0, cleanupActive: 0, started: 0, completed: 0, coordinators: 1, forceRequests: 0 }; + retiredState(zeroExecution, { forced: false }); + assert.throws(() => retiredState(zeroExecution, { executionWitness: true }), /recorded task start/); +}); + +test('delivery allowance cannot admit over-budget first or repeated shutdown measurements', () => { + for (const scenario of ['late', 'retained']) { + const budget = shutdownTiming(scenario); + assert.ok(budget.delivery > budget.maximum + budget.repeatedMaximum); + assertShutdownTiming(scenario, { elapsedMs: budget.maximum - 1, repeatedMs: 999 }); + assert.throws(() => assertShutdownTiming(scenario, { elapsedMs: budget.maximum, repeatedMs: 0 })); + assert.throws(() => assertShutdownTiming(scenario, { elapsedMs: budget.minimum, repeatedMs: 1000 })); + assert.throws(() => assertShutdownTiming(scenario, { elapsedMs: budget.minimum - 1, repeatedMs: 0 })); + } +}); + +test('native identity follows Duroxide exports, not the first cached addon', () => { + const loaderPath = path.resolve('index.js'); + const otherPath = path.resolve('unrelated.node'); + const nativePath = path.resolve('duroxide.test.node'); + const bindings = { JsRuntime: class {}, JsClient: class {}, JsSqliteProvider: class {} }; + const intended = { filename: nativePath, exports: bindings }; + const cache = { + [otherPath]: { filename: otherPath, exports: null }, + [loaderPath]: { exports: bindings }, + [nativePath]: intended, + }; + assert.equal(identifyNative(loaderPath, cache), intended); + delete cache[nativePath]; + assert.throws(() => identifyNative(loaderPath, cache), /exactly one native module/); +}); + +test('ordered-race failures preserve cause, iteration and early armed-gate cleanup', async () => { + const original = new Error('deliberate second gate failure'); + const released = []; + let stopped = 0; + const hooks = { + hold(gate) { if (gate === 'force') throw original; }, + release(gate) { released.push(gate); }, + snapshot() { throw new Error('diagnostics must not call snapshots after failure'); }, + }; + const runtime = { + async start() { throw new Error('must not reach startup'); }, + async shutdown() { stopped++; }, + }; + await assert.rejects(orderedRaceAttempt(runtime, hooks, true, 41), error => { + assert.equal(error.cause, original); + assert.match(error.message, /force-before-retirement iteration 42\/100 phase=arm:force lastSnapshot=null/); + return true; + }); + assert.deepEqual(released, ['blocking-provider']); + assert.equal(stopped, 1); +}); + +test('probe failures retain bounded stdout, stderr, protocol phase and original cause', async () => { + const probe = startProbe('deliberate-failure', { + script: `console.log('x'.repeat(${OUTPUT_LIMIT * 2})); console.error('diagnostic-stderr'); + console.log('LIFECYCLE_PROBE ' + JSON.stringify({ phase: 'FAILED', error: 'deliberate child stack' }));`, + }); + try { + await assert.rejects(probe.next('STOPPING', 3000), error => { + assert.match(error.message, /scenario=deliberate-failure expected=STOPPING last=FAILED exit=/); + assert.match(error.message, /diagnostic-stderr/); + assert.match(error.cause.message, /deliberate child stack/); + assert.ok(error.message.length < OUTPUT_LIMIT * 2 + 1000); + return true; + }); + } finally { await probe.dispose(); } +}); + +test('malformed child protocol retains its parse failure and reliable cleanup', async () => { + const probe = startProbe('malformed', { script: "console.log('LIFECYCLE_PROBE {broken');" }); + try { + await assert.rejects(probe.next('STOPPING', 3000), error => { + assert.ok(error.cause instanceof SyntaxError); + assert.match(error.message, /expected=STOPPING last=spawn/); + assert.match(error.message, /\{broken/); + return true; + }); + } finally { await probe.dispose(); } +}); + +test('instrumented CI rejects skips, absent required cases, empty selection and aborted totals', () => { + const tap = `# Subtest: [instrumented] required +ok 1 - [instrumented] required +# Subtest: production +ok 2 - production +# tests 2 +# pass 2 +# fail 0 +# skipped 0 +# cancelled 0 +# todo 0 +`; + assert.equal(lifecycleReport(tap, 'instrumented').requiredHookCases, 1); + const skipped = tap.replace('ok 1 - [instrumented] required', 'ok 1 - [instrumented] required # SKIP') + .replace('# pass 2', '# pass 1').replace('# skipped 0', '# skipped 1'); + assert.throws(() => lifecycleReport(skipped, 'instrumented'), /must execute, not skip/); + assert.equal(lifecycleReport(skipped, 'production').skipped, 1); + assert.throws(() => lifecycleReport(tap.replaceAll('[instrumented]', 'unmarked'), 'instrumented'), /not selected/); + assert.throws(() => lifecycleReport(tap.replace('# tests 2', '# tests 0'), 'instrumented'), /selection/); + assert.throws(() => lifecycleReport(tap.replace('# cancelled 0', '# cancelled 1'), 'instrumented'), /aborted/); +}); + +test('native provenance records survive TAP escaping of Windows paths', () => { + const record = { path: 'C:\\native\\duroxide.win32-x64-msvc.node', sha256: 'a'.repeat(64), instrumented: false }; + const encoded = Buffer.from(JSON.stringify(record)).toString('base64'); + assert.deepEqual(encodedRecords(`# LIFECYCLE_NATIVE_RECORD ${encoded}\r\n`, 'LIFECYCLE_NATIVE_RECORD'), [record]); + assert.deepEqual(encodedRecords('# unrelated\n', 'LIFECYCLE_NATIVE_RECORD'), []); + assert.throws(() => encodedRecords('# LIFECYCLE_NATIVE_RECORD e2Jyb2tlbg==\n', 'LIFECYCLE_NATIVE_RECORD')); +}); + +test('stop observer natural-exit oracle rejects ended sources, settled promises and retired notifications', () => { + const pending = { + source: { state: 'Requested', closed: false }, + settled: false, + execution: { started: 1, completed: 1, active: 0, cleanupActive: 0, coordinators: 1, observerPending: 1 }, + }; + pendingObserverExit(pending); + assert.throws(() => pendingObserverExit({ ...pending, source: { ...pending.source, closed: true } }), /ended source/); + assert.throws(() => pendingObserverExit({ ...pending, settled: true }), /settled promise/); + assert.throws(() => pendingObserverExit({ ...pending, execution: { ...pending.execution, observerPending: 0 } }), /still be owned/); + assert.throws(() => pendingObserverExit({ ...pending, execution: { ...pending.execution, active: 1 } })); +}); diff --git a/__tests__/lifecycle.test.js b/__tests__/lifecycle.test.js new file mode 100644 index 0000000..e1343d3 --- /dev/null +++ b/__tests__/lifecycle.test.js @@ -0,0 +1,776 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { performance } = require('node:perf_hooks'); +const { getEventListeners } = require('node:events'); +const { createHook } = require('node:async_hooks'); +const { SqliteProvider, Client, Runtime, RuntimeStopObserver } = require('../lib/duroxide.js'); +const { nativeProvenance } = require('../ci/smoke/native-provenance.cjs'); +const { startProbe } = require('./helpers/lifecycle-probe.cjs'); +const { + diagnostic, knownQuiescentCoreFailure, retiredState, + shutdownTiming, assertShutdownTiming, orderedRaceAttempt, + bounded, until, pendingObserverExit, observerRaceAttempt, +} = require('./helpers/lifecycle-oracles.cjs'); + +const root = path.resolve(__dirname, '..'); +const instrumented = process.env.DUROXIDE_LIFECYCLE_TEST_HOOKS === '1'; +const { native, provenance } = nativeProvenance(root, { + instrumented, manifestPath: process.env.DUROXIDE_NATIVE_MANIFEST, +}); +const SECRET = 'Password=lifecycle-secret-sentinel;Host=private-sentinel'; +const MAX_GRACE_MS = Number(((2n ** 64n) - 1n) / 1_000_000n) - 5000; + +const hookTest = (name, body) => test(`[instrumented] ${name}`, { skip: !instrumented }, body); +const record = (name, value) => console.log(`${name} ${Buffer.from(JSON.stringify(value)).toString('base64')}`); + +async function fixture(options) { + const provider = await SqliteProvider.inMemory(); + const runtime = new Runtime(provider, { dispatcherPollIntervalMs: 5, ...options }); + return { provider, runtime, hooks: instrumented ? native._lifecycleTestHooks(runtime._native) : null }; +} + +function retired(hooks, options) { + retiredState(JSON.parse(hooks.snapshot()), options); +} + +test('native provenance and production export boundary', () => { + console.log('LIFECYCLE_NATIVE ' + JSON.stringify(provenance)); + record('LIFECYCLE_NATIVE_RECORD', provenance); +}); + +test('invalid durations preserve registration, startup, and repeated-stop ownership', async () => { + const { runtime, hooks } = await fixture(); + for (const value of [-1, 0.5, NaN, Infinity, -Infinity, Number.MAX_SAFE_INTEGER + 1, '0', null, MAX_GRACE_MS + 1, Number.MAX_SAFE_INTEGER]) { + await assert.rejects(runtime.shutdown(value), diagnostic('lifecycle_invalid_timeout')); + } + runtime.registerOrchestration('LifecycleInvalidInput', function* () { return 'ok'; }); + await runtime.start(); + assert.notEqual(runtime.metricsSnapshot(), null); + await assert.rejects(runtime.shutdown(-1), diagnostic('lifecycle_invalid_timeout')); + assert.notEqual(runtime.metricsSnapshot(), null); + assert.equal(await runtime.shutdown(0), undefined); + assert.equal(runtime.metricsSnapshot(), null); + await assert.rejects(runtime.shutdown(Number.MAX_SAFE_INTEGER), diagnostic('lifecycle_invalid_timeout')); + assert.equal(await runtime.shutdown(), undefined); + if (hooks) retired(hooks); +}); + +test('direct-native negative integers do not consume registered work or stop ownership', async () => { + const { runtime, provider, hooks } = await fixture(); + const client = new Client(provider); + runtime.registerActivity('NativeNegativeActivity', async (_, input) => `original:${input}`); + runtime.registerOrchestration('NativeNegativeFlow', function* (context, input) { + return yield context.scheduleActivity('NativeNegativeActivity', input); + }); + const rejectsNative = value => assert.rejects( + async () => runtime._native.shutdown(value), diagnostic('lifecycle_invalid_timeout')); + for (const value of [-1, -1000, -Number.MAX_SAFE_INTEGER]) await rejectsNative(value); + await runtime.start(); + try { + await rejectsNative(-1); + assert.notEqual(runtime.metricsSnapshot(), null); + await client.startOrchestration('native-negative-first', 'NativeNegativeFlow', 'before-stop'); + const result = await client.waitForOrchestration('native-negative-first', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, 'original:before-stop'); + } finally { await runtime.shutdown(100); } + await rejectsNative(-1); + assert.equal(await runtime._native.shutdown(0), undefined); + assert.equal(await runtime.shutdown(), undefined); + if (hooks) retired(hooks, { executionWitness: true }); +}); + +test('maximum representable default-total boundary is checked on no-work and repeated stop', async () => { + const { runtime } = await fixture(); + assert.equal(await runtime.shutdown(MAX_GRACE_MS), undefined); + await assert.rejects(runtime.shutdown(MAX_GRACE_MS + 1), diagnostic('lifecycle_invalid_timeout')); + assert.equal(await runtime.shutdown(), undefined); +}); + +test('pre-start shutdown is a terminal no-work success', async () => { + const { runtime } = await fixture(); + assert.equal(await runtime.shutdown(0), undefined); + assert.equal(await runtime.shutdown(), undefined); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + assert.throws(() => runtime.registerActivity('late', async () => 'late'), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestration('late', function* () {}), /lifecycle_terminal/); + assert.throws(() => runtime.registerOrchestrationVersioned('late', '1.0.0', function* () {}), /lifecycle_terminal/); + assert.equal(runtime.metricsSnapshot(), null); +}); + +for (const typed of [false, true]) { + test(`rejected ${typed ? 'typed' : 'raw'} registrations cannot replace live handlers or shadow plain fallback`, async () => { + const { runtime: live, provider } = await fixture(); + const terminal = new Runtime(provider); + const client = new Client(provider); + const suffix = typed ? 'Typed' : 'Raw'; + const plain = `RegistrationOriginal${suffix}`; + const versioned = `RegistrationVersioned${suffix}`; + const fallback = `RegistrationFallback${suffix}`; + const register = typed ? 'registerOrchestrationTyped' : 'registerOrchestration'; + const registerVersioned = typed ? 'registerOrchestrationVersionedTyped' : 'registerOrchestrationVersioned'; + live[register](plain, function* (_, input) { return `plain:${input}`; }); + live[registerVersioned](versioned, '1.0.0', function* (_, input) { return `versioned:${input}`; }); + live[register](fallback, function* (_, input) { return `fallback:${input}`; }); + await terminal.shutdown(0); + await live.start(); + try { + const replacement = function* () { return 'rejected-replacement'; }; + assert.throws(() => terminal[register](plain, replacement), /lifecycle_terminal/); + assert.throws(() => terminal[registerVersioned](versioned, '1.0.0', replacement), /lifecycle_terminal/); + assert.throws(() => terminal[registerVersioned](fallback, '1.0.0', replacement), /lifecycle_terminal/); + for (const [name, expected, version] of [ + [plain, 'plain:fresh', undefined], + [versioned, 'versioned:fresh', '1.0.0'], + [fallback, 'fallback:fresh', '1.0.0'], + ]) { + const id = `rejected-registration-${name}`; + if (version) await client.startOrchestrationVersioned(id, name, 'fresh', version); + else await client.startOrchestration(id, name, 'fresh'); + const result = await client.waitForOrchestration(id, 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, expected, `${name} must resolve its original handler after rejection`); + } + } finally { + await live.shutdown(100); + await terminal.shutdown(0); + } + }); +} + +test('fallible preparation rejects invalid options without an escaping panic', async () => { + const { runtime } = await fixture({ sessionIdleTimeoutMs: 0 }); + await assert.rejects(runtime.start(), diagnostic('lifecycle_start_failed')); + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_start_failed')); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + assert.equal(runtime.metricsSnapshot(), null); +}); + +for (const scenario of ['registry-invalid', 'registry-descending']) { + test(`pre-core startup failure is ordinary and retained: ${scenario}`, async () => { + const probe = startProbe(scenario); + try { + await probe.successfulExit(5000); + assert.ok(probe.messages.some(message => message.phase === 'REGISTRY_FAILURE_RETAINED'), probe.describe()); + } finally { await probe.dispose(); } + }); +} + +test('ascending and duplicate version registration keep their existing successful startup policy', async () => { + const { runtime } = await fixture(); + for (const version of ['1.0.0', '1.0.0', '2.0.0']) { + runtime.registerOrchestrationVersioned('LifecycleVersionPolicy', version, function* () { return 'ok'; }); + } + assert.equal(await runtime.start(), undefined); + assert.equal(await runtime.shutdown(100), undefined); + assert.equal(await runtime.shutdown(0), undefined); +}); + +test('idle shutdown short-circuits long grace and preserves successful shapes', async () => { + const { runtime, hooks } = await fixture(); + await runtime.start(); + const start = performance.now(); + assert.equal(await runtime.shutdown(30_000), undefined); + assert.ok(performance.now() - start < 2000); + assert.equal(await runtime.shutdown(0), undefined); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + if (hooks) retired(hooks, { forced: false }); +}); + +test('SQLite public work and independent provider/client lifetimes survive worker shutdown', async () => { + const { runtime, provider, hooks } = await fixture(); + const client = new Client(provider); + runtime.registerActivity('LifecycleEcho', async (_, input) => input); + runtime.registerOrchestration('LifecycleEchoFlow', function* (context, input) { + return yield context.scheduleActivity('LifecycleEcho', input); + }); + await runtime.start(); + try { + await client.startOrchestration('lifecycle-echo', 'LifecycleEchoFlow', 'hello'); + const result = await client.waitForOrchestration('lifecycle-echo', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, 'hello'); + } finally { + await runtime.shutdown(100); + } + if (hooks) retired(hooks, { executionWitness: true }); + assert.equal((await client.waitForOrchestration('lifecycle-echo', 5000)).output, 'hello'); + const replacement = new Runtime(provider); + replacement.registerOrchestration('LifecycleIndependentProvider', function* (_, input) { return input; }); + await replacement.start(); + try { + await client.startOrchestration('lifecycle-replacement', 'LifecycleIndependentProvider', 'still-open'); + const result = await client.waitForOrchestration('lifecycle-replacement', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output, 'still-open'); + } finally { await replacement.shutdown(100); } +}); + +for (const typed of [false, true]) { + test(`SQLite ${typed ? 'typed' : 'raw'} races preserve failures, values, and shutdown`, async () => { + const { runtime, provider, hooks } = await fixture(); + const client = new Client(provider); + runtime.registerActivity('LifecycleRaceFail', async () => { throw new Error('race-failure'); }); + runtime.registerActivity('LifecycleRaceValue', async () => ({ err: 'data', ok: true })); + runtime.registerOrchestration('LifecycleRaceFlow', function* (context) { + const schedule = (typed ? context.scheduleActivityTyped : context.scheduleActivity).bind(context); + const race = (typed ? context.raceTyped : context.race).bind(context); + let direct; + try { + yield schedule('LifecycleRaceFail', null); + } catch (error) { + direct = { type: error.constructor.name, message: error.message }; + } + let raced; + try { + yield race(context.scheduleTimer(60_000), schedule('LifecycleRaceFail', null)); + } catch (error) { + raced = { type: error.constructor.name, message: error.message }; + } + const winner = yield race(schedule('LifecycleRaceValue', null), context.scheduleTimer(60_000)); + return { direct, raced, winner }; + }); + await runtime.start(); + try { + await client.startOrchestration('lifecycle-race', 'LifecycleRaceFlow', null); + const result = await client.waitForOrchestration('lifecycle-race', 5000); + assert.equal(result.status, 'Completed'); + assert.equal(result.output.direct.type, 'Error'); + assert.match(result.output.direct.message, /race-failure/); + assert.deepEqual(result.output.raced, result.output.direct); + assert.deepEqual(result.output.winner, { index: 0, value: { err: 'data', ok: true } }); + } finally { + await runtime.shutdown(100); + } + assert.equal(await runtime.shutdown(0), undefined); + if (hooks) retired(hooks, { executionWitness: true }); + }); +} + +hookTest('partial-startup failure retains descendants until actual rollback', async () => { + const { runtime, hooks } = await fixture(); + hooks.hold('partial-startup'); + hooks.hold('blocking-provider'); + hooks.fail('partial-startup', SECRET); + const startup = runtime.start(); + const failed = assert.rejects(startup, diagnostic('lifecycle_start_failed')); + try { + await hooks.waitEntered('partial-startup'); + await hooks.waitEntered('blocking-provider'); + hooks.release('partial-startup'); + await failed; + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + await assert.rejects(runtime.start(), diagnostic('lifecycle_terminal')); + } finally { + hooks.release('partial-startup'); + hooks.release('blocking-provider'); + } + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + await assert.rejects(runtime.shutdown(30_000), knownQuiescentCoreFailure); + retired(hooks, { rollback: true, executionWitness: true }); +}); + +hookTest('contained operational failure is preserved by repeated real shutdown calls', async () => { + const { runtime, hooks } = await fixture(); + hooks.hold('worker'); + hooks.fail('worker', SECRET); + await runtime.start(); + await hooks.waitEntered('worker'); + hooks.release('worker'); + await assert.rejects(runtime.shutdown(100), knownQuiescentCoreFailure); + await assert.rejects(runtime.shutdown(30_000), knownQuiescentCoreFailure); + retired(hooks, { rollback: true, executionWitness: true }); +}); + +hookTest('cancellation-safe provider fetch retires without waiting for its gate', async () => { + const { runtime, hooks } = await fixture(); + hooks.hold('provider'); + await runtime.start(); + await hooks.waitEntered('provider'); + try { + await runtime.shutdown(1000); + retired(hooks, { forced: false, executionWitness: true }); + } finally { + hooks.release('provider'); + } +}); + +for (const force of [false, true]) { + hookTest(`ordered ${force ? 'force-before-retirement' : 'retirement-before-force'} 100 times`, async () => { + for (let iteration = 0; iteration < 100; iteration++) { + const { runtime, hooks } = await fixture(); + await orderedRaceAttempt(runtime, hooks, force, iteration); + } + }); +} + +test('retained completed Runtime does not prevent natural process exit', async () => { + const probe = startProbe('exit'); + try { + await probe.successfulExit(3000); + assert.ok(probe.messages.some(message => message.phase === 'EXIT_READY'), probe.describe()); + } finally { await probe.dispose(); } +}); + +for (const scenario of ['late', 'retained']) { + hookTest(`isolated non-yielding provider timeout: ${scenario}`, async () => { + const probe = startProbe(scenario, { input: true }); + try { + await probe.next('STOPPING', 15_000); + const message = await probe.next('TIMED_OUT', shutdownTiming(scenario).delivery); + assertShutdownTiming(scenario, message); + probe.child.stdin.write(scenario === 'late' ? 'release\n' : 'retain\n'); + const completed = await probe.next(scenario === 'late' ? 'RECLAIMED' : 'RETAINED', 3000); + record('LIFECYCLE_PROBE_RECORD', { scenario, ...message, finalPhase: completed.phase }); + } catch (error) { + throw new Error(probe.describe(), { cause: error }); + } finally { await probe.dispose(); } + }); +} + +hookTest('incomplete cleanup keeps the event loop live without stdin or referenced timers', async () => { + const probe = startProbe('incomplete-liveness'); + try { + await probe.next('STOPPING', 15_000); + const message = await probe.next('TIMED_OUT', shutdownTiming('retained').delivery); + assertShutdownTiming('retained', message); + for (let pulse = 1; pulse <= 3; pulse++) { + const alive = await probe.next('ALIVE', 2000); + assert.equal(alive.pulse, pulse); + assert.ok(alive.state.active > 0); + assert.ok(alive.state.started > 0); + assert.equal(alive.state.coordinators, 1); + } + record('LIFECYCLE_PROBE_RECORD', { scenario: 'incomplete-liveness', ...message, livePulses: 3 }); + } catch (error) { + throw new Error(probe.describe(), { cause: error }); + } finally { await probe.dispose(); } +}); + +test('activity-only self-capturing callbacks collect after successful shutdown in a live process', async () => { + const probe = startProbe('completed', { fixture: 'lifecycle-collection-probe.cjs' }); + try { + const result = await probe.next('COLLECTION', 10_000); + assert.ok(result.controlCollected); + assert.ok(result.capturedCollected); + record('LIFECYCLE_PROBE_RECORD', result); + await probe.successfulExit(3000); + } finally { await probe.dispose(); } +}); + +for (const scenario of ['quiescent-failure', 'late', 'late-unobserved']) { + hookTest(`activity-only callbacks collect after ${scenario} cleanup`, async () => { + const probe = startProbe(scenario, { fixture: 'lifecycle-collection-probe.cjs' }); + try { + const result = await probe.next('COLLECTION', 15_000); + assert.ok(result.controlCollected); + assert.ok(result.capturedCollected); + if (scenario.startsWith('late')) assert.ok(result.retainedWhileIncomplete); + record('LIFECYCLE_PROBE_RECORD', result); + await probe.successfulExit(3000); + } finally { await probe.dispose(); } + }); +} + +test('stop observer arguments and already-aborted signals do not activate or stop the worker', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + try { + assert.ok(observer instanceof RuntimeStopObserver); + assert.throws(() => new RuntimeStopObserver(), TypeError); + for (const status of [undefined, null, 0, 'requested', {}, new String('Requested')]) { + await assert.rejects(observer.waitForChange(status), TypeError); + } + for (const options of [null, false, 1, 'options']) { + await assert.rejects(observer.waitForChange('NotRequested', options), TypeError); + } + for (const signal of [null, {}, false, new AbortController()]) { + await assert.rejects(observer.waitForChange('NotRequested', { signal }), TypeError); + } + for (const reason of [new Error('abort-reason'), null, false, 'abort-reason']) { + const abort = new AbortController(); + abort.abort(reason); + const observed = await observer.waitForChange('NotRequested', { signal: abort.signal }) + .then(() => assert.fail('an already-aborted wait must reject'), error => error); + assert.equal(observed, reason); + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + } + const ordinaryAbort = new AbortController(); + ordinaryAbort.abort(); + await assert.rejects(observer.waitForChange('NotRequested', { signal: ordinaryAbort.signal }), { name: 'AbortError' }); + assert.equal(observer.state, 'NotRequested'); + assert.equal(runtime.metricsSnapshot(), null); + if (hooks) { + const state = JSON.parse(hooks.snapshot()); + assert.equal(state.started, 0); + assert.equal(state.coordinators, 0); + assert.equal(state.observerPending, 0); + } + await runtime.start(); + assert.equal(observer.state, 'NotRequested'); + observer.close(); + assert.notEqual(runtime.metricsSnapshot(), null); + } finally { + observer.close(); + await runtime.shutdown(0); + } +}); + +test('stop observer waits isolate abort, overlap and terminal close and deliver unseen final state', async () => { + const { runtime } = await fixture(); + const first = runtime.stopObserver(); + const second = runtime.stopObserver(); + const closed = runtime.stopObserver(); + try { + const abort = new AbortController(); + const pending = first.waitForChange('NotRequested', { signal: abort.signal }); + pending.catch(() => {}); + const other = second.waitForChange('NotRequested'); + other.catch(() => {}); + const closingSignal = new AbortController().signal; + const closeWait = closed.waitForChange('NotRequested', { signal: closingSignal }); + closeWait.catch(() => {}); + await assert.rejects(first.waitForChange('NotRequested'), /stop_observer_busy/); + abort.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + assert.equal(first.state, 'NotRequested'); + closed.close(); + closed.close(); + await assert.rejects(closeWait, /stop_observer_closed/); + assert.equal(getEventListeners(closingSignal, 'abort').length, 0); + assert.throws(() => closed.state, /stop_observer_closed/); + await assert.rejects(closed.waitForChange('NotRequested'), /stop_observer_closed/); + const nextAbort = new AbortController(); + const retry = first.waitForChange('NotRequested', { signal: nextAbort.signal }); + retry.catch(() => {}); + await runtime.shutdown(0); + assert.deepEqual(await bounded(Promise.all([retry, other])), ['Requested', 'Requested']); + assert.equal(getEventListeners(nextAbort.signal, 'abort').length, 0); + nextAbort.abort(); + assert.equal(await retry, 'Requested'); + assert.equal(await first.waitForChange('NotRequested'), 'Requested'); + assert.equal(await first.waitForChange('Requested'), null); + assert.equal(first.state, 'Requested'); + const late = runtime.stopObserver(); + try { + assert.equal(await late.waitForChange('NotRequested'), 'Requested'); + assert.equal(await late.waitForChange('Requested'), null); + first.close(); + assert.equal(late.state, 'Requested'); + } finally { late.close(); } + } finally { + first.close(); + second.close(); + closed.close(); + await runtime.shutdown(0); + } +}); + +for (const action of ['abort', 'close', 'overlap']) { + test(`stop observer registration remains isolated under reentrant ${action}`, async () => { + for (const point of ['options', 'promise', 'native']) { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + const abort = new AbortController(); + const otherAbort = new AbortController(); + const reason = new Error(`registration-${action}-${point}`); + const resources = new Set(); + const destroyed = new Set(); + let entered = false; + let promises = 0; + let other; + const reenter = () => { + if (entered) return; + entered = true; + if (action === 'abort') abort.abort(reason); + else if (action === 'close') observer.close(); + else { + other = observer.waitForChange('NotRequested', { signal: otherAbort.signal }); + other.catch(() => {}); + } + }; + // Async hooks can run while the Promise or native notification is being constructed. + const hook = createHook({ + init(id, type) { + if (type === 'napi_rs_threadsafe_function') resources.add(id); + if (point === 'promise' && type === 'PROMISE' && ++promises === 2) reenter(); + if (point === 'native' && type === 'napi_rs_threadsafe_function') reenter(); + }, + destroy(id) { if (resources.has(id)) destroyed.add(id); }, + }); + try { + hook.enable(); + const options = point === 'options' + ? { get signal() { reenter(); return abort.signal; } } + : { signal: abort.signal }; + const pending = observer.waitForChange('NotRequested', options); + pending.catch(() => {}); + assert.ok(entered, `${action}: ${point}`); + if (action === 'abort') { + await bounded(assert.rejects(pending, error => error === reason)); + } else if (action === 'close') { + await bounded(assert.rejects(pending, /stop_observer_closed/)); + assert.throws(() => observer.state, /stop_observer_closed/); + } else { + const winner = point === 'native' ? pending : other; + const overlap = point === 'native' ? other : pending; + await bounded(assert.rejects(overlap, /stop_observer_busy/)); + (point === 'native' ? abort : otherAbort).abort(reason); + await bounded(assert.rejects(winner, error => error === reason)); + } + assert.equal(getEventListeners(abort.signal, 'abort').length, 0); + assert.equal(getEventListeners(otherAbort.signal, 'abort').length, 0); + await until(() => [...resources].every(id => destroyed.has(id))); + assert.equal(runtime.metricsSnapshot(), null); + if (action !== 'close') assert.equal(observer.state, 'NotRequested'); + if (hooks) { + const state = JSON.parse(hooks.snapshot()); + assert.equal(state.observerPending, 0); + assert.equal(state.started, 0); + assert.equal(state.coordinators, 0); + } + } finally { + hook.disable(); + observer.close(); + abort.abort(reason); + otherAbort.abort(reason); + await runtime.shutdown(0); + } + } + }); +} + +test('stop observer does not manufacture Faulted from a pre-core startup error', async () => { + const { runtime } = await fixture({ sessionIdleTimeoutMs: 0 }); + const observer = runtime.stopObserver(); + try { + await assert.rejects(runtime.start(), diagnostic('lifecycle_start_failed')); + assert.equal(observer.state, 'NotRequested'); + const pending = observer.waitForChange('NotRequested'); + pending.catch(() => {}); + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_start_failed')); + assert.equal(await bounded(pending), 'Requested'); + assert.equal(await observer.waitForChange('Requested'), null); + } finally { + observer.close(); + await assert.rejects(runtime.shutdown(0), diagnostic('lifecycle_start_failed')); + } +}); + +hookTest('stop observer transport errors are explicit, sanitized and independent of stop state', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + const signal = new AbortController().signal; + try { + hooks.fail('observer-delivery', SECRET); + const wait = observer.waitForChange('NotRequested', { signal }); + const rejected = assert.rejects(wait, diagnostic('stop_observation_failed')); + await runtime.shutdown(0); + await bounded(rejected); + assert.equal(observer.state, 'Requested'); + assert.equal(getEventListeners(signal, 'abort').length, 0); + assert.equal(await observer.waitForChange('NotRequested'), 'Requested'); + assert.equal(await observer.waitForChange('Requested'), null); + } finally { + observer.close(); + await runtime.shutdown(0); + } +}); + +hookTest('stop observer sees a real retained fault before held cleanup without an SDK stop', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + hooks.hold('worker'); + hooks.hold('blocking-provider'); + hooks.fail('worker', SECRET); + try { + const fault = observer.waitForChange('NotRequested'); + fault.catch(() => {}); + await runtime.start(); + await hooks.waitEntered('worker'); + await hooks.waitEntered('blocking-provider'); + hooks.release('worker'); + assert.equal(await bounded(fault), 'Faulted'); + assert.equal(observer.state, 'Faulted'); + assert.notEqual(runtime.metricsSnapshot(), null, 'observation must not call SDK shutdown'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + const late = runtime.stopObserver(); + try { + assert.equal(late.state, 'Faulted'); + assert.equal(await late.waitForChange('NotRequested'), 'Faulted'); + const abort = new AbortController(); + const pending = late.waitForChange('Faulted', { signal: abort.signal }); + pending.catch(() => {}); + abort.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(late.state, 'Faulted'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } finally { late.close(); } + } finally { + observer.close(); + hooks.release('worker'); + hooks.release('blocking-provider'); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + } +}); + +hookTest('stop observer sees a real fault while a foreign callback remains pending without an SDK stop', async () => { + const { runtime, provider, hooks } = await fixture(); + const client = new Client(provider); + let enter, release; + const entered = new Promise(resolve => { enter = resolve; }); + const retirement = new Promise(resolve => { release = resolve; }); + let completed = false; + runtime.registerActivity('ObserverFaultActivity', async (_, input) => { + enter(); + await retirement; + completed = true; + return input; + }); + runtime.registerOrchestration('ObserverFaultFlow', function* (context, input) { + return yield context.scheduleActivity('ObserverFaultActivity', input); + }); + const observer = runtime.stopObserver(); + try { + const fault = observer.waitForChange('NotRequested'); + fault.catch(() => {}); + await runtime.start(); + await client.startOrchestration('observer-foreign-fault', 'ObserverFaultFlow', 'held'); + await bounded(entered); + hooks.fail('provider', SECRET); + await client.startOrchestration('observer-foreign-fault-trigger', 'ObserverFaultFlow', 'trigger'); + assert.equal(await bounded(fault), 'Faulted'); + assert.equal(observer.state, 'Faulted'); + assert.equal(completed, false, 'the real foreign callback must remain held at fault observation'); + assert.notEqual(runtime.metricsSnapshot(), null, 'observation must not call SDK shutdown'); + const pending = JSON.parse(hooks.snapshot()); + const late = runtime.stopObserver(); + try { + assert.equal(late.state, 'Faulted'); + assert.equal(await late.waitForChange('NotRequested'), 'Faulted'); + const abort = new AbortController(); + const pending = late.waitForChange('Faulted', { signal: abort.signal }); + pending.catch(() => {}); + abort.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(late.state, 'Faulted'); + assert.equal(completed, false); + } finally { late.close(); } + // Native retirement does not imply completion of unregistered JS continuations. + record('LIFECYCLE_PROBE_RECORD', { + scenario: 'observer-foreign-fault', native: provenance, + faultWithoutSdkStop: true, callbackHeldAtFault: !completed, + state: observer.state, execution: pending, + }); + } finally { + observer.close(); + release(); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + assert.equal(completed, true); + } +}); + +hookTest('stop observer publisher retirement freezes the final state without retiring execution', async () => { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + hooks.hold('worker'); + hooks.hold('blocking-provider'); + hooks.fail('worker', SECRET); + try { + await runtime.start(); + await hooks.waitEntered('worker'); + await hooks.waitEntered('blocking-provider'); + const ended = observer.waitForChange('NotRequested'); + ended.catch(() => {}); + hooks.retireObservationPublisher(runtime._native); + assert.equal(await bounded(ended), null); + assert.equal(observer.state, 'NotRequested'); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + assert.equal(JSON.parse(hooks.snapshot()).coordinators, 0); + hooks.release('worker'); + await until(() => JSON.parse(hooks.snapshot()).coordinators === 1); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + assert.equal(observer.state, 'NotRequested', 'the closed source must not follow later core changes'); + assert.equal(await observer.waitForChange('NotRequested'), null); + } finally { + observer.close(); + hooks.release('worker'); + hooks.release('blocking-provider'); + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + } +}); + +for (const changeFirst of [false, true]) { + hookTest(`stop observer ordered attachment/change/abort/close races 100 times: ${changeFirst ? 'change-first' : 'cancel-first'}`, async () => { + for (let iteration = 0; iteration < 100; iteration++) { + const { runtime, hooks } = await fixture(); + await observerRaceAttempt(runtime, hooks, changeFirst, iteration); + } + }); +} + +hookTest('stop observer Requested-to-Faulted escalation stays monotonic through 100 ordered shutdowns', async () => { + for (let iteration = 0; iteration < 100; iteration++) { + const { runtime, hooks } = await fixture(); + const observer = runtime.stopObserver(); + hooks.hold('worker'); + hooks.hold('blocking-provider'); + hooks.fail('worker', SECRET); + let stop; + try { + await runtime.start(); + await hooks.waitEntered('worker'); + await hooks.waitEntered('blocking-provider'); + const requested = observer.waitForChange('NotRequested'); + requested.catch(() => {}); + stop = runtime.shutdown(30_000); + stop.catch(() => {}); + assert.equal(await bounded(requested), 'Requested'); + const fault = observer.waitForChange('Requested'); + fault.catch(() => {}); + hooks.release('worker'); + assert.equal(await bounded(fault), 'Faulted', `iteration ${iteration + 1}/100`); + assert.ok(JSON.parse(hooks.snapshot()).active > 0); + } finally { + observer.close(); + hooks.release('worker'); + hooks.release('blocking-provider'); + await assert.rejects(stop ?? runtime.shutdown(0), knownQuiescentCoreFailure); + } + await assert.rejects(runtime.shutdown(0), knownQuiescentCoreFailure); + const late = runtime.stopObserver(); + assert.equal(late.state, 'Faulted'); + late.close(); + retired(hooks, { rollback: true, executionWitness: true }); + } +}); + +test('stop observer stays pending through natural exit after real worker cleanup', async () => { + const probe = startProbe('observer-exit'); + try { + const pending = await probe.next('OBSERVER_EXIT_READY', 5000); + pendingObserverExit(pending); + await probe.successfulExit(3000); + const exiting = probe.messages.find(message => message.phase === 'OBSERVER_BEFORE_EXIT'); + assert.ok(exiting, probe.describe()); + pendingObserverExit(exiting); + record('LIFECYCLE_PROBE_RECORD', { scenario: 'observer-exit', ...exiting }); + } finally { await probe.dispose(); } +}); + +for (const scenario of ['observer-source-end', 'retained-observer']) { + test(`stop observer metadata does not retain callback/owner roots: ${scenario}`, async () => { + const probe = startProbe(scenario, { fixture: 'lifecycle-collection-probe.cjs' }); + try { + const result = await probe.next('OBSERVER_COLLECTION', 10_000); + assert.ok(result.collected, JSON.stringify(result)); + record('LIFECYCLE_PROBE_RECORD', result); + await probe.successfulExit(3000); + } finally { await probe.dispose(); } + }); +} diff --git a/ci/lifecycle-report.cjs b/ci/lifecycle-report.cjs new file mode 100644 index 0000000..bd8cce9 --- /dev/null +++ b/ci/lifecycle-report.cjs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); + +function lifecycleReport(stdout, mode) { + assert.ok(['production', 'instrumented'].includes(mode), 'explicit lifecycle mode is required'); + const count = label => { + const found = [...stdout.matchAll(new RegExp(`^# ${label} (\\d+)\\s*$`, 'gm'))]; + assert.equal(found.length, 1, `missing or ambiguous TAP total: ${label}`); + return Number(found[0][1]); + }; + const result = { + selected: count('tests'), passed: count('pass'), failed: count('fail'), + skipped: count('skipped'), aborted: count('cancelled'), todo: count('todo'), + requiredHookCases: [...stdout.matchAll(/^# Subtest: \[instrumented\] /gm)].length, + }; + assert.ok(result.selected > 0 && result.passed > 0, 'lifecycle selection must execute tests'); + assert.ok(result.requiredHookCases > 0, 'required instrumented cases were not selected'); + assert.equal(result.failed, 0, 'lifecycle tests failed'); + assert.equal(result.aborted, 0, 'lifecycle tests aborted'); + assert.equal(result.todo, 0, 'lifecycle tests may not be TODO'); + const skips = stdout.split(/\r?\n/).filter(line => /^ok \d+ - .* # SKIP(?:\s|$)/.test(line)); + assert.equal(skips.length, result.skipped, 'skip accounting mismatch'); + if (mode === 'instrumented') { + assert.equal(result.skipped, 0, 'required instrumented lifecycle cases must execute, not skip'); + } else { + assert.equal(result.skipped, result.requiredHookCases, 'production may skip only hook-dependent cases'); + assert.ok(skips.every(line => /^ok \d+ - \[instrumented\] /.test(line)), 'non-hook lifecycle test was skipped'); + } + assert.equal(result.selected, result.passed + result.skipped, 'incomplete lifecycle result totals'); + return result; +} + +function encodedRecords(stdout, name) { + assert.match(name, /^[A-Z_]+$/); + return [...stdout.matchAll(new RegExp(`^# ${name} ([A-Za-z0-9+/=]+)\\r?$`, 'gm'))] + .map(match => JSON.parse(Buffer.from(match[1], 'base64').toString('utf8'))); +} + +module.exports = { lifecycleReport, encodedRecords }; diff --git a/ci/run-lifecycle.cjs b/ci/run-lifecycle.cjs new file mode 100644 index 0000000..760c299 --- /dev/null +++ b/ci/run-lifecycle.cjs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const { readFileSync, writeFileSync } = require('node:fs'); +const path = require('node:path'); +const { lifecycleReport, encodedRecords } = require('./lifecycle-report.cjs'); + +const [mode, directory] = process.argv.slice(2); +assert.ok(['production', 'instrumented'].includes(mode) && directory, + 'usage: node ci/run-lifecycle.cjs '); +const stage = path.resolve(directory); +const manifestPath = path.join(stage, 'lifecycle-native.json'); +const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); +assert.equal(manifest.instrumented, mode === 'instrumented', 'stage mode differs from requested validation'); +assert.equal(manifest.sourceRoot, path.join(stage, 'node_modules', 'duroxide'), 'unexpected staged source root'); +const args = ['--test', '--test-reporter=tap', + path.join(manifest.sourceRoot, '__tests__', 'lifecycle-oracles.test.js'), + path.join(manifest.sourceRoot, '__tests__', 'lifecycle.test.js')]; +const child = spawnSync(process.execPath, args, { + cwd: manifest.sourceRoot, encoding: 'utf8', timeout: 180_000, maxBuffer: 16 * 1024 * 1024, + env: { + ...process.env, + DUROXIDE_LIFECYCLE_TEST_HOOKS: mode === 'instrumented' ? '1' : '0', + DUROXIDE_NATIVE_MANIFEST: manifestPath, + }, +}); +const stdout = child.stdout ?? ''; +const stderr = child.stderr ?? ''; +writeFileSync(path.join(stage, 'lifecycle.tap'), stdout); +writeFileSync(path.join(stage, 'lifecycle.stderr.log'), stderr); +process.stdout.write(stdout); +process.stderr.write(stderr); +const evidence = { + mode, native: manifest, command: [process.execPath, ...args], cwd: manifest.sourceRoot, + exit: child.status, signal: child.signal, error: child.error?.message, +}; +try { + assert.ifError(child.error); + assert.equal(child.status, 0, 'lifecycle child must exit successfully'); + assert.equal(child.signal, null, 'lifecycle child must not be terminated'); + evidence.counts = lifecycleReport(stdout, mode); + const records = encodedRecords(stdout, 'LIFECYCLE_NATIVE_RECORD'); + assert.equal(records.length, 1, 'lifecycle suite must report its native artifact'); + const actual = records[0]; + assert.equal(actual.path, manifest.path); + assert.equal(actual.sha256, manifest.sha256); + assert.equal(actual.instrumented, manifest.instrumented); + evidence.actualNative = actual; + evidence.probes = encodedRecords(stdout, 'LIFECYCLE_PROBE_RECORD'); +} catch (error) { + evidence.validationFailure = error.stack; + process.stderr.write(`${error.stack}\n`); + process.exitCode = 1; +} finally { + writeFileSync(path.join(stage, 'lifecycle-results.json'), JSON.stringify(evidence, null, 2) + '\n'); +} diff --git a/ci/smoke/README.md b/ci/smoke/README.md index ad12a05..1279e52 100644 --- a/ci/smoke/README.md +++ b/ci/smoke/README.md @@ -15,9 +15,15 @@ back to the local `.node` binary when run from the repo. | File | Purpose | |--------------------------|-------------------------------------------------| | `smoke.mjs` | Hello-world orchestration using `SqliteProvider`. | +| `native-provenance.cjs` | Verify the actual Duroxide binding, installed platform identity, production exports and SHA-256. | | `run-local.sh` / `.ps1` | Install from local tarballs (pre-publish). | | `run-registry.sh` / `.ps1` | Install from npm registry (post-publish). | +The launchers copy both the smoke script and its provenance helper. +Verification matches the loader's native class identities, not the first cached addon. +The installed smoke must resolve a declared Duroxide platform subpackage at the SDK's version. +That package must not expose lifecycle instrumentation. + ## Parity The smoke script MUST stay semantically in sync with the equivalent script in diff --git a/ci/smoke/native-provenance.cjs b/ci/smoke/native-provenance.cjs new file mode 100644 index 0000000..f37e3a0 --- /dev/null +++ b/ci/smoke/native-provenance.cjs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const { readFileSync, realpathSync } = require('node:fs'); +const path = require('node:path'); + +function identifyNative(loaderPath, cache = require.cache) { + const loader = cache[loaderPath]; + assert.ok(loader, `Duroxide loader was not loaded: ${loaderPath}`); + assert.equal(typeof loader.exports.JsRuntime, 'function', 'Duroxide loader must expose its native Runtime'); + const candidates = Object.values(cache).filter(module => + module.filename?.endsWith('.node') && + module.exports?.JsRuntime === loader.exports.JsRuntime && + module.exports?.JsClient === loader.exports.JsClient && + module.exports?.JsSqliteProvider === loader.exports.JsSqliteProvider); + assert.equal(candidates.length, 1, 'expected exactly one native module supplying the Duroxide bindings'); + const module = candidates[0]; + assert.match(path.basename(module.filename), /^duroxide\..+\.node$/); + return module; +} + +function nativeProvenance(root, { installed = false, instrumented = false, manifestPath } = {}) { + root = realpathSync(root); + const module = identifyNative(require.resolve(path.join(root, 'index.js'))); + const nativePath = realpathSync(module.filename); + const native = module.exports; + assert.equal('_lifecycleTestHooks' in native, instrumented, 'native hook export boundary mismatch'); + assert.equal('LifecycleTestHooks' in native, instrumented, 'native hook class export boundary mismatch'); + assert.equal(typeof native._lifecycleTestHooks === 'function', instrumented, 'native hook mode mismatch'); + assert.equal(typeof native.LifecycleTestHooks === 'function', instrumented, 'native hook class mode mismatch'); + if (installed) { + const sdkPackage = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')); + const platformPackage = JSON.parse(readFileSync(path.join(path.dirname(nativePath), 'package.json'), 'utf8')); + assert.equal(sdkPackage.name, 'duroxide'); + assert.ok(nativePath.includes(`${path.sep}node_modules${path.sep}`), 'smoke requires an installed platform package'); + assert.ok(Object.hasOwn(sdkPackage.optionalDependencies, platformPackage.name), 'native module is not a declared Duroxide platform package'); + assert.equal(platformPackage.version, sdkPackage.version, 'SDK/platform package version mismatch'); + assert.notEqual(path.dirname(nativePath), root, 'smoke must not use a repository-local addon'); + } else { + assert.equal(path.dirname(nativePath), root, 'lifecycle tests require the intended local addon, not an optional platform package'); + } + const provenance = { + path: nativePath, + sha256: createHash('sha256').update(readFileSync(nativePath)).digest('hex'), + instrumented, + }; + if (manifestPath) { + const expected = JSON.parse(readFileSync(manifestPath, 'utf8')); + assert.equal(realpathSync(expected.path), nativePath, 'native artifact path differs from staged build'); + assert.equal(expected.sha256, provenance.sha256, 'native artifact hash differs from staged build'); + assert.equal(expected.instrumented, instrumented, 'staged build mode mismatch'); + assert.deepEqual(expected.features, instrumented ? ['test-hooks'] : [], 'staged feature declaration mismatch'); + } + return { native, provenance }; +} + +module.exports = { identifyNative, nativeProvenance }; diff --git a/ci/smoke/run-local.ps1 b/ci/smoke/run-local.ps1 index 0290891..f7cbcac 100644 --- a/ci/smoke/run-local.ps1 +++ b/ci/smoke/run-local.ps1 @@ -21,6 +21,7 @@ try { if ($LASTEXITCODE -ne 0) { throw "npm install failed ($LASTEXITCODE)" } Copy-Item $env:SMOKE_SCRIPT ./smoke.mjs + Copy-Item (Join-Path (Split-Path $env:SMOKE_SCRIPT) 'native-provenance.cjs') ./native-provenance.cjs node ./smoke.mjs if ($LASTEXITCODE -ne 0) { throw "node smoke.mjs failed ($LASTEXITCODE)" } Write-Host "[smoke] local smoke OK" diff --git a/ci/smoke/run-local.sh b/ci/smoke/run-local.sh index aeb7fa3..362726e 100755 --- a/ci/smoke/run-local.sh +++ b/ci/smoke/run-local.sh @@ -29,5 +29,6 @@ npm init -y >/dev/null npm install --no-audit --no-fund --ignore-scripts "$TARBALL_PLATFORM" "$TARBALL_MAIN" cp "$SMOKE_SCRIPT" ./smoke.mjs +cp "$(dirname "$SMOKE_SCRIPT")/native-provenance.cjs" ./native-provenance.cjs node ./smoke.mjs echo "[smoke] local smoke OK" diff --git a/ci/smoke/run-registry.ps1 b/ci/smoke/run-registry.ps1 index ffc113c..3db1801 100644 --- a/ci/smoke/run-registry.ps1 +++ b/ci/smoke/run-registry.ps1 @@ -27,6 +27,7 @@ try { } Copy-Item $env:SMOKE_SCRIPT ./smoke.mjs + Copy-Item (Join-Path (Split-Path $env:SMOKE_SCRIPT) 'native-provenance.cjs') ./native-provenance.cjs node ./smoke.mjs if ($LASTEXITCODE -ne 0) { throw "node smoke.mjs failed ($LASTEXITCODE)" } Write-Host "[smoke] registry smoke OK" diff --git a/ci/smoke/run-registry.sh b/ci/smoke/run-registry.sh index cd23de9..d1832bc 100755 --- a/ci/smoke/run-registry.sh +++ b/ci/smoke/run-registry.sh @@ -36,5 +36,6 @@ for i in $(seq 1 "$attempts"); do done cp "$SMOKE_SCRIPT" ./smoke.mjs +cp "$(dirname "$SMOKE_SCRIPT")/native-provenance.cjs" ./native-provenance.cjs node ./smoke.mjs echo "[smoke] registry smoke OK" diff --git a/ci/smoke/smoke.mjs b/ci/smoke/smoke.mjs index c93510a..4d4aaf4 100644 --- a/ci/smoke/smoke.mjs +++ b/ci/smoke/smoke.mjs @@ -17,8 +17,15 @@ import { SqliteProvider, Client, Runtime } from 'duroxide'; import { mkdtempSync } from 'node:fs'; +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; +import { nativeProvenance } from './native-provenance.cjs'; + +const require = createRequire(import.meta.url); +const { provenance } = nativeProvenance(dirname(require.resolve('duroxide/package.json')), { installed: true }); +console.log('[smoke] native=' + JSON.stringify(provenance)); const dir = mkdtempSync(join(tmpdir(), 'duroxide-smoke-')); const dbPath = join(dir, 'smoke.db'); @@ -51,3 +58,9 @@ try { } finally { await runtime.shutdown(200); } + +assert.equal(await runtime.shutdown(0), undefined); +assert.equal(runtime.metricsSnapshot(), null); +const unstarted = new Runtime(provider); +await unstarted.shutdown(0); +await assert.rejects(unstarted.start(), /lifecycle_terminal/); diff --git a/ci/stage-lifecycle.cjs b/ci/stage-lifecycle.cjs new file mode 100644 index 0000000..4cd6b67 --- /dev/null +++ b/ci/stage-lifecycle.cjs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const { cpSync, mkdirSync, readFileSync, readdirSync, realpathSync, writeFileSync } = require('node:fs'); +const path = require('node:path'); + +const [nativeDirectory, stageDirectory, mode] = process.argv.slice(2); +assert.ok(nativeDirectory && stageDirectory && ['production', 'instrumented'].includes(mode), + 'usage: node ci/stage-lifecycle.cjs '); +const root = path.resolve(__dirname, '..'); +const nativeRoot = realpathSync(nativeDirectory); +const stage = path.resolve(stageDirectory); +const target = path.join(root, 'target') + path.sep; +assert.ok(stage.startsWith(target), 'stage must be isolated under this repository target directory'); +assert.notEqual(stage, nativeRoot, 'native output and staged sources must use separate directories'); +const binaries = readdirSync(nativeRoot).filter(name => /^duroxide\..+\.node$/.test(name)); +assert.equal(binaries.length, 1, 'native output must contain exactly one intended platform addon'); +// Root-level `node --test` must not rediscover staged copies of the suite. +const sourceRoot = path.join(stage, 'node_modules', 'duroxide'); +mkdirSync(sourceRoot, { recursive: true }); +const existing = readdirSync(sourceRoot).filter(name => name.endsWith('.node')); +assert.ok(existing.every(name => name === binaries[0]), 'staging directory contains a different native addon'); +for (const name of ['index.js', 'index.d.ts', 'package.json', 'lib', '__tests__', 'ci']) { + cpSync(path.join(root, name), path.join(sourceRoot, name), { recursive: true }); +} +const nativePath = path.join(sourceRoot, binaries[0]); +cpSync(path.join(nativeRoot, binaries[0]), nativePath); +const hash = file => createHash('sha256').update(readFileSync(file)).digest('hex'); +const sources = {}; +function record(directory) { + for (const entry of readdirSync(path.join(root, directory), { withFileTypes: true })) { + const name = path.join(directory, entry.name); + if (entry.isDirectory()) record(name); + else sources[name] = hash(path.join(root, name)); + } +} +for (const name of ['src', 'lib', '__tests__', 'ci']) record(name); +for (const name of ['Cargo.toml', 'Cargo.lock', 'package.json', 'index.js']) sources[name] = hash(path.join(root, name)); +const manifest = { + sourceRoot, + path: realpathSync(nativePath), + sha256: hash(nativePath), + instrumented: mode === 'instrumented', + features: mode === 'instrumented' ? ['test-hooks'] : [], + sources, +}; +writeFileSync(path.join(stage, 'lifecycle-native.json'), JSON.stringify(manifest, null, 2) + '\n'); +console.log('LIFECYCLE_STAGE ' + JSON.stringify({ stage, path: manifest.path, sha256: manifest.sha256, mode })); diff --git a/docs/architecture.md b/docs/architecture.md index 8164707..3ff1adb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -377,7 +377,9 @@ runtime.registerOrchestration('Parent', function* (ctx, input) { }); ``` -This is the **only fundamental limitation** vs the Rust core — all other features (typed APIs, cancellation propagation, etc.) have full parity. See the `async_blocks.test.js` tests for 12 comprehensive examples of this pattern. +See `async_blocks.test.js` for examples of this pattern. Lifecycle ownership is +also a boundary: this adapter does not track arbitrary JavaScript continuations +or provide the .NET adapter's additional foreign-retirement barrier. ### select/race Supports 2 Tasks @@ -389,7 +391,80 @@ File-based SQLite can hit "database is locked" errors under concurrent orchestra ### Runtime Shutdown -`Runtime.shutdown(timeoutMs)` returns as soon as all runtime work has stopped. `timeoutMs` (default 1000) is the longest it waits for in-flight work before cancelling the rest; it is not a fixed delay. +In the unreleased lifecycle, `Runtime.shutdown(timeoutMs)` remains a `Promise`. +It waits for actual core retirement within supplied grace plus 5000 ms. +Omitted grace is 1000 ms. Idle execution can finish early. +Zero requests immediate force. Force does not prove that cleanup ended. +The first valid stop fixes both deadlines, including for repeated observations. + +The adapter stores the prepared core Arc before it awaits fallible startup. +Startup failure, stop, and timeout do not discard cleanup ownership. +One native cleanup future remains owned independently of bounded caller waits. +Upstream core tracking includes native task destruction. +Provider futures can be canceled under the provider's atomic acknowledgment and recovery contract. +Repeated shutdown observes the same operation. +It does not report success merely because the wrapper transferred its Arc. + +```mermaid +flowchart TD + Stop["First valid stop: fix grace and total"] --> Cleanup["Keep one native cleanup operation"] + Stop --> Wait["Wait within the original total"] + Cleanup -->|Cleanup proven| Release["Release callback and core holders on the JavaScript thread"] + Cleanup -->|Cleanup unproven| Retain["Retain ownership"] + Release -->|Release fails| Retain + Wait -->|Completion published| Result["Resolve undefined or reject with an error"] + Wait -->|Total expires first| Timeout["Reject timeout; keep cleanup owned"] + Timeout --> Terminate["Application or supervisor must terminate the process"] +``` + +The caller's wait and cleanup have separate lifetimes. +An actual cleanup result can arrive after the caller's timeout. + +After proven core cleanup, the owned resolver runs on the JavaScript thread. +It removes event-loop references from retained ThreadsafeFunctions. +After successful removal, it releases obsolete callback and core holders. +The terminal result and deadlines remain available. +Removing an event-loop reference alone does not abort or collect a callback. +A callback-release failure or unproven cleanup retains ownership and can keep the process alive. + +A valid pre-start stop is terminal. +Later registration and start calls fail. +Metrics remain unavailable after stop or accepted startup failure. +Serialize lifecycle and registration calls on the same instance. +Timeout requires application- or supervisor-owned process termination, even after late cleanup. +The SDK never kills the process. + +This contract covers core-owned execution, not every JavaScript task. +Independent Clients, providers, and provider token-refresh work keep their separate lifetimes. +The global orchestration-handler map also remains unchanged. +See the [lifecycle guide](user-guide.md#runtime-lifecycle) for argument +validation, errors, and release prerequisites. + +### Passive Stop Observation + +`Runtime` caches a receiver-only native source when it creates its wrapper. +`stopObserver()` uses that source rather than borrowing the mutable native +Runtime during startup or shutdown. Each observer owns metadata and one pending +wait slot, not the Runtime, provider, or callback registry. + +The SDK publisher freezes the final state and detaches its live core receiver +when the source ends. Reads use coherent status/closed metadata. +Core retirement alone is not SDK source end. A wait delivers an unseen final +state before `null`; local `close()` instead rejects pending and later use. + +Each wait owns cancellation metadata and an unreferenced observation +ThreadsafeFunction. Settlement removes its AbortSignal listener and clears +resolver/rejector fields. Registration rechecks reentrant state and closes any +native handle returned after settlement. An old callback cannot settle a later +wait on the same observer. Native delivery context remains valid through final +access without joining worker cleanup. + +Only observation delivery is unreferenced early. Existing worker/shutdown +callback release still requires its own cleanup proof. +Native queued-delivery counters are not JavaScript Promise-settlement counts. +Neither status, source end, nor these counters prove that an unregistered +JavaScript continuation ended. See the +[observer contract and example](user-guide.md#passive-stop-observation). ### Platform-Specific Binary diff --git a/docs/user-guide.md b/docs/user-guide.md index a04f51a..b8affc3 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -49,6 +49,202 @@ async function main() { main(); ``` +## Runtime Lifecycle + +This section requires the **unreleased** adapter and a matching core lifecycle build. +Current published packages and temporary local core overrides do not prove release readiness. + +Register handlers before the first `await runtime.start()`. +Startup returns after activation, not after shutdown. Failures reject the Promise. +Registry and preparation failures become ordinary `lifecycle_start_failed` errors. +These include invalid or descending versions. +Subsequent shutdown calls retain those startup errors. +Version ordering and duplicate-registration policy do not change. +Adapter error messages exclude panic payloads. +This does not suppress Rust's separate process-wide panic-hook output. + +`await runtime.shutdown()` uses one second of grace plus five seconds of total headroom. +`shutdown(0)` requests immediate force, with a five-second total. +The first stop fixes the deadlines. Repeated calls never receive a new budget. +Success remains `undefined`. +Errors use `lifecycle_start_failed`, `lifecycle_shutdown_timed_out`, or `lifecycle_shutdown_failed`. + +**Quiescence** means that tracked core execution and cleanup have ended. +The `lifecycle_shutdown_failed` category alone does **not** prove quiescence. +It covers core failures after proven cleanup and loss of the native shutdown owner without proof of cleanup. +It also covers failure to release a Node callback's event-loop reference after core cleanup. +That callback-release error message does not itself assert quiescence. +Tests for known-quiescent core failures apply only to that outcome. +Neither a generic failure-category check nor task-count telemetry permits release. + +Non-finite, fractional, negative, unsafe-integer, or unsupported native durations fail validation. +Validation failure does not consume registrations or accept stop. +Strings and `null` also fail. `undefined` selects the omitted grace. +Native validation extracts signed milliseconds and checks unsigned-nanosecond deadlines. +The maximum grace interval is `floor((2^64 - 1) / 1_000_000) - 5000` milliseconds. +It must also fit the platform's monotonic clock range. +Validation still runs on unused runtimes and repeated calls. + +Stopping before start is terminal. +Do not overlap lifecycle or registration calls on the same instance. +Metrics return `null` after stop. +Repeated shutdown can observe actual late completion, but a timeout permanently prevents reuse. +The application or supervisor must still terminate the process. +The SDK retains cleanup and never exits the process itself. + +This contract covers core-owned execution, not arbitrary unregistered JavaScript continuations. +It does not add the .NET adapter's additional foreign-continuation barrier. +Independent provider-owned background work has a separate lifetime. +Ignoring the returned Promise does not cancel the adapter-owned native cleanup future. +The adapter owns bounded waits and first-stop deadlines. +Native Rust shutdown waits for cleanup without a total limit. +Provider futures can be canceled under the provider's atomic acknowledgment and peek-lock recovery contract. + +Callback and core holders are separate from persistent shutdown results and deadlines. +Only proven native quiescence permits their release on the JavaScript thread. +This includes known-quiescent failures and actual late completion without another shutdown call. +Release breaks obsolete activity callbacks' self-capture cycles. +Removing an event-loop reference alone does not collect the callback. +Unproven cleanup, a lost cleanup owner, or a callback-release failure retains ownership and leaves the worker unusable. + +Application-held references and the global orchestration-handler map are outside this collection guarantee. +Stopping a worker does not dispose independent provider or Client ownership. +Late release never removes the application's prior termination requirement. + +This complete example has no admitted work and can finish before grace: + +```javascript +const { SqliteProvider, Runtime } = require('duroxide'); + +async function main() { + const provider = await SqliteProvider.inMemory(); + const runtime = new Runtime(provider); + await runtime.start(); + await runtime.shutdown(); // 1000 ms grace, 6000 ms total; can finish early + await runtime.shutdown(0); // Observes the same actual completion, not a new stop + console.log(runtime.metricsSnapshot()); // null: stopped worker, not active metrics +} +main().catch(error => { + console.error(error); + process.exitCode = 1; +}); +``` + +After an incomplete stop, setting an exit code alone might not terminate retained work. +Arrange process termination through the application's supervisor policy. +A caught error or successful late observation does not permit runtime reuse. +A finite grace does not guarantee bounded forced cleanup. + +### Sizing the supervisor budget + +Use the supervisor time **remaining at the first shutdown call**, not its original allocation: + +```text +remaining outer time > grace + 5000 ms + other shutdown work + delivery/exit margin +``` + +For example, assume that 30 seconds remain. +A 30-second grace requests up to 35 seconds of waiting and cannot fit. +A 20-second grace gives the SDK 25 seconds to wait. +That leaves five seconds for other work, scheduling, result delivery, and process exit. +Subtract time already spent before calling `shutdown`. +If less than five seconds remain, `shutdown(0)` alone cannot make the SDK's timeout fit. + +Omitted grace still means 1000 ms grace and 6000 ms total. +Zero grace still means 5000 ms total. +The first call fixes those deadlines. +Repeated calls cannot renew the budget or remove a previous termination requirement. +Native cleanup has no total bound and can remain pending indefinitely. +The application's parent or supervisor must allow time for process termination outside the SDK wait budget. +The SDK does not provide a process-kill or supervisor mechanism. + +### Passive stop observation + +`runtime.stopObserver()` creates an independent `RuntimeStopObserver`. +Do not construct the observer directly. Creating it, reading `state`, and +waiting do not start or stop execution. +Observation can run during native startup or shutdown; lifecycle and +registration calls themselves must still be serialized. + +| State | Meaning | +|-------|---------| +| `'NotRequested'` | No stop is recorded. The worker can be unconstructed, starting, or running; this is not readiness or health. | +| `'Requested'` | Stop was accepted, not necessarily received by native execution or completed. | +| `'Faulted'` | The core recorded a runtime-wide fault. It takes precedence and cannot regress. | + +These are string values; `RuntimeStopStatus` is a TypeScript type, not a +JavaScript enum. Read the initial snapshot before waiting. +A late observer can already be Faulted. Invalid input, pre-core validation +failure, cancellation, and observation errors do not create a runtime fault. + +This helper owns the observer passed to it. It reports the initial state and +each observed change, then ends its own observation after a fault. +Pass an observer rather than the Runtime so the pending function need not +retain the worker: + +```javascript +async function reportStopChanges(observer, signal) { + try { + let state = observer.state; + while (true) { + console.log('Runtime stop state:', state); + if (state === 'Faulted') return; + + const next = await observer.waitForChange(state, { signal }); + if (next === null) { + console.log('Stop observation source ended; cleanup is not proved.'); + return; + } + state = next; + } + } catch (error) { + if (signal?.aborted && Object.is(error, signal.reason)) { + console.log('Stop observation canceled; worker shutdown is unchanged.'); + } else if (error instanceof Error && error.message.startsWith('stop_observer_closed:')) { + console.log('Stop observation was closed.'); + } else { + throw error; + } + } finally { + observer.close(); + } +} +``` + +In an application that owns a Runtime, call +`reportStopChanges(runtime.stopObserver(), observationController.signal)`, +where `observationController` is an `AbortController`. +Observe the returned Promise and handle unexpected observation failures. +Cancel that controller to end only monitoring. +The application's lifecycle owner must still stop the Runtime normally. + +`waitForChange` compares against the supplied last-seen state; it is not an +event queue. Changes can coalesce. A final unseen state arrives before `null`. +The final snapshot stays readable until local close. +The source can remain open after worker cleanup. An unchanged `'Requested'` +or `'Faulted'` therefore still waits; cancel monitoring when no longer needed. + +Only one wait can be pending on an observer; overlap rejects with +`stop_observer_busy`. Separate observers do not consume each other's state. +An already-aborted or later-aborted signal rejects with its actual `reason`, +which is not necessarily an `AbortError`. Cancellation alone permits retry. +The helper above also closes its observer on exit. +`close()` is idempotent and terminal: pending and later operations fail with +`stop_observer_closed`. A settled Promise is not rewritten. + +The observer retains metadata, not Runtime/provider/registry ownership. +Native notification context stays valid through final access even after the +public wait settles. Closing observation does not join that tail or worker cleanup. +Observation delivery is unreferenced and cannot by itself keep Node alive. +A pending Promise alone is not a keepalive; leave the event loop free to deliver +changes while your application remains active. + +Neither a state nor source end proves quiescence or permits release. +A real core fault can be visible while an unregistered JavaScript callback +continues after native retirement. This API does not add ownership of such work. +It does not request shutdown, renew first-stop budgets, remove a termination +requirement, or select host/process policy. + ## Orchestration Patterns ### Sequential Steps diff --git a/index.d.ts b/index.d.ts index 9b8d35e..66f5fb3 100644 --- a/index.d.ts +++ b/index.d.ts @@ -102,6 +102,10 @@ export interface JsRuntimeOptions { /** Tag list for "tags" or "defaultAnd" filter modes. */ workerTagFilterTags?: Array } +export interface JsRuntimeStopSnapshot { + state: string + closed: boolean +} /** Orchestration status returned to JS. */ export interface JsOrchestrationStatus { status: string @@ -379,6 +383,7 @@ export declare class JsRuntime { * These three functions handle: creating generators, driving next steps, and disposing. */ setGeneratorDriver(createFn: (arg: string) => any, nextFn: (arg: string) => any, disposeFn: (arg: string) => any): void + stopSource(): JsRuntimeStopSource /** * Register a JavaScript activity function. * The JS function receives (contextInfoJson, input) and returns a Promise. @@ -400,11 +405,13 @@ export declare class JsRuntime { start(): Promise /** Get a snapshot of runtime metrics. */ metricsSnapshot(): JsMetricsSnapshot | null - /** - * Shutdown the runtime gracefully. - * - * # Safety - * Must not be called concurrently from multiple threads. - */ + /** Stop with bounded waiting. Do not overlap lifecycle/registration calls. */ shutdown(timeoutMs?: number | undefined | null): Promise } +export declare class JsRuntimeStopSource { + snapshot(): JsRuntimeStopSnapshot + waitForChange(lastSeen: string, callback: (err: Error | null, arg: string | undefined | null) => any): JsRuntimeStopWait +} +export declare class JsRuntimeStopWait { + close(): void +} diff --git a/index.js b/index.js index 9d65251..52583cd 100644 --- a/index.js +++ b/index.js @@ -313,12 +313,14 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { JsClient, JsPostgresProvider, JsSqliteProvider, JsRuntime, activityTraceLog, orchestrationTraceLog, activityIsCancelled, activityGetClient, activityTag, orchestrationSetCustomStatus, orchestrationResetCustomStatus, orchestrationGetCustomStatus, orchestrationSetValue, orchestrationGetValue, orchestrationClearValue, orchestrationClearAllValues, orchestrationGetKvAllValues, orchestrationGetKvAllKeys, orchestrationGetKvLength, orchestrationPruneKvValues, initTracing } = nativeBinding +const { JsClient, JsPostgresProvider, JsSqliteProvider, JsRuntime, JsRuntimeStopSource, JsRuntimeStopWait, activityTraceLog, orchestrationTraceLog, activityIsCancelled, activityGetClient, activityTag, orchestrationSetCustomStatus, orchestrationResetCustomStatus, orchestrationGetCustomStatus, orchestrationSetValue, orchestrationGetValue, orchestrationClearValue, orchestrationClearAllValues, orchestrationGetKvAllValues, orchestrationGetKvAllKeys, orchestrationGetKvLength, orchestrationPruneKvValues, initTracing } = nativeBinding module.exports.JsClient = JsClient module.exports.JsPostgresProvider = JsPostgresProvider module.exports.JsSqliteProvider = JsSqliteProvider module.exports.JsRuntime = JsRuntime +module.exports.JsRuntimeStopSource = JsRuntimeStopSource +module.exports.JsRuntimeStopWait = JsRuntimeStopWait module.exports.activityTraceLog = activityTraceLog module.exports.orchestrationTraceLog = orchestrationTraceLog module.exports.activityIsCancelled = activityIsCancelled diff --git a/lib/duroxide.d.ts b/lib/duroxide.d.ts index d59de45..79543fd 100644 --- a/lib/duroxide.d.ts +++ b/lib/duroxide.d.ts @@ -311,6 +311,41 @@ export interface RuntimeOptions extends JsRuntimeOptions { workerTagFilter?: TagFilter; } +/** + * Retained stop state, not readiness or proof of cleanup. + * NotRequested includes unconstructed/starting/running; Requested records accepted stop. + * Faulted is a sticky recorded core-wide fault, not an observation error. + * This is a type-only union, not a runtime enum. + */ +export type RuntimeStopStatus = 'NotRequested' | 'Requested' | 'Faulted'; + +/** + * Independent, owned observation that does not retain worker/provider/registry resources. + * Obtain it through Runtime.stopObserver() and read state before waiting. + * Neither status nor source end proves cleanup or changes shutdown budgets/policy. + */ +export declare class RuntimeStopObserver { + private constructor(); + /** Retained snapshot, including after normal source end. Fails after local close. */ + readonly state: RuntimeStopStatus; + /** + * Compare against lastSeen; return a different state or null at normal source end. + * Return an unseen final state before null. Intermediate changes can coalesce. + * An unchanged Requested/Faulted still waits while the source is open. + * One pending wait per observer; overlap rejects with stop_observer_busy. + * Abort rejects with signal.reason, affects only this wait and permits retry. + * Local close instead rejects with stop_observer_closed and prevents reuse. + * Observation errors propagate without manufacturing Faulted. + * Delivery requires event-loop progress but does not keep Node alive by itself. + */ + waitForChange(lastSeen: RuntimeStopStatus, options?: { signal?: AbortSignal }): Promise; + /** + * End observation and reject a pending wait. Idempotent; later reads and waits fail. + * Does not rewrite settled results or join worker cleanup/native final access. + */ + close(): void; +} + /** Durable execution runtime. */ export declare class Runtime { constructor(provider: SqliteProvider | PostgresProvider, options?: RuntimeOptions); @@ -321,8 +356,20 @@ export declare class Runtime { registerOrchestrationTyped(name: string, fn: (ctx: OrchestrationContext, input: TIn) => Generator): void; registerOrchestrationVersioned(name: string, version: string, fn: (ctx: OrchestrationContext, input: any) => Generator): void; registerOrchestrationVersionedTyped(name: string, version: string, fn: (ctx: OrchestrationContext, input: TIn) => Generator): void; + /** Start once and return after startup. Do not overlap lifecycle or registration calls. */ start(): Promise; + /** + * Stop permanently, including before start. Default grace is 1000 ms. Total waiting is grace + 5000 ms. + * Repeated calls keep the original deadlines and observe actual completion or errors. + * Timeout rejects while cleanup remains owned. The application or supervisor must terminate the process. + * timeoutMs must be a nonnegative safe integer within the native monotonic deadline range. + */ shutdown(timeoutMs?: number): Promise; + /** + * Create an independent observer, including during native startup/shutdown. + * Does not start/stop execution or retain this worker. The caller owns close(). + */ + stopObserver(): RuntimeStopObserver; metricsSnapshot(): JsMetricsSnapshot | null; } diff --git a/lib/duroxide.js b/lib/duroxide.js index 8d00044..962aba7 100644 --- a/lib/duroxide.js +++ b/lib/duroxide.js @@ -1077,6 +1077,128 @@ class Client { } } +const stopObserverToken = Symbol('RuntimeStopObserver'); +const stopStatuses = new Set(['NotRequested', 'Requested', 'Faulted']); +const signalAborted = Object.getOwnPropertyDescriptor(AbortSignal.prototype, 'aborted').get; +const signalReason = Object.getOwnPropertyDescriptor(AbortSignal.prototype, 'reason').get; + +/** + * Owned stop metadata. Obtain an independent observer through Runtime.stopObserver(). + * Read state before waiting: a late reader can already see Faulted. + * NotRequested includes unconstructed, starting and running workers; it is not health. + * Requested records accepted stop. Faulted records a sticky core-wide fault. + * Neither state nor source end proves cleanup or permits resource release. + * Observation retains no worker/provider/registry ownership and applies no shutdown policy. + */ +class RuntimeStopObserver { + #source; + #closed = false; + #pending; + + constructor(source, token) { + if (token !== stopObserverToken) throw new TypeError('Use Runtime.stopObserver() to obtain an observer'); + this.#source = source; + } + + /** + * Read retained state without starting or stopping execution. + * The final snapshot remains readable after source end, until local close. + * @returns {'NotRequested'|'Requested'|'Faulted'} + */ + get state() { + if (this.#closed) throw new Error('stop_observer_closed: this observer is closed'); + return this.#source.snapshot().state; + } + + #assertCanWait() { + if (this.#closed) throw new Error('stop_observer_closed: this observer is closed'); + if (this.#pending) throw new Error('stop_observer_busy: this observer already has a pending wait'); + } + + /** + * Wait for a different retained state, or null at normal source end. + * An unseen final state arrives before null; intermediate states can coalesce. + * An unchanged Requested or Faulted still waits while the source is open. + * Only one wait may be pending on this observer; overlap rejects with stop_observer_busy. + * Abort rejects with signal.reason, consumes no state and permits retry. + * Local close instead rejects with stop_observer_closed and prevents reuse. + * Observation errors do not manufacture Faulted or stop the worker. + * Delivery needs event-loop progress but does not keep Node alive by itself. + * @param {'NotRequested'|'Requested'|'Faulted'} lastSeen State previously observed from this source. + * @param {{signal?: AbortSignal}} [options] Cancellation for this wait only. + * @returns {Promise<'NotRequested'|'Requested'|'Faulted'|null>} + */ + async waitForChange(lastSeen, options = {}) { + this.#assertCanWait(); + if (!stopStatuses.has(lastSeen)) throw new TypeError('lastSeen must be NotRequested, Requested or Faulted'); + if (options === null || typeof options !== 'object') throw new TypeError('options must be an object'); + let signal = options.signal; + if (signal !== undefined) { + let aborted; + try { aborted = signalAborted.call(signal); } + catch { throw new TypeError('signal must be an AbortSignal'); } + if (aborted) throw signalReason.call(signal); + } + this.#assertCanWait(); + const snapshot = this.#source.snapshot(); + if (snapshot.state !== lastSeen) return snapshot.state; + if (snapshot.closed) return null; + + return new Promise((resolve, reject) => { + this.#assertCanWait(); + const wait = { resolve, reject, native: undefined, finish: undefined }; + const abort = () => { + if (this.#pending === wait) wait.finish(true, signalReason.call(signal)); + }; + wait.finish = (failed, value) => { + if (this.#pending !== wait) return; + this.#pending = undefined; + if (signal !== undefined) { + EventTarget.prototype.removeEventListener.call(signal, 'abort', abort); + signal = undefined; + } + wait.native?.close(); + wait.native = undefined; + const settle = failed ? wait.reject : wait.resolve; + wait.resolve = wait.reject = undefined; + settle(value); + }; + this.#pending = wait; + try { + if (signal !== undefined) { + EventTarget.prototype.addEventListener.call(signal, 'abort', abort, { once: true }); + } + if (this.#pending !== wait) return; + if (signal !== undefined && signalAborted.call(signal)) { + abort(); + return; + } + const native = this.#source.waitForChange(lastSeen, (error, status) => { + if (error != null) wait.finish(true, error); + else if (status === null || stopStatuses.has(status)) wait.finish(false, status); + else wait.finish(true, new Error('stop_observation_failed: invalid native notification')); + }); + if (this.#pending === wait) wait.native = native; + else native.close(); + } catch (error) { + wait.finish(true, error); + } + }); + } + + /** + * End this observer and reject its pending wait. Idempotent and terminal. + * Later reads/waits fail; a settled Promise is not rewritten. + * Does not stop the worker or join cleanup or native notification final access. + */ + close() { + if (this.#closed) return; + this.#closed = true; + this.#pending?.finish(true, new Error('stop_observer_closed: this observer is closed')); + this.#source = undefined; + } +} + /** * Durable execution runtime. */ @@ -1108,6 +1230,8 @@ class Runtime { } else { this._native = new JsRuntime(provider._native, nativeOptions); } + // Keep observation separate from the native owner's asynchronous mutable startup borrow. + this._stopSource = this._native.stopSource(); // Wire up the generator driver functions this._native.setGeneratorDriver(createGenerator, nextStep, disposeGenerator); } @@ -1146,8 +1270,8 @@ class Runtime { * @param {GeneratorFunction} fn - function*(ctx, input) { ... } */ registerOrchestration(name, fn) { - orchestrationFunctions.set(name, fn); this._native.registerOrchestration(name); + orchestrationFunctions.set(name, fn); } /** @@ -1188,27 +1312,43 @@ class Runtime { */ registerOrchestrationVersioned(name, version, fn) { const key = `${name}@${version}`; - orchestrationFunctions.set(key, fn); // Don't overwrite the plain name key — it serves as the fallback // for versions registered via registerOrchestration() (unversioned). this._native.registerOrchestrationVersioned(name, version); + orchestrationFunctions.set(key, fn); } /** - * Start the runtime. Blocks until shutdown is called. + * Start once and return after startup. Do not overlap lifecycle or registration calls. */ async start() { await this._native.start(); } /** - * Shutdown the runtime gracefully. - * @param {number} [timeoutMs] + * Stop permanently. Omitted grace is 1000 ms. Total waiting is grace plus 5000 ms. + * Repeated calls observe the original stop, including failures or actual late completion. + * Timeout leaves cleanup owned. The application or supervisor must terminate the process. + * Even shutdown before start is terminal. Do not overlap lifecycle or registration calls. + * @param {number} [timeoutMs] Nonnegative safe-integer milliseconds in the native deadline range. */ async shutdown(timeoutMs) { + if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0)) { + throw new RangeError('lifecycle_invalid_timeout: timeoutMs must be a nonnegative safe integer'); + } await this._native.shutdown(timeoutMs); } + /** + * Obtain an independent, metadata-only observer without starting or stopping execution. + * Safe to obtain during native startup/shutdown. The caller owns close(). + * Read its initial state before waiting. It does not retain this worker. + * @returns {RuntimeStopObserver} + */ + stopObserver() { + return new RuntimeStopObserver(this._stopSource, stopObserverToken); + } + /** * Get a snapshot of runtime metrics. * @returns {object|null} @@ -1298,6 +1438,7 @@ module.exports = { PostgresProvider, Client, Runtime, + RuntimeStopObserver, OrchestrationContext, ActivityContext, initTracing, diff --git a/package.json b/package.json index 8fd5944..e4890a0 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "artifacts": "napi artifacts", "build": "napi build --platform --release", "build:debug": "napi build --platform", - "test": "node --test __tests__/e2e.test.js __tests__/kv_store.test.js __tests__/races.test.js __tests__/admin_api.test.js __tests__/sessions.test.js __tests__/advanced_features.test.js __tests__/async_blocks.test.js __tests__/typed_apis.test.js __tests__/copilot_chat.test.js __tests__/replay_isolation.test.js __tests__/lock_options.test.js", + "test": "node --test __tests__/e2e.test.js __tests__/kv_store.test.js __tests__/races.test.js __tests__/admin_api.test.js __tests__/sessions.test.js __tests__/advanced_features.test.js __tests__/async_blocks.test.js __tests__/typed_apis.test.js __tests__/copilot_chat.test.js __tests__/lifecycle.test.js __tests__/replay_isolation.test.js __tests__/lock_options.test.js", + "test:lifecycle": "node --test __tests__/lifecycle-oracles.test.js __tests__/lifecycle.test.js", "test:races": "node --test __tests__/races.test.js", "test:admin": "node --test __tests__/admin_api.test.js", "test:scenarios": "node --test __tests__/scenarios/*.test.js", diff --git a/src/lib.rs b/src/lib.rs index 2539715..ef0a143 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,12 +12,18 @@ mod handlers; mod pg_provider; mod provider; mod runtime; +mod stop_observation; +#[cfg(feature = "test-hooks")] +mod test_hooks; mod types; pub use client::JsClient; pub use pg_provider::JsPostgresProvider; pub use provider::JsSqliteProvider; pub use runtime::{JsRuntime, JsRuntimeOptions}; +pub use stop_observation::{JsRuntimeStopSnapshot, JsRuntimeStopSource, JsRuntimeStopWait}; +#[cfg(feature = "test-hooks")] +pub use test_hooks::lifecycle_test_hooks; pub use types::{JsOrchestrationStatus, JsQueueDepths, JsSystemMetrics}; /// Emit an activity trace through the current Rust ActivityContext. diff --git a/src/runtime.rs b/src/runtime.rs index 8992b91..cfbfdd2 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -4,18 +4,21 @@ use napi::bindgen_prelude::*; use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction}; use napi_derive::napi; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; +use tokio::sync::{oneshot, watch}; -use duroxide::runtime::{self, OrchestrationRegistry}; use duroxide::runtime::OrchestrationHandler; +use duroxide::runtime::{self, OrchestrationRegistry}; -use duroxide::runtime::LogFormat; use duroxide::providers::TagFilter; +use duroxide::runtime::LogFormat; use crate::handlers::{JsActivityHandler, JsOrchestrationHandler}; -use crate::provider::JsSqliteProvider; use crate::pg_provider::JsPostgresProvider; +use crate::provider::JsSqliteProvider; +use crate::stop_observation::{JsRuntimeStopSource, StopSource}; use crate::types::JsMetricsSnapshot; /// Runtime options configurable from JavaScript. @@ -87,6 +90,14 @@ pub struct JsRuntime { dispose_fn: Option>, options: Option, inner: Option>, + callback_refs: Vec>, + started: bool, + shutdown_requested: bool, + stop: Option, + startup_error: Option, + pub(crate) stop_source: StopSource, + #[cfg(feature = "test-hooks")] + pub(crate) test_hooks: runtime::test_hooks::LifecycleHooks, } #[napi] @@ -102,6 +113,14 @@ impl JsRuntime { dispose_fn: None, options, inner: None, + callback_refs: Vec::new(), + started: false, + shutdown_requested: false, + stop: None, + startup_error: None, + stop_source: StopSource::new(), + #[cfg(feature = "test-hooks")] + test_hooks: runtime::test_hooks::LifecycleHooks::default(), } } @@ -117,6 +136,14 @@ impl JsRuntime { dispose_fn: None, options, inner: None, + callback_refs: Vec::new(), + started: false, + shutdown_requested: false, + stop: None, + startup_error: None, + stop_source: StopSource::new(), + #[cfg(feature = "test-hooks")] + test_hooks: runtime::test_hooks::LifecycleHooks::default(), } } @@ -128,10 +155,17 @@ impl JsRuntime { create_fn: ThreadsafeFunction, next_fn: ThreadsafeFunction, dispose_fn: ThreadsafeFunction, - ) { + ) -> Result<()> { + self.ensure_configurable()?; self.create_fn = Some(create_fn); self.next_fn = Some(next_fn); self.dispose_fn = Some(dispose_fn); + Ok(()) + } + + #[napi] + pub fn stop_source(&self) -> JsRuntimeStopSource { + self.stop_source.subscribe() } /// Register a JavaScript activity function. @@ -141,21 +175,31 @@ impl JsRuntime { &mut self, name: String, callback: ThreadsafeFunction, - ) { + ) -> Result<()> { + self.ensure_configurable()?; self.activity_builders.push((name, callback)); + Ok(()) } /// Register a JavaScript orchestration (generator function). /// The orchestration name is used for both registration and the generator function lookup. #[napi] - pub fn register_orchestration(&mut self, name: String) { + pub fn register_orchestration(&mut self, name: String) -> Result<()> { + self.ensure_configurable()?; self.orchestration_names.push((name, None)); + Ok(()) } /// Register a versioned JavaScript orchestration. #[napi] - pub fn register_orchestration_versioned(&mut self, name: String, version: String) { + pub fn register_orchestration_versioned( + &mut self, + name: String, + version: String, + ) -> Result<()> { + self.ensure_configurable()?; self.orchestration_names.push((name, Some(version))); + Ok(()) } /// Start the runtime. This processes orchestrations and activities until shutdown. @@ -164,175 +208,417 @@ impl JsRuntime { /// This is async and takes &mut self. napi-rs requires async &mut methods to be marked unsafe. #[napi] pub async unsafe fn start(&mut self) -> Result<()> { - // Build activity registry - let mut activity_builder = duroxide::runtime::registry::ActivityRegistry::builder(); - for (name, callback) in self.activity_builders.drain(..) { - let handler = Arc::new(JsActivityHandler::new(name.clone(), callback)); - activity_builder = activity_builder.register(&name, move |ctx, input| { - let h = handler.clone(); - async move { h.invoke(ctx, input).await } - }); - } - let activities = activity_builder.build(); - - // Build orchestration registry - let create_fn = self - .create_fn - .take() - .ok_or_else(|| Error::from_reason("Generator driver not set. Call setGeneratorDriver() before start()."))?; - let next_fn = self - .next_fn - .take() - .ok_or_else(|| Error::from_reason("Generator driver not set."))?; - let dispose_fn = self - .dispose_fn - .take() - .ok_or_else(|| Error::from_reason("Generator driver not set."))?; - - let mut orch_builder = OrchestrationRegistry::builder(); - for (name, version) in self.orchestration_names.drain(..) { - let handler = Arc::new(JsOrchestrationHandler::new( - create_fn.clone(), - next_fn.clone(), - dispose_fn.clone(), + self.ensure_configurable()?; + if self.create_fn.is_none() || self.next_fn.is_none() || self.dispose_fn.is_none() { + return Err(Error::from_reason( + "lifecycle_start_failed: generator driver not set", )); - if let Some(ver) = version { - orch_builder = orch_builder.register_versioned(&name, &ver, move |ctx, input| { - let h = handler.clone(); - async move { h.invoke(ctx, input).await } - }); - } else { - orch_builder = orch_builder.register(&name, move |ctx, input| { + } + self.started = true; + let prepared = catch_unwind(AssertUnwindSafe(|| { + let mut callback_refs: Vec<_> = self + .activity_builders + .iter() + .map(|(_, callback)| callback.clone()) + .collect(); + + // Build activity registry + let mut activity_builder = duroxide::runtime::registry::ActivityRegistry::builder(); + for (name, callback) in self.activity_builders.drain(..) { + let handler = Arc::new(JsActivityHandler::new(name.clone(), callback)); + activity_builder = activity_builder.register(&name, move |ctx, input| { let h = handler.clone(); async move { h.invoke(ctx, input).await } }); } - } - let orchestrations = orch_builder.build(); + let activities = activity_builder.build(); - // Build runtime options - let mut rt_options = runtime::RuntimeOptions::default(); - if let Some(ref opts) = self.options { - if let Some(c) = opts.orchestration_concurrency { - rt_options.orchestration_concurrency = c as usize; - } - if let Some(c) = opts.worker_concurrency { - rt_options.worker_concurrency = c as usize; - } - if let Some(ms) = opts.dispatcher_poll_interval_ms { - rt_options.dispatcher_min_poll_interval = Duration::from_millis(ms as u64); - } - if let Some(ms) = opts.worker_lock_timeout_ms { - rt_options.worker_lock_timeout = Duration::from_millis(ms as u64); - } - if let Some(ms) = opts.worker_lock_renewal_buffer_ms { - rt_options.worker_lock_renewal_buffer = Duration::from_millis(ms as u64); - } - if let Some(ms) = opts.orchestrator_lock_timeout_ms { - rt_options.orchestrator_lock_timeout = Duration::from_millis(ms as u64); - } - if let Some(ms) = opts.orchestrator_lock_renewal_buffer_ms { - rt_options.orchestrator_lock_renewal_buffer = Duration::from_millis(ms as u64); - } - if let Some(n) = opts.max_attempts { - rt_options.max_attempts = n as u32; - } - if let Some(ref fmt) = opts.log_format { - rt_options.observability.log_format = match fmt.as_str() { - "json" => LogFormat::Json, - "pretty" => LogFormat::Pretty, - _ => LogFormat::Compact, - }; - } - if let Some(ref level) = opts.log_level { - rt_options.observability.log_level = level.clone(); + // Build orchestration registry + let create_fn = self + .create_fn + .take() + .ok_or(runtime::RuntimeStartError::StartupFailed)?; + let next_fn = self + .next_fn + .take() + .ok_or(runtime::RuntimeStartError::StartupFailed)?; + let dispose_fn = self + .dispose_fn + .take() + .ok_or(runtime::RuntimeStartError::StartupFailed)?; + callback_refs.extend([create_fn.clone(), next_fn.clone(), dispose_fn.clone()]); + + let mut orch_builder = OrchestrationRegistry::builder(); + for (name, version) in self.orchestration_names.drain(..) { + let handler = Arc::new(JsOrchestrationHandler::new( + create_fn.clone(), + next_fn.clone(), + dispose_fn.clone(), + )); + if let Some(ver) = version { + orch_builder = + orch_builder.register_versioned(&name, &ver, move |ctx, input| { + let h = handler.clone(); + async move { h.invoke(ctx, input).await } + }); + } else { + orch_builder = orch_builder.register(&name, move |ctx, input| { + let h = handler.clone(); + async move { h.invoke(ctx, input).await } + }); + } } - if let Some(ref name) = opts.service_name { - rt_options.observability.service_name = name.clone(); + let orchestrations = orch_builder.build(); + + // Build runtime options + let mut rt_options = runtime::RuntimeOptions::default(); + if let Some(ref opts) = self.options { + if let Some(c) = opts.orchestration_concurrency { + rt_options.orchestration_concurrency = c as usize; + } + if let Some(c) = opts.worker_concurrency { + rt_options.worker_concurrency = c as usize; + } + if let Some(ms) = opts.dispatcher_poll_interval_ms { + rt_options.dispatcher_min_poll_interval = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.worker_lock_timeout_ms { + rt_options.worker_lock_timeout = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.worker_lock_renewal_buffer_ms { + rt_options.worker_lock_renewal_buffer = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.orchestrator_lock_timeout_ms { + rt_options.orchestrator_lock_timeout = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.orchestrator_lock_renewal_buffer_ms { + rt_options.orchestrator_lock_renewal_buffer = Duration::from_millis(ms as u64); + } + if let Some(n) = opts.max_attempts { + rt_options.max_attempts = n as u32; + } + if let Some(ref fmt) = opts.log_format { + rt_options.observability.log_format = match fmt.as_str() { + "json" => LogFormat::Json, + "pretty" => LogFormat::Pretty, + _ => LogFormat::Compact, + }; + } + if let Some(ref level) = opts.log_level { + rt_options.observability.log_level = level.clone(); + } + if let Some(ref name) = opts.service_name { + rt_options.observability.service_name = name.clone(); + } + if let Some(ref ver) = opts.service_version { + rt_options.observability.service_version = Some(ver.clone()); + } + if let Some(max) = opts.max_sessions_per_runtime { + rt_options.max_sessions_per_runtime = max as usize; + } + if let Some(ms) = opts.session_idle_timeout_ms { + rt_options.session_idle_timeout = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.session_lock_timeout_ms { + rt_options.session_lock_timeout = Duration::from_millis(ms as u64); + } + if let Some(ms) = opts.session_lock_renewal_buffer_ms { + rt_options.session_lock_renewal_buffer = Duration::from_millis(ms as u64); + } + if let Some(ref nid) = opts.worker_node_id { + rt_options.worker_node_id = Some(nid.clone()); + } + if let Some(ref filter) = opts.worker_tag_filter { + rt_options.worker_tag_filter = match filter.as_str() { + "defaultOnly" => TagFilter::DefaultOnly, + "any" => TagFilter::Any, + "none" => TagFilter::None, + "tags" => { + let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); + TagFilter::Tags(tags.into_iter().collect()) + } + "defaultAnd" => { + let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); + TagFilter::DefaultAnd(tags.into_iter().collect()) + } + _ => TagFilter::DefaultOnly, + }; + } } - if let Some(ref ver) = opts.service_version { - rt_options.observability.service_version = Some(ver.clone()); + + let provider = self.provider.clone(); + #[cfg(feature = "test-hooks")] + let provider = Arc::new(runtime::test_hooks::TestProvider::new( + provider, + self.test_hooks.clone(), + )); + let rt = runtime::Runtime::prepare(provider, activities, orchestrations, rt_options)?; + self.callback_refs = callback_refs; + Ok::<_, runtime::RuntimeStartError>(rt) + })); + let rt = match prepared { + Ok(result) => result, + Err(payload) => { + discard_panic_payload(payload); + Err(runtime::RuntimeStartError::StartupFailed) } - if let Some(max) = opts.max_sessions_per_runtime { - rt_options.max_sessions_per_runtime = max as usize; + } + .map_err(|error| { + self.startup_error = Some(error); + start_error(error) + })?; + self.stop_source.attach(rt.stop_observer()); + self.inner = Some(rt.clone()); + #[cfg(feature = "test-hooks")] + rt.set_lifecycle_hooks(self.test_hooks.clone()) + .map_err(|error| { + self.startup_error = Some(error); + start_error(error) + })?; + rt.start_execution().await.map_err(|error| { + self.startup_error = Some(error); + start_error(error) + }) + } + + /// Get a snapshot of runtime metrics. + #[napi] + pub fn metrics_snapshot(&self) -> Option { + if self.shutdown_requested || self.startup_error.is_some() { + return None; + } + self.inner + .as_ref()? + .metrics_snapshot() + .map(|m| JsMetricsSnapshot { + orch_starts: m.orch_starts as i64, + orch_completions: m.orch_completions as i64, + orch_failures: m.orch_failures as i64, + orch_application_errors: m.orch_application_errors as i64, + orch_infrastructure_errors: m.orch_infrastructure_errors as i64, + orch_configuration_errors: m.orch_configuration_errors as i64, + orch_poison: m.orch_poison as i64, + activity_success: m.activity_success as i64, + activity_app_errors: m.activity_app_errors as i64, + activity_infra_errors: m.activity_infra_errors as i64, + activity_config_errors: m.activity_config_errors as i64, + activity_poison: m.activity_poison as i64, + orch_dispatcher_items_fetched: m.orch_dispatcher_items_fetched as i64, + worker_dispatcher_items_fetched: m.worker_dispatcher_items_fetched as i64, + orch_continue_as_new: m.orch_continue_as_new as i64, + suborchestration_calls: m.suborchestration_calls as i64, + provider_errors: m.provider_errors as i64, + }) + } + + /// Stop with bounded waiting. Do not overlap lifecycle/registration calls. + #[napi(ts_return_type = "Promise")] + pub fn shutdown(&mut self, env: Env, timeout_ms: Option) -> Result { + let (grace_deadline, total_deadline) = shutdown_deadlines(timeout_ms)?; + self.shutdown_requested = true; + self.stop_source.requested(); + if self.stop.is_none() { + if let Some(runtime) = self.inner.take() { + runtime.request_shutdown(); + let (completed, completion) = watch::channel(None); + self.stop = Some(ShutdownOperation { + completion, + total_deadline, + }); + let owned_runtime = runtime.clone(); + let resources = ShutdownResources(Some(ExecutionResources { + runtime, + callbacks: std::mem::take(&mut self.callback_refs), + })); + let (native_completed, native_completion) = oneshot::channel(); + napi::bindgen_prelude::spawn(async move { + let result = owned_runtime.shutdown_until(grace_deadline).await; + let _ = native_completed.send(result); + }); + // This owned observer outlives ordinary bounded waits. Its resolver + // runs on the JS thread, including when cleanup completes late. + env.execute_tokio_future( + async move { + Ok(match native_completion.await { + Ok(result) => result.map_err(ShutdownWaitError::Core), + Err(_) => Err(ShutdownWaitError::OwnerLost), + }) + }, + move |env, result| { + completed.send_replace(Some(resources.complete(env, result))); + Ok(()) + }, + )?; } - if let Some(ms) = opts.session_idle_timeout_ms { - rt_options.session_idle_timeout = Duration::from_millis(ms as u64); + } + if self.inner.is_none() { + if self.stop.is_none() { + self.stop_source.close(); } - if let Some(ms) = opts.session_lock_timeout_ms { - rt_options.session_lock_timeout = Duration::from_millis(ms as u64); + self.activity_builders.clear(); + self.orchestration_names.clear(); + self.create_fn = None; + self.next_fn = None; + self.dispose_fn = None; + } + let stop = self.stop.clone(); + let startup_error = self.startup_error; + env.execute_tokio_future( + async move { + Ok(match stop { + Some(stop) => Some(stop.wait().await), + None => None, + }) + }, + move |_, result| match result { + Some(result) => result + .map_err(shutdown_error) + .and_then(|_| startup_error.map_or(Ok(()), |error| Err(start_error(error)))), + None => startup_error.map_or(Ok(()), |error| Err(start_error(error))), + }, + ) + } +} + +pub(crate) fn discard_panic_payload(payload: Box) { + if let Err(secondary) = catch_unwind(AssertUnwindSafe(|| drop(payload))) { + std::mem::forget(secondary); + } +} + +impl JsRuntime { + fn ensure_configurable(&self) -> Result<()> { + if self.started || self.shutdown_requested { + return Err(Error::from_reason( + "lifecycle_terminal: runtime has already started or shutdown was requested", + )); + } + Ok(()) + } +} + +fn start_error(error: runtime::RuntimeStartError) -> Error { + Error::from_reason(format!("lifecycle_start_failed: {error}")) +} + +#[derive(Clone)] +struct ShutdownOperation { + completion: watch::Receiver>, + total_deadline: Instant, +} + +type ShutdownResult = std::result::Result; + +struct ExecutionResources { + runtime: Arc, + callbacks: Vec>, +} + +struct ShutdownResources(Option); + +impl ShutdownResources { + fn complete(mut self, env: &Env, result: ShutdownResult) -> ShutdownResult { + let quiescent = match result { + Ok(_) => true, + Err(ShutdownWaitError::Core(error)) => error.is_quiescent(), + Err(_) => false, + }; + if quiescent { + if let Some(resources) = self.0.as_mut() { + for callback in &mut resources.callbacks { + callback + .unref(env) + .map_err(|_| ShutdownWaitError::CallbackReleaseFailed)?; + } } - if let Some(ms) = opts.session_lock_renewal_buffer_ms { - rt_options.session_lock_renewal_buffer = Duration::from_millis(ms as u64); + if let Some(resources) = self.0.take() { + drop(resources.runtime); + drop(resources.callbacks); } - if let Some(ref nid) = opts.worker_node_id { - rt_options.worker_node_id = Some(nid.clone()); + } + result + } +} + +impl Drop for ShutdownResources { + fn drop(&mut self) { + if let Some(resources) = self.0.take() { + // Owner loss, failed JS dispatch or an unproven core result cannot + // authorize release. Retain the graph for supervisor termination. + std::mem::forget(resources); + } + } +} + +#[derive(Clone, Copy)] +enum ShutdownWaitError { + TimedOut, + OwnerLost, + Core(runtime::RuntimeShutdownError), + CallbackReleaseFailed, +} + +impl ShutdownOperation { + async fn wait(mut self) -> ShutdownResult { + loop { + { + let completed = self.completion.borrow_and_update(); + if let Some(result) = *completed { + return result; + } + if Instant::now() >= self.total_deadline { + return Err(ShutdownWaitError::TimedOut); + } } - if let Some(ref filter) = opts.worker_tag_filter { - rt_options.worker_tag_filter = match filter.as_str() { - "defaultOnly" => TagFilter::DefaultOnly, - "any" => TagFilter::Any, - "none" => TagFilter::None, - "tags" => { - let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); - TagFilter::Tags(tags.into_iter().collect()) - } - "defaultAnd" => { - let tags = opts.worker_tag_filter_tags.clone().unwrap_or_default(); - TagFilter::DefaultAnd(tags.into_iter().collect()) + tokio::select! { + biased; + changed = self.completion.changed() => { + if changed.is_err() { + return Err(ShutdownWaitError::OwnerLost); } - _ => TagFilter::DefaultOnly, - }; + }, + () = tokio::time::sleep_until(self.total_deadline.into()) => {}, } } - - let provider = self.provider.clone(); - let rt = runtime::Runtime::start_with_options( - provider, - activities, - orchestrations, - rt_options, - ) - .await; - - self.inner = Some(rt); - Ok(()) } +} - /// Get a snapshot of runtime metrics. - #[napi] - pub fn metrics_snapshot(&self) -> Option { - self.inner.as_ref()?.metrics_snapshot().map(|m| JsMetricsSnapshot { - orch_starts: m.orch_starts as i64, - orch_completions: m.orch_completions as i64, - orch_failures: m.orch_failures as i64, - orch_application_errors: m.orch_application_errors as i64, - orch_infrastructure_errors: m.orch_infrastructure_errors as i64, - orch_configuration_errors: m.orch_configuration_errors as i64, - orch_poison: m.orch_poison as i64, - activity_success: m.activity_success as i64, - activity_app_errors: m.activity_app_errors as i64, - activity_infra_errors: m.activity_infra_errors as i64, - activity_config_errors: m.activity_config_errors as i64, - activity_poison: m.activity_poison as i64, - orch_dispatcher_items_fetched: m.orch_dispatcher_items_fetched as i64, - worker_dispatcher_items_fetched: m.worker_dispatcher_items_fetched as i64, - orch_continue_as_new: m.orch_continue_as_new as i64, - suborchestration_calls: m.suborchestration_calls as i64, - provider_errors: m.provider_errors as i64, - }) +fn shutdown_error(error: ShutdownWaitError) -> Error { + match error { + ShutdownWaitError::TimedOut => Error::from_reason( + "lifecycle_shutdown_timed_out: cleanup remains owned and incomplete; application or supervisor must terminate the process", + ), + ShutdownWaitError::OwnerLost => Error::from_reason( + "lifecycle_shutdown_failed: native shutdown owner failed; quiescence is unproven; application or supervisor must terminate the process", + ), + ShutdownWaitError::Core(error) => { + Error::from_reason(format!("lifecycle_shutdown_failed: {error}")) + } + ShutdownWaitError::CallbackReleaseFailed => Error::from_reason( + "lifecycle_shutdown_failed: callback event-loop reference release failed", + ), } +} - /// Shutdown the runtime gracefully. - /// - /// # Safety - /// Must not be called concurrently from multiple threads. - #[napi] - pub async unsafe fn shutdown(&mut self, timeout_ms: Option) -> Result<()> { - if let Some(rt) = self.inner.take() { - let timeout = timeout_ms.map(|ms| ms as u64); - rt.shutdown(timeout).await; - } - Ok(()) +fn shutdown_deadlines(timeout_ms: Option) -> Result<(Instant, Instant)> { + let invalid = || { + Error::from_reason( + "lifecycle_invalid_timeout: timeoutMs must be a nonnegative supported millisecond duration", + ) + }; + let millis = u64::try_from(timeout_ms.unwrap_or(1000)).map_err(|_| invalid())?; + let grace = Duration::from_millis(millis); + let total = grace + .checked_add(Duration::from_secs(5)) + .ok_or_else(invalid)?; + if total > Duration::from_nanos(u64::MAX) { + return Err(invalid()); } + let now = Instant::now(); + let grace_deadline = now.checked_add(grace).ok_or_else(invalid)?; + let total_deadline = now.checked_add(total).ok_or_else(invalid)?; + total_deadline + .checked_add(Duration::from_millis(1)) + .ok_or_else(invalid)?; + Ok((grace_deadline, total_deadline)) } diff --git a/src/stop_observation.rs b/src/stop_observation.rs new file mode 100644 index 0000000..3ed990c --- /dev/null +++ b/src/stop_observation.rs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use duroxide::runtime::{RuntimeStopObserver, RuntimeStopStatus}; +use napi::bindgen_prelude::*; +use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}; +use napi_derive::napi; +use tokio::sync::watch; + +#[derive(Clone)] +struct SourceState { + status: RuntimeStopStatus, + core: Option, + closed: bool, +} + +fn merge(left: RuntimeStopStatus, right: RuntimeStopStatus) -> RuntimeStopStatus { + use RuntimeStopStatus::*; + match (left, right) { + (Faulted, _) | (_, Faulted) => Faulted, + (Requested, _) | (_, Requested) => Requested, + _ => NotRequested, + } +} + +impl SourceState { + fn status(&self) -> RuntimeStopStatus { + merge( + self.status, + self.core.as_ref().map_or(self.status, |core| core.state()), + ) + } +} + +pub(crate) struct StopSource { + state: watch::Sender, + #[cfg(feature = "test-hooks")] + pub(crate) hooks: crate::test_hooks::ObservationHooks, +} + +impl StopSource { + pub fn new() -> Self { + Self { + state: watch::channel(SourceState { + status: RuntimeStopStatus::NotRequested, + core: None, + closed: false, + }) + .0, + #[cfg(feature = "test-hooks")] + hooks: crate::test_hooks::ObservationHooks::default(), + } + } + + pub fn subscribe(&self) -> JsRuntimeStopSource { + JsRuntimeStopSource { + state: self.state.subscribe(), + #[cfg(feature = "test-hooks")] + hooks: self.hooks.clone(), + } + } + + pub fn attach(&self, core: RuntimeStopObserver) { + self.state.send_modify(|state| { + assert!(!state.closed && state.core.is_none()); + state.status = merge(state.status, core.state()); + state.core = Some(core); + }); + } + + pub fn requested(&self) { + self.state.send_if_modified(|state| { + let status = merge(state.status, RuntimeStopStatus::Requested); + if state.closed || state.status == status { + return false; + } + state.status = status; + true + }); + } + + pub fn close(&self) { + self.state.send_modify(|state| { + state.status = state.status(); + state.core = None; + state.closed = true; + }); + } +} + +impl Drop for StopSource { + fn drop(&mut self) { + // Dropping only the sender would leave a live core receiver behind. + self.close(); + } +} + +fn encode(status: RuntimeStopStatus) -> String { + match status { + RuntimeStopStatus::NotRequested => "NotRequested", + RuntimeStopStatus::Requested => "Requested", + RuntimeStopStatus::Faulted => "Faulted", + } + .to_owned() +} + +fn decode(status: &str) -> Result { + match status { + "NotRequested" => Ok(RuntimeStopStatus::NotRequested), + "Requested" => Ok(RuntimeStopStatus::Requested), + "Faulted" => Ok(RuntimeStopStatus::Faulted), + _ => Err(Error::new( + Status::InvalidArg, + "stop_observation_invalid_status: invalid last-seen status", + )), + } +} + +#[napi(object)] +pub struct JsRuntimeStopSnapshot { + pub state: String, + pub closed: bool, +} + +#[napi] +pub struct JsRuntimeStopSource { + state: watch::Receiver, + #[cfg(feature = "test-hooks")] + hooks: crate::test_hooks::ObservationHooks, +} + +#[napi] +impl JsRuntimeStopSource { + #[napi] + pub fn snapshot(&self) -> JsRuntimeStopSnapshot { + let state = self.state.borrow(); + JsRuntimeStopSnapshot { + state: encode(state.status()), + closed: state.closed, + } + } + + #[napi] + pub fn wait_for_change( + &self, + env: Env, + last_seen: String, + mut callback: ThreadsafeFunction>, + ) -> Result { + let last_seen = decode(&last_seen)?; + callback.unref(&env)?; + let mut source = self.state.clone(); + let (cancel, mut canceled) = watch::channel(()); + #[cfg(feature = "test-hooks")] + let hooks = self.hooks.clone(); + #[cfg(feature = "test-hooks")] + let pending = hooks.register(); + spawn(async move { + let result = match tokio::spawn(async move { + tokio::select! { + biased; + _ = canceled.changed() => Err(Error::from_reason( + "stop_observation_canceled: notification was canceled")), + status = changed(&mut source, last_seen) => Ok(status.map(encode)), + } + }) + .await + { + Ok(result) => result, + Err(error) => { + if error.is_panic() { + crate::runtime::discard_panic_payload(error.into_panic()); + } + Err(Error::from_reason( + "stop_observation_failed: native observation failed", + )) + } + }; + #[cfg(feature = "test-hooks")] + let result = if hooks.checkpoint("observer-delivery").await { + Err(Error::from_reason( + "stop_observation_failed: native observation failed", + )) + } else { + result + }; + let status = callback.call(result, ThreadsafeFunctionCallMode::NonBlocking); + if status != Status::Ok && status != Status::Closing { + eprintln!("stop_observation_failed: notification could not be delivered"); + } + #[cfg(feature = "test-hooks")] + { + hooks.delivered(status == Status::Ok); + hooks.checkpoint("observer-final-access").await; + } + drop(callback); + #[cfg(feature = "test-hooks")] + drop(pending); + }); + Ok(JsRuntimeStopWait { + cancel: Some(cancel), + }) + } +} + +async fn changed( + source: &mut watch::Receiver, + last_seen: RuntimeStopStatus, +) -> Option { + loop { + let (state, status) = { + let state = source.borrow_and_update(); + (state.clone(), state.status()) + }; + if status != last_seen { + return Some(status); + } + if state.closed { + return None; + } + if let Some(mut core) = state.core { + let native_status = core.state(); + if merge(state.status, native_status) != last_seen { + continue; + } + tokio::select! { + _ = source.changed() => {}, + result = core.changed(native_status) => { + if result.is_none() { + // The SDK publisher owns source lifetime, independently of core cleanup. + let _ = source.changed().await; + } + }, + } + } else { + let _ = source.changed().await; + } + } +} + +#[napi] +pub struct JsRuntimeStopWait { + cancel: Option>, +} + +#[napi] +impl JsRuntimeStopWait { + #[napi] + pub fn close(&mut self) { + self.cancel.take(); + } +} diff --git a/src/test_hooks.rs b/src/test_hooks.rs new file mode 100644 index 0000000..da366da --- /dev/null +++ b/src/test_hooks.rs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use duroxide::runtime::test_hooks::{ + LifecycleHooks, LifecyclePoint, LifecycleTask, ProviderOperation, +}; +use napi::{Error, Result}; +use napi_derive::napi; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::watch; + +use crate::runtime::JsRuntime; +use crate::stop_observation::StopSource; + +#[napi] +pub struct LifecycleTestHooks { + hooks: LifecycleHooks, + observation: ObservationHooks, +} + +#[napi(js_name = "_lifecycleTestHooks")] +pub fn lifecycle_test_hooks(runtime: &JsRuntime) -> LifecycleTestHooks { + LifecycleTestHooks { + hooks: runtime.test_hooks.clone(), + observation: runtime.stop_source.hooks.clone(), + } +} + +#[napi] +impl LifecycleTestHooks { + #[napi] + pub fn hold(&self, point: String) -> Result<()> { + if point.starts_with("observer-") { + return self.observation.hold(&point); + } + let parsed = parse_point(&point)?; + let gate = if point == "blocking-provider" { + self.hooks.hold_blocking(parsed) + } else { + self.hooks.hold(parsed) + }; + gate.map(|_| ()) + .map_err(|error| Error::from_reason(format!("test gate: {error:?}"))) + } + + #[napi] + pub fn release(&self, point: String) -> Result<()> { + if point.starts_with("observer-") { + self.observation.gate(&point)?.released.send_replace(true); + return Ok(()); + } + self.hooks + .gate(parse_point(&point)?) + .ok_or_else(|| Error::from_reason("test gate not armed"))? + .release(); + Ok(()) + } + + #[napi] + pub async fn wait_entered(&self, point: String) -> Result<()> { + if point.starts_with("observer-") { + let gate = self.observation.gate(&point)?; + return tokio::time::timeout( + Duration::from_secs(3), + signaled(gate.entered.subscribe()), + ) + .await + .map_err(|_| Error::from_reason("test gate entry timed out")); + } + let gate = self + .hooks + .gate(parse_point(&point)?) + .ok_or_else(|| Error::from_reason("test gate not armed"))?; + tokio::time::timeout(Duration::from_secs(3), gate.entered()) + .await + .map_err(|_| Error::from_reason("test gate entry timed out")) + } + + #[napi] + pub fn fail(&self, point: String, message: String) -> Result<()> { + if point == "observer-delivery" { + self.observation.0.lock().failures.insert(point); + return Ok(()); + } + self.hooks + .fail_once_with_message(parse_point(&point)?, message) + .map_err(|error| Error::from_reason(format!("test fault: {error:?}"))) + } + + #[napi] + pub fn retire_observation_publisher(&self, runtime: &mut JsRuntime) -> Result<()> { + if !Arc::ptr_eq(&self.observation.0, &runtime.stop_source.hooks.0) { + return Err(Error::from_reason( + "test observation publisher does not match", + )); + } + // Retain the worker so closure controls can test a later, real core fault. + drop(std::mem::replace( + &mut runtime.stop_source, + StopSource::new(), + )); + Ok(()) + } + + #[napi] + pub fn snapshot(&self) -> String { + let counts = self.hooks.counts(); + let observation = self.observation.0.lock(); + serde_json::json!({ + "started": counts.iter().map(|count| count.started).sum::(), + "completed": counts.iter().map(|count| count.completed).sum::(), + "active": counts.iter().map(|count| count.active).sum::(), + "cleanupActive": self.hooks.cleanup_counts().active, + "coordinators": self.hooks.hits(LifecyclePoint::ShutdownCoordinator), + "forceRequests": self.hooks.hits(LifecyclePoint::ForceRequested), + "observerPending": observation.pending, + "observerDeliveries": observation.deliveries, + }) + .to_string() + } +} + +#[derive(Clone, Default)] +pub(crate) struct ObservationHooks(Arc>); + +#[derive(Default)] +struct ObservationState { + gates: HashMap>, + failures: HashSet, + pending: usize, + deliveries: usize, +} + +struct ObservationGate { + entered: watch::Sender, + released: watch::Sender, +} + +async fn signaled(mut signal: watch::Receiver) { + while !*signal.borrow_and_update() { + signal + .changed() + .await + .expect("test gate sender is retained"); + } +} + +impl ObservationHooks { + fn hold(&self, point: &str) -> Result<()> { + if !matches!(point, "observer-delivery" | "observer-final-access") { + return Err(Error::from_reason("unknown observation test point")); + } + let mut state = self.0.lock(); + if state + .gates + .get(point) + .is_some_and(|gate| !*gate.released.borrow()) + { + return Err(Error::from_reason("observation test gate already armed")); + } + state.gates.insert( + point.to_owned(), + Arc::new(ObservationGate { + entered: watch::channel(false).0, + released: watch::channel(false).0, + }), + ); + Ok(()) + } + + fn gate(&self, point: &str) -> Result> { + self.0 + .lock() + .gates + .get(point) + .cloned() + .ok_or_else(|| Error::from_reason("observation test gate not armed")) + } + + pub(crate) async fn checkpoint(&self, point: &str) -> bool { + let (gate, failed) = { + let mut state = self.0.lock(); + ( + state.gates.get(point).cloned(), + state.failures.remove(point), + ) + }; + if let Some(gate) = gate { + gate.entered.send_replace(true); + signaled(gate.released.subscribe()).await; + } + failed + } + + pub(crate) fn register(&self) -> ObservationTask { + self.0.lock().pending += 1; + ObservationTask(self.clone()) + } + + pub(crate) fn delivered(&self, queued: bool) { + if queued { + self.0.lock().deliveries += 1; + } + } +} + +pub(crate) struct ObservationTask(ObservationHooks); + +impl Drop for ObservationTask { + fn drop(&mut self) { + self.0 .0.lock().pending -= 1; + } +} + +fn parse_point(point: &str) -> Result { + match point { + "startup" => Ok(LifecyclePoint::StartupBeforeIo), + "partial-startup" => Ok(LifecyclePoint::StartupAfterSpawn( + LifecycleTask::OrchestrationSlot, + )), + "provider" | "blocking-provider" => Ok(LifecyclePoint::ProviderReturn( + ProviderOperation::FetchOrchestration, + )), + "worker" => Ok(LifecyclePoint::ParentWork(LifecycleTask::WorkerSlot)), + "force" => Ok(LifecyclePoint::ForceRequested), + "coordinator" => Ok(LifecyclePoint::ShutdownCoordinator), + _ => Err(Error::from_reason("unknown lifecycle test point")), + } +}