From 76a304e0aa1e8090b521050c5b741b613494b34c Mon Sep 17 00:00:00 2001 From: drewstone Date: Sat, 17 Oct 2026 15:06:38 -0600 Subject: [PATCH 1/5] test: add opt-in real local OpenCode Runtime proof --- tests/runtime-consumer/opencode-live.mjs | 177 +++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 tests/runtime-consumer/opencode-live.mjs diff --git a/tests/runtime-consumer/opencode-live.mjs b/tests/runtime-consumer/opencode-live.mjs new file mode 100644 index 0000000..2c14632 --- /dev/null +++ b/tests/runtime-consumer/opencode-live.mjs @@ -0,0 +1,177 @@ +#!/usr/bin/env node +/** Opt-in real Runtime -> CLI-Bridge -> OpenCode proof. No mock execution path. + * Run from an installed cli-bridge checkout. See opencode-live.md beside this file. + */ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { randomBytes, createHash } from 'node:crypto' +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { parseArgs, promisify } from 'node:util' + +const exec = promisify(execFile) +const { values } = parseArgs({ options: { + run: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false }, + model: { type: 'string' }, + out: { type: 'string' }, + 'timeout-ms': { type: 'string', default: '120000' }, +} }) +if (values.help) { + console.log('node tests/runtime-consumer/opencode-live.mjs [--run] --model [--out NEW_DIRECTORY]\nWithout --run: preflight only; no model inference. BRIDGE_URL defaults to http://127.0.0.1:3344. Set BRIDGE_BEARER when required. The exact model is required; there is no fallback.') + process.exit(0) +} +const timeoutMs = Number(values['timeout-ms']) +assert(Number.isSafeInteger(timeoutMs) && timeoutMs >= 1000 && timeoutMs <= 600000, 'timeout-ms must be 1000..600000') +const bridge = new URL(process.env.BRIDGE_URL || 'http://127.0.0.1:3344') +assert(['http:', 'https:'].includes(bridge.protocol) && ['127.0.0.1', '[::1]', 'localhost'].includes(bridge.hostname) + && !bridge.username && !bridge.password && !bridge.search && !bridge.hash && bridge.pathname === '/', + 'Use a credential-free loopback BRIDGE_URL origin; this proof runs on the bridge host') +const bearer = process.env.BRIDGE_BEARER || '' +const selected = values.model || process.env.OPENCODE_MODEL || '' +assert(!selected || /^[A-Za-z0-9_.-]+\/[^\s]+$/.test(selected), 'model must be OpenCode provider/model, without the outer bridge prefix') +const evidenceRoot = values.out ? resolve(values.out) : await mkdtemp(join(tmpdir(), 'cli-bridge-proof-')) +if (values.out) await mkdir(evidenceRoot, { recursive: false, mode: 0o700 }) +const hash = value => createHash('sha256').update(value).digest('hex') +const redact = value => bearer ? value.replaceAll(bearer, '[REDACTED]') : value +const save = (name, value) => writeFile(join(evidenceRoot, name), redact(JSON.stringify(value, null, 2)) + '\n', { mode: 0o600 }) +const report = { + version: 1, startedAt: new Date().toISOString(), status: 'preflight', + requestedModel: selected || null, bridgeOrigin: bridge.origin, + sourceSha256: hash(await readFile(fileURLToPath(import.meta.url))), + checks: [], turns: [], inferenceCompleted: false, + scope: 'Synthetic local Runtime/CLI-Bridge/OpenCode execution only. Not Casework, browser, managed sandbox, legal calibration, or containment proof.', +} +async function check(name, fn) { + try { const detail = await fn(); report.checks.push({ name, ok: true, detail }); return detail } + catch (error) { report.checks.push({ name, ok: false, error: redact(error.message) }); return null } +} +let activeExecutor, workspace +async function main() { + await check('node', async () => { + const [major, minor] = process.versions.node.split('.').map(Number) + assert(major > 22 || (major === 22 && minor >= 13), 'Node 22.13+ required') + return process.versions.node + }) + await check('explicit-model', async () => { assert(selected, 'Select an exact provider/model from opencode models; no automatic paid or free fallback'); return selected }) + const bin = process.env.OPENCODE_BIN || 'opencode' + const cliVersion = await check('opencode-installed', async () => { + const { stdout } = await exec(bin, ['--version'], { timeout: 15000, maxBuffer: 65536 }) + return stdout.trim() + }) + if (cliVersion && selected) await check('model-listed', async () => { + const { stdout } = await exec(bin, ['models'], { timeout: 30000, maxBuffer: 2 * 1024 * 1024 }) + assert(stdout.split(/\r?\n/).map(x => x.trim()).includes(selected), 'Selected model is not in opencode models; no substitution allowed') + return { selected, catalogueSha256: hash(stdout), pricing: 'not inferred from model name or catalogue presence' } + }) + const runtimeEntry = await check('installed-runtime', async () => { + const entry = fileURLToPath(import.meta.resolve('@tangle-network/agent-runtime/kernel')) + let directory = dirname(entry) + for (;;) { + try { + const pkg = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')) + if (pkg.name === '@tangle-network/agent-runtime') { report.runtimeVersion = pkg.version; break } + } catch (error) { if (error.code !== 'ENOENT') throw error } + const parent = dirname(directory); assert(parent !== directory, 'Cannot identify installed Runtime version'); directory = parent + } + return entry + }) + await check('bridge-reachable', async () => { + const response = await fetch(new URL('/health', bridge), { + headers: bearer ? { Authorization: `Bearer ${bearer}` } : {}, + redirect: 'error', signal: AbortSignal.timeout(5000), + }) + assert(response.ok, `Bridge health HTTP ${response.status}`) + await response.arrayBuffer() + return { httpStatus: response.status } // Readiness is not inference success. + }) + if (report.checks.some(item => !item.ok)) { + report.status = 'blocked'; process.exitCode = 2; return + } + if (!values.run) { report.status = 'ready-not-run'; return } + + // Existing published executor, not an OpenAI shim or replacement tool loop. + const { createExecutor } = await import(pathToFileURL(runtimeEntry).href) + workspace = await mkdtemp(join(tmpdir(), 'opencode-synthetic-matter-')) + report.workspace = workspace + const [provider, ...modelParts] = selected.split('/') + const profile = { + name: 'local-opencode-live-proof', harness: 'opencode', + model: { provider, default: modelParts.join('/') }, + tools: { '*': false, read: true }, + prompt: { systemPrompt: 'You are checking fictional test evidence. Use only the read tool and only files in the supplied working directory. Never use shell, network tools, or other directories. Follow the requested JSON response exactly. A figure stated in a source is not a recovery forecast. Do not accept a case or invent outcomes.' }, + } + report.profile = profile + report.sessionId = `local-proof-${randomBytes(12).toString('hex')}` + const matterCode = randomBytes(16).toString('hex') + const revisionCode = randomBytes(16).toString('hex') + const file = join(workspace, 'matter.txt') + const initial = `FICTIONAL TEST RECORD\nMatter code: ${matterCode}\nAmount claimed: USD 120000\nPossible contractual cap: USD 25000\nOnly an unsigned agreement copy is available. Applicability is unverified. This is not a recovery forecast.\n` + await writeFile(file, initial, { mode: 0o600 }) + await save('fixture-initial.json', { content: initial, sha256: hash(initial) }) + async function turn(name, prompt, validate) { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(new Error('Local proof deadline exceeded')), timeoutMs + 5000) + const record = { name, prompt, startedAt: new Date().toISOString(), ok: false } + report.turns.push(record) + try { + activeExecutor = createExecutor({ backend: 'bridge', bridgeUrl: bridge.origin, + bridgeBearer: bearer, cwd: workspace, sessionId: report.sessionId, timeoutMs, + })({ profile, harness: null }, { signal: controller.signal, seams: {} }) + const stream = activeExecutor.execute(prompt, controller.signal) + assert(stream && typeof stream[Symbol.asyncIterator] === 'function', 'Runtime did not return streaming execution') + let count = 0 + for await (const event of stream) { count++; if (count > 50000) throw new Error('Unbounded event stream') } + const artifact = activeExecutor.resultArtifact() + await save(`${name}.json`, { artifact, eventCount: count, assertions: 'evaluated in proof.json' }) + record.eventCount = count + record.model = artifact.out?.model ?? null + record.spend = artifact.spent + record.toolCalls = artifact.out?.toolCalls ?? null + assert(artifact.spent?.tokensKnown !== false && artifact.spent?.tokens?.input > 0 + && artifact.spent?.tokens?.output > 0, 'Missing measured input/output token usage') + assert(record.model, 'Missing observed model identity') + assert([selected, modelParts.join('/'), `opencode/${selected}`].includes(record.model), 'Observed model does not match requested model') + assert(typeof artifact.out?.content === 'string' && artifact.out.content.trim(), 'Empty completed answer') + const json = artifact.out.content.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '') + const answer = JSON.parse(json) + validate(answer, artifact) + record.answer = answer; record.ok = true + } finally { + clearTimeout(timer) + // Let Runtime cancel/settle its owned jobs; never kill an unrelated daemon. + if (activeExecutor) { + const disposed = activeExecutor; activeExecutor = null + const cleanup = await disposed.teardown(5000) + assert(cleanup.destroyed, 'Runtime could not confirm termination of its owned bridge run') + } + record.completedAt = new Date().toISOString() + } + } + await turn('01-read', 'Read matter.txt from the working directory using your read tool. Return only JSON: {"matterCode":"exact code from the file","claimed":number,"cap":number,"applicability":"unverified"}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode, claimed: 120000, cap: 25000, applicability: 'unverified' }) + assert(artifact.out.toolCalls > 0, 'No observed tool execution for the file read') + }) + await turn('02-resume', 'Without using any tools or reading files, repeat only the matterCode from your previous answer as JSON: {"matterCode":"..."}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode }); assert.equal(artifact.out.toolCalls, 0, 'Continuation used a tool instead of retained conversation') + }) + const updated = initial + `\nNEW SIGNED AMENDMENT\nRevision code: ${revisionCode}\nThe cap for this fictional shipment is amended to USD 50000. Recovery remains unassessed.\n` + await writeFile(file, updated, { mode: 0o600 }) + await save('fixture-updated.json', { content: updated, sha256: hash(updated) }) + await turn('03-new-evidence', 'matter.txt has changed. Read it again with the read tool. Return only JSON: {"matterCode":"...","revisionCode":"exact new code","previousCap":number,"amendedCap":number,"recovery":"not_assessed"}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode, revisionCode, previousCap: 25000, amendedCap: 50000, recovery: 'not_assessed' }) + assert(artifact.out.toolCalls > 0, 'New evidence was not retrieved through an observed tool call') + }) + report.status = 'passed'; report.inferenceCompleted = true +} +try { await main() } +catch (error) { report.status = 'failed'; report.error = redact(error.message); process.exitCode = 1 } +finally { + report.completedAt = new Date().toISOString() + await save('proof.json', report) + console.log(JSON.stringify({ status: report.status, inferenceCompleted: report.inferenceCompleted, + checks: report.checks.map(({ name, ok, error }) => ({ name, ok, ...(error ? { error } : {}) })), + turns: report.turns.map(({ name, ok }) => ({ name, ok })), proof: join(evidenceRoot, 'proof.json') }, null, 2)) +} From 4908e120477be04e9735760d8dcaf798c4300c86 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sat, 17 Oct 2026 15:06:58 -0600 Subject: [PATCH 2/5] docs: document local OpenCode proof and its boundaries --- tests/runtime-consumer/opencode-live.md | 41 +++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/runtime-consumer/opencode-live.md diff --git a/tests/runtime-consumer/opencode-live.md b/tests/runtime-consumer/opencode-live.md new file mode 100644 index 0000000..a5f78ee --- /dev/null +++ b/tests/runtime-consumer/opencode-live.md @@ -0,0 +1,41 @@ +# Real local OpenCode check + +Use the existing stack, not a new server or simulated model: + +``` +Runtime bridge executor -> localhost:3344 -> OpenCode -> selected provider/model +``` + +Run on the same machine as CLI-Bridge. Install the repo's pinned dependencies and OpenCode normally. Select an exact model from `opencode models` and verify its access and pricing in your own account. There is no default model and no fallback. Hosting the harness locally does not host its model weights locally. + +From the repo root, in the first terminal: + +```sh +pnpm install --frozen-lockfile +opencode models +BRIDGE_HOST=127.0.0.1 BRIDGE_PORT=3344 BRIDGE_BACKENDS=opencode \ + OPENCODE_EXECUTOR=host BRIDGE_DATA_DIR="$(mktemp -d)" pnpm start +``` + +In a second terminal, select the same exact `provider/model` shown by OpenCode: + +```sh +export OPENCODE_MODEL='opencode/glm-5.3-flash' # example; verify access/price first +# Supply BRIDGE_BEARER here if the bridge requires it. Never commit that value. +node tests/runtime-consumer/opencode-live.mjs +node tests/runtime-consumer/opencode-live.mjs --run +``` + +The first command checks prerequisites without model inference. Only `--run` authorizes inference, and the selected account may be charged. Each turn has a 120-second deadline, configurable with `--timeout-ms`; this is not a hard dollar cap. Existing operator jail and credential policy is not disabled or modified. No production service is contacted except the selected model provider through OpenCode. Run in a dedicated development account/environment, never against client matters. + +The live path imports the Runtime already pinned by this consumer workspace. It makes three real turns with no transport interception or fake backend: + +1. Read a temporary synthetic matter through OpenCode's native read tool. The file contains a random matter code not supplied in the prompt. +2. Resume the same bridge session and return that code without calling tools or replaying the file in the new prompt. +3. Read updated evidence containing another random code. Report the old and amended caps separately, leaving recovery unassessed. + +A pass requires matching answers, actual tool-call counts, nonzero measured input/output tokens, and a matching reported model. Unknown dollar cost stays unknown. Gateway-reported identity is not independent verification of provider model weights. A successful `/health` alone never passes the test. Errors, missing dependencies, model substitutions, and blocked prerequisites return nonzero. + +The command prints its private evidence directory with a `proof.json`, full Runtime artifacts, fixture hashes, script hash, and package/CLI identity. It preserves the synthetic working directory and bridge session for inspection; remove them after review. Run `--help` for options. The script does not install dependencies, modify credentials, start an extra execution framework, or run on ordinary CI automatically. + +This proves a narrow local execution path when successful. It does **not** prove the authenticated Casework app, background reassessment, browser streaming/reload/stop, a managed sandbox, or the design preview's backend integration. Those must use the application's real producers and case tools; changing a model URL alone does not move a sandbox-owned workflow onto the local bridge. From 470dd28ecb88dcfa698af5bc6c8713aac2d458a7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 22 Oct 2026 15:12:18 -0600 Subject: [PATCH 3/5] test: integrate three-turn evidence with jailed live proof --- tests/runtime-consumer/live-opencode.mjs | 174 +++++++++++++++++----- tests/runtime-consumer/opencode-live.md | 86 ++++++----- tests/runtime-consumer/opencode-live.mjs | 177 ----------------------- 3 files changed, 190 insertions(+), 247 deletions(-) delete mode 100644 tests/runtime-consumer/opencode-live.mjs diff --git a/tests/runtime-consumer/live-opencode.mjs b/tests/runtime-consumer/live-opencode.mjs index ba86e62..a8a0c7c 100644 --- a/tests/runtime-consumer/live-opencode.mjs +++ b/tests/runtime-consumer/live-opencode.mjs @@ -15,7 +15,10 @@ import { setTimeout as delay } from 'node:timers/promises' const exec = promisify(execFile) const repository = fileURLToPath(new URL('../../', import.meta.url)) const nativeModel = process.env.CLI_BRIDGE_LIVE_MODEL ?? 'opencode/glm-5.3-flash-free' -const receipt = { schema: 'cli-bridge.live-proof.v1', status: 'failed', model: nativeModel, stages: [] } +const scenario = process.env.CLI_BRIDGE_LIVE_SCENARIO ?? 'filesystem' +const evidenceScenario = scenario === 'evidence' +const timeoutMs = evidenceScenario ? 120_000 : 180_000 +const receipt = { schema: 'cli-bridge.live-proof.v1', status: 'failed', model: nativeModel, scenario, stages: [] } const hash = bytes => createHash('sha256').update(bytes).digest('hex') let root, evidence, child, childClosed, childOutcome, bearer = '', bridgeLog = '', origin, env let ownsEvidence = false @@ -81,7 +84,9 @@ async function session(id) { async function main() { assert.equal(process.env.CLI_BRIDGE_REAL_OPENCODE_TESTS, '1', 'Opt in with CLI_BRIDGE_REAL_OPENCODE_TESTS=1; this invokes a real external model.') + assert.ok(['filesystem', 'evidence'].includes(scenario), 'Select filesystem or evidence; scenarios never run together.') assert.match(nativeModel, /^opencode\/[a-z0-9.-]+-free$/, 'Select an explicitly free OpenCode model. No paid model or silent fallback is allowed.') + if (evidenceScenario) assert.equal(nativeModel, 'opencode/glm-5.3-flash-free', 'The three-turn proof only admits the explicitly authorized anonymous model.') assert.equal(process.platform, 'linux', 'This proof requires the existing Linux filesystem jail; it never downgrades confinement.') const [major, minor] = process.versions.node.split('.').map(Number) assert.ok(major > 22 || (major === 22 && minor >= 19), 'The live Runtime proof requires Node 22.19 or newer') @@ -100,7 +105,7 @@ async function main() { XDG_CACHE_HOME: join(root, 'cache'), XDG_STATE_HOME: join(root, 'state'), BRIDGE_HOST: '127.0.0.1', BRIDGE_BEARER: bearer, BRIDGE_DATA_DIR: join(root, 'bridge'), BRIDGE_BACKENDS: 'opencode', OPENCODE_EXECUTOR: 'host', OPENCODE_BIN: opencode, - OPENCODE_TIMEOUT_MS: '180000', OPENCODE_DISABLE_AUTOUPDATE: '1', OPENCODE_DISABLE_SHARE: '1', + OPENCODE_TIMEOUT_MS: String(timeoutMs), OPENCODE_DISABLE_AUTOUPDATE: '1', OPENCODE_DISABLE_SHARE: '1', OPENCODE_INSTALL_DIR: dirname(dirname(opencode)), BRIDGE_JAIL_MODE: 'fs-jail', BRIDGE_RAW_CAPTURE_DIR: join(evidence, 'raw'), @@ -108,12 +113,40 @@ async function main() { for (const key of ['HTTPS_PROXY', 'HTTP_PROXY', 'NO_PROXY', 'SSL_CERT_FILE', 'NODE_EXTRA_CA_CERTS']) { if (process.env[key]) env[key] = process.env[key] } + if (evidenceScenario) { + // These fields and auxiliary agent overrides are supported by the pinned OpenCode source: + // https://github.com/anomalyco/opencode/tree/e3c4eb242585dcd77be1a867391b7c25fd47ce52/packages/core/src/v1/config + // Whitelist one model so a title/summary/compaction cannot select a paid fallback. + env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ + model: nativeModel, small_model: nativeModel, enabled_providers: ['opencode'], + provider: { opencode: { whitelist: [nativeModel.slice('opencode/'.length)] } }, + agent: Object.fromEntries(['title', 'summary', 'compaction'].map(name => [name, { model: nativeModel }])), + }) + } receipt.opencodeVersion = (await exec(opencode, ['--version'], { env, cwd: join(root, 'task'), timeout: 20_000 })).stdout.trim() receipt.bridgeCommit = (await exec('git', ['rev-parse', 'HEAD'], { cwd: repository, timeout: 5_000 })).stdout.trim() receipt.nodeVersion = process.version + if (evidenceScenario) { + // The installer workflow owns the version pin; do not duplicate its changing value here. + const workflow = await readFile(join(repository, '.github/workflows/live-inference.yml'), 'utf8') + const pinnedVersion = /\/opencode\/releases\/download\/v([^/]+)\//.exec(workflow)?.[1] + assert.ok(pinnedVersion, 'Cannot read the maintained OpenCode installer pin') + assert.equal(receipt.opencodeVersion, pinnedVersion, 'Use the checksum-pinned OpenCode from the maintained installer') + const config = JSON.parse((await exec(opencode, ['debug', 'config'], { env, cwd: join(root, 'task'), timeout: 30_000, maxBuffer: 2 * 1024 * 1024 })).stdout) + assert.equal(config.model, nativeModel, 'OpenCode changed the selected main model') + assert.equal(config.small_model, nativeModel, 'OpenCode changed the auxiliary model') + assert.deepEqual(config.enabled_providers, ['opencode'], 'OpenCode enabled another provider') + assert.deepEqual(Object.keys(config.provider?.opencode ?? {}).sort(), ['whitelist'], 'Anonymous inference refuses configured provider credentials or endpoint overrides') + assert.deepEqual(config.provider?.opencode?.whitelist, [nativeModel.slice('opencode/'.length)], 'OpenCode admitted another model') + for (const name of ['title', 'summary', 'compaction']) assert.equal(config.agent?.[name]?.model, nativeModel, `OpenCode changed the ${name} model`) + receipt.modelPolicy = JSON.parse(env.OPENCODE_CONFIG_CONTENT) + receipt.scriptSha256 = hash(await readFile(fileURLToPath(import.meta.url))) + } // This is availability, not successful inference. The later filesystem checks prove execution. const models = (await exec(opencode, ['models', 'opencode'], { env, cwd: join(root, 'task'), timeout: 60_000, maxBuffer: 4 * 1024 * 1024 })).stdout - assert.ok(models.split(/\r?\n/).map(value => value.trim()).includes(nativeModel), `Requested model is absent from OpenCode: ${nativeModel}`) + const listedModels = models.split(/\r?\n/).map(value => value.trim()).filter(Boolean) + assert.ok(listedModels.includes(nativeModel), `Requested model is absent from OpenCode: ${nativeModel}`) + if (evidenceScenario) assert.deepEqual(listedModels, [nativeModel], 'OpenCode admitted a fallback model') receipt.stages.push({ name: 'model-discovery', status: 'passed' }) const socket = createServer() await new Promise((resolveListen, reject) => { socket.once('error', reject); socket.listen(0, '127.0.0.1', resolveListen) }) @@ -122,17 +155,23 @@ async function main() { await new Promise((resolveClose, reject) => socket.close(error => error ? reject(error) : resolveClose())) const { createExecutor } = await import('@tangle-network/agent-runtime/kernel') const sessionId = `live-${randomUUID()}` - const profile = { name: 'local-inference-proof', harness: 'opencode', model: { provider: 'opencode', default: nativeModel.slice('opencode/'.length) } } - const marker = randomBytes(16).toString('hex') - const input = randomBytes(64).toString('hex') + '\n' - await writeFile(join(root, 'task/input.txt'), input, { mode: 0o600 }) + const profile = { + name: evidenceScenario ? 'local-evidence-proof' : 'local-inference-proof', harness: 'opencode', + model: { provider: 'opencode', default: nativeModel.slice('opencode/'.length) }, + ...(evidenceScenario ? { + tools: { '*': false, read: true }, + prompt: { systemPrompt: 'You are checking fictional test evidence. Use only the read tool and only files in the supplied working directory. Never use shell, network tools, or other directories. Follow the requested JSON response exactly. A figure stated in a source is not a recovery forecast. Do not accept a case or invent outcomes.' }, + } : {}), + } await startBridge() - executor = createExecutor({ backend: 'bridge', bridgeUrl: origin, bridgeBearer: bearer, - cwd: join(root, 'task'), sessionId, timeoutMs: 180_000, - })({ profile, harness: null }, { signal: AbortSignal.timeout(420_000), seams: {} }) - async function turn(name, prompt) { - const signal = AbortSignal.timeout(180_000) + const makeExecutor = () => createExecutor({ backend: 'bridge', bridgeUrl: origin, bridgeBearer: bearer, + cwd: join(root, 'task'), sessionId, timeoutMs, + })({ profile, harness: null }, { signal: AbortSignal.timeout(evidenceScenario ? 130_000 : 420_000), seams: {} }) + if (!evidenceScenario) executor = makeExecutor() + async function turn(name, prompt, validate) { + if (evidenceScenario) executor = makeExecutor() + const signal = AbortSignal.timeout(timeoutMs) const events = [] const stage = { name, status: 'failed', eventCount: 0 } receipt.stages.push(stage) @@ -147,40 +186,101 @@ async function main() { } artifact = executor.resultArtifact() assert.notEqual(artifact.verdict?.valid, false, 'Runtime reported an invalid result') - stage.status = 'executed' + if (evidenceScenario) { + stage.model = artifact.out?.model ?? null + stage.spend = artifact.spent + stage.toolCalls = artifact.out?.toolCalls ?? null + assert(artifact.spent?.tokensKnown !== false && artifact.spent?.tokens?.input > 0 + && artifact.spent?.tokens?.output > 0, 'Missing measured input/output token usage') + assert([nativeModel, nativeModel.slice('opencode/'.length), `opencode/${nativeModel}`].includes(stage.model), 'Observed model does not match requested model') + assert(typeof artifact.out?.content === 'string' && artifact.out.content.trim(), 'Empty completed answer') + const json = artifact.out.content.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '') + stage.answer = JSON.parse(json) + validate(stage.answer, artifact) + } + stage.status = evidenceScenario ? 'passed' : 'executed' } finally { - stage.durationMs = Date.now() - started - stage.eventCount = events.length - await save(`${name}.json`, { events, artifact: artifact ?? null }) + try { + if (evidenceScenario && executor) { + const cleanup = await executor.teardown(5_000) + stage.teardown = cleanup + assert(cleanup.destroyed, 'Runtime could not confirm termination of its owned bridge run') + executor = undefined + } + } catch (error) { + stage.status = 'failed' + throw error + } finally { + stage.durationMs = Date.now() - started + stage.eventCount = events.length + await save(`${name}.json`, { events, artifact: artifact ?? null, stage }) + } } } - await turn('filesystem-task', `Work only in your current directory. Read input.txt with a tool and compute the SHA-256 of its exact bytes using a command. Create answer.txt containing only the 64-character hexadecimal digest and a newline. Do not change input.txt. Remember this conversation marker for our next turn: ${marker}. Do not write the marker to any file. Reply briefly when finished.`) - assert.equal(await readFile(join(root, 'task/input.txt'), 'utf8'), input, 'The task input must remain unchanged') - assert.equal((await readFile(join(root, 'task/answer.txt'), 'utf8')).trim(), hash(input), 'The model must really compute the independently checked digest') - const first = await session(sessionId) - await save('session-before-restart.json', first) - receipt.stages.at(-1).status = 'passed' + if (evidenceScenario) { + const matterCode = randomBytes(16).toString('hex') + const revisionCode = randomBytes(16).toString('hex') + const file = join(root, 'task/matter.txt') + const initial = `FICTIONAL TEST RECORD\nMatter code: ${matterCode}\nAmount claimed: USD 120000\nPossible contractual cap: USD 25000\nOnly an unsigned agreement copy is available. Applicability is unverified. This is not a recovery forecast.\n` + await writeFile(file, initial, { mode: 0o600 }) + await save('fixture-initial.json', { content: initial, sha256: hash(initial) }) + await turn('01-read', 'Read matter.txt from the working directory using your read tool. Return only JSON: {"matterCode":"exact code from the file","claimed":number,"cap":number,"applicability":"unverified"}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode, claimed: 120000, cap: 25000, applicability: 'unverified' }) + assert(artifact.out.toolCalls > 0, 'No observed tool execution for the file read') + }) + const first = await session(sessionId) + await save('session-before-restart.json', first) + await stopBridge() + await startBridge() + const restored = await session(sessionId) + assert.equal(restored.internalId, first.internalId, 'Native session identity must survive bridge restart') + await turn('02-resume', 'Without using any tools or reading files, repeat only the matterCode from your previous answer as JSON: {"matterCode":"..."}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode }) + assert.equal(artifact.out.toolCalls, 0, 'Continuation used a tool instead of retained conversation') + }) + const updated = initial + `\nNEW SIGNED AMENDMENT\nRevision code: ${revisionCode}\nThe cap for this fictional shipment is amended to USD 50000. Recovery remains unassessed.\n` + await writeFile(file, updated, { mode: 0o600 }) + await save('fixture-updated.json', { content: updated, sha256: hash(updated) }) + await turn('03-new-evidence', 'matter.txt has changed. Read it again with the read tool. Return only JSON: {"matterCode":"...","revisionCode":"exact new code","previousCap":number,"amendedCap":number,"recovery":"not_assessed"}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode, revisionCode, previousCap: 25000, amendedCap: 50000, recovery: 'not_assessed' }) + assert(artifact.out.toolCalls > 0, 'New evidence was not retrieved through an observed tool call') + }) + const resumed = await session(sessionId) + assert.equal(resumed.internalId, first.internalId, 'All three turns must use the same native OpenCode session') + await save('session-after-restart.json', resumed) + receipt.nativeSessionId = first.internalId + } else { + const marker = randomBytes(16).toString('hex') + const input = randomBytes(64).toString('hex') + '\n' + await writeFile(join(root, 'task/input.txt'), input, { mode: 0o600 }) + await turn('filesystem-task', `Work only in your current directory. Read input.txt with a tool and compute the SHA-256 of its exact bytes using a command. Create answer.txt containing only the 64-character hexadecimal digest and a newline. Do not change input.txt. Remember this conversation marker for our next turn: ${marker}. Do not write the marker to any file. Reply briefly when finished.`) + assert.equal(await readFile(join(root, 'task/input.txt'), 'utf8'), input, 'The task input must remain unchanged') + assert.equal((await readFile(join(root, 'task/answer.txt'), 'utf8')).trim(), hash(input), 'The model must really compute the independently checked digest') + const first = await session(sessionId) + await save('session-before-restart.json', first) + receipt.stages.at(-1).status = 'passed' - // Restart the actual server; do not replay the first prompt to manufacture continuity. - await stopBridge() - await startBridge() - const restored = await session(sessionId) - assert.equal(restored.internalId, first.internalId, 'Native session identity must survive bridge restart') - await turn('session-resume', 'Continue our previous conversation. Write its conversation marker, and nothing else except a newline, to resume.txt in the current directory. Do not read any other files or use search to find the marker; use your conversation context. Reply briefly when done.') - assert.equal((await readFile(join(root, 'task/resume.txt'), 'utf8')).trim(), marker, 'Resume must retain context without replaying the first prompt') - const resumed = await session(sessionId) - assert.equal(resumed.internalId, first.internalId, 'Resume must use the same native OpenCode session') - receipt.stages.at(-1).status = 'passed' - await save('session-after-restart.json', resumed) + // Restart the actual server; do not replay the first prompt to manufacture continuity. + await stopBridge() + await startBridge() + const restored = await session(sessionId) + assert.equal(restored.internalId, first.internalId, 'Native session identity must survive bridge restart') + await turn('session-resume', 'Continue our previous conversation. Write its conversation marker, and nothing else except a newline, to resume.txt in the current directory. Do not read any other files or use search to find the marker; use your conversation context. Reply briefly when done.') + assert.equal((await readFile(join(root, 'task/resume.txt'), 'utf8')).trim(), marker, 'Resume must retain context without replaying the first prompt') + const resumed = await session(sessionId) + assert.equal(resumed.internalId, first.internalId, 'Resume must use the same native OpenCode session') + receipt.stages.at(-1).status = 'passed' + await save('session-after-restart.json', resumed) + receipt.nativeSessionId = first.internalId + receipt.answerSha256 = hash(await readFile(join(root, 'task/answer.txt'))) + receipt.resumeSha256 = hash(await readFile(join(root, 'task/resume.txt'))) + } await stopBridge() const traces = await readFile(join(root, 'bridge/traces/spans.jsonl'), 'utf8') const spans = traces.trim().split('\n').map(line => JSON.parse(line)) - assert.ok(spans.length >= 2, 'Both invocations must leave real trace records') + assert.ok(spans.length >= (evidenceScenario ? 3 : 2), 'Every invocation must leave real trace records') await save('spans.jsonl', traces) - receipt.nativeSessionId = first.internalId receipt.traceRecords = spans.length - receipt.answerSha256 = hash(await readFile(join(root, 'task/answer.txt'))) - receipt.resumeSha256 = hash(await readFile(join(root, 'task/resume.txt'))) receipt.scope = 'Local Runtime executor, bridge HTTP/process path, filesystem tools, native session restart/resume, and local trace retention. Not Agent App UI, cloud R2 custody, disaster recovery, or training eligibility.' receipt.status = 'passed' } diff --git a/tests/runtime-consumer/opencode-live.md b/tests/runtime-consumer/opencode-live.md index a5f78ee..803d801 100644 --- a/tests/runtime-consumer/opencode-live.md +++ b/tests/runtime-consumer/opencode-live.md @@ -1,41 +1,61 @@ -# Real local OpenCode check +# Three-turn local evidence proof -Use the existing stack, not a new server or simulated model: +The opt-in `evidence` scenario uses the maintained live proof's authenticated +loopback HTTP bridge, fresh HOME/XDG directories, Linux filesystem jail, evidence +capture and owned-process cleanup. It does not use host model credentials or an +externally started daemon. The default `filesystem` scenario still makes its +existing two turns; selecting `evidence` runs only the three turns below. -``` -Runtime bridge executor -> localhost:3344 -> OpenCode -> selected provider/model -``` - -Run on the same machine as CLI-Bridge. Install the repo's pinned dependencies and OpenCode normally. Select an exact model from `opencode models` and verify its access and pricing in your own account. There is no default model and no fallback. Hosting the harness locally does not host its model weights locally. +Use a supported Linux host with working bubblewrap and the checksum-pinned +OpenCode installer in `.github/workflows/live-inference.yml`. That workflow owns +the executable version and checksum. `OPENCODE_BIN` selects the installed binary; +the scenario checks its reported version against that pin before inference. +Install this repository's pinned dependencies through the admitted gate path. -From the repo root, in the first terminal: - -```sh -pnpm install --frozen-lockfile -opencode models -BRIDGE_HOST=127.0.0.1 BRIDGE_PORT=3344 BRIDGE_BACKENDS=opencode \ - OPENCODE_EXECUTOR=host BRIDGE_DATA_DIR="$(mktemp -d)" pnpm start -``` +The source supports the named anonymous `opencode/glm-5.3-flash-free` route only. +Before executing it, obtain the operator's supported-host and centralized-slot +admission and retain current [OpenCode pricing](https://opencode.ai/v2/docs/console/models/) +showing that the route and every token category are free. If that evidence is +unavailable or the provider changes its price, stop before inference. A model +name or catalogue listing alone does not establish the permitted cost. -In a second terminal, select the same exact `provider/model` shown by OpenCode: +After those admissions, from the repository root: ```sh -export OPENCODE_MODEL='opencode/glm-5.3-flash' # example; verify access/price first -# Supply BRIDGE_BEARER here if the bridge requires it. Never commit that value. -node tests/runtime-consumer/opencode-live.mjs -node tests/runtime-consumer/opencode-live.mjs --run +CLI_BRIDGE_REAL_OPENCODE_TESTS=1 \ +CLI_BRIDGE_LIVE_SCENARIO=evidence \ +CLI_BRIDGE_LIVE_MODEL=opencode/glm-5.3-flash-free \ +pnpm test:local-inference ``` -The first command checks prerequisites without model inference. Only `--run` authorizes inference, and the selected account may be charged. Each turn has a 120-second deadline, configurable with `--timeout-ms`; this is not a hard dollar cap. Existing operator jail and credential policy is not disabled or modified. No production service is contacted except the selected model provider through OpenCode. Run in a dedicated development account/environment, never against client matters. - -The live path imports the Runtime already pinned by this consumer workspace. It makes three real turns with no transport interception or fake backend: - -1. Read a temporary synthetic matter through OpenCode's native read tool. The file contains a random matter code not supplied in the prompt. -2. Resume the same bridge session and return that code without calling tools or replaying the file in the new prompt. -3. Read updated evidence containing another random code. Report the old and amended caps separately, leaving recovery unassessed. - -A pass requires matching answers, actual tool-call counts, nonzero measured input/output tokens, and a matching reported model. Unknown dollar cost stays unknown. Gateway-reported identity is not independent verification of provider model weights. A successful `/health` alone never passes the test. Errors, missing dependencies, model substitutions, and blocked prerequisites return nonzero. - -The command prints its private evidence directory with a `proof.json`, full Runtime artifacts, fixture hashes, script hash, and package/CLI identity. It preserves the synthetic working directory and bridge session for inspection; remove them after review. Run `--help` for options. The script does not install dependencies, modify credentials, start an extra execution framework, or run on ordinary CI automatically. - -This proves a narrow local execution path when successful. It does **not** prove the authenticated Casework app, background reassessment, browser streaming/reload/stop, a managed sandbox, or the design preview's backend integration. Those must use the application's real producers and case tools; changing a model URL alone does not move a sandbox-owned workflow onto the local bridge. +`CLI_BRIDGE_LIVE_OUT` selects a new private evidence directory. Each turn has a +120-second deadline. This is a three-user-turn allowance, not permission for the +default scenario as an additional run. Main, small, title, summary and compaction +models are pinned to the same selected route, and the provider model whitelist +contains only that model. There is no paid fallback. These config fields and +agent overrides were checked against the installer's pinned OpenCode +[schema](https://github.com/anomalyco/opencode/tree/e3c4eb242585dcd77be1a867391b7c25fd47ce52/packages/core/src/v1/config) +and [agent implementation](https://github.com/anomalyco/opencode/blob/e3c4eb242585dcd77be1a867391b7c25fd47ce52/packages/opencode/src/agent/agent.ts#L267). +The scenario checks the effective config and available model list before starting +the bridge. Auxiliary work, when OpenCode performs it, uses that same route. + +1. Read synthetic evidence through the native read tool, returning its random + matter code, claimed amount, possible cap and unverified applicability. +2. After restarting the actual bridge, resume the same native session and return + the matter code without tools, source replay or the code in the new prompt. +3. Read amended evidence with a new random revision code, distinguishing the + previous cap, amended cap and still-unassessed recovery. + +Every turn requires nonzero measured input/output tokens, matching reported model +identity and a nonempty parseable answer. Turns one and three require observed +tool calls; turn two requires zero. Every turn tears down its owned Runtime run +and asserts `destroyed`. The receipt also verifies native session continuity, +retains fixture hashes and Runtime artifacts, then cleans owned scratch after +process shutdown. Unknown billed cost stays unknown rather than being invented +from model selection. A health check, blocked preflight or catalogue listing +never counts as a successful proof. + +This qualifies only synthetic local Runtime/CLI-Bridge/OpenCode execution when +it passes. It does not qualify Casework, authenticated Agent App browser flows, +legal calibration, managed sandboxes, cloud trace custody or design preview. +The source integration itself is not evidence that inference has completed. diff --git a/tests/runtime-consumer/opencode-live.mjs b/tests/runtime-consumer/opencode-live.mjs deleted file mode 100644 index 2c14632..0000000 --- a/tests/runtime-consumer/opencode-live.mjs +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env node -/** Opt-in real Runtime -> CLI-Bridge -> OpenCode proof. No mock execution path. - * Run from an installed cli-bridge checkout. See opencode-live.md beside this file. - */ -import assert from 'node:assert/strict' -import { execFile } from 'node:child_process' -import { randomBytes, createHash } from 'node:crypto' -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { dirname, join, resolve } from 'node:path' -import { fileURLToPath, pathToFileURL } from 'node:url' -import { parseArgs, promisify } from 'node:util' - -const exec = promisify(execFile) -const { values } = parseArgs({ options: { - run: { type: 'boolean', default: false }, - help: { type: 'boolean', default: false }, - model: { type: 'string' }, - out: { type: 'string' }, - 'timeout-ms': { type: 'string', default: '120000' }, -} }) -if (values.help) { - console.log('node tests/runtime-consumer/opencode-live.mjs [--run] --model [--out NEW_DIRECTORY]\nWithout --run: preflight only; no model inference. BRIDGE_URL defaults to http://127.0.0.1:3344. Set BRIDGE_BEARER when required. The exact model is required; there is no fallback.') - process.exit(0) -} -const timeoutMs = Number(values['timeout-ms']) -assert(Number.isSafeInteger(timeoutMs) && timeoutMs >= 1000 && timeoutMs <= 600000, 'timeout-ms must be 1000..600000') -const bridge = new URL(process.env.BRIDGE_URL || 'http://127.0.0.1:3344') -assert(['http:', 'https:'].includes(bridge.protocol) && ['127.0.0.1', '[::1]', 'localhost'].includes(bridge.hostname) - && !bridge.username && !bridge.password && !bridge.search && !bridge.hash && bridge.pathname === '/', - 'Use a credential-free loopback BRIDGE_URL origin; this proof runs on the bridge host') -const bearer = process.env.BRIDGE_BEARER || '' -const selected = values.model || process.env.OPENCODE_MODEL || '' -assert(!selected || /^[A-Za-z0-9_.-]+\/[^\s]+$/.test(selected), 'model must be OpenCode provider/model, without the outer bridge prefix') -const evidenceRoot = values.out ? resolve(values.out) : await mkdtemp(join(tmpdir(), 'cli-bridge-proof-')) -if (values.out) await mkdir(evidenceRoot, { recursive: false, mode: 0o700 }) -const hash = value => createHash('sha256').update(value).digest('hex') -const redact = value => bearer ? value.replaceAll(bearer, '[REDACTED]') : value -const save = (name, value) => writeFile(join(evidenceRoot, name), redact(JSON.stringify(value, null, 2)) + '\n', { mode: 0o600 }) -const report = { - version: 1, startedAt: new Date().toISOString(), status: 'preflight', - requestedModel: selected || null, bridgeOrigin: bridge.origin, - sourceSha256: hash(await readFile(fileURLToPath(import.meta.url))), - checks: [], turns: [], inferenceCompleted: false, - scope: 'Synthetic local Runtime/CLI-Bridge/OpenCode execution only. Not Casework, browser, managed sandbox, legal calibration, or containment proof.', -} -async function check(name, fn) { - try { const detail = await fn(); report.checks.push({ name, ok: true, detail }); return detail } - catch (error) { report.checks.push({ name, ok: false, error: redact(error.message) }); return null } -} -let activeExecutor, workspace -async function main() { - await check('node', async () => { - const [major, minor] = process.versions.node.split('.').map(Number) - assert(major > 22 || (major === 22 && minor >= 13), 'Node 22.13+ required') - return process.versions.node - }) - await check('explicit-model', async () => { assert(selected, 'Select an exact provider/model from opencode models; no automatic paid or free fallback'); return selected }) - const bin = process.env.OPENCODE_BIN || 'opencode' - const cliVersion = await check('opencode-installed', async () => { - const { stdout } = await exec(bin, ['--version'], { timeout: 15000, maxBuffer: 65536 }) - return stdout.trim() - }) - if (cliVersion && selected) await check('model-listed', async () => { - const { stdout } = await exec(bin, ['models'], { timeout: 30000, maxBuffer: 2 * 1024 * 1024 }) - assert(stdout.split(/\r?\n/).map(x => x.trim()).includes(selected), 'Selected model is not in opencode models; no substitution allowed') - return { selected, catalogueSha256: hash(stdout), pricing: 'not inferred from model name or catalogue presence' } - }) - const runtimeEntry = await check('installed-runtime', async () => { - const entry = fileURLToPath(import.meta.resolve('@tangle-network/agent-runtime/kernel')) - let directory = dirname(entry) - for (;;) { - try { - const pkg = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')) - if (pkg.name === '@tangle-network/agent-runtime') { report.runtimeVersion = pkg.version; break } - } catch (error) { if (error.code !== 'ENOENT') throw error } - const parent = dirname(directory); assert(parent !== directory, 'Cannot identify installed Runtime version'); directory = parent - } - return entry - }) - await check('bridge-reachable', async () => { - const response = await fetch(new URL('/health', bridge), { - headers: bearer ? { Authorization: `Bearer ${bearer}` } : {}, - redirect: 'error', signal: AbortSignal.timeout(5000), - }) - assert(response.ok, `Bridge health HTTP ${response.status}`) - await response.arrayBuffer() - return { httpStatus: response.status } // Readiness is not inference success. - }) - if (report.checks.some(item => !item.ok)) { - report.status = 'blocked'; process.exitCode = 2; return - } - if (!values.run) { report.status = 'ready-not-run'; return } - - // Existing published executor, not an OpenAI shim or replacement tool loop. - const { createExecutor } = await import(pathToFileURL(runtimeEntry).href) - workspace = await mkdtemp(join(tmpdir(), 'opencode-synthetic-matter-')) - report.workspace = workspace - const [provider, ...modelParts] = selected.split('/') - const profile = { - name: 'local-opencode-live-proof', harness: 'opencode', - model: { provider, default: modelParts.join('/') }, - tools: { '*': false, read: true }, - prompt: { systemPrompt: 'You are checking fictional test evidence. Use only the read tool and only files in the supplied working directory. Never use shell, network tools, or other directories. Follow the requested JSON response exactly. A figure stated in a source is not a recovery forecast. Do not accept a case or invent outcomes.' }, - } - report.profile = profile - report.sessionId = `local-proof-${randomBytes(12).toString('hex')}` - const matterCode = randomBytes(16).toString('hex') - const revisionCode = randomBytes(16).toString('hex') - const file = join(workspace, 'matter.txt') - const initial = `FICTIONAL TEST RECORD\nMatter code: ${matterCode}\nAmount claimed: USD 120000\nPossible contractual cap: USD 25000\nOnly an unsigned agreement copy is available. Applicability is unverified. This is not a recovery forecast.\n` - await writeFile(file, initial, { mode: 0o600 }) - await save('fixture-initial.json', { content: initial, sha256: hash(initial) }) - async function turn(name, prompt, validate) { - const controller = new AbortController() - const timer = setTimeout(() => controller.abort(new Error('Local proof deadline exceeded')), timeoutMs + 5000) - const record = { name, prompt, startedAt: new Date().toISOString(), ok: false } - report.turns.push(record) - try { - activeExecutor = createExecutor({ backend: 'bridge', bridgeUrl: bridge.origin, - bridgeBearer: bearer, cwd: workspace, sessionId: report.sessionId, timeoutMs, - })({ profile, harness: null }, { signal: controller.signal, seams: {} }) - const stream = activeExecutor.execute(prompt, controller.signal) - assert(stream && typeof stream[Symbol.asyncIterator] === 'function', 'Runtime did not return streaming execution') - let count = 0 - for await (const event of stream) { count++; if (count > 50000) throw new Error('Unbounded event stream') } - const artifact = activeExecutor.resultArtifact() - await save(`${name}.json`, { artifact, eventCount: count, assertions: 'evaluated in proof.json' }) - record.eventCount = count - record.model = artifact.out?.model ?? null - record.spend = artifact.spent - record.toolCalls = artifact.out?.toolCalls ?? null - assert(artifact.spent?.tokensKnown !== false && artifact.spent?.tokens?.input > 0 - && artifact.spent?.tokens?.output > 0, 'Missing measured input/output token usage') - assert(record.model, 'Missing observed model identity') - assert([selected, modelParts.join('/'), `opencode/${selected}`].includes(record.model), 'Observed model does not match requested model') - assert(typeof artifact.out?.content === 'string' && artifact.out.content.trim(), 'Empty completed answer') - const json = artifact.out.content.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '') - const answer = JSON.parse(json) - validate(answer, artifact) - record.answer = answer; record.ok = true - } finally { - clearTimeout(timer) - // Let Runtime cancel/settle its owned jobs; never kill an unrelated daemon. - if (activeExecutor) { - const disposed = activeExecutor; activeExecutor = null - const cleanup = await disposed.teardown(5000) - assert(cleanup.destroyed, 'Runtime could not confirm termination of its owned bridge run') - } - record.completedAt = new Date().toISOString() - } - } - await turn('01-read', 'Read matter.txt from the working directory using your read tool. Return only JSON: {"matterCode":"exact code from the file","claimed":number,"cap":number,"applicability":"unverified"}.', (answer, artifact) => { - assert.deepEqual(answer, { matterCode, claimed: 120000, cap: 25000, applicability: 'unverified' }) - assert(artifact.out.toolCalls > 0, 'No observed tool execution for the file read') - }) - await turn('02-resume', 'Without using any tools or reading files, repeat only the matterCode from your previous answer as JSON: {"matterCode":"..."}.', (answer, artifact) => { - assert.deepEqual(answer, { matterCode }); assert.equal(artifact.out.toolCalls, 0, 'Continuation used a tool instead of retained conversation') - }) - const updated = initial + `\nNEW SIGNED AMENDMENT\nRevision code: ${revisionCode}\nThe cap for this fictional shipment is amended to USD 50000. Recovery remains unassessed.\n` - await writeFile(file, updated, { mode: 0o600 }) - await save('fixture-updated.json', { content: updated, sha256: hash(updated) }) - await turn('03-new-evidence', 'matter.txt has changed. Read it again with the read tool. Return only JSON: {"matterCode":"...","revisionCode":"exact new code","previousCap":number,"amendedCap":number,"recovery":"not_assessed"}.', (answer, artifact) => { - assert.deepEqual(answer, { matterCode, revisionCode, previousCap: 25000, amendedCap: 50000, recovery: 'not_assessed' }) - assert(artifact.out.toolCalls > 0, 'New evidence was not retrieved through an observed tool call') - }) - report.status = 'passed'; report.inferenceCompleted = true -} -try { await main() } -catch (error) { report.status = 'failed'; report.error = redact(error.message); process.exitCode = 1 } -finally { - report.completedAt = new Date().toISOString() - await save('proof.json', report) - console.log(JSON.stringify({ status: report.status, inferenceCompleted: report.inferenceCompleted, - checks: report.checks.map(({ name, ok, error }) => ({ name, ok, ...(error ? { error } : {}) })), - turns: report.turns.map(({ name, ok }) => ({ name, ok })), proof: join(evidenceRoot, 'proof.json') }, null, 2)) -} From 436937b4c42bf43ea115cbf0f6163c36c1a79eeb Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 22 Oct 2026 17:53:51 -0600 Subject: [PATCH 4/5] fix(test): honor the admitted Vitest worker budget --- vitest.config.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vitest.config.ts b/vitest.config.ts index 1ab1d2f..dad4604 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,7 +1,15 @@ import { defineConfig } from 'vitest/config' +// The gate owns this budget; Vitest otherwise derives workers from host parallelism. +const workerBudget = process.env.VITEST_MAX_WORKERS +const maxWorkers = workerBudget === undefined ? undefined : Number(workerBudget) +if (maxWorkers !== undefined && (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1)) { + throw new Error('VITEST_MAX_WORKERS must be a positive integer') +} + export default defineConfig({ test: { + maxWorkers, exclude: [ '**/node_modules/**', '**/.git/**', From d7549875916d90b391725b6f31a92cf0e0b4ffcb Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 22 Oct 2026 17:55:02 -0600 Subject: [PATCH 5/5] fix(test): use supported OpenCode profile instructions --- tests/runtime-consumer/live-opencode.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtime-consumer/live-opencode.mjs b/tests/runtime-consumer/live-opencode.mjs index a8a0c7c..ccca8ff 100644 --- a/tests/runtime-consumer/live-opencode.mjs +++ b/tests/runtime-consumer/live-opencode.mjs @@ -160,7 +160,7 @@ async function main() { model: { provider: 'opencode', default: nativeModel.slice('opencode/'.length) }, ...(evidenceScenario ? { tools: { '*': false, read: true }, - prompt: { systemPrompt: 'You are checking fictional test evidence. Use only the read tool and only files in the supplied working directory. Never use shell, network tools, or other directories. Follow the requested JSON response exactly. A figure stated in a source is not a recovery forecast. Do not accept a case or invent outcomes.' }, + prompt: { instructions: 'You are checking fictional test evidence. Use only the read tool and only files in the supplied working directory. Never use shell, network tools, or other directories. Follow the requested JSON response exactly. A figure stated in a source is not a recovery forecast. Do not accept a case or invent outcomes.' }, } : {}), } await startBridge()