diff --git a/tests/runtime-consumer/live-opencode.mjs b/tests/runtime-consumer/live-opencode.mjs index ba86e62..ccca8ff 100644 --- a/tests/runtime-consumer/live-opencode.mjs +++ b/tests/runtime-consumer/live-opencode.mjs @@ -15,7 +15,10 @@ import { setTimeout as delay } from 'node:timers/promises' const exec = promisify(execFile) const repository = fileURLToPath(new URL('../../', import.meta.url)) const nativeModel = process.env.CLI_BRIDGE_LIVE_MODEL ?? 'opencode/glm-5.3-flash-free' -const receipt = { schema: 'cli-bridge.live-proof.v1', status: 'failed', model: nativeModel, stages: [] } +const scenario = process.env.CLI_BRIDGE_LIVE_SCENARIO ?? 'filesystem' +const evidenceScenario = scenario === 'evidence' +const timeoutMs = evidenceScenario ? 120_000 : 180_000 +const receipt = { schema: 'cli-bridge.live-proof.v1', status: 'failed', model: nativeModel, scenario, stages: [] } const hash = bytes => createHash('sha256').update(bytes).digest('hex') let root, evidence, child, childClosed, childOutcome, bearer = '', bridgeLog = '', origin, env let ownsEvidence = false @@ -81,7 +84,9 @@ async function session(id) { async function main() { assert.equal(process.env.CLI_BRIDGE_REAL_OPENCODE_TESTS, '1', 'Opt in with CLI_BRIDGE_REAL_OPENCODE_TESTS=1; this invokes a real external model.') + assert.ok(['filesystem', 'evidence'].includes(scenario), 'Select filesystem or evidence; scenarios never run together.') assert.match(nativeModel, /^opencode\/[a-z0-9.-]+-free$/, 'Select an explicitly free OpenCode model. No paid model or silent fallback is allowed.') + if (evidenceScenario) assert.equal(nativeModel, 'opencode/glm-5.3-flash-free', 'The three-turn proof only admits the explicitly authorized anonymous model.') assert.equal(process.platform, 'linux', 'This proof requires the existing Linux filesystem jail; it never downgrades confinement.') const [major, minor] = process.versions.node.split('.').map(Number) assert.ok(major > 22 || (major === 22 && minor >= 19), 'The live Runtime proof requires Node 22.19 or newer') @@ -100,7 +105,7 @@ async function main() { XDG_CACHE_HOME: join(root, 'cache'), XDG_STATE_HOME: join(root, 'state'), BRIDGE_HOST: '127.0.0.1', BRIDGE_BEARER: bearer, BRIDGE_DATA_DIR: join(root, 'bridge'), BRIDGE_BACKENDS: 'opencode', OPENCODE_EXECUTOR: 'host', OPENCODE_BIN: opencode, - OPENCODE_TIMEOUT_MS: '180000', OPENCODE_DISABLE_AUTOUPDATE: '1', OPENCODE_DISABLE_SHARE: '1', + OPENCODE_TIMEOUT_MS: String(timeoutMs), OPENCODE_DISABLE_AUTOUPDATE: '1', OPENCODE_DISABLE_SHARE: '1', OPENCODE_INSTALL_DIR: dirname(dirname(opencode)), BRIDGE_JAIL_MODE: 'fs-jail', BRIDGE_RAW_CAPTURE_DIR: join(evidence, 'raw'), @@ -108,12 +113,40 @@ async function main() { for (const key of ['HTTPS_PROXY', 'HTTP_PROXY', 'NO_PROXY', 'SSL_CERT_FILE', 'NODE_EXTRA_CA_CERTS']) { if (process.env[key]) env[key] = process.env[key] } + if (evidenceScenario) { + // These fields and auxiliary agent overrides are supported by the pinned OpenCode source: + // https://github.com/anomalyco/opencode/tree/e3c4eb242585dcd77be1a867391b7c25fd47ce52/packages/core/src/v1/config + // Whitelist one model so a title/summary/compaction cannot select a paid fallback. + env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ + model: nativeModel, small_model: nativeModel, enabled_providers: ['opencode'], + provider: { opencode: { whitelist: [nativeModel.slice('opencode/'.length)] } }, + agent: Object.fromEntries(['title', 'summary', 'compaction'].map(name => [name, { model: nativeModel }])), + }) + } receipt.opencodeVersion = (await exec(opencode, ['--version'], { env, cwd: join(root, 'task'), timeout: 20_000 })).stdout.trim() receipt.bridgeCommit = (await exec('git', ['rev-parse', 'HEAD'], { cwd: repository, timeout: 5_000 })).stdout.trim() receipt.nodeVersion = process.version + if (evidenceScenario) { + // The installer workflow owns the version pin; do not duplicate its changing value here. + const workflow = await readFile(join(repository, '.github/workflows/live-inference.yml'), 'utf8') + const pinnedVersion = /\/opencode\/releases\/download\/v([^/]+)\//.exec(workflow)?.[1] + assert.ok(pinnedVersion, 'Cannot read the maintained OpenCode installer pin') + assert.equal(receipt.opencodeVersion, pinnedVersion, 'Use the checksum-pinned OpenCode from the maintained installer') + const config = JSON.parse((await exec(opencode, ['debug', 'config'], { env, cwd: join(root, 'task'), timeout: 30_000, maxBuffer: 2 * 1024 * 1024 })).stdout) + assert.equal(config.model, nativeModel, 'OpenCode changed the selected main model') + assert.equal(config.small_model, nativeModel, 'OpenCode changed the auxiliary model') + assert.deepEqual(config.enabled_providers, ['opencode'], 'OpenCode enabled another provider') + assert.deepEqual(Object.keys(config.provider?.opencode ?? {}).sort(), ['whitelist'], 'Anonymous inference refuses configured provider credentials or endpoint overrides') + assert.deepEqual(config.provider?.opencode?.whitelist, [nativeModel.slice('opencode/'.length)], 'OpenCode admitted another model') + for (const name of ['title', 'summary', 'compaction']) assert.equal(config.agent?.[name]?.model, nativeModel, `OpenCode changed the ${name} model`) + receipt.modelPolicy = JSON.parse(env.OPENCODE_CONFIG_CONTENT) + receipt.scriptSha256 = hash(await readFile(fileURLToPath(import.meta.url))) + } // This is availability, not successful inference. The later filesystem checks prove execution. const models = (await exec(opencode, ['models', 'opencode'], { env, cwd: join(root, 'task'), timeout: 60_000, maxBuffer: 4 * 1024 * 1024 })).stdout - assert.ok(models.split(/\r?\n/).map(value => value.trim()).includes(nativeModel), `Requested model is absent from OpenCode: ${nativeModel}`) + const listedModels = models.split(/\r?\n/).map(value => value.trim()).filter(Boolean) + assert.ok(listedModels.includes(nativeModel), `Requested model is absent from OpenCode: ${nativeModel}`) + if (evidenceScenario) assert.deepEqual(listedModels, [nativeModel], 'OpenCode admitted a fallback model') receipt.stages.push({ name: 'model-discovery', status: 'passed' }) const socket = createServer() await new Promise((resolveListen, reject) => { socket.once('error', reject); socket.listen(0, '127.0.0.1', resolveListen) }) @@ -122,17 +155,23 @@ async function main() { await new Promise((resolveClose, reject) => socket.close(error => error ? reject(error) : resolveClose())) const { createExecutor } = await import('@tangle-network/agent-runtime/kernel') const sessionId = `live-${randomUUID()}` - const profile = { name: 'local-inference-proof', harness: 'opencode', model: { provider: 'opencode', default: nativeModel.slice('opencode/'.length) } } - const marker = randomBytes(16).toString('hex') - const input = randomBytes(64).toString('hex') + '\n' - await writeFile(join(root, 'task/input.txt'), input, { mode: 0o600 }) + const profile = { + name: evidenceScenario ? 'local-evidence-proof' : 'local-inference-proof', harness: 'opencode', + model: { provider: 'opencode', default: nativeModel.slice('opencode/'.length) }, + ...(evidenceScenario ? { + tools: { '*': false, read: true }, + prompt: { instructions: 'You are checking fictional test evidence. Use only the read tool and only files in the supplied working directory. Never use shell, network tools, or other directories. Follow the requested JSON response exactly. A figure stated in a source is not a recovery forecast. Do not accept a case or invent outcomes.' }, + } : {}), + } await startBridge() - executor = createExecutor({ backend: 'bridge', bridgeUrl: origin, bridgeBearer: bearer, - cwd: join(root, 'task'), sessionId, timeoutMs: 180_000, - })({ profile, harness: null }, { signal: AbortSignal.timeout(420_000), seams: {} }) - async function turn(name, prompt) { - const signal = AbortSignal.timeout(180_000) + const makeExecutor = () => createExecutor({ backend: 'bridge', bridgeUrl: origin, bridgeBearer: bearer, + cwd: join(root, 'task'), sessionId, timeoutMs, + })({ profile, harness: null }, { signal: AbortSignal.timeout(evidenceScenario ? 130_000 : 420_000), seams: {} }) + if (!evidenceScenario) executor = makeExecutor() + async function turn(name, prompt, validate) { + if (evidenceScenario) executor = makeExecutor() + const signal = AbortSignal.timeout(timeoutMs) const events = [] const stage = { name, status: 'failed', eventCount: 0 } receipt.stages.push(stage) @@ -147,40 +186,101 @@ async function main() { } artifact = executor.resultArtifact() assert.notEqual(artifact.verdict?.valid, false, 'Runtime reported an invalid result') - stage.status = 'executed' + if (evidenceScenario) { + stage.model = artifact.out?.model ?? null + stage.spend = artifact.spent + stage.toolCalls = artifact.out?.toolCalls ?? null + assert(artifact.spent?.tokensKnown !== false && artifact.spent?.tokens?.input > 0 + && artifact.spent?.tokens?.output > 0, 'Missing measured input/output token usage') + assert([nativeModel, nativeModel.slice('opencode/'.length), `opencode/${nativeModel}`].includes(stage.model), 'Observed model does not match requested model') + assert(typeof artifact.out?.content === 'string' && artifact.out.content.trim(), 'Empty completed answer') + const json = artifact.out.content.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '') + stage.answer = JSON.parse(json) + validate(stage.answer, artifact) + } + stage.status = evidenceScenario ? 'passed' : 'executed' } finally { - stage.durationMs = Date.now() - started - stage.eventCount = events.length - await save(`${name}.json`, { events, artifact: artifact ?? null }) + try { + if (evidenceScenario && executor) { + const cleanup = await executor.teardown(5_000) + stage.teardown = cleanup + assert(cleanup.destroyed, 'Runtime could not confirm termination of its owned bridge run') + executor = undefined + } + } catch (error) { + stage.status = 'failed' + throw error + } finally { + stage.durationMs = Date.now() - started + stage.eventCount = events.length + await save(`${name}.json`, { events, artifact: artifact ?? null, stage }) + } } } - await turn('filesystem-task', `Work only in your current directory. Read input.txt with a tool and compute the SHA-256 of its exact bytes using a command. Create answer.txt containing only the 64-character hexadecimal digest and a newline. Do not change input.txt. Remember this conversation marker for our next turn: ${marker}. Do not write the marker to any file. Reply briefly when finished.`) - assert.equal(await readFile(join(root, 'task/input.txt'), 'utf8'), input, 'The task input must remain unchanged') - assert.equal((await readFile(join(root, 'task/answer.txt'), 'utf8')).trim(), hash(input), 'The model must really compute the independently checked digest') - const first = await session(sessionId) - await save('session-before-restart.json', first) - receipt.stages.at(-1).status = 'passed' + if (evidenceScenario) { + const matterCode = randomBytes(16).toString('hex') + const revisionCode = randomBytes(16).toString('hex') + const file = join(root, 'task/matter.txt') + const initial = `FICTIONAL TEST RECORD\nMatter code: ${matterCode}\nAmount claimed: USD 120000\nPossible contractual cap: USD 25000\nOnly an unsigned agreement copy is available. Applicability is unverified. This is not a recovery forecast.\n` + await writeFile(file, initial, { mode: 0o600 }) + await save('fixture-initial.json', { content: initial, sha256: hash(initial) }) + await turn('01-read', 'Read matter.txt from the working directory using your read tool. Return only JSON: {"matterCode":"exact code from the file","claimed":number,"cap":number,"applicability":"unverified"}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode, claimed: 120000, cap: 25000, applicability: 'unverified' }) + assert(artifact.out.toolCalls > 0, 'No observed tool execution for the file read') + }) + const first = await session(sessionId) + await save('session-before-restart.json', first) + await stopBridge() + await startBridge() + const restored = await session(sessionId) + assert.equal(restored.internalId, first.internalId, 'Native session identity must survive bridge restart') + await turn('02-resume', 'Without using any tools or reading files, repeat only the matterCode from your previous answer as JSON: {"matterCode":"..."}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode }) + assert.equal(artifact.out.toolCalls, 0, 'Continuation used a tool instead of retained conversation') + }) + const updated = initial + `\nNEW SIGNED AMENDMENT\nRevision code: ${revisionCode}\nThe cap for this fictional shipment is amended to USD 50000. Recovery remains unassessed.\n` + await writeFile(file, updated, { mode: 0o600 }) + await save('fixture-updated.json', { content: updated, sha256: hash(updated) }) + await turn('03-new-evidence', 'matter.txt has changed. Read it again with the read tool. Return only JSON: {"matterCode":"...","revisionCode":"exact new code","previousCap":number,"amendedCap":number,"recovery":"not_assessed"}.', (answer, artifact) => { + assert.deepEqual(answer, { matterCode, revisionCode, previousCap: 25000, amendedCap: 50000, recovery: 'not_assessed' }) + assert(artifact.out.toolCalls > 0, 'New evidence was not retrieved through an observed tool call') + }) + const resumed = await session(sessionId) + assert.equal(resumed.internalId, first.internalId, 'All three turns must use the same native OpenCode session') + await save('session-after-restart.json', resumed) + receipt.nativeSessionId = first.internalId + } else { + const marker = randomBytes(16).toString('hex') + const input = randomBytes(64).toString('hex') + '\n' + await writeFile(join(root, 'task/input.txt'), input, { mode: 0o600 }) + await turn('filesystem-task', `Work only in your current directory. Read input.txt with a tool and compute the SHA-256 of its exact bytes using a command. Create answer.txt containing only the 64-character hexadecimal digest and a newline. Do not change input.txt. Remember this conversation marker for our next turn: ${marker}. Do not write the marker to any file. Reply briefly when finished.`) + assert.equal(await readFile(join(root, 'task/input.txt'), 'utf8'), input, 'The task input must remain unchanged') + assert.equal((await readFile(join(root, 'task/answer.txt'), 'utf8')).trim(), hash(input), 'The model must really compute the independently checked digest') + const first = await session(sessionId) + await save('session-before-restart.json', first) + receipt.stages.at(-1).status = 'passed' - // Restart the actual server; do not replay the first prompt to manufacture continuity. - await stopBridge() - await startBridge() - const restored = await session(sessionId) - assert.equal(restored.internalId, first.internalId, 'Native session identity must survive bridge restart') - await turn('session-resume', 'Continue our previous conversation. Write its conversation marker, and nothing else except a newline, to resume.txt in the current directory. Do not read any other files or use search to find the marker; use your conversation context. Reply briefly when done.') - assert.equal((await readFile(join(root, 'task/resume.txt'), 'utf8')).trim(), marker, 'Resume must retain context without replaying the first prompt') - const resumed = await session(sessionId) - assert.equal(resumed.internalId, first.internalId, 'Resume must use the same native OpenCode session') - receipt.stages.at(-1).status = 'passed' - await save('session-after-restart.json', resumed) + // Restart the actual server; do not replay the first prompt to manufacture continuity. + await stopBridge() + await startBridge() + const restored = await session(sessionId) + assert.equal(restored.internalId, first.internalId, 'Native session identity must survive bridge restart') + await turn('session-resume', 'Continue our previous conversation. Write its conversation marker, and nothing else except a newline, to resume.txt in the current directory. Do not read any other files or use search to find the marker; use your conversation context. Reply briefly when done.') + assert.equal((await readFile(join(root, 'task/resume.txt'), 'utf8')).trim(), marker, 'Resume must retain context without replaying the first prompt') + const resumed = await session(sessionId) + assert.equal(resumed.internalId, first.internalId, 'Resume must use the same native OpenCode session') + receipt.stages.at(-1).status = 'passed' + await save('session-after-restart.json', resumed) + receipt.nativeSessionId = first.internalId + receipt.answerSha256 = hash(await readFile(join(root, 'task/answer.txt'))) + receipt.resumeSha256 = hash(await readFile(join(root, 'task/resume.txt'))) + } await stopBridge() const traces = await readFile(join(root, 'bridge/traces/spans.jsonl'), 'utf8') const spans = traces.trim().split('\n').map(line => JSON.parse(line)) - assert.ok(spans.length >= 2, 'Both invocations must leave real trace records') + assert.ok(spans.length >= (evidenceScenario ? 3 : 2), 'Every invocation must leave real trace records') await save('spans.jsonl', traces) - receipt.nativeSessionId = first.internalId receipt.traceRecords = spans.length - receipt.answerSha256 = hash(await readFile(join(root, 'task/answer.txt'))) - receipt.resumeSha256 = hash(await readFile(join(root, 'task/resume.txt'))) receipt.scope = 'Local Runtime executor, bridge HTTP/process path, filesystem tools, native session restart/resume, and local trace retention. Not Agent App UI, cloud R2 custody, disaster recovery, or training eligibility.' receipt.status = 'passed' } diff --git a/tests/runtime-consumer/opencode-live.md b/tests/runtime-consumer/opencode-live.md new file mode 100644 index 0000000..803d801 --- /dev/null +++ b/tests/runtime-consumer/opencode-live.md @@ -0,0 +1,61 @@ +# Three-turn local evidence proof + +The opt-in `evidence` scenario uses the maintained live proof's authenticated +loopback HTTP bridge, fresh HOME/XDG directories, Linux filesystem jail, evidence +capture and owned-process cleanup. It does not use host model credentials or an +externally started daemon. The default `filesystem` scenario still makes its +existing two turns; selecting `evidence` runs only the three turns below. + +Use a supported Linux host with working bubblewrap and the checksum-pinned +OpenCode installer in `.github/workflows/live-inference.yml`. That workflow owns +the executable version and checksum. `OPENCODE_BIN` selects the installed binary; +the scenario checks its reported version against that pin before inference. +Install this repository's pinned dependencies through the admitted gate path. + +The source supports the named anonymous `opencode/glm-5.3-flash-free` route only. +Before executing it, obtain the operator's supported-host and centralized-slot +admission and retain current [OpenCode pricing](https://opencode.ai/v2/docs/console/models/) +showing that the route and every token category are free. If that evidence is +unavailable or the provider changes its price, stop before inference. A model +name or catalogue listing alone does not establish the permitted cost. + +After those admissions, from the repository root: + +```sh +CLI_BRIDGE_REAL_OPENCODE_TESTS=1 \ +CLI_BRIDGE_LIVE_SCENARIO=evidence \ +CLI_BRIDGE_LIVE_MODEL=opencode/glm-5.3-flash-free \ +pnpm test:local-inference +``` + +`CLI_BRIDGE_LIVE_OUT` selects a new private evidence directory. Each turn has a +120-second deadline. This is a three-user-turn allowance, not permission for the +default scenario as an additional run. Main, small, title, summary and compaction +models are pinned to the same selected route, and the provider model whitelist +contains only that model. There is no paid fallback. These config fields and +agent overrides were checked against the installer's pinned OpenCode +[schema](https://github.com/anomalyco/opencode/tree/e3c4eb242585dcd77be1a867391b7c25fd47ce52/packages/core/src/v1/config) +and [agent implementation](https://github.com/anomalyco/opencode/blob/e3c4eb242585dcd77be1a867391b7c25fd47ce52/packages/opencode/src/agent/agent.ts#L267). +The scenario checks the effective config and available model list before starting +the bridge. Auxiliary work, when OpenCode performs it, uses that same route. + +1. Read synthetic evidence through the native read tool, returning its random + matter code, claimed amount, possible cap and unverified applicability. +2. After restarting the actual bridge, resume the same native session and return + the matter code without tools, source replay or the code in the new prompt. +3. Read amended evidence with a new random revision code, distinguishing the + previous cap, amended cap and still-unassessed recovery. + +Every turn requires nonzero measured input/output tokens, matching reported model +identity and a nonempty parseable answer. Turns one and three require observed +tool calls; turn two requires zero. Every turn tears down its owned Runtime run +and asserts `destroyed`. The receipt also verifies native session continuity, +retains fixture hashes and Runtime artifacts, then cleans owned scratch after +process shutdown. Unknown billed cost stays unknown rather than being invented +from model selection. A health check, blocked preflight or catalogue listing +never counts as a successful proof. + +This qualifies only synthetic local Runtime/CLI-Bridge/OpenCode execution when +it passes. It does not qualify Casework, authenticated Agent App browser flows, +legal calibration, managed sandboxes, cloud trace custody or design preview. +The source integration itself is not evidence that inference has completed. diff --git a/vitest.config.ts b/vitest.config.ts index 1ab1d2f..dad4604 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,7 +1,15 @@ import { defineConfig } from 'vitest/config' +// The gate owns this budget; Vitest otherwise derives workers from host parallelism. +const workerBudget = process.env.VITEST_MAX_WORKERS +const maxWorkers = workerBudget === undefined ? undefined : Number(workerBudget) +if (maxWorkers !== undefined && (!Number.isSafeInteger(maxWorkers) || maxWorkers < 1)) { + throw new Error('VITEST_MAX_WORKERS must be a positive integer') +} + export default defineConfig({ test: { + maxWorkers, exclude: [ '**/node_modules/**', '**/.git/**',