From faba90dad09d5612cda1c1da8be2424efaefbf35 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 19 Oct 2026 04:59:33 -0600 Subject: [PATCH] fix(opencode): end a turn whose provider asked for a long retry wait, instead of sleeping in it opencode retries a refused model call itself and sleeps the response's retry-after, capped only at 2^31 ms, while `run --format json` prints nothing (it never emits the session.status retry event). The free seat (opencode/glm-5.3-flash-free) answers a spent quota with 429 and retry-after: 47331 (13.1 h). magnetar-rust-20261019b's root slept from 06:28:51Z until this bridge's 4 h turn cap aborted it; agent-runtime classed the abort terminal and the 48 h run ended. A per-turn opencode plugin in the profile's private config directory prints opencode's own retry status as one stdout line. A wait over OPENCODE_RETRY_WAIT_MAX_MS (default 60 s) ends the turn with error type upstream_unavailable (a code agent-runtime pauses on, with no status inferred) and the reset time, which agent-runtime 0.326.0 holds for when it is inside its hold window. --- src/backends/opencode.ts | 78 +++++++++++++++++++++ tests/opencode-retry-wait-real.test.ts | 95 ++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 tests/opencode-retry-wait-real.test.ts diff --git a/src/backends/opencode.ts b/src/backends/opencode.ts index 2f6958a..195019b 100644 --- a/src/backends/opencode.ts +++ b/src/backends/opencode.ts @@ -33,6 +33,7 @@ import { } from './profile-support.js' import { registerJailReadable } from '../jail/index.js' import { dirname, join } from 'node:path' +import { mkdirSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { contentToText } from './content.js' import { scopedHostSpawner } from '../executors/scoped-host.js' @@ -96,6 +97,12 @@ export class OpencodeBackend implements Backend { cwd, profileExecutionIdentity(req, session, 'opencode', variant), ) + try { + installRetrySignal(provisioned.env.OPENCODE_CONFIG_DIR) + } catch (error) { + provisioned.cleanup?.() + throw error + } // Materialize the one selected MCP source into a temporary OpenCode config. // opencode-cli has no per-invocation `--mcp-config-file` flag — @@ -276,6 +283,13 @@ export class OpencodeBackend implements Backend { } const type = String(ev.type ?? '') + if (type === RETRY_SIGNAL_TYPE) { + const refusal = longRetryWait(ev) + if (refusal === null) continue + // The process is still asleep in its retry; the `finally` below stops it. + yield { ...terminalOutcome('opencode', refusal, emittedToolCall), internal_session_id: internalSessionId } + return + } const eventUsage = extractUsage(ev) if (eventUsage) { if (isStepReceipt(ev)) stepUsage = collectUsage(stepUsage, eventUsage) @@ -594,3 +608,67 @@ function providerStatus(value: unknown): number | undefined { ? value : undefined } + +/** + * A provider's retry wait, made visible. + * + * opencode retries a refused model call itself (`session/retry.ts`). A response with a + * `retry-after` header sets the wait, capped only at 2^31 ms, and `run --format json` prints + * nothing while it waits: it never emits the `session.status` retry event. The free seat answers + * a spent quota with `429` and `retry-after: 47331` (13.1 h), so magnetar-rust-20261019b's root + * slept in silence until this bridge's 4 h turn cap aborted it, and agent-runtime ended a 48 h run + * on that abort. The plugin below prints opencode's own retry status as one stdout line, which + * the event loop reads with the rest of the stream. + */ +const RETRY_SIGNAL_TYPE = 'cli-bridge.opencode-retry' +const RETRY_SIGNAL_PLUGIN = `// Written by cli-bridge for this turn. Prints opencode's retry status, which \`run --format json\` omits. +export const CliBridgeRetrySignal = async () => ({ + event: async ({ event }) => { + const status = event?.properties?.status + if (event?.type !== 'session.status' || status?.type !== 'retry') return + process.stdout.write(JSON.stringify({ + type: ${JSON.stringify(RETRY_SIGNAL_TYPE)}, + sessionID: event.properties.sessionID, + attempt: status.attempt, + message: status.message, + next: status.next, + timestamp: Date.now(), + }) + '\\n') + }, +}) +` + +/** + * The longest retry wait a turn sits through. opencode's own backoff without a `retry-after` + * stays at or under 32 s; anything longer is the provider naming a reset, which agent-runtime + * holds for without spending the process or an attempt. + */ +const RETRY_WAIT_LIMIT_MS = Math.max(1_000, Number(process.env.OPENCODE_RETRY_WAIT_MAX_MS ?? 60_000)) + +/** + * Put the plugin in the turn's private config directory. A turn without an AgentProfile has + * none: opencode would read the operator's own config directory, which this bridge never writes. + */ +function installRetrySignal(configDir: string | undefined): void { + if (configDir === undefined) return + const pluginDir = join(configDir, 'plugin') + mkdirSync(pluginDir, { recursive: true, mode: 0o755 }) + writeFileSync(join(pluginDir, 'cli-bridge-retry-signal.js'), RETRY_SIGNAL_PLUGIN, { mode: 0o644 }) +} + +/** + * The refusal a retry wait longer than {@link RETRY_WAIT_LIMIT_MS} stands for, or `null` for a + * short wait opencode should sit out. It carries `upstream_unavailable` rather than a status: + * opencode's retry status names no HTTP status, and the bridge never infers one. agent-runtime + * pauses on that code, and a `reset at` time within its hold window holds the run until then. + */ +function longRetryWait(ev: Record): BackendFailureReason | null { + const next = typeof ev.next === 'number' ? ev.next : NaN + const now = typeof ev.timestamp === 'number' ? ev.timestamp : Date.now() + if (!Number.isFinite(next) || next - now <= RETRY_WAIT_LIMIT_MS) return null + const message = typeof ev.message === 'string' && ev.message.length > 0 ? ev.message : 'the provider refused the call' + return { + message: `${message}; the provider asked opencode to wait ${Math.round((next - now) / 1000)} s before retrying, reset at ${new Date(next).toISOString()}`, + type: 'upstream_unavailable', + } +} diff --git a/tests/opencode-retry-wait-real.test.ts b/tests/opencode-retry-wait-real.test.ts new file mode 100644 index 0000000..53c4b44 --- /dev/null +++ b/tests/opencode-retry-wait-real.test.ts @@ -0,0 +1,95 @@ +/** + * A provider's long `retry-after` ends the turn instead of hanging it, against the REAL opencode. + * + * opencode sleeps a refused call's `retry-after` itself (capped only at 2^31 ms) and + * `run --format json` prints nothing while it waits. The free seat answers a spent quota with + * `429` and `retry-after: 47331`, and magnetar-rust-20261019b's root slept until the bridge's + * 4 h turn cap. A local endpoint answers every call the same way here; the turn must end within + * the bridge's retry-wait limit, coded `upstream_unavailable` and naming the reset time. + * + * Skipped when the binary is absent, and by `CLI_BRIDGE_SKIP_REAL_OPENCODE=1`. + * Measured against opencode 1.18.33. + */ +import { createServer, type Server } from 'node:http' +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import type { AddressInfo } from 'node:net' +import { homedir, tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { OpencodeBackend } from '../src/backends/opencode.js' +import type { ChatDelta, ChatRequest } from '../src/backends/types.js' +import { hostSpawner } from '../src/executors/host.js' + +const BIN = process.env.OPENCODE_BIN ?? join(homedir(), '.opencode/bin/opencode') +const ENABLED = existsSync(BIN) && process.env.CLI_BRIDGE_SKIP_REAL_OPENCODE !== '1' +const RETRY_AFTER_S = 3600 + +describe.skipIf(!ENABLED)('a real opencode turn refused with a long retry-after', () => { + let server: Server + let calls = 0 + let baseURL = '' + const scratch = mkdtempSync(join(tmpdir(), 'opencode-retry-wait-')) + const saved = { ...process.env } + + beforeAll(async () => { + server = createServer((req, res) => { + calls++ + req.resume() + res.writeHead(429, { 'content-type': 'application/json', 'retry-after': String(RETRY_AFTER_S) }) + res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'Rate limit exceeded. Please try again later.' } })) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + baseURL = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1` + Object.assign(process.env, { + XDG_CONFIG_HOME: join(scratch, 'config'), + XDG_DATA_HOME: join(scratch, 'data'), + XDG_CACHE_HOME: join(scratch, 'cache'), + XDG_STATE_HOME: join(scratch, 'state'), + OPENCODE_CONFIG_CONTENT: JSON.stringify({ + provider: { + probe: { + npm: '@ai-sdk/openai-compatible', + name: 'probe', + options: { baseURL, apiKey: 'probe' }, + models: { m: { name: 'm' } }, + }, + }, + }), + }) + }) + + afterAll(async () => { + for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key] + Object.assign(process.env, saved) + await new Promise((resolve) => server.close(() => resolve())) + rmSync(scratch, { recursive: true, force: true }) + }) + + it('ends the turn coded upstream_unavailable with the reset time, well inside the wait', async () => { + const cwd = mkdtempSync(join(scratch, 'cwd-')) + const backend = new OpencodeBackend({ bin: BIN, timeoutMs: 300_000, spawner: hostSpawner }) + const req: ChatRequest = { + cwd, + session_id: 'retry-wait-real', + mode: 'byob', + model: 'opencode/probe/m', + messages: [{ role: 'user', content: 'say ok' }], + agent_profile: { name: 'director', harness: 'opencode', prompt: { instructions: ['You are the director.'] } }, + } as ChatRequest + + const started = Date.now() + let last: ChatDelta | undefined + for await (const delta of backend.chat(req, null, new AbortController().signal)) { + if (delta.finish_reason !== undefined) last = delta + } + const elapsedMs = Date.now() - started + + expect(calls).toBeGreaterThan(0) + expect(last?.finish_reason).toBe('error') + expect(last?.error?.type).toBe('upstream_unavailable') + expect(last?.error?.status).toBeUndefined() + expect(last?.error?.message).toMatch(/Rate limit exceeded/) + expect(last?.error?.message).toMatch(/wait 3\d{3} s before retrying, reset at \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z/) + expect(elapsedMs).toBeLessThan(120_000) + }, 180_000) +})