From 2c0ae290906d4d23623b12aef73dba53e63b6135 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:37:52 +0800 Subject: [PATCH 1/7] fix(persona): keep a single system message at the assistant boundary --- src/lib/identity/crs-persona.mjs | 28 ++++++++++++++++++++++------ test/unit/crs-persona.test.mjs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/lib/identity/crs-persona.mjs b/src/lib/identity/crs-persona.mjs index 6607c0958..109163e65 100644 --- a/src/lib/identity/crs-persona.mjs +++ b/src/lib/identity/crs-persona.mjs @@ -1784,13 +1784,30 @@ function firstUserIndex(messages) { return messages.findIndex((message) => message?.role === 'user') } +function insertSystemAfterUser(messages, user, content) { + if (user < 0) return messages + const assistant = messages.findIndex((message, index) => index > user && message?.role === 'assistant') + const at = assistant < 0 ? messages.length : assistant + const previous = messages[at - 1] + const existing = previous?.content + if (previous?.role === 'system' && (Array.isArray(existing) ? existing.length : String(existing || '').trim())) { + const joined = + typeof existing === 'string' && typeof content === 'string' + ? `${existing}\n\n${content}` + : [ + ...(Array.isArray(existing) ? existing : [{ type: 'text', text: existing }]), + ...(Array.isArray(content) ? content : [{ type: 'text', text: content }]), + ] + return [...messages.slice(0, at - 1), { ...previous, content: joined }, ...messages.slice(at)] + } + return [...messages.slice(0, at), { role: 'system', content }, ...messages.slice(at)] +} + /** 2.1.263 leftover: after the first user, not inside system[]. */ export function insertMidConversationSystem(messages, text) { const t = String(text || '').trim() if (!t || !Array.isArray(messages)) return messages - const idx = firstUserIndex(messages) - if (idx < 0) return messages - return [...messages.slice(0, idx + 1), { role: 'system', content: t }, ...messages.slice(idx + 1)] + return insertSystemAfterUser(messages, firstUserIndex(messages), t) } /** Caller text stays in system segment 5. Do not lift it into role=system. */ @@ -2182,9 +2199,8 @@ function applyTemplatePersona( // without this injected block, so a tail position shifts every turn and // breaks the cache prefix from that point on. Zero templates keep the // caller-visible tail placement. - const at = zero ? outMessages.findLastIndex((message) => message?.role === 'user') : firstUserIndex(outMessages) - if (at >= 0) - outMessages = [...outMessages.slice(0, at + 1), { role: 'system', content }, ...outMessages.slice(at + 1)] + const user = zero ? outMessages.findLastIndex((message) => message?.role === 'user') : firstUserIndex(outMessages) + outMessages = insertSystemAfterUser(outMessages, user, content) } } if (midSystem) { diff --git a/test/unit/crs-persona.test.mjs b/test/unit/crs-persona.test.mjs index 8cf42c2f3..45d9e1088 100644 --- a/test/unit/crs-persona.test.mjs +++ b/test/unit/crs-persona.test.mjs @@ -1870,3 +1870,33 @@ test('official prompt env message stays at a stable position across turns so the assert.deepEqual(turn2.messages.slice(0, turn1.messages.length), turn1.messages) assert.ok(systems(turn1).length > 0) }) + +test('persona environment respects directive-only turns and extends an existing system turn', () => { + const directive = { role: 'system', content: [], output_config: { effort: 'high' } } + const caller = { role: 'system', content: 'caller environment' } + const user = { role: 'user', content: 'first' } + const assistant = { role: 'assistant', content: 'answer' } + const render = (messages) => + applyCrsUnofficialPersona( + { model: 'claude-opus-5-5', system: 'caller rules', messages }, + { mode: 'official_prompt', park: false }, + ).messages + const withDirective = render([user, directive, assistant, user]) + assert.deepEqual( + withDirective.map(({ role }) => role), + ['user', 'system', 'system', 'assistant', 'user'], + ) + assert.deepEqual(withDirective[1], directive) + assert.match(withDirective[2].content, /Environment/) + const withCaller = render([user, caller, assistant, user]) + assert.deepEqual( + withCaller.map(({ role }) => role), + ['user', 'system', 'assistant', 'user'], + ) + assert.match(withCaller[1].content, /^caller environment\n\n[\s\S]*Environment/) + assert.equal(caller.content, 'caller environment') + assert.deepEqual( + render([user, user]).map(({ role }) => role), + ['user', 'user', 'system'], + ) +}) From 8f9205a89a9f92f55e743c67eac0b8c739bfb8a2 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:37:53 +0800 Subject: [PATCH 2/7] fix(protocol): align names in inline tool additions and removals --- src/lib/protocol/anthropic-policy.mjs | 8 ++++++++ test/unit/anthropic-policy.test.mjs | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/lib/protocol/anthropic-policy.mjs b/src/lib/protocol/anthropic-policy.mjs index 1d766ba1c..913a5d8b9 100644 --- a/src/lib/protocol/anthropic-policy.mjs +++ b/src/lib/protocol/anthropic-policy.mjs @@ -706,6 +706,14 @@ function scrubTextBlock(block) { function scrubToolReferenceFields(block, rename) { if (!block || typeof block !== 'object') return { block, changed: false } + if ( + (block.type === 'tool_addition' || block.type === 'tool_removal') && + block.tool?.type === 'tool_reference' && + typeof block.tool.name === 'string' + ) { + const name = rename(block.tool.name) + if (name !== block.tool.name) return { block: { ...block, tool: { ...block.tool, name } }, changed: true } + } if (block.type === 'tool_result' && Array.isArray(block.content)) { let changed = false const content = block.content.map((inner) => { diff --git a/test/unit/anthropic-policy.test.mjs b/test/unit/anthropic-policy.test.mjs index 3b387a81b..94a47687e 100644 --- a/test/unit/anthropic-policy.test.mjs +++ b/test/unit/anthropic-policy.test.mjs @@ -814,6 +814,33 @@ test('#953 third-party scrub rewrites tool_reference inside tool_result history' assert.equal(result.reverse.mcp__WebSearch, 'WebSearch') }) +test('third-party tool additions and removals follow the declared tool name', () => { + const body = { + tools: [{ name: 'lookup_doc', input_schema: { type: 'object' } }], + messages: [ + { role: 'user', content: 'lookup' }, + { + role: 'system', + content: ['tool_addition', 'tool_removal'].map((type) => ({ + type, + tool: { type: 'tool_reference', name: 'lookup_doc' }, + })), + }, + ], + } + const normalized = toStandardAnthropicBody(body).body + assert.equal(normalized.tools[0].name, 'mcp__lookup_doc') + assert.deepEqual( + normalized.messages[1].content.map((block) => block.tool.name), + ['mcp__lookup_doc', 'mcp__lookup_doc'], + ) + assert.deepEqual( + body.messages[1].content.map((block) => block.tool.name), + ['lookup_doc', 'lookup_doc'], + ) + assert.deepEqual(toStandardAnthropicBody(body, { official: true }).body.messages[1], body.messages[1]) +}) + test('#953 client-supplied mcp__ names in tool_reference stay unchanged', () => { const result = scrubUpstreamClientBody({ tools: [ From 0f98ec357fa7e6cca61bc5641c527d132ca45cd1 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:37:53 +0800 Subject: [PATCH 3/7] fix(crag): clamp session limits to physical worker capacity --- src/lib/transport/kernel-watchdog.mjs | 8 +-- src/lib/transport/rust-kernel-supervisor.mjs | 56 ++++++++++++++------ test/unit/kernel-router.test.mjs | 19 +++++++ 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/lib/transport/kernel-watchdog.mjs b/src/lib/transport/kernel-watchdog.mjs index 1a4b1692a..a241bf99b 100644 --- a/src/lib/transport/kernel-watchdog.mjs +++ b/src/lib/transport/kernel-watchdog.mjs @@ -19,6 +19,7 @@ import { ensureRustKernel, dataplaneUsesCcNode, kernelSlotsConverge, + requestedCragSessionSlots, wrapSlotCount, slotContainerName, } from './rust-kernel-supervisor.mjs' @@ -144,10 +145,11 @@ export function watchdogProbeVm(vm, { relayPaths = clusterRelayPaths, exists = f } } -function kernelSlotMismatch(exec, health) { +function kernelSlotMismatch(exec, health, routing) { const configPath = rustKernelPaths(exec).configPath if (!configPath) return false - return kernelSlotsConverge(configPath, health, wrapSlotCount(exec.vm)) + const requested = routing?.inference ? requestedCragSessionSlots(exec.vm, routing) : undefined + return kernelSlotsConverge(configPath, health, wrapSlotCount(exec.vm), requested) } export function createKernelWatchdog({ @@ -312,7 +314,7 @@ export function createKernelWatchdog({ vmTimeoutTracker?.reset?.(vm.id) } // Config convergence, not a failure: outside the restart budget. - if (kernelSlotMismatch(exec, current)) { + if (kernelSlotMismatch(exec, current, routing)) { const observed = await observeRuntime(vm) if (!held.has(vm.id) && observed?.state === 'running') await ensure(exec, { timeoutMs: config.timeout_ms }) } diff --git a/src/lib/transport/rust-kernel-supervisor.mjs b/src/lib/transport/rust-kernel-supervisor.mjs index e0b412440..bcff99fdc 100644 --- a/src/lib/transport/rust-kernel-supervisor.mjs +++ b/src/lib/transport/rust-kernel-supervisor.mjs @@ -13,7 +13,7 @@ import { rustKernelReachable, rustKernelBusy, } from './rust-kernel-client.mjs' -import { maxRequestBytes, streamIdleTimeoutMs } from '../core/config.mjs' +import { maxRequestBytes, readRoutingConfigFile, streamIdleTimeoutMs } from '../core/config.mjs' import { profileForPreset } from '../identity/official-profiles.mjs' import { resolveWorkstationProfile } from '../identity/workstation-profile.mjs' import { cacheTtlFromRouting, normalizeCacheTtl } from '../protocol/cache-ttl.mjs' @@ -125,7 +125,7 @@ export function wrapSlotCount(vm = {}, routing) { * pending idle reload carries a VM snapshot that a live capacity raise can * outdate; sizing from it would shrink the kernel back to the old limits. */ -export function expectedKernelSlots(exec = {}) { +function kernelVm(exec) { const root = projectRootFromExec(exec) const id = exec?.vmId || exec?.vm?.id let vm = exec?.vm @@ -134,7 +134,18 @@ export function expectedKernelSlots(exec = {}) { vm = getVm(root, id) || vm } catch {} } - return wrapSlotCount(vm || {}) + return vm || {} +} + +export function expectedKernelSlots(exec = {}) { + return wrapSlotCount(kernelVm(exec)) +} + +export function requestedCragSessionSlots(vm, routing, fallback) { + const followGlobal = + routing?.inference?.session_slots != null && + (vm?.policy?.sessionSlotsOverride ?? vm?.session_slots_override) === false + return resolveSessionSlots(followGlobal ? {} : vm, routing ?? fallback) } function startCurrent(control) { @@ -152,11 +163,17 @@ export function slotContainerName(exec = {}) { return vmId ? `kin-${String(vmId).replace(/^vm-/, '')}` : '' } -export function syncKernelSlotCount(configPath, expected = WRAP_SLOT_MAX) { +export function syncKernelSlotCount(configPath, expected = WRAP_SLOT_MAX, requested) { const prev = readExistingKernelConfig(configPath) if (!prev || typeof prev !== 'object') return false - if (Number(prev.slots_per_worker) === expected) return false + const sameCount = Number(prev.slots_per_worker) === expected + const sessionSlots = + prev.runtime_variant === 'crag-v2' && Number.isInteger(prev.session_slots) + ? Math.min(!sameCount && Number.isInteger(requested) ? requested : prev.session_slots, expected) + : prev.session_slots + if (sameCount && prev.session_slots === sessionSlots) return false prev.slots_per_worker = expected + if (sessionSlots !== undefined) prev.session_slots = sessionSlots try { fs.writeFileSync(configPath, JSON.stringify(prev, null, 2) + '\n', { mode: 0o600 }) return true @@ -191,9 +208,9 @@ export function kernelSlotsDiffer(configPath, health, expected) { * count (an older release or a hand edit), only the file is rewritten so a * later crash restart reads the right count; that needs no reload. */ -export function kernelSlotsConverge(configPath, health, expected) { +export function kernelSlotsConverge(configPath, health, expected, requested) { if (kernelSlotsDiffer(configPath, health, expected)) return true - syncKernelSlotCount(configPath, expected) + syncKernelSlotCount(configPath, expected, requested) return false } @@ -270,7 +287,10 @@ async function acceptNodeClaudeSetup(exec, runDockerExec) { if (!vmId) return null const last = nodeSetupAcceptedAt.get(vmId) || 0 if (Date.now() - last < NODE_SETUP_RECHECK_MS) return null - const result = await runDockerExec(execArgv(exec, ['python3', '-c', NODE_CLAUDE_SETUP_PY], { user: host.execUser(exec.vm) }), { timeoutMs: 8000 }) + const result = await runDockerExec( + execArgv(exec, ['python3', '-c', NODE_CLAUDE_SETUP_PY], { user: host.execUser(exec.vm) }), + { timeoutMs: 8000 }, + ) if (!result?.ok) return 'node_claude_setup_failed' nodeSetupAcceptedAt.set(vmId, Date.now()) return String(result.stdout || '').includes('patched') ? 'node_claude_setup' : null @@ -893,14 +913,23 @@ async function startRustKernel( const existing = await rustKernelHealth(exec, { timeoutMs: healthTimeoutMs }) if (!startCurrent(control)) return { ok: false, reason: 'start_cancelled' } const staleWrap = wrapNewerThanKernel(exec) - const expectedSlots = expectedKernelSlots(exec) + const currentVm = kernelVm(exec) + const expectedSlots = wrapSlotCount(currentVm) const slotMismatch = !!paths.configPath && kernelSlotsDiffer(paths.configPath, existing, expectedSlots) + let requestedSessionSlots + if (slotMismatch && readExistingKernelConfig(paths.configPath)?.runtime_variant === 'crag-v2') { + try { + requestedSessionSlots = requestedCragSessionSlots(currentVm, readRoutingConfigFile(projectRootFromExec(exec))) + } catch (error) { + if (error.cause?.code !== 'ENOENT') console.warn('[kernel] could not read routing for CRAG session capacity') + } + } const occupied = rustKernelBusy(existing) || wrapHopInflight(exec) > 0 const host = slotHost(exec?.vm) // A slot-count change is config convergence: never kill live sessions for it. // Write the file now and let the idle reload pick it up. if (slotMismatch && occupied && !force && !staleWrap && !prepareGuestAppArmor && !allowMissingCredential) { - syncKernelSlotCount(paths.configPath, expectedSlots) + syncKernelSlotCount(paths.configPath, expectedSlots, requestedSessionSlots) requestKernelReloadWhenIdle(exec) const reconcile = await reconcileCliHopRuntime(exec, { runDockerExec }) return { ok: true, reason: 'busy', slot_reload: 'pending', health: existing, reconcile } @@ -921,7 +950,7 @@ async function startRustKernel( return { ok: true, reason: 'already_up', health: existing, reconcile } } if (!paths.configPath || !fs.existsSync(paths.configPath)) return { ok: false, reason: 'config_missing' } - syncKernelSlotCount(paths.configPath, expectedSlots) + syncKernelSlotCount(paths.configPath, expectedSlots, requestedSessionSlots) try { await host.syncRun(exec.vm, path.dirname(paths.runDir)) } catch (error) { @@ -1195,10 +1224,7 @@ export function writeKernelConfig( ...(!host.native && !isCodexVm(record) ? { telemetry: buildNativeTelemetry(record, projectRoot, routing) } : {}), } if (dataplane === 'crag' && resolveCragKernelVersion(target, kernelRouting) === 'v2' && !host.native) { - const followGlobal = - routing?.inference?.session_slots != null && - (target.policy?.sessionSlotsOverride ?? target.session_slots_override) === false - config.session_slots = resolveSessionSlots(followGlobal ? {} : target, kernelRouting) + config.session_slots = Math.min(requestedCragSessionSlots(target, routing, kernelRouting), config.slots_per_worker) config.prewarm_enabled = kernelRouting.inference?.prewarm_enabled === true config.crag_kernel_version = 'v2' config.runtime_variant = 'crag-v2' diff --git a/test/unit/kernel-router.test.mjs b/test/unit/kernel-router.test.mjs index 33f273e1b..9bcda05bc 100644 --- a/test/unit/kernel-router.test.mjs +++ b/test/unit/kernel-router.test.mjs @@ -1269,6 +1269,12 @@ test('CRAG session policy follows the panel unless the VM pins its seat limit', const pinned = write({ ...vm, policy: { sessionSlots: 4, sessionSlotsOverride: true } }, { routing }) assert.equal(pinned.session_slots, 4) assert.equal(pinned.prewarm_enabled, false) + const capped = write( + { ...vm, machine: { memory: '2g', pids_limit: 448 }, claude: { account_tier: 'pro' } }, + { routing: { ...routing, slot_capacity: { enabled: true } } }, + ) + assert.equal(capped.slots_per_worker, 5) + assert.equal(capped.session_slots, 5) assert.equal(write({ ...vm, dataplane: 'wrap' }, { routing }).prewarm_enabled, undefined) } finally { fs.rmSync(root, { recursive: true, force: true }) @@ -2174,6 +2180,19 @@ unixTest('Rust supervisor does not recycle a busy CLI with ready_slots=0', async } }) +test('CRAG slot convergence repairs an over-cap session limit even when the slot count is unchanged', (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-crag-session-cap-')) + t.after(() => fs.rmSync(root, { recursive: true, force: true })) + const file = path.join(root, 'kernel.json') + const config = { slots_per_worker: 5, session_slots: 20, runtime_variant: 'crag-v2' } + fs.writeFileSync(file, JSON.stringify(config)) + const health = { ok: true, engine: 'rust', healthy: true, capacity: 5, cli_pid: 1 } + assert.equal(kernelSlotsConverge(file, health, 5), false) + assert.deepEqual(JSON.parse(fs.readFileSync(file, 'utf8')), { ...config, session_slots: 5 }) + assert.equal(kernelSlotsConverge(file, { ...health, capacity: 12 }, 12, 20), false) + assert.deepEqual(JSON.parse(fs.readFileSync(file, 'utf8')), { ...config, slots_per_worker: 12, session_slots: 12 }) +}) + test('kernel slot count compares the live count before the file', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'kin-kernel-slot-count-')) const file = path.join(root, 'kernel.json') From f80d39426e8eeb0230bbe71f319fb2ee0cc56ca0 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:37:53 +0800 Subject: [PATCH 4/7] fix(crag): stop failover on structured cache policy rejection --- src/lib/pool/schedule-state.mjs | 2 +- src/lib/pool/upstream-error-policy.mjs | 9 ++++++ src/lib/transport/go-worker-client.mjs | 40 ++++++++++++++++++++++++-- test/unit/native-handoff.test.mjs | 33 +++++++++++++++++++++ 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/lib/pool/schedule-state.mjs b/src/lib/pool/schedule-state.mjs index 5db060138..bf85424be 100644 --- a/src/lib/pool/schedule-state.mjs +++ b/src/lib/pool/schedule-state.mjs @@ -88,7 +88,7 @@ const RESTRICTED_NEW_SESSION = [ * request waits, and local transport errors. */ const STATELESS = [ - /^(distill_blocked|policy_blocked|refusal_guard|hard_regex.*|jev.*|decision_model.*)$/, + /^(distill_blocked|policy_blocked|crag_policy_blocked|crag_opaque_429|refusal_guard|hard_regex.*|jev.*|decision_model.*)$/, /^(content_filter_refusal|content_policy_refusal|usage_policy_refusal|invalid_request|entitlement_required|http_\d+)$/, /_repairable$/, /^(concurrency_limit|concurrency_disabled|rpm_limit|slot_busy|quota_band_inflight|refresh_pending)$/, diff --git a/src/lib/pool/upstream-error-policy.mjs b/src/lib/pool/upstream-error-policy.mjs index 999cac709..b843ca61b 100644 --- a/src/lib/pool/upstream-error-policy.mjs +++ b/src/lib/pool/upstream-error-policy.mjs @@ -341,6 +341,15 @@ function classifyUpstreamResultRaw( retrySameAccount: false, } } + if (result.cragPolicyBlock === true && !result.committed) { + return { + scope: 'request', + action: 'stop', + reason: 'crag_policy_blocked', + cooldownUntil: null, + retrySameAccount: false, + } + } const handoffCode = result?.body?.error?.code if ( result.nativeHandoff === true && diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index 9f7d76beb..10d4aab79 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -800,7 +800,39 @@ function mockScenario(exec) { } } -export async function callGoWorker({ +function restoreCragPolicyBlock(result) { + if (!result || result.ok || result.committed || isClientCancelledResult(result)) return result + const facts = result.providerResult + if ( + facts?.schema_version !== 2 || + facts.source !== 'crag' || + facts.provider_complete !== false || + facts.observation?.policy_blocked !== true || + facts.observation?.exception?.code !== 'CRAG_CACHE_CONTROL' + ) + return result + return { + ...result, + status: 502, + terminalState: 'rejected', + cragPolicyBlock: true, + body: { + type: 'error', + error: { + type: 'api_error', + code: 'crag_policy_blocked', + message: 'Gateway kernel rejected the request cache controls before dispatch (CRAG_CACHE_CONTROL)', + details: { retryable: false }, + }, + }, + } +} + +export async function callGoWorker(options) { + return restoreCragPolicyBlock(await callGoWorkerRaw(options)) +} + +async function callGoWorkerRaw({ requestContext = null, exec, body, @@ -1003,7 +1035,11 @@ export async function callGoWorker({ } } -export async function streamGoWorker({ +export async function streamGoWorker(options) { + return restoreCragPolicyBlock(await streamGoWorkerRaw(options)) +} + +async function streamGoWorkerRaw({ requestContext = null, exec, body, diff --git a/test/unit/native-handoff.test.mjs b/test/unit/native-handoff.test.mjs index 9d491f5bb..1841f863a 100644 --- a/test/unit/native-handoff.test.mjs +++ b/test/unit/native-handoff.test.mjs @@ -6,6 +6,7 @@ import os from 'node:os' import path from 'node:path' import { createRespond, readBody } from '../../src/lib/http/respond.mjs' import { FailoverRunner } from '../../src/lib/pool/failover-runner.mjs' +import { classifyUpstreamResult } from '../../src/lib/pool/upstream-error-policy.mjs' import { RateLimitService } from '../../src/lib/pool/rate-limit-service.mjs' import { createHandleProtocol } from '../../src/lib/protocol/handle-protocol.mjs' import { callGoWorker, streamGoWorker } from '../../src/lib/transport/go-worker-client.mjs' @@ -545,6 +546,38 @@ for (const stream of [false, true]) { }) } +unixTest('CRAG cache policy rejection is returned without replay and without exposing diagnostics', async (t) => { + const rejection = { type: 'error', error: { type: 'api_error', message: 'Kernel rejected the request' } } + const facts = { + type: 'kin_provider_result', + schema_version: 2, + source: 'crag', + provider_complete: false, + usage_complete: false, + public_status: 502, + error: rejection.error, + observation: { policy_blocked: true, exception: { code: 'CRAG_CACHE_CONTROL', message: 'private observation' } }, + } + const fx = await fixture( + (_req, res) => { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.end(resultFrame(facts) + frame(rejection)) + }, + { capabilities: ['native_handoff_v1', 'provider_result_v2'] }, + ) + t.after(() => fx.close()) + const body = { model: 'claude-opus-5-5', messages: [{ role: 'user', content: 'hello' }] } + const result = await streamGoWorker(options(fx, { envelope: { body, headers: {}, stream: true } })) + assert.equal(result.terminalState, 'rejected') + assert.equal(result.body.error.code, 'crag_policy_blocked') + assert.equal(result.body.error.details.retryable, false) + assert.doesNotMatch(JSON.stringify(result.body), /private observation/) + const policy = classifyUpstreamResult(result) + assert.equal(policy.action, 'stop') + assert.equal(policy.cooldownUntil, null) + assert.equal(policy.scope, 'request') +}) + unixTest('#1139: opaque CRAG 429 returns once without restricting the account', async (t) => { const error = { type: 'rate_limit_error', message: 'Error' } const rejection = { type: 'error', error, request_id: 'req_compact_fixture' } From 060802f50e2a0bbb89aa6841b13ff129b3d3a069 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:37:53 +0800 Subject: [PATCH 5/7] fix(pool): return a rate limit for proven quota-only waits --- src/lib/pool/pool-scheduler.mjs | 8 +++++++- test/unit/pool-scheduler.test.mjs | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index 37752704b..9ccc3b089 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -352,8 +352,14 @@ function selectionSnapshot(candidates = [], available = [], extras = {}) { const waitPool = extras.waitPool || candidates const waitReasons = [...new Set((waitPool || []).map((candidate) => candidate.waitReason).filter(Boolean))] const soonest = (waitPool || []).map((candidate) => Number(candidate.availableAt) || 0).filter((value) => value > now) + const quotaLimited = + extras.reason === 'all_accounts_busy' && + waitPool.length > 0 && + waitPool.every( + (candidate) => candidate.waitReason === 'account_cooldown' && isQuotaWindowReason(candidate.cooldownReason), + ) return { - reason: extras.reason, + reason: quotaLimited ? 'pool_rate_limited' : extras.reason, wait_ms: extras.waitMs ?? 0, soonest_available_ms: soonest.length ? Math.min(...soonest) - now : null, wait_reasons: waitReasons, diff --git a/test/unit/pool-scheduler.test.mjs b/test/unit/pool-scheduler.test.mjs index 44d4f2e10..65088f0f1 100644 --- a/test/unit/pool-scheduler.test.mjs +++ b/test/unit/pool-scheduler.test.mjs @@ -547,6 +547,35 @@ test('long cooldown beyond wait budget fails immediately', async (t) => { assert.equal(selected.available, 0) }) +test('pinned quota cooldown returns rate limited with retry time without dispatching', async (t) => { + const root = project() + let pool + t.after(() => cleanupProject(root, pool)) + const repo = new RuntimeRepo() + repo.markCooldown('account-1', { vmId: 'vm-01', until: Date.now() + 60_000, reason: 'quota_5h_header' }) + pool = new PoolScheduler({ + projectRoot: root, + runtimeRepo: repo, + accountQuota: { canAccept: () => ({ ok: true }) }, + workerHealth: async () => ({ ok: true, credential: { generation: 1, has_access: true } }), + }) + let dispatched = false + const result = await new FailoverRunner({ scheduler: pool }).run({ + requestId: 'quota-cooldown-pin', + model: 'claude-test', + pinVmId: 'vm-01', + callAttempt: async () => { + dispatched = true + throw new Error('must not dispatch') + }, + }) + assert.equal(dispatched, false) + assert.equal(result.status, 429) + assert.equal(result.body.error.code, 'pool_rate_limited') + assert.ok(result.retryAfterSec > 0) + assert.deepEqual(result.body.error.details.wait_reasons, ['account_cooldown']) +}) + test('wait timeouts clamp to 1s–120s and invalid values fall back', () => { const pool = new PoolScheduler({ config: { sticky_wait_timeout_ms: 500, fallback_wait_timeout_ms: 999999 }, From 7d36b95445adc433daedcd315445f62a038517a4 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:42:55 +0800 Subject: [PATCH 6/7] fix(pool): skip quota probes for request-scoped rejections --- src/lib/pool/rate-limit-service.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/pool/rate-limit-service.mjs b/src/lib/pool/rate-limit-service.mjs index 8e9df2deb..447e5ce5b 100755 --- a/src/lib/pool/rate-limit-service.mjs +++ b/src/lib/pool/rate-limit-service.mjs @@ -15,7 +15,7 @@ export const DEFAULT_RATE_LIMIT = Object.freeze({ fallback_cooldown_min: 30, overload_cooldown_min: 10, empty_response_cooldown_sec: 60, - // Every upstream 429 asks /usage which window (if any) is spent; one ask per account per this many seconds. + // Account or model 429s can ask /usage, at most once per account per interval. usage_probe_min_interval_sec: 180, }) @@ -104,13 +104,13 @@ export class RateLimitService { * sub2api HandleUpstreamError: writes the hard column for this failure. Returns the block, or null. * The classifier decides the scope; a model / Fable / RPM 429 stays a short cooldown. * An unknown bare 429 writes no hard block: the scope is unproven. - * Every upstream 429 asks the official usage probe (throttled per account) + * A 429 not scoped to the request asks the official usage probe (throttled per account) * which window is spent: a `rejected` window then keeps the scheduler off the * account until its reset, a short limit just passes. Retry-After alone * never proves a plan window. */ handleUpstreamError({ accountId, vmId, result = {}, policy = null, now = Date.now() } = {}) { - if (!accountId || !this.runtimeRepo || result?.committed) return null + if (!accountId || !this.runtimeRepo || result?.committed || policy?.scope === 'request') return null const reason = String(policy?.reason || '') if (Number(result?.status) === 429) { this.requestUsageProbe({ accountId, vmId, now }) From efdbd4227637432dce6a1d207ebe70e9bcd61314 Mon Sep 17 00:00:00 2001 From: dofastted Date: Tue, 17 Nov 2026 05:50:11 +0800 Subject: [PATCH 7/7] fix(pool): bound account hops and stop on repeated bare 429s --- docs/PANEL_API.md | 2 ++ src/lib/pool/failover-runner.mjs | 27 ++++++++++++++- src/lib/pool/schedule-state.mjs | 2 +- src/lib/pool/upstream-error-policy.mjs | 9 ++--- test/unit/failover-runner.test.mjs | 44 ++++++++++++++++++++++++ test/unit/upstream-error-policy.test.mjs | 37 +++++++++++++++----- 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/docs/PANEL_API.md b/docs/PANEL_API.md index 6725490f4..f859ab715 100644 --- a/docs/PANEL_API.md +++ b/docs/PANEL_API.md @@ -344,6 +344,8 @@ Analytics 按官方 [`codex_thread_initialized`](https://github.com/openai/codex `failover.stream_idle_timeout_ms`(毫秒,30 秒–60 分钟,默认 180 秒):上游流连续无事件即判超时;网关流路径、各槽 `kernel.json` 的 `idle_timeout_seconds` 与 job 看门狗共用。优先级:面板该字段 → 启动时写入 `cfg.limits.stream_idle_timeout_ms` → 控制面 `.env` 的 `KIN_STREAM_IDLE_TIMEOUT` → 180 秒。保存后热写各槽 `kernel.json`;运行中 kernel 在无任务时自动重启以读取新看门狗值。 +`failover.max_accounts_per_request`(正整数,默认 `0` 不限制):单请求实际执行过的不同账号数达到上限后,不再向新账号发起推理;没执行的准入失败不计入。无配额、模型或重试时间证据的上游 429 不直接冷却账号或 Fable;同一请求在两个账号收到此类 429 时保留上游错误并停止调度。 + `compatibility.persona_preset`(`official` / `official_full` / `zero` / `custom`)和 `compatibility.cache_ttl`(`5m` / `1h`)保存后投影到每个 Claude 槽的 `vms//run/kernel.json`:`persona_preset`、`system_layout`(`zero`→`zero`,其余→`identity`)、`default_cache_ttl`。响应 `kernel_persona.updated` 是本次字节有变化的槽数。`PATCH /vms/:id` 的 `timezone` / `timezone_follow_proxy` 另把 `timezone` 热写进该槽 `kernel.json`,`timezone_sync.kernel_hot` 表示文件有变化。容器 `TZ` 不在这次写入里。Codex 槽不写。 `compatibility.crag_persona_preset`(`official_full` / `zero`,缺省 `official_full`)只给 CRAG v2 print 清洗,不投影进 `kernel.json`。v2 worker 不读 `persona_preset` 或 `CLAUDE_CODE_SYSTEM_LAYOUT`。`official_full` 只 append 调用方尾巴;`zero` 送 `crag_prompt.mode=replace`。其它值保存时 400。槽位显式 `zero` 覆盖该档;其它槽位人设在 print 上收成 append。这不是 TUI:不关内置工具,也不加 `--no-chrome`。 diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 1e7f3b017..76c29995a 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -20,7 +20,7 @@ import { hasRefreshPresence, readWorkerCredentialFile } from '../oauth/oauth-cre import { ensureWorkerCredential } from '../transport/go-worker-client.mjs' import { kernelFaults } from '../transport/rust-kernel-client.mjs' import { resolveOfficialCcInference } from '../vm/slot-engine.mjs' -import { AttemptCoordinator } from './unit-decision.mjs' +import { attachFailureDecision, AttemptCoordinator } from './unit-decision.mjs' import { SOFT_COOLDOWN_REASONS } from './pool-scheduler.mjs' // Seat wait for the one-shot repair hop when the rejected hop already spent the deadline. @@ -50,6 +50,7 @@ export const vmTimeoutTracker = { const DEFAULTS = { max_account_switches: 10, max_total_attempts: 12, + max_accounts_per_request: 0, total_retry_deadline_ms: 120000, // Same-session serialization waits out the previous turn. Cap a wedged tail // so the queue cannot sit forever. 0 disables the cap. Long enough that a @@ -684,6 +685,8 @@ export class FailoverRunner { }) const deadline = budget.deadline const excluded = budget.excluded + const accountCap = Number(this.config.max_accounts_per_request) + const bare429Accounts = new Set() // Accounts left only because the kernel had no free slot; their pins stay. const spilled = budget.spilled // session_affinity_first: a turn that spilled to another unit wrote the @@ -885,6 +888,16 @@ export class FailoverRunner { if (signal?.aborted) { return { ...clientCancelledResult(), via: 'pool-failover', ...attribution() } } + if ( + selected?.ok && + Number.isSafeInteger(accountCap) && + accountCap > 0 && + budget.executions.size >= accountCap && + !budget.executions.has(selected.accountId) + ) { + releaseStickySlots() + return preferLastResult(lastResult, lastPolicy, null, { ...attribution(), account_cap: accountCap }) + } if (selected?.familyGated && familyKey && familyVmId) { // The family home left the pool (quota, credential): the family moves // with the next bind. An exclusion this request made itself is not proof. @@ -1017,6 +1030,18 @@ export class FailoverRunner { }, this.scheduler?.accountQuota, ) + if ( + !pinVmId && + selected.accountId && + policy.reason === 'rate_limited_unknown' && + policy.action === 'continue' && + executionsOf(result) > 0 + ) { + bare429Accounts.add(selected.accountId) + if (bare429Accounts.size >= 2) { + policy = attachFailureDecision({ scope: 'request', action: 'stop', reason: 'rate_limited_request' }) + } + } if (knownSearchRepair && policy.reason === 'search_repairable') { // Repeating the identical flatten cannot fix a second rejection. this.searchRepairs.delete(repairKey) diff --git a/src/lib/pool/schedule-state.mjs b/src/lib/pool/schedule-state.mjs index bf85424be..130303aef 100644 --- a/src/lib/pool/schedule-state.mjs +++ b/src/lib/pool/schedule-state.mjs @@ -93,7 +93,7 @@ const STATELESS = [ /_repairable$/, /^(concurrency_limit|concurrency_disabled|rpm_limit|slot_busy|quota_band_inflight|refresh_pending)$/, /^(model_cooldown|fable_cooldown|fable_concurrency|fable_timeout|fable_plan_denied|fable_rate_limited|fable_requires_max|model_not_allowed|platform_mismatch|codex_vm)$/, - /^(quota_guard_request_too_large|no_eligible_accounts|pool_overload_cooldown|pool_rate_limited|vm_unschedulable)$/, + /^(quota_guard_request_too_large|no_eligible_accounts|pool_overload_cooldown|pool_rate_limited|rate_limited_request|vm_unschedulable)$/, /^(wrap_connection_error|worker_transport_failure|worker_timeout|worker_process_exit|upstream_network_error|stream_interrupted|empty_response|incomplete_assistant|downstream_committed_or_incomplete|provider_timeout|client_cancelled|oauth_unconfirmed)$/, // Seat placement labels (why this VM was picked), not a slot condition. /^sticky$/, diff --git a/src/lib/pool/upstream-error-policy.mjs b/src/lib/pool/upstream-error-policy.mjs index b843ca61b..c797d79f3 100644 --- a/src/lib/pool/upstream-error-policy.mjs +++ b/src/lib/pool/upstream-error-policy.mjs @@ -656,8 +656,9 @@ function classifyUpstreamResultRaw( cooldownUntil: accountLimitUntil(reset, usage, now, message), } } - if (isFableWindowLimit(result.headers) || isFableModel(model)) { - const fableReset = fableResetFromHeaders(result.headers, now) || reset + if (isFableWindowLimit(result.headers) || isFableWindowLimit(upstreamHeaders)) { + const fableReset = + fableResetFromHeaders(result.headers, now) || fableResetFromHeaders(upstreamHeaders, now) || reset return { scope: 'model', action: 'continue-and-cooldown', @@ -692,9 +693,9 @@ function classifyUpstreamResultRaw( } return { scope: 'account', - action: 'continue-and-cooldown', + action: reset > now ? 'continue-and-cooldown' : 'continue', reason: 'rate_limited_unknown', - cooldownUntil: reset || now + 60_000, + cooldownUntil: reset > now ? reset : null, retrySameAccount: false, } } diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index 55b5898f7..a8380c4a6 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -566,6 +566,50 @@ test('request-scoped entitlement error does not walk the pool', async () => { assert.equal(scheduler.cooldowns.length, 0) }) +test('bare 429s on two accounts stop without cooling accounts or Fable', async () => { + for (const model of ['claude-sonnet-test', 'claude-fable-test']) { + const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3)]) + const runner = new FailoverRunner({ scheduler }) + const result = await runner.run({ + requestId: `req-bare-${model}`, + model, + callAttempt: () => ({ + ok: false, + status: 429, + terminalState: 'rejected', + body: { error: { type: 'rate_limit_error', message: 'Error' } }, + }), + }) + assert.equal(result.status, 429) + assert.equal(result.policy.reason, 'rate_limited_request') + assert.equal(result.attemptCount, 2) + assert.equal(scheduler.selectCalls, 2) + assert.deepEqual(scheduler.cooldowns, []) + assert.deepEqual(scheduler.releasedAccounts, []) + } +}) + +test('the optional account cap counts executions, not admission misses', async () => { + const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3), candidate(4)]) + const runner = new FailoverRunner({ scheduler, config: { max_accounts_per_request: 1 } }) + const seen = [] + const result = await runner.run({ + requestId: 'req-account-cap', + model: 'claude-sonnet-test', + callAttempt: ({ candidate: selected }) => { + seen.push(selected.accountId) + if (selected.accountId === 'account-1') { + return { ok: false, status: 503, upstreamExecutions: 0, body: { error: { code: 'slot_busy' } } } + } + return { ok: false, status: 529, terminalState: 'rejected', body: { error: { message: 'Overloaded' } } } + }, + }) + assert.deepEqual(seen, ['account-1', 'account-2']) + assert.equal(result.account_cap, 1) + assert.equal(result.attemptCount, 1) + assert.equal(result.status, 529) +}) + test('committed realtime stream failure never switches accounts', async () => { const scheduler = new Scheduler([candidate(1), candidate(2)]) const runner = new FailoverRunner({ scheduler }) diff --git a/test/unit/upstream-error-policy.test.mjs b/test/unit/upstream-error-policy.test.mjs index 517c7ef0d..4fd591232 100644 --- a/test/unit/upstream-error-policy.test.mjs +++ b/test/unit/upstream-error-policy.test.mjs @@ -96,19 +96,26 @@ test('unified account 429 cools account until authoritative reset', () => { }) test('#163: a bare 429 is unknown for every model; the model name never decides scope', () => { - for (const model of ['claude-sonnet-4-5-20250929', 'claude-haiku-4-5', 'claude-opus-4-6', 'unknown-model']) { + for (const model of [ + 'claude-sonnet-4-5-20250929', + 'claude-haiku-4-5', + 'claude-opus-4-6', + 'claude-fable-5-2', + 'unknown-model', + ]) { const policy = classifyUpstreamResult( { status: 429, body: { error: { type: 'rate_limit_error', message: 'Rate limited' } }, headers: {} }, { model, now: 1000 }, ) assert.equal(policy.reason, 'rate_limited_unknown', model) assert.equal(policy.scope, 'account', model) - assert.equal(policy.cooldownUntil, 61_000, model) + assert.equal(policy.action, 'continue', model) + assert.equal(policy.cooldownUntil, null, model) assert.equal(policy.circuit, undefined, model) } }) -test('#1139: real Fable rate evidence and unrelated bare 429s still cool the model', () => { +test('#1139: only a proven Fable window cools the Fable family', () => { const error = { type: 'rate_limit_error', message: 'Error' } const body = { type: 'error', error } const facts = { @@ -119,21 +126,33 @@ test('#1139: real Fable rate evidence and unrelated bare 429s still cool the mod error, } const result = { status: 429, body, providerResult: facts } - for (const [label, failure] of [ + for (const [label, failure, reason, action] of [ [ 'native limit header', { ...result, providerResult: { ...facts, response_headers: { 'Anthropic-RateLimit-Unified-7d_oi-Status': 'rejected' } }, }, + 'fable_rate_limited', + 'continue-and-cooldown', + ], + [ + 'forwarded Retry-After', + { ...result, headers: { 'Retry-After': '7' } }, + 'rate_limited_unknown', + 'continue-and-cooldown', + ], + ['no native facts', { status: 429, body }, 'rate_limited_unknown', 'continue'], + [ + 'different error text', + { ...result, body: { ...body, error: { ...error, message: 'Rate limited' } } }, + 'rate_limited_unknown', + 'continue', ], - ['forwarded Retry-After', { ...result, headers: { 'Retry-After': '7' } }], - ['no native facts', { status: 429, body }], - ['different error text', { ...result, body: { ...body, error: { ...error, message: 'Rate limited' } } }], ]) { const policy = classifyUpstreamResult(failure, { model: 'claude-fable-5-2', now: 1000 }) - assert.equal(policy.reason, 'fable_rate_limited', label) - assert.equal(policy.action, 'continue-and-cooldown', label) + assert.equal(policy.reason, reason, label) + assert.equal(policy.action, action, label) } })