From b69e3178b73e84b83dc97e09a270523f76dd171b Mon Sep 17 00:00:00 2001 From: panjinhe Date: Mon, 16 Nov 2026 01:42:51 -0800 Subject: [PATCH 1/3] fix(crag): drop caller cache_control on model-emitted history blocks CRAG 3.0.7 blocks a breakpoint on an assistant block that carries `caller` (CRAG_CACHE_CONTROL 'Caller cache anchor changed in native history'). Claude Code 2.1.325 puts one on the previous tool_use next to the trailing tool_result; the official API accepts it. Drop only that anchor until the kernel accepts it. --- src/lib/protocol/crag-print-prompt.mjs | 28 +++++++++++++ test/unit/crag-native-anchor.test.mjs | 54 ++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 test/unit/crag-native-anchor.test.mjs diff --git a/src/lib/protocol/crag-print-prompt.mjs b/src/lib/protocol/crag-print-prompt.mjs index 577c7d338..6833f066b 100644 --- a/src/lib/protocol/crag-print-prompt.mjs +++ b/src/lib/protocol/crag-print-prompt.mjs @@ -389,6 +389,33 @@ function foldSummarySystemTurns(messages) { return { messages: [...folded.messages, ...tail], outputConfig } } +/** + * Temporary: CRAG 3.0.7 blocks a caller cache_control placed on a model-emitted + * history block (one carrying `caller`, e.g. tool_use) with CRAG_CACHE_CONTROL + * "Caller cache anchor changed in native history". Claude Code 2.1.325 puts a + * breakpoint on the previous assistant tool_use as well as the trailing + * tool_result; the official API accepts both. Drop only the one on the native + * block: the trailing breakpoint still covers the whole prefix. + */ +export function dropNativeBlockCacheAnchors(messages) { + if (!Array.isArray(messages)) return messages + let changed = false + const out = messages.map((message) => { + if (message?.role !== 'assistant' || !Array.isArray(message.content)) return message + if (!message.content.some((block) => block?.caller != null && block.cache_control != null)) return message + changed = true + return { + ...message, + content: message.content.map((block) => { + if (block?.caller == null || block.cache_control == null) return block + const { cache_control: _dropped, ...rest } = block + return rest + }), + } + }) + return changed ? out : messages +} + /** * @param {object} body caller body, before cli-node persona injection * @param {{ preset?: 'official_full' | 'zero', officialPassthrough?: boolean, capabilities?: string[], agentSummary?: boolean, foldSystemMessages?: boolean }} [options] @@ -404,6 +431,7 @@ export function prepareCragPrintBody( } = {}, ) { const out = { ...(body || {}) } + if (Array.isArray(out.messages)) out.messages = dropNativeBlockCacheAnchors(out.messages) if (agentSummary) out.messages = readOnlySummaryResults(out.messages) if (foldSystemMessages) { const folded = agentSummary ? foldSummarySystemTurns(out.messages) : foldSystemTurns(out.messages) diff --git a/test/unit/crag-native-anchor.test.mjs b/test/unit/crag-native-anchor.test.mjs new file mode 100644 index 000000000..210d05122 --- /dev/null +++ b/test/unit/crag-native-anchor.test.mjs @@ -0,0 +1,54 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { dropNativeBlockCacheAnchors, prepareCragPrintBody } from '../../src/lib/protocol/crag-print-prompt.mjs' + +const CC = { type: 'ephemeral' } +const history = () => [ + { role: 'user', content: [{ type: 'text', text: 'run ls' }] }, + { + role: 'assistant', + content: [ + { type: 'thinking', thinking: 'x', signature: 'sig' }, + { + type: 'tool_use', + id: 'toolu_1', + name: 'Bash', + input: { command: 'ls' }, + caller: { type: 'direct' }, + cache_control: CC, + }, + ], + }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'a', cache_control: CC }] }, +] + +test('drops cache_control only on model-emitted (caller) assistant blocks', () => { + const input = history() + const out = dropNativeBlockCacheAnchors(input) + const toolUse = out[1].content[1] + assert.equal(toolUse.cache_control, undefined) + assert.deepEqual(toolUse.caller, { type: 'direct' }) + assert.equal(toolUse.id, 'toolu_1') + assert.deepEqual(out[1].content[0], input[1].content[0]) + assert.deepEqual(out[2].content[0].cache_control, CC, 'trailing tool_result breakpoint stays') + assert.equal(input[1].content[1].cache_control, CC, 'input not mutated') +}) + +test('leaves blocks without caller and non-assistant turns alone', () => { + const input = history() + delete input[1].content[1].caller + assert.equal(dropNativeBlockCacheAnchors(input), input) + const user = [{ role: 'user', content: [{ type: 'text', text: 'x', caller: { type: 'direct' }, cache_control: CC }] }] + assert.equal(dropNativeBlockCacheAnchors(user), user) +}) + +test('prepareCragPrintBody applies it', () => { + const out = prepareCragPrintBody( + { model: 'claude-opus-5-5', max_tokens: 10, messages: history() }, + { foldSystemMessages: false }, + ) + const toolUse = out.messages[1].content.find((block) => block.type === 'tool_use') + assert.equal(toolUse.id, 'toolu_1') + assert.equal(toolUse.cache_control, undefined) + assert.deepEqual(out.messages[2].content[0].cache_control, CC) +}) From 73806c319543eca5805d7ad513e66016d7bd7ed6 Mon Sep 17 00:00:00 2001 From: panjinhe Date: Mon, 16 Nov 2026 01:30:20 -0800 Subject: [PATCH 2/3] fix(pool): stop failover on CRAG CRAG_CACHE_CONTROL policy block CRAG blocks the body before dispatch ('Caller cache anchor changed in native history'). vm2api saw an empty incomplete hop and replayed it on up to 17 slots, all of which run the same check. Surface the block as crag_policy_blocked and classify it as a request-level stop. --- src/lib/pool/schedule-state.mjs | 2 +- src/lib/pool/upstream-error-policy.mjs | 10 ++++ src/lib/transport/go-worker-client.mjs | 57 +++++++++++++++++++++- test/unit/crag-policy-block.test.mjs | 65 ++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 test/unit/crag-policy-block.test.mjs diff --git a/src/lib/pool/schedule-state.mjs b/src/lib/pool/schedule-state.mjs index 5db060138..6352f6371 100644 --- a/src/lib/pool/schedule-state.mjs +++ b/src/lib/pool/schedule-state.mjs @@ -88,7 +88,7 @@ const RESTRICTED_NEW_SESSION = [ * request waits, and local transport errors. */ const STATELESS = [ - /^(distill_blocked|policy_blocked|refusal_guard|hard_regex.*|jev.*|decision_model.*)$/, + /^(distill_blocked|policy_blocked|crag_policy_blocked|refusal_guard|hard_regex.*|jev.*|decision_model.*)$/, /^(content_filter_refusal|content_policy_refusal|usage_policy_refusal|invalid_request|entitlement_required|http_\d+)$/, /_repairable$/, /^(concurrency_limit|concurrency_disabled|rpm_limit|slot_busy|quota_band_inflight|refresh_pending)$/, diff --git a/src/lib/pool/upstream-error-policy.mjs b/src/lib/pool/upstream-error-policy.mjs index ab759a15f..c56e54d51 100644 --- a/src/lib/pool/upstream-error-policy.mjs +++ b/src/lib/pool/upstream-error-policy.mjs @@ -341,6 +341,16 @@ function classifyUpstreamResultRaw( retrySameAccount: false, } } + if (result?.cragPolicyBlock && !result.committed) { + // CRAG rejected the body itself before dispatch; another slot runs the same check. + return { + scope: 'request', + action: 'stop', + reason: 'crag_policy_blocked', + cooldownUntil: null, + retrySameAccount: false, + } + } const handoffCode = result?.body?.error?.code if ( result.nativeHandoff === true && diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index d1881aacd..a21cf4121 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -800,7 +800,7 @@ function mockScenario(exec) { } } -export async function callGoWorker({ +async function callGoWorkerRaw({ requestContext = null, exec, body, @@ -1002,7 +1002,7 @@ export async function callGoWorker({ } } -export async function streamGoWorker({ +async function streamGoWorkerRaw({ requestContext = null, exec, body, @@ -1959,3 +1959,56 @@ export async function countTokensViaWorker(exec, { body, headers = {}, timeoutMs } return runSlotOauth(exec, 'count-tokens', { body, headers, timeoutMs }) } + +/** CRAG policy blocks that depend only on the request body: every slot rejects them the same way. */ +const DETERMINISTIC_CRAG_BLOCKS = new Set(['CRAG_CACHE_CONTROL']) + +export function cragDeterministicBlock(result) { + const facts = result?.providerResult + const exception = facts?.observation?.exception + if ( + !result || + result.ok || + result.committed || + facts?.schema_version !== 2 || + facts.source !== 'crag' || + facts.provider_complete === true || + facts.observation?.policy_blocked !== true || + !DETERMINISTIC_CRAG_BLOCKS.has(exception?.code) + ) + return null + return { code: exception.code, message: String(exception.message || '').slice(0, 300) } +} + +/** + * Surface a deterministic CRAG policy block as itself instead of an empty hop, + * so the pool stops instead of replaying the same body on every slot. + */ +export function surfaceCragPolicyBlock(result) { + const block = cragDeterministicBlock(result) + if (!block) return result + return { + ...result, + ok: false, + status: 502, + terminalState: 'rejected', + cragPolicyBlock: block, + body: { + type: 'error', + error: { + type: 'api_error', + code: 'crag_policy_blocked', + message: `Gateway kernel rejected the request before dispatch (${block.code}: ${block.message})`, + details: { retryable: false }, + }, + }, + } +} + +export async function callGoWorker(opts = {}) { + return surfaceCragPolicyBlock(await callGoWorkerRaw(opts)) +} + +export async function streamGoWorker(opts = {}) { + return surfaceCragPolicyBlock(await streamGoWorkerRaw(opts)) +} diff --git a/test/unit/crag-policy-block.test.mjs b/test/unit/crag-policy-block.test.mjs new file mode 100644 index 000000000..c972975c5 --- /dev/null +++ b/test/unit/crag-policy-block.test.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { surfaceCragPolicyBlock } from '../../src/lib/transport/go-worker-client.mjs' +import { classifyUpstreamResult, shouldContinue } from '../../src/lib/pool/upstream-error-policy.mjs' + +const blocked = (over = {}, result = {}) => ({ + ok: false, + status: 502, + via: 'go-worker-stream', + committed: false, + terminalState: 'incomplete', + body: { type: 'message', role: 'assistant', content: [] }, + nativeHandoff: false, + providerResult: { + _internal_v2: true, + schema_version: 2, + type: 'kin_provider_result', + source: 'crag', + provider_complete: false, + usage_complete: false, + public_status: 502, + delivery_error: { status: 502, type: 'crag_policy_error' }, + error: { type: 'crag_policy_error', message: 'Request was blocked before provider dispatch' }, + observation: { + source: 'crag', + policy_blocked: true, + capture_failed: false, + exception: { + name: 'Error', + code: 'CRAG_CACHE_CONTROL', + message: 'Caller cache anchor changed in native history', + }, + }, + ...over, + }, + ...result, +}) + +test('CRAG cache-anchor block is surfaced and stops failover', () => { + const before = classifyUpstreamResult(blocked()) + assert.equal(shouldContinue(before), true, 'baseline: an unmarked block looks like a retryable empty hop') + + const out = surfaceCragPolicyBlock(blocked()) + assert.equal(out.status, 502) + assert.equal(out.terminalState, 'rejected') + assert.equal(out.body.error.code, 'crag_policy_blocked') + assert.match(out.body.error.message, /CRAG_CACHE_CONTROL: Caller cache anchor changed/) + const policy = classifyUpstreamResult(out) + assert.equal(policy.action, 'stop') + assert.equal(policy.reason, 'crag_policy_blocked') + assert.equal(shouldContinue(policy), false) +}) + +test('other results are untouched', () => { + for (const input of [ + blocked({ observation: { policy_blocked: true, exception: { code: 'CRAG_OTHER' } } }), + blocked({ source: 'provider_sse' }), + blocked({ provider_complete: true }), + blocked({}, { committed: true }), + blocked({}, { ok: true, status: 200 }), + blocked({}, { providerResult: null }), + ]) { + assert.equal(surfaceCragPolicyBlock(input), input) + } +}) From a9e7b6f8278f308e5e8ec02f246a8b9dfca9459b Mon Sep 17 00:00:00 2001 From: panjinhe Date: Mon, 16 Nov 2026 09:21:44 -0800 Subject: [PATCH 3/3] fix(crag): drop cache anchor on a tool_reference tool_result before CRAG print CRAG 3.0.7 rejects a breakpoint on a ToolSearch result (tool_result holding a tool_reference) with CRAG_CACHE_CONTROL "Caller cache anchor changed in native history", same as on a caller-marked assistant block. Replayed the blocked request: 502 as sent, 200 with that breakpoint removed. --- src/lib/protocol/crag-print-prompt.mjs | 22 +++++++++++++-- test/unit/crag-native-anchor.test.mjs | 38 ++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/lib/protocol/crag-print-prompt.mjs b/src/lib/protocol/crag-print-prompt.mjs index 6833f066b..c5269f593 100644 --- a/src/lib/protocol/crag-print-prompt.mjs +++ b/src/lib/protocol/crag-print-prompt.mjs @@ -396,18 +396,34 @@ function foldSummarySystemTurns(messages) { * breakpoint on the previous assistant tool_use as well as the trailing * tool_result; the official API accepts both. Drop only the one on the native * block: the trailing breakpoint still covers the whole prefix. + * The same block hits a breakpoint on a tool_result whose content holds a + * tool_reference (a ToolSearch result): CRAG treats it as native history too. + * That one is dropped as well; the turn then writes no new cache entry, and the + * next turn's trailing breakpoint covers it again. */ +function isNativeAnchorBlock(role, block) { + if (block?.cache_control == null) return false + if (role === 'assistant') return block.caller != null + return ( + role === 'user' && + block.type === 'tool_result' && + Array.isArray(block.content) && + block.content.some((item) => item?.type === 'tool_reference') + ) +} + export function dropNativeBlockCacheAnchors(messages) { if (!Array.isArray(messages)) return messages let changed = false const out = messages.map((message) => { - if (message?.role !== 'assistant' || !Array.isArray(message.content)) return message - if (!message.content.some((block) => block?.caller != null && block.cache_control != null)) return message + const role = message?.role + if (!Array.isArray(message?.content)) return message + if (!message.content.some((block) => isNativeAnchorBlock(role, block))) return message changed = true return { ...message, content: message.content.map((block) => { - if (block?.caller == null || block.cache_control == null) return block + if (!isNativeAnchorBlock(role, block)) return block const { cache_control: _dropped, ...rest } = block return rest }), diff --git a/test/unit/crag-native-anchor.test.mjs b/test/unit/crag-native-anchor.test.mjs index 210d05122..2ea22d91c 100644 --- a/test/unit/crag-native-anchor.test.mjs +++ b/test/unit/crag-native-anchor.test.mjs @@ -52,3 +52,41 @@ test('prepareCragPrintBody applies it', () => { assert.equal(toolUse.cache_control, undefined) assert.deepEqual(out.messages[2].content[0].cache_control, CC) }) + +test('drops cache_control on a tool_result that carries a tool_reference', () => { + const input = [ + { role: 'user', content: [{ type: 'text', text: 'plan' }] }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'toolu_s', name: 'ToolSearch', input: { query: 'x' }, caller: { type: 'direct' } }, + ], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_s', + content: [{ type: 'tool_reference', tool_name: 'ExitPlanMode' }], + cache_control: CC, + }, + ], + }, + ] + const out = dropNativeBlockCacheAnchors(input) + assert.equal(out[2].content[0].cache_control, undefined) + assert.deepEqual(out[2].content[0].content, input[2].content[0].content) + assert.deepEqual(input[2].content[0].cache_control, CC, 'input not mutated') +}) + +test('keeps cache_control on an ordinary tool_result and on user text', () => { + const input = [ + { + role: 'user', + content: [{ type: 'tool_result', tool_use_id: 't', content: [{ type: 'text', text: 'a' }], cache_control: CC }], + }, + { role: 'user', content: [{ type: 'text', text: 'go', cache_control: CC }] }, + ] + assert.equal(dropNativeBlockCacheAnchors(input), input) +})