diff --git a/src/lib/pool/schedule-state.mjs b/src/lib/pool/schedule-state.mjs index 5db060138..6352f6371 100644 --- a/src/lib/pool/schedule-state.mjs +++ b/src/lib/pool/schedule-state.mjs @@ -88,7 +88,7 @@ const RESTRICTED_NEW_SESSION = [ * request waits, and local transport errors. */ const STATELESS = [ - /^(distill_blocked|policy_blocked|refusal_guard|hard_regex.*|jev.*|decision_model.*)$/, + /^(distill_blocked|policy_blocked|crag_policy_blocked|refusal_guard|hard_regex.*|jev.*|decision_model.*)$/, /^(content_filter_refusal|content_policy_refusal|usage_policy_refusal|invalid_request|entitlement_required|http_\d+)$/, /_repairable$/, /^(concurrency_limit|concurrency_disabled|rpm_limit|slot_busy|quota_band_inflight|refresh_pending)$/, diff --git a/src/lib/pool/upstream-error-policy.mjs b/src/lib/pool/upstream-error-policy.mjs index ab759a15f..c56e54d51 100644 --- a/src/lib/pool/upstream-error-policy.mjs +++ b/src/lib/pool/upstream-error-policy.mjs @@ -341,6 +341,16 @@ function classifyUpstreamResultRaw( retrySameAccount: false, } } + if (result?.cragPolicyBlock && !result.committed) { + // CRAG rejected the body itself before dispatch; another slot runs the same check. + return { + scope: 'request', + action: 'stop', + reason: 'crag_policy_blocked', + cooldownUntil: null, + retrySameAccount: false, + } + } const handoffCode = result?.body?.error?.code if ( result.nativeHandoff === true && diff --git a/src/lib/protocol/crag-print-prompt.mjs b/src/lib/protocol/crag-print-prompt.mjs index 577c7d338..c5269f593 100644 --- a/src/lib/protocol/crag-print-prompt.mjs +++ b/src/lib/protocol/crag-print-prompt.mjs @@ -389,6 +389,49 @@ function foldSummarySystemTurns(messages) { return { messages: [...folded.messages, ...tail], outputConfig } } +/** + * Temporary: CRAG 3.0.7 blocks a caller cache_control placed on a model-emitted + * history block (one carrying `caller`, e.g. tool_use) with CRAG_CACHE_CONTROL + * "Caller cache anchor changed in native history". Claude Code 2.1.325 puts a + * breakpoint on the previous assistant tool_use as well as the trailing + * tool_result; the official API accepts both. Drop only the one on the native + * block: the trailing breakpoint still covers the whole prefix. + * The same block hits a breakpoint on a tool_result whose content holds a + * tool_reference (a ToolSearch result): CRAG treats it as native history too. + * That one is dropped as well; the turn then writes no new cache entry, and the + * next turn's trailing breakpoint covers it again. + */ +function isNativeAnchorBlock(role, block) { + if (block?.cache_control == null) return false + if (role === 'assistant') return block.caller != null + return ( + role === 'user' && + block.type === 'tool_result' && + Array.isArray(block.content) && + block.content.some((item) => item?.type === 'tool_reference') + ) +} + +export function dropNativeBlockCacheAnchors(messages) { + if (!Array.isArray(messages)) return messages + let changed = false + const out = messages.map((message) => { + const role = message?.role + if (!Array.isArray(message?.content)) return message + if (!message.content.some((block) => isNativeAnchorBlock(role, block))) return message + changed = true + return { + ...message, + content: message.content.map((block) => { + if (!isNativeAnchorBlock(role, block)) return block + const { cache_control: _dropped, ...rest } = block + return rest + }), + } + }) + return changed ? out : messages +} + /** * @param {object} body caller body, before cli-node persona injection * @param {{ preset?: 'official_full' | 'zero', officialPassthrough?: boolean, capabilities?: string[], agentSummary?: boolean, foldSystemMessages?: boolean }} [options] @@ -404,6 +447,7 @@ export function prepareCragPrintBody( } = {}, ) { const out = { ...(body || {}) } + if (Array.isArray(out.messages)) out.messages = dropNativeBlockCacheAnchors(out.messages) if (agentSummary) out.messages = readOnlySummaryResults(out.messages) if (foldSystemMessages) { const folded = agentSummary ? foldSummarySystemTurns(out.messages) : foldSystemTurns(out.messages) diff --git a/src/lib/transport/go-worker-client.mjs b/src/lib/transport/go-worker-client.mjs index d1881aacd..a21cf4121 100644 --- a/src/lib/transport/go-worker-client.mjs +++ b/src/lib/transport/go-worker-client.mjs @@ -800,7 +800,7 @@ function mockScenario(exec) { } } -export async function callGoWorker({ +async function callGoWorkerRaw({ requestContext = null, exec, body, @@ -1002,7 +1002,7 @@ export async function callGoWorker({ } } -export async function streamGoWorker({ +async function streamGoWorkerRaw({ requestContext = null, exec, body, @@ -1959,3 +1959,56 @@ export async function countTokensViaWorker(exec, { body, headers = {}, timeoutMs } return runSlotOauth(exec, 'count-tokens', { body, headers, timeoutMs }) } + +/** CRAG policy blocks that depend only on the request body: every slot rejects them the same way. */ +const DETERMINISTIC_CRAG_BLOCKS = new Set(['CRAG_CACHE_CONTROL']) + +export function cragDeterministicBlock(result) { + const facts = result?.providerResult + const exception = facts?.observation?.exception + if ( + !result || + result.ok || + result.committed || + facts?.schema_version !== 2 || + facts.source !== 'crag' || + facts.provider_complete === true || + facts.observation?.policy_blocked !== true || + !DETERMINISTIC_CRAG_BLOCKS.has(exception?.code) + ) + return null + return { code: exception.code, message: String(exception.message || '').slice(0, 300) } +} + +/** + * Surface a deterministic CRAG policy block as itself instead of an empty hop, + * so the pool stops instead of replaying the same body on every slot. + */ +export function surfaceCragPolicyBlock(result) { + const block = cragDeterministicBlock(result) + if (!block) return result + return { + ...result, + ok: false, + status: 502, + terminalState: 'rejected', + cragPolicyBlock: block, + body: { + type: 'error', + error: { + type: 'api_error', + code: 'crag_policy_blocked', + message: `Gateway kernel rejected the request before dispatch (${block.code}: ${block.message})`, + details: { retryable: false }, + }, + }, + } +} + +export async function callGoWorker(opts = {}) { + return surfaceCragPolicyBlock(await callGoWorkerRaw(opts)) +} + +export async function streamGoWorker(opts = {}) { + return surfaceCragPolicyBlock(await streamGoWorkerRaw(opts)) +} diff --git a/test/unit/crag-native-anchor.test.mjs b/test/unit/crag-native-anchor.test.mjs new file mode 100644 index 000000000..2ea22d91c --- /dev/null +++ b/test/unit/crag-native-anchor.test.mjs @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { dropNativeBlockCacheAnchors, prepareCragPrintBody } from '../../src/lib/protocol/crag-print-prompt.mjs' + +const CC = { type: 'ephemeral' } +const history = () => [ + { role: 'user', content: [{ type: 'text', text: 'run ls' }] }, + { + role: 'assistant', + content: [ + { type: 'thinking', thinking: 'x', signature: 'sig' }, + { + type: 'tool_use', + id: 'toolu_1', + name: 'Bash', + input: { command: 'ls' }, + caller: { type: 'direct' }, + cache_control: CC, + }, + ], + }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'a', cache_control: CC }] }, +] + +test('drops cache_control only on model-emitted (caller) assistant blocks', () => { + const input = history() + const out = dropNativeBlockCacheAnchors(input) + const toolUse = out[1].content[1] + assert.equal(toolUse.cache_control, undefined) + assert.deepEqual(toolUse.caller, { type: 'direct' }) + assert.equal(toolUse.id, 'toolu_1') + assert.deepEqual(out[1].content[0], input[1].content[0]) + assert.deepEqual(out[2].content[0].cache_control, CC, 'trailing tool_result breakpoint stays') + assert.equal(input[1].content[1].cache_control, CC, 'input not mutated') +}) + +test('leaves blocks without caller and non-assistant turns alone', () => { + const input = history() + delete input[1].content[1].caller + assert.equal(dropNativeBlockCacheAnchors(input), input) + const user = [{ role: 'user', content: [{ type: 'text', text: 'x', caller: { type: 'direct' }, cache_control: CC }] }] + assert.equal(dropNativeBlockCacheAnchors(user), user) +}) + +test('prepareCragPrintBody applies it', () => { + const out = prepareCragPrintBody( + { model: 'claude-opus-5-5', max_tokens: 10, messages: history() }, + { foldSystemMessages: false }, + ) + const toolUse = out.messages[1].content.find((block) => block.type === 'tool_use') + assert.equal(toolUse.id, 'toolu_1') + assert.equal(toolUse.cache_control, undefined) + assert.deepEqual(out.messages[2].content[0].cache_control, CC) +}) + +test('drops cache_control on a tool_result that carries a tool_reference', () => { + const input = [ + { role: 'user', content: [{ type: 'text', text: 'plan' }] }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'toolu_s', name: 'ToolSearch', input: { query: 'x' }, caller: { type: 'direct' } }, + ], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_s', + content: [{ type: 'tool_reference', tool_name: 'ExitPlanMode' }], + cache_control: CC, + }, + ], + }, + ] + const out = dropNativeBlockCacheAnchors(input) + assert.equal(out[2].content[0].cache_control, undefined) + assert.deepEqual(out[2].content[0].content, input[2].content[0].content) + assert.deepEqual(input[2].content[0].cache_control, CC, 'input not mutated') +}) + +test('keeps cache_control on an ordinary tool_result and on user text', () => { + const input = [ + { + role: 'user', + content: [{ type: 'tool_result', tool_use_id: 't', content: [{ type: 'text', text: 'a' }], cache_control: CC }], + }, + { role: 'user', content: [{ type: 'text', text: 'go', cache_control: CC }] }, + ] + assert.equal(dropNativeBlockCacheAnchors(input), input) +}) diff --git a/test/unit/crag-policy-block.test.mjs b/test/unit/crag-policy-block.test.mjs new file mode 100644 index 000000000..c972975c5 --- /dev/null +++ b/test/unit/crag-policy-block.test.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { surfaceCragPolicyBlock } from '../../src/lib/transport/go-worker-client.mjs' +import { classifyUpstreamResult, shouldContinue } from '../../src/lib/pool/upstream-error-policy.mjs' + +const blocked = (over = {}, result = {}) => ({ + ok: false, + status: 502, + via: 'go-worker-stream', + committed: false, + terminalState: 'incomplete', + body: { type: 'message', role: 'assistant', content: [] }, + nativeHandoff: false, + providerResult: { + _internal_v2: true, + schema_version: 2, + type: 'kin_provider_result', + source: 'crag', + provider_complete: false, + usage_complete: false, + public_status: 502, + delivery_error: { status: 502, type: 'crag_policy_error' }, + error: { type: 'crag_policy_error', message: 'Request was blocked before provider dispatch' }, + observation: { + source: 'crag', + policy_blocked: true, + capture_failed: false, + exception: { + name: 'Error', + code: 'CRAG_CACHE_CONTROL', + message: 'Caller cache anchor changed in native history', + }, + }, + ...over, + }, + ...result, +}) + +test('CRAG cache-anchor block is surfaced and stops failover', () => { + const before = classifyUpstreamResult(blocked()) + assert.equal(shouldContinue(before), true, 'baseline: an unmarked block looks like a retryable empty hop') + + const out = surfaceCragPolicyBlock(blocked()) + assert.equal(out.status, 502) + assert.equal(out.terminalState, 'rejected') + assert.equal(out.body.error.code, 'crag_policy_blocked') + assert.match(out.body.error.message, /CRAG_CACHE_CONTROL: Caller cache anchor changed/) + const policy = classifyUpstreamResult(out) + assert.equal(policy.action, 'stop') + assert.equal(policy.reason, 'crag_policy_blocked') + assert.equal(shouldContinue(policy), false) +}) + +test('other results are untouched', () => { + for (const input of [ + blocked({ observation: { policy_blocked: true, exception: { code: 'CRAG_OTHER' } } }), + blocked({ source: 'provider_sse' }), + blocked({ provider_complete: true }), + blocked({}, { committed: true }), + blocked({}, { ok: true, status: 200 }), + blocked({}, { providerResult: null }), + ]) { + assert.equal(surfaceCragPolicyBlock(input), input) + } +})