From fd260624e6cfd52049735bb7ce6bef2a0aeaafb5 Mon Sep 17 00:00:00 2001 From: panjinhe Date: Mon, 16 Nov 2026 01:43:18 -0800 Subject: [PATCH] fix(pool): cap accounts per request, scope bare 429 by evidence, keep failed hops and classifier bodies in debug log 08:50 incident: one auto-mode classifier request got a bare 429 on 8 accounts in a row; each 429 cooled its whole account for 60s and the pool went eligible=0. - failover.max_accounts_per_request (0 = unlimited): hard stop after N accounts ran - bare 429 on bare_429_request_scope_after (2) accounts of one request -> request-scope stop, nobody cooled - a bare 429 cools the account only after bare_429_account_sessions (2) distinct sessions hit it within bare_429_window_ms; otherwise it only steps aside for the request - failed hops (status, rate-limit headers, body, provider facts) go into the debug record as hop_errors - classifier requests keep inbound/outbound bodies in debug mode --- src/lib/admin/request-log.mjs | 27 +++-- src/lib/pool/failover-runner.mjs | 155 ++++++++++++++++++++++++++++- test/unit/failover-runner.test.mjs | 96 ++++++++++++++++++ test/unit/request-log.test.mjs | 26 +++-- 4 files changed, 288 insertions(+), 16 deletions(-) diff --git a/src/lib/admin/request-log.mjs b/src/lib/admin/request-log.mjs index 0fd05e71f..8aa133042 100644 --- a/src/lib/admin/request-log.mjs +++ b/src/lib/admin/request-log.mjs @@ -27,6 +27,16 @@ import { disjointOpenaiUsage } from '../protocol/openai-usage.mjs' import { REASONING_EFFORTS, sessionIdForLog } from '../protocol/log-fields.mjs' /** Persist the token a client actually sent — only for failed ingress auth. */ + +/** Failed upstream hops recorded by failover-runner, keyed by request id (shared via globalThis). */ +function takeHopErrors(requestId) { + const store = globalThis[Symbol.for('vm2api.failover.hopErrors')] + if (!requestId || !(store instanceof Map)) return null + const list = store.get(requestId) || null + store.delete(requestId) + return list +} + export function presentedApiKeyForLog(token) { const t = String(token || '').trim() if (!t || t.startsWith('kin-panel-')) return '' @@ -613,18 +623,19 @@ export class RequestLogStore { this.repo.insertSummaryIfAbsent(summary) } catch {} if (this.jsonlMirror) this._appendJsonl(summary) - if (ctx.mode !== 'debug') return + if (ctx.mode !== 'debug') { + takeHopErrors(summary.request_id) + return + } const debugRec = { ...summary, headers: ctx.headers, inbound_summary: extra.inbound_summary || summarizeBody(extra.inbound_body), - request_body_snapshot: - !extra.classifier && extra.inbound_body != null ? sanitizeRequestBodySnapshot(extra.inbound_body) : null, - inbound_body: - !extra.classifier && extra.inbound_body != null - ? clampBody(extra.inbound_body, this.maxDebugBodyChars) - : null, + request_body_snapshot: extra.inbound_body != null ? sanitizeRequestBodySnapshot(extra.inbound_body) : null, + inbound_body: extra.inbound_body != null ? clampBody(extra.inbound_body, this.maxDebugBodyChars) : null, hop_meta: extra.hop_meta || null, + // Every failed upstream hop with its status, headers and body (failover-runner). + hop_errors: takeHopErrors(summary.request_id) || null, via: extra.via || extra.hop_meta?.via || null, upstream_status: extra.upstream_status ?? null, outbound_summary: extra.outbound_summary || null, @@ -639,7 +650,7 @@ export class RequestLogStore { ? redactHeaders(extra.worker_envelope_headers) : null, outbound_body: - ctx.capture_outbound && !extra.classifier && extra.outbound_body != null + ctx.capture_outbound && extra.outbound_body != null ? clampBody(extra.outbound_body, this.maxDebugBodyChars) : null, outbound_body_source: ctx.capture_outbound ? extra.outbound_body_source || null : null, diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index c4851fed3..d52ea23b0 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -60,6 +60,83 @@ const DEFAULTS = { same_account_retry_delay_ms: 500, same_account_retry_max_hop_ms: 10_000, signature_repair: false, + // Hard cap on distinct accounts one request may run on (0 = unlimited). + // The switch/total caps above still let untried units through; this one does not. + max_accounts_per_request: 0, + // A bare 429 (no quota/RPM/model proof) on this many accounts in one request + // means the request, not the accounts, is the problem: stop (0 = off). + bare_429_request_scope_after: 2, + // A bare 429 cools an account only after this many distinct sessions hit one + // there within bare_429_window_ms; until then it only steps aside for the request. + bare_429_account_sessions: 2, + bare_429_window_ms: 120_000, +} + +const HOP_ERRORS_KEY = Symbol.for('vm2api.failover.hopErrors') +const HOP_ERRORS_MAX = 2000 +const HOP_ERROR_TEXT_MAX = 4000 +const HOP_ERROR_HEADER_RE = /ratelimit|retry-after|request-id|x-should-retry|^cf-|^server$|^via$/i + +/** Failed hops by request id; request-log takes them into the debug record. */ +function hopErrorStore() { + if (!(globalThis[HOP_ERRORS_KEY] instanceof Map)) globalThis[HOP_ERRORS_KEY] = new Map() + return globalThis[HOP_ERRORS_KEY] +} + +function clipText(value, max = HOP_ERROR_TEXT_MAX) { + if (value == null) return null + let text + try { + text = typeof value === 'string' ? value : JSON.stringify(value) + } catch { + text = String(value) + } + return text.length > max ? `${text.slice(0, max)}…[+${text.length - max}]` : text +} + +function pickHopHeaders(headers) { + if (!headers || typeof headers !== 'object') return null + const out = {} + for (const [key, value] of Object.entries(headers)) { + if (HOP_ERROR_HEADER_RE.test(key)) out[key] = clipText(value, 300) + } + return Object.keys(out).length ? out : null +} + +export function recordHopError(requestId, entry) { + if (!requestId) return + const store = hopErrorStore() + const list = store.get(requestId) || [] + if (!store.has(requestId)) { + while (store.size >= HOP_ERRORS_MAX) store.delete(store.keys().next().value) + store.set(requestId, list) + } + if (list.length < 20) list.push(entry) +} + +export function takeHopErrors(requestId) { + const store = hopErrorStore() + const list = store.get(requestId) || null + store.delete(requestId) + return list +} + +function hopErrorEntry(attemptNo, selected, result, policy, latencyMs) { + return { + attempt: attemptNo, + at: new Date().toISOString(), + vm_id: selected?.vmId || null, + account_id: selected?.accountId || null, + status: result?.status ?? null, + terminal_state: result?.terminalState || null, + scope: policy?.scope || null, + action: policy?.action || null, + reason: policy?.reason || null, + latency_ms: latencyMs ?? null, + headers: pickHopHeaders(result?.headers), + body: clipText(result?.body), + provider_result: clipText(result?.providerResult, 2000), + } } function clone(value) { @@ -404,6 +481,20 @@ function incompleteHopResult(result, policy, extras = {}) { } } +/** max_accounts_per_request spent: answer with the last hop (a bare 429 stays a 429). */ +function stopAtAccountCap(result, policy, accounts, hops, extras = {}) { + const capped = { account_cap: accounts, attempt_count: hops } + if (Number(result?.status) === 429) { + return { ...result, ok: false, finalState: 'exhausted', policy, ...capped, ...extras } + } + return preferLastResult( + result, + policy, + poolError('account_pool_exhausted', 'Per-request account cap reached', capped), + { ...capped, ...extras }, + ) +} + /** Selection found no seat. Busy until the wait ran out is capacity; nothing eligible is not. */ function selectionFailure(selected, { excluded, lastPolicy, lastResult, hops }) { const reason = selected?.reason || 'no_eligible_accounts' @@ -521,6 +612,45 @@ export class FailoverRunner { } } + accountCapReached(ranAccounts) { + const cap = Number(this.config.max_accounts_per_request ?? 0) + return cap > 0 && ranAccounts.size >= cap + } + + /** + * Bare 429 = scope unproven. Same request on N accounts → the request is the + * problem: stop and cool nothing. Otherwise the account only steps aside for + * this request until distinct sessions confirm it (then the normal cooldown). + */ + gradeBare429(selected, policy, { accounts, key }) { + const accountId = selected?.accountId + if (accountId) accounts.add(accountId) + const after = Number(this.config.bare_429_request_scope_after ?? 2) + if (after > 0 && accounts.size >= after) { + return { + scope: 'request', + action: 'stop', + reason: 'rate_limited_request', + cooldownUntil: null, + retrySameAccount: false, + } + } + const need = Number(this.config.bare_429_account_sessions ?? 2) + if (!(need > 1) || !accountId) return policy + const windowMs = Number(this.config.bare_429_window_ms ?? 120_000) + const now = Date.now() + if (!this.bare429Hits) this.bare429Hits = new Map() + const hits = (this.bare429Hits.get(accountId) || []).filter((hit) => now - hit.at < windowMs) + hits.push({ key: String(key || ''), at: now }) + this.bare429Hits.set(accountId, hits) + if (this.bare429Hits.size > 500) { + for (const [id, list] of this.bare429Hits) + if (!list.some((hit) => now - hit.at < windowMs)) this.bare429Hits.delete(id) + } + if (new Set(hits.map((hit) => hit.key)).size >= need) return policy + return { ...policy, action: 'continue', cooldownUntil: null } + } + /** * An empty hop leaves the outbound session behind for a fresh-session retry. * Client cancellation only ends its job; kernel cancel_ack fences slot reuse. @@ -665,6 +795,11 @@ export class FailoverRunner { // Accounts left only because the kernel had no free slot; their pins stay. const spilled = budget.spilled const bindKeys = uniqueStickyKeys(stickyKey, stickyKeys) + // Accounts that answered this request with a bare 429. + const bare429Accounts = new Set() + // Accounts that really ran this request upstream (admission misses excluded). + const ranAccounts = new Set() + const bare429Key = stickyKey || requestId const pinKeys = familyKey ? [...bindKeys, familyKey] : bindKeys // Pins as this request last saw them. A pin moved by someone else since // (quota migration, credential death) is never written back by a late hop. @@ -852,7 +987,10 @@ export class FailoverRunner { if (hopNoted) return hopNoted = true const executions = executionsOf(value) - if (executions > 0) budget.noteHop(selected.accountId, selected.vmId, executions) + if (executions > 0) { + budget.noteHop(selected.accountId, selected.vmId, executions) + if (selected.accountId) ranAccounts.add(selected.accountId) + } } try { if (signal?.aborted) { @@ -986,6 +1124,9 @@ export class FailoverRunner { }, this.scheduler?.accountQuota, ) + if (policy.reason === 'rate_limited_unknown' && !pinVmId && !result?.committed) { + policy = this.gradeBare429(selected, policy, { accounts: bare429Accounts, key: bare429Key }) + } if (knownSearchRepair && policy.reason === 'search_repairable') { // Repeating the identical flatten cannot fix a second rejection. this.searchRepairs.delete(repairKey) @@ -1037,6 +1178,9 @@ export class FailoverRunner { ttftMs: result?.ttftMs ?? null, latencyMs: Date.now() - attemptStarted, }) + if (!result?.ok) { + recordHopError(requestId, hopErrorEntry(attemptNo, selected, result, policy, Date.now() - attemptStarted)) + } if (typeof onAttempt === 'function') { await onAttempt({ attemptNo, selected, result, policy }) } @@ -1111,6 +1255,10 @@ export class FailoverRunner { avoid = new Set([selected.accountId, selected.vmId]) continue } + if (this.accountCapReached(ranAccounts)) { + releaseStickySlots() + return stopAtAccountCap(result, policy, ranAccounts.size, budget.hops, attribution()) + } budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) } catch (error) { if (attemptStarted === undefined) throw error @@ -1209,6 +1357,7 @@ export class FailoverRunner { terminalState: result.terminalState, latencyMs: Date.now() - attemptStarted, }) + recordHopError(requestId, hopErrorEntry(attemptNo, selected, result, policy, Date.now() - attemptStarted)) if (committed || !shouldContinue(policy)) { releaseStickySlots() return { ...result, ...attribution(), finalState: result.terminalState, policy } @@ -1228,6 +1377,10 @@ export class FailoverRunner { avoid = new Set([selected.accountId, selected.vmId]) continue } + if (this.accountCapReached(ranAccounts)) { + releaseStickySlots() + return stopAtAccountCap(result, policy, ranAccounts.size, budget.hops, attribution()) + } budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' }) } finally { // Setup and attempt share the same permit lifetime. diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index 1867fafef..f0ce2e736 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -2122,3 +2122,99 @@ test('a slot that cannot refresh its credential hops and cools down', async () = assert.equal(scheduler.cooldowns[0].update.reason, 'credential_refresh_failed') assert.equal(scheduler.cooldowns[0].update.status, 'cooldown') }) + +function bare429() { + return { + ok: false, + status: 429, + committed: false, + terminalState: 'incomplete', + headers: { 'request-id': 'req_up_1', 'x-unrelated': 'drop' }, + body: { type: 'error', error: { type: 'rate_limit_error', message: 'Error' } }, + } +} + +test('bare 429 on two accounts of one request stops as request scope and cools nobody', async () => { + const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3), candidate(4)]) + const runner = new FailoverRunner({ scheduler }) + const seen = [] + const result = await runner.run({ + requestId: 'req-bare-429', + stickyKey: 'session-a', + canonicalBody: { model: 'claude-sonnet-test' }, + model: 'claude-sonnet-test', + callAttempt: ({ candidate: selected }) => { + seen.push(selected.accountId) + return bare429() + }, + }) + assert.deepEqual(seen, ['account-1', 'account-2']) + assert.equal(result.status, 429) + assert.equal(result.policy.reason, 'rate_limited_request') + assert.equal(scheduler.cooldowns.length, 0) +}) + +test('bare 429 cools an account only once a second session hits it', async () => { + const scheduler = new Scheduler([candidate(1), candidate(2)]) + const runner = new FailoverRunner({ scheduler }) + const call = ({ candidate: selected }) => (selected.accountId === 'account-1' ? bare429() : success()) + const base = { canonicalBody: { model: 'claude-sonnet-test' }, model: 'claude-sonnet-test', callAttempt: call } + const first = await runner.run({ ...base, requestId: 'req-s1', stickyKey: 'session-1' }) + assert.equal(first.ok, true) + assert.equal(scheduler.cooldowns.length, 0) + const again = await runner.run({ ...base, requestId: 'req-s1b', stickyKey: 'session-1' }) + assert.equal(again.ok, true) + assert.equal(scheduler.cooldowns.length, 0) + const second = await runner.run({ ...base, requestId: 'req-s2', stickyKey: 'session-2' }) + assert.equal(second.ok, true) + assert.equal(scheduler.cooldowns.length, 1) + assert.equal(scheduler.cooldowns[0].candidate.accountId, 'account-1') + assert.equal(scheduler.cooldowns[0].update.reason, 'rate_limited_unknown') +}) + +test('max_accounts_per_request stops after that many accounts ran', async () => { + const scheduler = new Scheduler([1, 2, 3, 4, 5, 6].map(candidate)) + const runner = new FailoverRunner({ + scheduler, + config: { max_accounts_per_request: 3, same_account_retry_delay_ms: 0 }, + }) + const seen = [] + const result = await runner.run({ + requestId: 'req-cap', + canonicalBody: { model: 'claude-sonnet-test' }, + model: 'claude-sonnet-test', + callAttempt: ({ candidate: selected }) => { + seen.push(selected.accountId) + return { + ok: false, + status: 401, + committed: false, + terminalState: 'rejected', + body: { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key' } }, + } + }, + }) + assert.equal(new Set(seen).size, 3) + assert.equal(result.ok, false) + assert.equal(result.account_cap, 3) +}) + +test('failed hops are kept for the request log', async () => { + const { takeHopErrors } = await import('../../src/lib/pool/failover-runner.mjs') + const scheduler = new Scheduler([candidate(1), candidate(2)]) + const runner = new FailoverRunner({ scheduler }) + await runner.run({ + requestId: 'req-hop-log', + stickyKey: 'session-log', + canonicalBody: { model: 'claude-sonnet-test' }, + model: 'claude-sonnet-test', + callAttempt: ({ candidate: selected }) => (selected.accountId === 'account-1' ? bare429() : success()), + }) + const hops = takeHopErrors('req-hop-log') + assert.equal(hops.length, 1) + assert.equal(hops[0].status, 429) + assert.equal(hops[0].vm_id, 'vm-01') + assert.deepEqual(hops[0].headers, { 'request-id': 'req_up_1' }) + assert.match(hops[0].body, /rate_limit_error/) + assert.equal(takeHopErrors('req-hop-log'), null) +}) diff --git a/test/unit/request-log.test.mjs b/test/unit/request-log.test.mjs index 39252f4f6..fa13c5655 100644 --- a/test/unit/request-log.test.mjs +++ b/test/unit/request-log.test.mjs @@ -684,7 +684,7 @@ test('off mode writes nothing', () => { assert.equal(store.listNormal({ limit: 5 }).length, 0) }) -test('classifier debug diagnostics omit system and transcript bodies', () => { +test('classifier debug records keep the request body like any other request', () => { const store = tmpStore('debug') const ctx = store.start( { method: 'POST', headers: {}, socket: {} }, @@ -692,8 +692,8 @@ test('classifier debug diagnostics omit system and transcript bodies', () => { ) const body = { model: 'claude-sonnet-4-6', - system: 'sensitive policy', - messages: [{ role: 'user', content: 'sensitive transcript' }], + system: 'classifier policy', + messages: [{ role: 'user', content: 'classifier transcript' }], } const classifier = { purpose: 'auto_mode_classifier', @@ -705,10 +705,22 @@ test('classifier debug diagnostics omit system and transcript bodies', () => { store.finish(ctx, { status: 200, classifier, inbound_body: body, outbound_body: body }) const [debug] = store.listDebug({ limit: 1 }) assert.deepEqual(debug.classifier, classifier) - assert.equal(debug.request_body_snapshot, null) - assert.equal(debug.inbound_body, null) - assert.equal(debug.outbound_body, null) - assert.equal(JSON.stringify(debug).includes('sensitive transcript'), false) + assert.equal(JSON.stringify(debug.inbound_body).includes('classifier transcript'), true) +}) + +test('debug records carry the failed upstream hops failover-runner kept', () => { + const store = tmpStore('debug') + const ctx = store.start( + { method: 'POST', headers: {}, socket: {} }, + { protocol: 'anthropic.messages', pathName: '/v1/messages' }, + ) + const key = Symbol.for('vm2api.failover.hopErrors') + if (!(globalThis[key] instanceof Map)) globalThis[key] = new Map() + globalThis[key].set(ctx.request_id, [{ attempt: 1, status: 429, body: '{"error":"x"}' }]) + store.finish(ctx, { status: 429 }) + const debug = store.listDebug({ limit: 5 }).find((item) => item.request_id === ctx.request_id) + assert.equal(debug.hop_errors[0].status, 429) + assert.equal(globalThis[key].has(ctx.request_id), false) }) function logOne(store, extra = {}) {