Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 19 additions & 8 deletions src/lib/admin/request-log.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@ import { disjointOpenaiUsage } from '../protocol/openai-usage.mjs'
import { REASONING_EFFORTS, sessionIdForLog } from '../protocol/log-fields.mjs'

/** Persist the token a client actually sent — only for failed ingress auth. */

/** Failed upstream hops recorded by failover-runner, keyed by request id (shared via globalThis). */
function takeHopErrors(requestId) {
const store = globalThis[Symbol.for('vm2api.failover.hopErrors')]
if (!requestId || !(store instanceof Map)) return null
const list = store.get(requestId) || null
store.delete(requestId)
return list
}

export function presentedApiKeyForLog(token) {
const t = String(token || '').trim()
if (!t || t.startsWith('kin-panel-')) return ''
Expand Down Expand Up @@ -613,18 +623,19 @@ export class RequestLogStore {
this.repo.insertSummaryIfAbsent(summary)
} catch {}
if (this.jsonlMirror) this._appendJsonl(summary)
if (ctx.mode !== 'debug') return
if (ctx.mode !== 'debug') {
takeHopErrors(summary.request_id)
return
}
const debugRec = {
...summary,
headers: ctx.headers,
inbound_summary: extra.inbound_summary || summarizeBody(extra.inbound_body),
request_body_snapshot:
!extra.classifier && extra.inbound_body != null ? sanitizeRequestBodySnapshot(extra.inbound_body) : null,
inbound_body:
!extra.classifier && extra.inbound_body != null
? clampBody(extra.inbound_body, this.maxDebugBodyChars)
: null,
request_body_snapshot: extra.inbound_body != null ? sanitizeRequestBodySnapshot(extra.inbound_body) : null,
inbound_body: extra.inbound_body != null ? clampBody(extra.inbound_body, this.maxDebugBodyChars) : null,
hop_meta: extra.hop_meta || null,
// Every failed upstream hop with its status, headers and body (failover-runner).
hop_errors: takeHopErrors(summary.request_id) || null,
via: extra.via || extra.hop_meta?.via || null,
upstream_status: extra.upstream_status ?? null,
outbound_summary: extra.outbound_summary || null,
Expand All @@ -639,7 +650,7 @@ export class RequestLogStore {
? redactHeaders(extra.worker_envelope_headers)
: null,
outbound_body:
ctx.capture_outbound && !extra.classifier && extra.outbound_body != null
ctx.capture_outbound && extra.outbound_body != null
? clampBody(extra.outbound_body, this.maxDebugBodyChars)
: null,
outbound_body_source: ctx.capture_outbound ? extra.outbound_body_source || null : null,
Expand Down
155 changes: 154 additions & 1 deletion src/lib/pool/failover-runner.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,83 @@ const DEFAULTS = {
same_account_retry_delay_ms: 500,
same_account_retry_max_hop_ms: 10_000,
signature_repair: false,
// Hard cap on distinct accounts one request may run on (0 = unlimited).
// The switch/total caps above still let untried units through; this one does not.
max_accounts_per_request: 0,
// A bare 429 (no quota/RPM/model proof) on this many accounts in one request
// means the request, not the accounts, is the problem: stop (0 = off).
bare_429_request_scope_after: 2,
// A bare 429 cools an account only after this many distinct sessions hit one
// there within bare_429_window_ms; until then it only steps aside for the request.
bare_429_account_sessions: 2,
bare_429_window_ms: 120_000,
}

const HOP_ERRORS_KEY = Symbol.for('vm2api.failover.hopErrors')
const HOP_ERRORS_MAX = 2000
const HOP_ERROR_TEXT_MAX = 4000
const HOP_ERROR_HEADER_RE = /ratelimit|retry-after|request-id|x-should-retry|^cf-|^server$|^via$/i

/** Failed hops by request id; request-log takes them into the debug record. */
function hopErrorStore() {
if (!(globalThis[HOP_ERRORS_KEY] instanceof Map)) globalThis[HOP_ERRORS_KEY] = new Map()
return globalThis[HOP_ERRORS_KEY]
}

function clipText(value, max = HOP_ERROR_TEXT_MAX) {
if (value == null) return null
let text
try {
text = typeof value === 'string' ? value : JSON.stringify(value)
} catch {
text = String(value)
}
return text.length > max ? `${text.slice(0, max)}…[+${text.length - max}]` : text
}

function pickHopHeaders(headers) {
if (!headers || typeof headers !== 'object') return null
const out = {}
for (const [key, value] of Object.entries(headers)) {
if (HOP_ERROR_HEADER_RE.test(key)) out[key] = clipText(value, 300)
}
return Object.keys(out).length ? out : null
}

export function recordHopError(requestId, entry) {
if (!requestId) return
const store = hopErrorStore()
const list = store.get(requestId) || []
if (!store.has(requestId)) {
while (store.size >= HOP_ERRORS_MAX) store.delete(store.keys().next().value)
store.set(requestId, list)
}
if (list.length < 20) list.push(entry)
}

export function takeHopErrors(requestId) {
const store = hopErrorStore()
const list = store.get(requestId) || null
store.delete(requestId)
return list
}

function hopErrorEntry(attemptNo, selected, result, policy, latencyMs) {
return {
attempt: attemptNo,
at: new Date().toISOString(),
vm_id: selected?.vmId || null,
account_id: selected?.accountId || null,
status: result?.status ?? null,
terminal_state: result?.terminalState || null,
scope: policy?.scope || null,
action: policy?.action || null,
reason: policy?.reason || null,
latency_ms: latencyMs ?? null,
headers: pickHopHeaders(result?.headers),
body: clipText(result?.body),
provider_result: clipText(result?.providerResult, 2000),
Comment on lines +137 to +138

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Redact credential-shaped strings before storing hop errors

When an upstream or CLI error contains a bearer token or sk-ant-... credential, clipText only truncates it, and RequestLogStore.finish persists the resulting hop_errors verbatim. Unlike the existing clampBody and debugResponse paths, neither this helper nor the new persistence path applies redactSecrets. Consequently, even caller-forced debug mode can leave usable credentials in SQLite and the panel's debug records. Apply the existing secret masking to the hop body and provider facts before retaining them.

Useful? React with 👍 / 👎.

}
}

function clone(value) {
Expand Down Expand Up @@ -404,6 +481,20 @@ function incompleteHopResult(result, policy, extras = {}) {
}
}

/** max_accounts_per_request spent: answer with the last hop (a bare 429 stays a 429). */
function stopAtAccountCap(result, policy, accounts, hops, extras = {}) {
const capped = { account_cap: accounts, attempt_count: hops }
if (Number(result?.status) === 429) {
return { ...result, ok: false, finalState: 'exhausted', policy, ...capped, ...extras }
}
return preferLastResult(
result,
policy,
poolError('account_pool_exhausted', 'Per-request account cap reached', capped),
{ ...capped, ...extras },
)
}

/** Selection found no seat. Busy until the wait ran out is capacity; nothing eligible is not. */
function selectionFailure(selected, { excluded, lastPolicy, lastResult, hops }) {
const reason = selected?.reason || 'no_eligible_accounts'
Expand Down Expand Up @@ -521,6 +612,45 @@ export class FailoverRunner {
}
}

accountCapReached(ranAccounts) {
const cap = Number(this.config.max_accounts_per_request ?? 0)
return cap > 0 && ranAccounts.size >= cap
}

/**
* Bare 429 = scope unproven. Same request on N accounts → the request is the
* problem: stop and cool nothing. Otherwise the account only steps aside for
* this request until distinct sessions confirm it (then the normal cooldown).
*/
gradeBare429(selected, policy, { accounts, key }) {
const accountId = selected?.accountId
if (accountId) accounts.add(accountId)
const after = Number(this.config.bare_429_request_scope_after ?? 2)
if (after > 0 && accounts.size >= after) {
return {
scope: 'request',
action: 'stop',
reason: 'rate_limited_request',
cooldownUntil: null,
retrySameAccount: false,
}
}
const need = Number(this.config.bare_429_account_sessions ?? 2)
if (!(need > 1) || !accountId) return policy
const windowMs = Number(this.config.bare_429_window_ms ?? 120_000)
const now = Date.now()
if (!this.bare429Hits) this.bare429Hits = new Map()
const hits = (this.bare429Hits.get(accountId) || []).filter((hit) => now - hit.at < windowMs)
hits.push({ key: String(key || ''), at: now })
this.bare429Hits.set(accountId, hits)
Comment on lines +644 to +645

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Discard cooldown evidence from request-scoped 429s

With the defaults, let one session receive bare 429s from accounts A and B, then let a second session receive the same responses within 120 seconds. Both requests end as rate_limited_request, but the first request leaves a hit for A in this map. The second request's first hop therefore triggers applyCooldown(A) before B produces the request-scope verdict, and neither the hit nor the cooldown is rolled back. Repeated request-scoped failures consequently still remove accounts from the pool despite the intended "cool nobody" behavior. Defer or discard account cooldown evidence for requests that receive a request-scope verdict.

Useful? React with 👍 / 👎.

if (this.bare429Hits.size > 500) {
for (const [id, list] of this.bare429Hits)
if (!list.some((hit) => now - hit.at < windowMs)) this.bare429Hits.delete(id)
}
if (new Set(hits.map((hit) => hit.key)).size >= need) return policy
return { ...policy, action: 'continue', cooldownUntil: null }
}

/**
* An empty hop leaves the outbound session behind for a fresh-session retry.
* Client cancellation only ends its job; kernel cancel_ack fences slot reuse.
Expand Down Expand Up @@ -665,6 +795,11 @@ export class FailoverRunner {
// Accounts left only because the kernel had no free slot; their pins stay.
const spilled = budget.spilled
const bindKeys = uniqueStickyKeys(stickyKey, stickyKeys)
// Accounts that answered this request with a bare 429.
const bare429Accounts = new Set()
// Accounts that really ran this request upstream (admission misses excluded).
const ranAccounts = new Set()
const bare429Key = stickyKey || requestId
const pinKeys = familyKey ? [...bindKeys, familyKey] : bindKeys
// Pins as this request last saw them. A pin moved by someone else since
// (quota migration, credential death) is never written back by a late hop.
Expand Down Expand Up @@ -852,7 +987,10 @@ export class FailoverRunner {
if (hopNoted) return
hopNoted = true
const executions = executionsOf(value)
if (executions > 0) budget.noteHop(selected.accountId, selected.vmId, executions)
if (executions > 0) {
budget.noteHop(selected.accountId, selected.vmId, executions)
if (selected.accountId) ranAccounts.add(selected.accountId)
}
}
try {
if (signal?.aborted) {
Expand Down Expand Up @@ -986,6 +1124,9 @@ export class FailoverRunner {
},
this.scheduler?.accountQuota,
)
if (policy.reason === 'rate_limited_unknown' && !pinVmId && !result?.committed) {
policy = this.gradeBare429(selected, policy, { accounts: bare429Accounts, key: bare429Key })
}
if (knownSearchRepair && policy.reason === 'search_repairable') {
// Repeating the identical flatten cannot fix a second rejection.
this.searchRepairs.delete(repairKey)
Expand Down Expand Up @@ -1037,6 +1178,9 @@ export class FailoverRunner {
ttftMs: result?.ttftMs ?? null,
latencyMs: Date.now() - attemptStarted,
})
if (!result?.ok) {
recordHopError(requestId, hopErrorEntry(attemptNo, selected, result, policy, Date.now() - attemptStarted))
}
if (typeof onAttempt === 'function') {
await onAttempt({ attemptNo, selected, result, policy })
}
Expand Down Expand Up @@ -1111,6 +1255,10 @@ export class FailoverRunner {
avoid = new Set([selected.accountId, selected.vmId])
continue
}
if (this.accountCapReached(ranAccounts)) {
releaseStickySlots()
return stopAtAccountCap(result, policy, ranAccounts.size, budget.hops, attribution())
Comment on lines +1258 to +1260

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Enforce the account cap on retry continuations too

With max_accounts_per_request: 1 and two available accounts, an uncommitted empty/incomplete hop takes the preceding allowSameUnit continuation before reaching this guard. That continuation sets avoid to the first account, so the next selection deliberately prefers another account and executes the request on two distinct accounts despite the cap of one. The exception path has the same ordering. Enforce the cap when selecting the next account, including retry continuations, rather than only in the ordinary switch path.

Useful? React with 👍 / 👎.

}
budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' })
} catch (error) {
if (attemptStarted === undefined) throw error
Expand Down Expand Up @@ -1209,6 +1357,7 @@ export class FailoverRunner {
terminalState: result.terminalState,
latencyMs: Date.now() - attemptStarted,
})
recordHopError(requestId, hopErrorEntry(attemptNo, selected, result, policy, Date.now() - attemptStarted))
if (committed || !shouldContinue(policy)) {
releaseStickySlots()
return { ...result, ...attribution(), finalState: result.terminalState, policy }
Expand All @@ -1228,6 +1377,10 @@ export class FailoverRunner {
avoid = new Set([selected.accountId, selected.vmId])
continue
}
if (this.accountCapReached(ranAccounts)) {
releaseStickySlots()
return stopAtAccountCap(result, policy, ranAccounts.size, budget.hops, attribution())
}
budget.noteSwitch(selected.accountId, selected.vmId, { spill: policy.reason === 'slot_busy' })
} finally {
// Setup and attempt share the same permit lifetime.
Expand Down
96 changes: 96 additions & 0 deletions test/unit/failover-runner.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2122,3 +2122,99 @@ test('a slot that cannot refresh its credential hops and cools down', async () =
assert.equal(scheduler.cooldowns[0].update.reason, 'credential_refresh_failed')
assert.equal(scheduler.cooldowns[0].update.status, 'cooldown')
})

function bare429() {
return {
ok: false,
status: 429,
committed: false,
terminalState: 'incomplete',
headers: { 'request-id': 'req_up_1', 'x-unrelated': 'drop' },
body: { type: 'error', error: { type: 'rate_limit_error', message: 'Error' } },
}
}

test('bare 429 on two accounts of one request stops as request scope and cools nobody', async () => {
const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3), candidate(4)])
const runner = new FailoverRunner({ scheduler })
const seen = []
const result = await runner.run({
requestId: 'req-bare-429',
stickyKey: 'session-a',
canonicalBody: { model: 'claude-sonnet-test' },
model: 'claude-sonnet-test',
callAttempt: ({ candidate: selected }) => {
seen.push(selected.accountId)
return bare429()
},
})
assert.deepEqual(seen, ['account-1', 'account-2'])
assert.equal(result.status, 429)
assert.equal(result.policy.reason, 'rate_limited_request')
assert.equal(scheduler.cooldowns.length, 0)
})

test('bare 429 cools an account only once a second session hits it', async () => {
const scheduler = new Scheduler([candidate(1), candidate(2)])
const runner = new FailoverRunner({ scheduler })
const call = ({ candidate: selected }) => (selected.accountId === 'account-1' ? bare429() : success())
const base = { canonicalBody: { model: 'claude-sonnet-test' }, model: 'claude-sonnet-test', callAttempt: call }
const first = await runner.run({ ...base, requestId: 'req-s1', stickyKey: 'session-1' })
assert.equal(first.ok, true)
assert.equal(scheduler.cooldowns.length, 0)
const again = await runner.run({ ...base, requestId: 'req-s1b', stickyKey: 'session-1' })
assert.equal(again.ok, true)
assert.equal(scheduler.cooldowns.length, 0)
const second = await runner.run({ ...base, requestId: 'req-s2', stickyKey: 'session-2' })
assert.equal(second.ok, true)
assert.equal(scheduler.cooldowns.length, 1)
assert.equal(scheduler.cooldowns[0].candidate.accountId, 'account-1')
assert.equal(scheduler.cooldowns[0].update.reason, 'rate_limited_unknown')
})

test('max_accounts_per_request stops after that many accounts ran', async () => {
const scheduler = new Scheduler([1, 2, 3, 4, 5, 6].map(candidate))
const runner = new FailoverRunner({
scheduler,
config: { max_accounts_per_request: 3, same_account_retry_delay_ms: 0 },
})
const seen = []
const result = await runner.run({
requestId: 'req-cap',
canonicalBody: { model: 'claude-sonnet-test' },
model: 'claude-sonnet-test',
callAttempt: ({ candidate: selected }) => {
seen.push(selected.accountId)
return {
ok: false,
status: 401,
committed: false,
terminalState: 'rejected',
body: { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key' } },
}
},
})
assert.equal(new Set(seen).size, 3)
assert.equal(result.ok, false)
assert.equal(result.account_cap, 3)
})

test('failed hops are kept for the request log', async () => {
const { takeHopErrors } = await import('../../src/lib/pool/failover-runner.mjs')
const scheduler = new Scheduler([candidate(1), candidate(2)])
const runner = new FailoverRunner({ scheduler })
await runner.run({
requestId: 'req-hop-log',
stickyKey: 'session-log',
canonicalBody: { model: 'claude-sonnet-test' },
model: 'claude-sonnet-test',
callAttempt: ({ candidate: selected }) => (selected.accountId === 'account-1' ? bare429() : success()),
})
const hops = takeHopErrors('req-hop-log')
assert.equal(hops.length, 1)
assert.equal(hops[0].status, 429)
assert.equal(hops[0].vm_id, 'vm-01')
assert.deepEqual(hops[0].headers, { 'request-id': 'req_up_1' })
assert.match(hops[0].body, /rate_limit_error/)
assert.equal(takeHopErrors('req-hop-log'), null)
})
Loading
Loading