Skip to content
Merged
2 changes: 2 additions & 0 deletions docs/PANEL_API.md
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,8 @@ Analytics 按官方 [`codex_thread_initialized`](https://github.com/openai/codex

`failover.stream_idle_timeout_ms`(毫秒,30 秒–60 分钟,默认 180 秒):上游流连续无事件即判超时;网关流路径、各槽 `kernel.json` 的 `idle_timeout_seconds` 与 job 看门狗共用。优先级:面板该字段 → 启动时写入 `cfg.limits.stream_idle_timeout_ms` → 控制面 `.env` 的 `KIN_STREAM_IDLE_TIMEOUT` → 180 秒。保存后热写各槽 `kernel.json`;运行中 kernel 在无任务时自动重启以读取新看门狗值。

`failover.max_accounts_per_request`(正整数,默认 `0` 不限制):单请求实际执行过的不同账号数达到上限后,不再向新账号发起推理;没执行的准入失败不计入。无配额、模型或重试时间证据的上游 429 不直接冷却账号或 Fable;同一请求在两个账号收到此类 429 时保留上游错误并停止调度。

`compatibility.persona_preset`(`official` / `official_full` / `zero` / `custom`)和 `compatibility.cache_ttl`(`5m` / `1h`)保存后投影到每个 Claude 槽的 `vms/<id>/run/kernel.json`:`persona_preset`、`system_layout`(`zero`→`zero`,其余→`identity`)、`default_cache_ttl`。响应 `kernel_persona.updated` 是本次字节有变化的槽数。`PATCH /vms/:id` 的 `timezone` / `timezone_follow_proxy` 另把 `timezone` 热写进该槽 `kernel.json`,`timezone_sync.kernel_hot` 表示文件有变化。容器 `TZ` 不在这次写入里。Codex 槽不写。

`compatibility.crag_persona_preset`(`official_full` / `zero`,缺省 `official_full`)只给 CRAG v2 print 清洗,不投影进 `kernel.json`。v2 worker 不读 `persona_preset` 或 `CLAUDE_CODE_SYSTEM_LAYOUT`。`official_full` 只 append 调用方尾巴;`zero` 送 `crag_prompt.mode=replace`。其它值保存时 400。槽位显式 `zero` 覆盖该档;其它槽位人设在 print 上收成 append。这不是 TUI:不关内置工具,也不加 `--no-chrome`。
Expand Down
27 changes: 26 additions & 1 deletion src/lib/pool/failover-runner.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { hasRefreshPresence, readWorkerCredentialFile } from '../oauth/oauth-cre
import { ensureWorkerCredential } from '../transport/go-worker-client.mjs'
import { kernelFaults } from '../transport/rust-kernel-client.mjs'
import { resolveOfficialCcInference } from '../vm/slot-engine.mjs'
import { AttemptCoordinator } from './unit-decision.mjs'
import { attachFailureDecision, AttemptCoordinator } from './unit-decision.mjs'
import { SOFT_COOLDOWN_REASONS } from './pool-scheduler.mjs'

// Seat wait for the one-shot repair hop when the rejected hop already spent the deadline.
Expand Down Expand Up @@ -50,6 +50,7 @@ export const vmTimeoutTracker = {
const DEFAULTS = {
max_account_switches: 10,
max_total_attempts: 12,
max_accounts_per_request: 0,
total_retry_deadline_ms: 120000,
// Same-session serialization waits out the previous turn. Cap a wedged tail
// so the queue cannot sit forever. 0 disables the cap. Long enough that a
Expand Down Expand Up @@ -684,6 +685,8 @@ export class FailoverRunner {
})
const deadline = budget.deadline
const excluded = budget.excluded
const accountCap = Number(this.config.max_accounts_per_request)
const bare429Accounts = new Set()
// Accounts left only because the kernel had no free slot; their pins stay.
const spilled = budget.spilled
// session_affinity_first: a turn that spilled to another unit wrote the
Expand Down Expand Up @@ -885,6 +888,16 @@ export class FailoverRunner {
if (signal?.aborted) {
return { ...clientCancelledResult(), via: 'pool-failover', ...attribution() }
}
if (
selected?.ok &&
Number.isSafeInteger(accountCap) &&
accountCap > 0 &&
budget.executions.size >= accountCap &&
!budget.executions.has(selected.accountId)
) {
releaseStickySlots()
return preferLastResult(lastResult, lastPolicy, null, { ...attribution(), account_cap: accountCap })
}
if (selected?.familyGated && familyKey && familyVmId) {
// The family home left the pool (quota, credential): the family moves
// with the next bind. An exclusion this request made itself is not proof.
Expand Down Expand Up @@ -1017,6 +1030,18 @@ export class FailoverRunner {
},
this.scheduler?.accountQuota,
)
if (
!pinVmId &&
selected.accountId &&
policy.reason === 'rate_limited_unknown' &&
policy.action === 'continue' &&
executionsOf(result) > 0
) {
bare429Accounts.add(selected.accountId)
if (bare429Accounts.size >= 2) {
policy = attachFailureDecision({ scope: 'request', action: 'stop', reason: 'rate_limited_request' })
}
}
if (knownSearchRepair && policy.reason === 'search_repairable') {
// Repeating the identical flatten cannot fix a second rejection.
this.searchRepairs.delete(repairKey)
Expand Down
2 changes: 1 addition & 1 deletion src/lib/pool/schedule-state.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ const STATELESS = [
/_repairable$/,
/^(concurrency_limit|concurrency_disabled|rpm_limit|slot_busy|quota_band_inflight|refresh_pending)$/,
/^(model_cooldown|fable_cooldown|fable_concurrency|fable_timeout|fable_plan_denied|fable_rate_limited|fable_requires_max|model_not_allowed|platform_mismatch|codex_vm)$/,
/^(quota_guard_request_too_large|no_eligible_accounts|pool_overload_cooldown|pool_rate_limited|vm_unschedulable)$/,
/^(quota_guard_request_too_large|no_eligible_accounts|pool_overload_cooldown|pool_rate_limited|rate_limited_request|vm_unschedulable)$/,
/^(wrap_connection_error|worker_transport_failure|worker_timeout|worker_process_exit|upstream_network_error|stream_interrupted|empty_response|incomplete_assistant|downstream_committed_or_incomplete|provider_timeout|client_cancelled|oauth_unconfirmed)$/,
// Seat placement labels (why this VM was picked), not a slot condition.
/^sticky$/,
Expand Down
9 changes: 5 additions & 4 deletions src/lib/pool/upstream-error-policy.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -656,8 +656,9 @@ function classifyUpstreamResultRaw(
cooldownUntil: accountLimitUntil(reset, usage, now, message),
}
}
if (isFableWindowLimit(result.headers) || isFableModel(model)) {
const fableReset = fableResetFromHeaders(result.headers, now) || reset
if (isFableWindowLimit(result.headers) || isFableWindowLimit(upstreamHeaders)) {
const fableReset =
fableResetFromHeaders(result.headers, now) || fableResetFromHeaders(upstreamHeaders, now) || reset
return {
scope: 'model',
action: 'continue-and-cooldown',
Expand Down Expand Up @@ -692,9 +693,9 @@ function classifyUpstreamResultRaw(
}
return {
scope: 'account',
action: 'continue-and-cooldown',
action: reset > now ? 'continue-and-cooldown' : 'continue',
reason: 'rate_limited_unknown',
cooldownUntil: reset || now + 60_000,
cooldownUntil: reset > now ? reset : null,
retrySameAccount: false,
}
}
Expand Down
44 changes: 44 additions & 0 deletions test/unit/failover-runner.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,50 @@ test('request-scoped entitlement error does not walk the pool', async () => {
assert.equal(scheduler.cooldowns.length, 0)
})

test('bare 429s on two accounts stop without cooling accounts or Fable', async () => {
for (const model of ['claude-sonnet-test', 'claude-fable-test']) {
const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3)])
const runner = new FailoverRunner({ scheduler })
const result = await runner.run({
requestId: `req-bare-${model}`,
model,
callAttempt: () => ({
ok: false,
status: 429,
terminalState: 'rejected',
body: { error: { type: 'rate_limit_error', message: 'Error' } },
}),
})
assert.equal(result.status, 429)
assert.equal(result.policy.reason, 'rate_limited_request')
assert.equal(result.attemptCount, 2)
assert.equal(scheduler.selectCalls, 2)
assert.deepEqual(scheduler.cooldowns, [])
assert.deepEqual(scheduler.releasedAccounts, [])
}
})

test('the optional account cap counts executions, not admission misses', async () => {
const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3), candidate(4)])
const runner = new FailoverRunner({ scheduler, config: { max_accounts_per_request: 1 } })
const seen = []
const result = await runner.run({
requestId: 'req-account-cap',
model: 'claude-sonnet-test',
callAttempt: ({ candidate: selected }) => {
seen.push(selected.accountId)
if (selected.accountId === 'account-1') {
return { ok: false, status: 503, upstreamExecutions: 0, body: { error: { code: 'slot_busy' } } }
}
return { ok: false, status: 529, terminalState: 'rejected', body: { error: { message: 'Overloaded' } } }
},
})
assert.deepEqual(seen, ['account-1', 'account-2'])
assert.equal(result.account_cap, 1)
assert.equal(result.attemptCount, 1)
assert.equal(result.status, 529)
})

test('committed realtime stream failure never switches accounts', async () => {
const scheduler = new Scheduler([candidate(1), candidate(2)])
const runner = new FailoverRunner({ scheduler })
Expand Down
37 changes: 28 additions & 9 deletions test/unit/upstream-error-policy.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -96,19 +96,26 @@ test('unified account 429 cools account until authoritative reset', () => {
})

test('#163: a bare 429 is unknown for every model; the model name never decides scope', () => {
for (const model of ['claude-sonnet-4-5-20250929', 'claude-haiku-4-5', 'claude-opus-4-6', 'unknown-model']) {
for (const model of [
'claude-sonnet-4-5-20250929',
'claude-haiku-4-5',
'claude-opus-4-6',
'claude-fable-5-2',
'unknown-model',
]) {
const policy = classifyUpstreamResult(
{ status: 429, body: { error: { type: 'rate_limit_error', message: 'Rate limited' } }, headers: {} },
{ model, now: 1000 },
)
assert.equal(policy.reason, 'rate_limited_unknown', model)
assert.equal(policy.scope, 'account', model)
assert.equal(policy.cooldownUntil, 61_000, model)
assert.equal(policy.action, 'continue', model)
assert.equal(policy.cooldownUntil, null, model)
assert.equal(policy.circuit, undefined, model)
}
})

test('#1139: real Fable rate evidence and unrelated bare 429s still cool the model', () => {
test('#1139: only a proven Fable window cools the Fable family', () => {
const error = { type: 'rate_limit_error', message: 'Error' }
const body = { type: 'error', error }
const facts = {
Expand All @@ -119,21 +126,33 @@ test('#1139: real Fable rate evidence and unrelated bare 429s still cool the mod
error,
}
const result = { status: 429, body, providerResult: facts }
for (const [label, failure] of [
for (const [label, failure, reason, action] of [
[
'native limit header',
{
...result,
providerResult: { ...facts, response_headers: { 'Anthropic-RateLimit-Unified-7d_oi-Status': 'rejected' } },
},
'fable_rate_limited',
'continue-and-cooldown',
],
[
'forwarded Retry-After',
{ ...result, headers: { 'Retry-After': '7' } },
'rate_limited_unknown',
'continue-and-cooldown',
],
['no native facts', { status: 429, body }, 'rate_limited_unknown', 'continue'],
[
'different error text',
{ ...result, body: { ...body, error: { ...error, message: 'Rate limited' } } },
'rate_limited_unknown',
'continue',
],
['forwarded Retry-After', { ...result, headers: { 'Retry-After': '7' } }],
['no native facts', { status: 429, body }],
['different error text', { ...result, body: { ...body, error: { ...error, message: 'Rate limited' } } }],
]) {
const policy = classifyUpstreamResult(failure, { model: 'claude-fable-5-2', now: 1000 })
assert.equal(policy.reason, 'fable_rate_limited', label)
assert.equal(policy.action, 'continue-and-cooldown', label)
assert.equal(policy.reason, reason, label)
assert.equal(policy.action, action, label)
}
})

Expand Down
Loading