diff --git a/docs/PANEL_API.md b/docs/PANEL_API.md index 6725490f4..f859ab715 100644 --- a/docs/PANEL_API.md +++ b/docs/PANEL_API.md @@ -344,6 +344,8 @@ Analytics 按官方 [`codex_thread_initialized`](https://github.com/openai/codex `failover.stream_idle_timeout_ms`(毫秒,30 秒–60 分钟,默认 180 秒):上游流连续无事件即判超时;网关流路径、各槽 `kernel.json` 的 `idle_timeout_seconds` 与 job 看门狗共用。优先级:面板该字段 → 启动时写入 `cfg.limits.stream_idle_timeout_ms` → 控制面 `.env` 的 `KIN_STREAM_IDLE_TIMEOUT` → 180 秒。保存后热写各槽 `kernel.json`;运行中 kernel 在无任务时自动重启以读取新看门狗值。 +`failover.max_accounts_per_request`(正整数,默认 `0` 不限制):单请求实际执行过的不同账号数达到上限后,不再向新账号发起推理;没执行的准入失败不计入。无配额、模型或重试时间证据的上游 429 不直接冷却账号或 Fable;同一请求在两个账号收到此类 429 时保留上游错误并停止调度。 + `compatibility.persona_preset`(`official` / `official_full` / `zero` / `custom`)和 `compatibility.cache_ttl`(`5m` / `1h`)保存后投影到每个 Claude 槽的 `vms//run/kernel.json`:`persona_preset`、`system_layout`(`zero`→`zero`,其余→`identity`)、`default_cache_ttl`。响应 `kernel_persona.updated` 是本次字节有变化的槽数。`PATCH /vms/:id` 的 `timezone` / `timezone_follow_proxy` 另把 `timezone` 热写进该槽 `kernel.json`,`timezone_sync.kernel_hot` 表示文件有变化。容器 `TZ` 不在这次写入里。Codex 槽不写。 `compatibility.crag_persona_preset`(`official_full` / `zero`,缺省 `official_full`)只给 CRAG v2 print 清洗,不投影进 `kernel.json`。v2 worker 不读 `persona_preset` 或 `CLAUDE_CODE_SYSTEM_LAYOUT`。`official_full` 只 append 调用方尾巴;`zero` 送 `crag_prompt.mode=replace`。其它值保存时 400。槽位显式 `zero` 覆盖该档;其它槽位人设在 print 上收成 append。这不是 TUI:不关内置工具,也不加 `--no-chrome`。 diff --git a/src/lib/pool/failover-runner.mjs b/src/lib/pool/failover-runner.mjs index 1e7f3b017..76c29995a 100644 --- a/src/lib/pool/failover-runner.mjs +++ b/src/lib/pool/failover-runner.mjs @@ -20,7 +20,7 @@ import { hasRefreshPresence, readWorkerCredentialFile } from '../oauth/oauth-cre import { ensureWorkerCredential } from '../transport/go-worker-client.mjs' import { kernelFaults } from '../transport/rust-kernel-client.mjs' import { resolveOfficialCcInference } from '../vm/slot-engine.mjs' -import { AttemptCoordinator } from './unit-decision.mjs' +import { attachFailureDecision, AttemptCoordinator } from './unit-decision.mjs' import { SOFT_COOLDOWN_REASONS } from './pool-scheduler.mjs' // Seat wait for the one-shot repair hop when the rejected hop already spent the deadline. @@ -50,6 +50,7 @@ export const vmTimeoutTracker = { const DEFAULTS = { max_account_switches: 10, max_total_attempts: 12, + max_accounts_per_request: 0, total_retry_deadline_ms: 120000, // Same-session serialization waits out the previous turn. Cap a wedged tail // so the queue cannot sit forever. 0 disables the cap. Long enough that a @@ -684,6 +685,8 @@ export class FailoverRunner { }) const deadline = budget.deadline const excluded = budget.excluded + const accountCap = Number(this.config.max_accounts_per_request) + const bare429Accounts = new Set() // Accounts left only because the kernel had no free slot; their pins stay. const spilled = budget.spilled // session_affinity_first: a turn that spilled to another unit wrote the @@ -885,6 +888,16 @@ export class FailoverRunner { if (signal?.aborted) { return { ...clientCancelledResult(), via: 'pool-failover', ...attribution() } } + if ( + selected?.ok && + Number.isSafeInteger(accountCap) && + accountCap > 0 && + budget.executions.size >= accountCap && + !budget.executions.has(selected.accountId) + ) { + releaseStickySlots() + return preferLastResult(lastResult, lastPolicy, null, { ...attribution(), account_cap: accountCap }) + } if (selected?.familyGated && familyKey && familyVmId) { // The family home left the pool (quota, credential): the family moves // with the next bind. An exclusion this request made itself is not proof. @@ -1017,6 +1030,18 @@ export class FailoverRunner { }, this.scheduler?.accountQuota, ) + if ( + !pinVmId && + selected.accountId && + policy.reason === 'rate_limited_unknown' && + policy.action === 'continue' && + executionsOf(result) > 0 + ) { + bare429Accounts.add(selected.accountId) + if (bare429Accounts.size >= 2) { + policy = attachFailureDecision({ scope: 'request', action: 'stop', reason: 'rate_limited_request' }) + } + } if (knownSearchRepair && policy.reason === 'search_repairable') { // Repeating the identical flatten cannot fix a second rejection. this.searchRepairs.delete(repairKey) diff --git a/src/lib/pool/schedule-state.mjs b/src/lib/pool/schedule-state.mjs index bf85424be..130303aef 100644 --- a/src/lib/pool/schedule-state.mjs +++ b/src/lib/pool/schedule-state.mjs @@ -93,7 +93,7 @@ const STATELESS = [ /_repairable$/, /^(concurrency_limit|concurrency_disabled|rpm_limit|slot_busy|quota_band_inflight|refresh_pending)$/, /^(model_cooldown|fable_cooldown|fable_concurrency|fable_timeout|fable_plan_denied|fable_rate_limited|fable_requires_max|model_not_allowed|platform_mismatch|codex_vm)$/, - /^(quota_guard_request_too_large|no_eligible_accounts|pool_overload_cooldown|pool_rate_limited|vm_unschedulable)$/, + /^(quota_guard_request_too_large|no_eligible_accounts|pool_overload_cooldown|pool_rate_limited|rate_limited_request|vm_unschedulable)$/, /^(wrap_connection_error|worker_transport_failure|worker_timeout|worker_process_exit|upstream_network_error|stream_interrupted|empty_response|incomplete_assistant|downstream_committed_or_incomplete|provider_timeout|client_cancelled|oauth_unconfirmed)$/, // Seat placement labels (why this VM was picked), not a slot condition. /^sticky$/, diff --git a/src/lib/pool/upstream-error-policy.mjs b/src/lib/pool/upstream-error-policy.mjs index b843ca61b..c797d79f3 100644 --- a/src/lib/pool/upstream-error-policy.mjs +++ b/src/lib/pool/upstream-error-policy.mjs @@ -656,8 +656,9 @@ function classifyUpstreamResultRaw( cooldownUntil: accountLimitUntil(reset, usage, now, message), } } - if (isFableWindowLimit(result.headers) || isFableModel(model)) { - const fableReset = fableResetFromHeaders(result.headers, now) || reset + if (isFableWindowLimit(result.headers) || isFableWindowLimit(upstreamHeaders)) { + const fableReset = + fableResetFromHeaders(result.headers, now) || fableResetFromHeaders(upstreamHeaders, now) || reset return { scope: 'model', action: 'continue-and-cooldown', @@ -692,9 +693,9 @@ function classifyUpstreamResultRaw( } return { scope: 'account', - action: 'continue-and-cooldown', + action: reset > now ? 'continue-and-cooldown' : 'continue', reason: 'rate_limited_unknown', - cooldownUntil: reset || now + 60_000, + cooldownUntil: reset > now ? reset : null, retrySameAccount: false, } } diff --git a/test/unit/failover-runner.test.mjs b/test/unit/failover-runner.test.mjs index 55b5898f7..a8380c4a6 100644 --- a/test/unit/failover-runner.test.mjs +++ b/test/unit/failover-runner.test.mjs @@ -566,6 +566,50 @@ test('request-scoped entitlement error does not walk the pool', async () => { assert.equal(scheduler.cooldowns.length, 0) }) +test('bare 429s on two accounts stop without cooling accounts or Fable', async () => { + for (const model of ['claude-sonnet-test', 'claude-fable-test']) { + const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3)]) + const runner = new FailoverRunner({ scheduler }) + const result = await runner.run({ + requestId: `req-bare-${model}`, + model, + callAttempt: () => ({ + ok: false, + status: 429, + terminalState: 'rejected', + body: { error: { type: 'rate_limit_error', message: 'Error' } }, + }), + }) + assert.equal(result.status, 429) + assert.equal(result.policy.reason, 'rate_limited_request') + assert.equal(result.attemptCount, 2) + assert.equal(scheduler.selectCalls, 2) + assert.deepEqual(scheduler.cooldowns, []) + assert.deepEqual(scheduler.releasedAccounts, []) + } +}) + +test('the optional account cap counts executions, not admission misses', async () => { + const scheduler = new Scheduler([candidate(1), candidate(2), candidate(3), candidate(4)]) + const runner = new FailoverRunner({ scheduler, config: { max_accounts_per_request: 1 } }) + const seen = [] + const result = await runner.run({ + requestId: 'req-account-cap', + model: 'claude-sonnet-test', + callAttempt: ({ candidate: selected }) => { + seen.push(selected.accountId) + if (selected.accountId === 'account-1') { + return { ok: false, status: 503, upstreamExecutions: 0, body: { error: { code: 'slot_busy' } } } + } + return { ok: false, status: 529, terminalState: 'rejected', body: { error: { message: 'Overloaded' } } } + }, + }) + assert.deepEqual(seen, ['account-1', 'account-2']) + assert.equal(result.account_cap, 1) + assert.equal(result.attemptCount, 1) + assert.equal(result.status, 529) +}) + test('committed realtime stream failure never switches accounts', async () => { const scheduler = new Scheduler([candidate(1), candidate(2)]) const runner = new FailoverRunner({ scheduler }) diff --git a/test/unit/upstream-error-policy.test.mjs b/test/unit/upstream-error-policy.test.mjs index 517c7ef0d..4fd591232 100644 --- a/test/unit/upstream-error-policy.test.mjs +++ b/test/unit/upstream-error-policy.test.mjs @@ -96,19 +96,26 @@ test('unified account 429 cools account until authoritative reset', () => { }) test('#163: a bare 429 is unknown for every model; the model name never decides scope', () => { - for (const model of ['claude-sonnet-4-5-20250929', 'claude-haiku-4-5', 'claude-opus-4-6', 'unknown-model']) { + for (const model of [ + 'claude-sonnet-4-5-20250929', + 'claude-haiku-4-5', + 'claude-opus-4-6', + 'claude-fable-5-2', + 'unknown-model', + ]) { const policy = classifyUpstreamResult( { status: 429, body: { error: { type: 'rate_limit_error', message: 'Rate limited' } }, headers: {} }, { model, now: 1000 }, ) assert.equal(policy.reason, 'rate_limited_unknown', model) assert.equal(policy.scope, 'account', model) - assert.equal(policy.cooldownUntil, 61_000, model) + assert.equal(policy.action, 'continue', model) + assert.equal(policy.cooldownUntil, null, model) assert.equal(policy.circuit, undefined, model) } }) -test('#1139: real Fable rate evidence and unrelated bare 429s still cool the model', () => { +test('#1139: only a proven Fable window cools the Fable family', () => { const error = { type: 'rate_limit_error', message: 'Error' } const body = { type: 'error', error } const facts = { @@ -119,21 +126,33 @@ test('#1139: real Fable rate evidence and unrelated bare 429s still cool the mod error, } const result = { status: 429, body, providerResult: facts } - for (const [label, failure] of [ + for (const [label, failure, reason, action] of [ [ 'native limit header', { ...result, providerResult: { ...facts, response_headers: { 'Anthropic-RateLimit-Unified-7d_oi-Status': 'rejected' } }, }, + 'fable_rate_limited', + 'continue-and-cooldown', + ], + [ + 'forwarded Retry-After', + { ...result, headers: { 'Retry-After': '7' } }, + 'rate_limited_unknown', + 'continue-and-cooldown', + ], + ['no native facts', { status: 429, body }, 'rate_limited_unknown', 'continue'], + [ + 'different error text', + { ...result, body: { ...body, error: { ...error, message: 'Rate limited' } } }, + 'rate_limited_unknown', + 'continue', ], - ['forwarded Retry-After', { ...result, headers: { 'Retry-After': '7' } }], - ['no native facts', { status: 429, body }], - ['different error text', { ...result, body: { ...body, error: { ...error, message: 'Rate limited' } } }], ]) { const policy = classifyUpstreamResult(failure, { model: 'claude-fable-5-2', now: 1000 }) - assert.equal(policy.reason, 'fable_rate_limited', label) - assert.equal(policy.action, 'continue-and-cooldown', label) + assert.equal(policy.reason, reason, label) + assert.equal(policy.action, action, label) } })