diff --git a/src/lib/pool/pool-scheduler.mjs b/src/lib/pool/pool-scheduler.mjs index ac183d8d7..5d4c3e534 100644 --- a/src/lib/pool/pool-scheduler.mjs +++ b/src/lib/pool/pool-scheduler.mjs @@ -235,8 +235,14 @@ function selectionSnapshot(candidates = [], available = [], extras = {}) { const waitPool = extras.waitPool || candidates const waitReasons = [...new Set((waitPool || []).map((candidate) => candidate.waitReason).filter(Boolean))] const soonest = (waitPool || []).map((candidate) => Number(candidate.availableAt) || 0).filter((value) => value > now) + const quotaLimited = + extras.reason === 'all_accounts_busy' && + waitPool.length > 0 && + waitPool.every( + (candidate) => candidate.waitReason === 'account_cooldown' && isQuotaWindowReason(candidate.cooldownReason), + ) return { - reason: extras.reason, + reason: quotaLimited ? 'pool_rate_limited' : extras.reason, wait_ms: extras.waitMs ?? 0, soonest_available_ms: soonest.length ? Math.min(...soonest) - now : null, wait_reasons: waitReasons, diff --git a/src/lib/pool/quota-window.mjs b/src/lib/pool/quota-window.mjs index f1f6f7199..9c24e4bbd 100644 --- a/src/lib/pool/quota-window.mjs +++ b/src/lib/pool/quota-window.mjs @@ -97,13 +97,20 @@ export function isPlanLimitMessage(message) { /** * Whether a finished request proves the account's own plan window is spent. - * A 429 the pool synthesized itself (`fable_requires_max`, `pool_rate_limited`) - * never reached the provider, so it says nothing about the account's quota. + * Only explicit plan-limit text or rejected account-window headers prove + * exhaustion. A bare 429 can be a model or RPM limit and must not invent a + * persistent 5h rejection for the entire account. */ export function isAccountQuotaExhausted(result = {}, limitText = '') { if (!result || result.ok) return false if (isPlanLimitMessage(limitText)) return true - return Number(result.status) === 429 && result.via !== 'pool-failover' + if (Number(result.status) !== 429 || result.via === 'pool-failover') return false + const headers = result.headers || {} + const entries = typeof headers.entries === 'function' ? [...headers.entries()] : Object.entries(headers) + return entries.some( + ([key, value]) => + /^anthropic-ratelimit-unified-(5h|7d)-status$/i.test(key) && /^(rejected|rate_limited)$/i.test(String(value)), + ) } /** diff --git a/test/unit/account-quota.test.mjs b/test/unit/account-quota.test.mjs index 744dead6d..b1d22e854 100644 --- a/test/unit/account-quota.test.mjs +++ b/test/unit/account-quota.test.mjs @@ -4,7 +4,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { AccountQuota, extraHeadersFromLimitError } from '../../src/lib/pool/account-quota.mjs' -import { parseResetMs } from '../../src/lib/pool/quota-window.mjs' +import { parseResetMs, isAccountQuotaExhausted } from '../../src/lib/pool/quota-window.mjs' function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-')) @@ -923,6 +923,34 @@ test('429 unified exhausted marks headers rejected without inventing official 10 assert.equal(q.canAccept('acc-429').ok, false) }) +test('provider Fable 429 without quota evidence preserves the open account windows', () => { + const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) + q.ingestHeaders('acc-fable-429', { + 'anthropic-ratelimit-unified-5h-utilization': '0.02', + 'anthropic-ratelimit-unified-5h-status': 'allowed', + 'anthropic-ratelimit-unified-7d-utilization': '0.07', + 'anthropic-ratelimit-unified-7d-status': 'allowed', + }) + const result = { + ok: false, + status: 429, + via: 'rust-kernel-stream', + headers: {}, + body: { error: { type: 'rate_limit_error', message: 'Error' } }, + } + const text = result.body.error.message + q.ingestHeaders('acc-fable-429', extraHeadersFromLimitError(text, result.headers), null, { + exhausted: isAccountQuotaExhausted(result, text), + status: result.status, + }) + const account = q.repo.get('acc-fable-429') + assert.equal(account.unified.headers['5h'].status, 'allowed') + assert.equal(account.unified.headers['5h'].utilization, 0.02) + assert.equal(account.unified.headers['7d'].status, 'allowed') + assert.equal(account.unified.headers.exhausted_at ?? null, null) + assert.equal(q.canAccept('acc-fable-429').ok, true) +}) + test('local 429 without quota evidence does not reject the 5h window', () => { const q = new AccountQuota({ dataDir: tmpDir(), config: {} }) q.ingestHeaders('acc-local-429', { diff --git a/test/unit/pool-scheduler.test.mjs b/test/unit/pool-scheduler.test.mjs index 2a50c2bc4..0ba983852 100644 --- a/test/unit/pool-scheduler.test.mjs +++ b/test/unit/pool-scheduler.test.mjs @@ -505,6 +505,40 @@ test('wait timeout on cooldown returns busy rather than empty pool', async (t) = assert.ok(selected.wait_reasons.includes('account_cooldown')) }) +test('pinned quota cooldown returns rate limited rather than pool overload', async (t) => { + const root = project() + let pool + t.after(() => cleanupProject(root, pool)) + const repo = new RuntimeRepo() + repo.markCooldown('account-1', { + vmId: 'vm-01', + until: Date.now() + 60_000, + reason: 'quota_5h_header', + }) + pool = new PoolScheduler({ + projectRoot: root, + runtimeRepo: repo, + accountQuota: { canAccept: () => ({ ok: true }) }, + workerHealth: async () => ({ ok: true, credential: { generation: 1, has_access: true } }), + }) + let dispatched = false + const runner = new FailoverRunner({ scheduler: pool }) + const result = await runner.run({ + requestId: 'quota-cooldown-pin', + model: 'claude-test', + pinVmId: 'vm-01', + callAttempt: async () => { + dispatched = true + throw new Error('must not dispatch') + }, + }) + assert.equal(dispatched, false) + assert.equal(result.status, 429) + assert.equal(result.body.error.code, 'pool_rate_limited') + assert.ok(result.retryAfterSec > 0) + assert.deepEqual(result.body.error.details.wait_reasons, ['account_cooldown']) +}) + test('long cooldown beyond wait budget fails immediately', async (t) => { const root = project() t.after(() => fs.rmSync(root, { recursive: true, force: true })) diff --git a/test/unit/quota-window.test.mjs b/test/unit/quota-window.test.mjs index e92add6b0..970c22f6b 100644 --- a/test/unit/quota-window.test.mjs +++ b/test/unit/quota-window.test.mjs @@ -253,11 +253,36 @@ test('isAccountQuotaExhausted ignores 429s the pool synthesized itself', () => { assert.equal(isAccountQuotaExhausted(poolRateLimited, 'rate limited'), false) }) -test('isAccountQuotaExhausted keeps provider 429s and plan-limit text', () => { - assert.equal(isAccountQuotaExhausted({ ok: false, status: 429, via: 'go-worker-stream' }, ''), true) - assert.equal(isAccountQuotaExhausted({ ok: false, status: 429 }, ''), true) +test('isAccountQuotaExhausted requires quota evidence rather than a bare provider 429', () => { + assert.equal(isAccountQuotaExhausted({ ok: false, status: 429, via: 'go-worker-stream' }, ''), false) + assert.equal(isAccountQuotaExhausted({ ok: false, status: 429 }, ''), false) const planText = "You've hit your limit ยท resets 3pm" assert.equal(isAccountQuotaExhausted({ ok: false, status: 400, via: 'pool-failover' }, planText), true) assert.equal(isAccountQuotaExhausted({ ok: true, status: 200 }, planText), false) assert.equal(isAccountQuotaExhausted({ ok: false, status: 503, via: 'go-worker' }, ''), false) }) + +test('account-window rejection headers prove quota while Fable-only rejection does not', () => { + for (const window of ['5h', '7d']) { + assert.equal( + isAccountQuotaExhausted({ + ok: false, + status: 429, + headers: { + [`Anthropic-Ratelimit-Unified-${window}-Status`]: 'rejected', + }, + }), + true, + ) + } + assert.equal( + isAccountQuotaExhausted({ + ok: false, + status: 429, + headers: { + 'anthropic-ratelimit-unified-7d_oi-status': 'rejected', + }, + }), + false, + ) +})