Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/lib/pool/pool-scheduler.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,14 @@ function selectionSnapshot(candidates = [], available = [], extras = {}) {
const waitPool = extras.waitPool || candidates
const waitReasons = [...new Set((waitPool || []).map((candidate) => candidate.waitReason).filter(Boolean))]
const soonest = (waitPool || []).map((candidate) => Number(candidate.availableAt) || 0).filter((value) => value > now)
const quotaLimited =
extras.reason === 'all_accounts_busy' &&
waitPool.length > 0 &&
waitPool.every(
(candidate) => candidate.waitReason === 'account_cooldown' && isQuotaWindowReason(candidate.cooldownReason),
)
return {
reason: extras.reason,
reason: quotaLimited ? 'pool_rate_limited' : extras.reason,
wait_ms: extras.waitMs ?? 0,
soonest_available_ms: soonest.length ? Math.min(...soonest) - now : null,
wait_reasons: waitReasons,
Expand Down
13 changes: 10 additions & 3 deletions src/lib/pool/quota-window.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,20 @@ export function isPlanLimitMessage(message) {

/**
* Whether a finished request proves the account's own plan window is spent.
* A 429 the pool synthesized itself (`fable_requires_max`, `pool_rate_limited`)
* never reached the provider, so it says nothing about the account's quota.
* Only explicit plan-limit text or rejected account-window headers prove
* exhaustion. A bare 429 can be a model or RPM limit and must not invent a
* persistent 5h rejection for the entire account.
*/
export function isAccountQuotaExhausted(result = {}, limitText = '') {
if (!result || result.ok) return false
if (isPlanLimitMessage(limitText)) return true
return Number(result.status) === 429 && result.via !== 'pool-failover'
if (Number(result.status) !== 429 || result.via === 'pool-failover') return false
const headers = result.headers || {}
const entries = typeof headers.entries === 'function' ? [...headers.entries()] : Object.entries(headers)
return entries.some(
([key, value]) =>
/^anthropic-ratelimit-unified-(5h|7d)-status$/i.test(key) && /^(rejected|rate_limited)$/i.test(String(value)),
)
}

/**
Expand Down
30 changes: 29 additions & 1 deletion test/unit/account-quota.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { AccountQuota, extraHeadersFromLimitError } from '../../src/lib/pool/account-quota.mjs'
import { parseResetMs } from '../../src/lib/pool/quota-window.mjs'
import { parseResetMs, isAccountQuotaExhausted } from '../../src/lib/pool/quota-window.mjs'

function tmpDir() {
return fs.mkdtempSync(path.join(os.tmpdir(), 'kin-quota-'))
Expand Down Expand Up @@ -923,6 +923,34 @@ test('429 unified exhausted marks headers rejected without inventing official 10
assert.equal(q.canAccept('acc-429').ok, false)
})

test('provider Fable 429 without quota evidence preserves the open account windows', () => {
const q = new AccountQuota({ dataDir: tmpDir(), config: {} })
q.ingestHeaders('acc-fable-429', {
'anthropic-ratelimit-unified-5h-utilization': '0.02',
'anthropic-ratelimit-unified-5h-status': 'allowed',
'anthropic-ratelimit-unified-7d-utilization': '0.07',
'anthropic-ratelimit-unified-7d-status': 'allowed',
})
const result = {
ok: false,
status: 429,
via: 'rust-kernel-stream',
headers: {},
body: { error: { type: 'rate_limit_error', message: 'Error' } },
}
const text = result.body.error.message
q.ingestHeaders('acc-fable-429', extraHeadersFromLimitError(text, result.headers), null, {
exhausted: isAccountQuotaExhausted(result, text),
status: result.status,
})
const account = q.repo.get('acc-fable-429')
assert.equal(account.unified.headers['5h'].status, 'allowed')
assert.equal(account.unified.headers['5h'].utilization, 0.02)
assert.equal(account.unified.headers['7d'].status, 'allowed')
assert.equal(account.unified.headers.exhausted_at ?? null, null)
assert.equal(q.canAccept('acc-fable-429').ok, true)
})

test('local 429 without quota evidence does not reject the 5h window', () => {
const q = new AccountQuota({ dataDir: tmpDir(), config: {} })
q.ingestHeaders('acc-local-429', {
Expand Down
34 changes: 34 additions & 0 deletions test/unit/pool-scheduler.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,40 @@ test('wait timeout on cooldown returns busy rather than empty pool', async (t) =
assert.ok(selected.wait_reasons.includes('account_cooldown'))
})

test('pinned quota cooldown returns rate limited rather than pool overload', async (t) => {
const root = project()
let pool
t.after(() => cleanupProject(root, pool))
const repo = new RuntimeRepo()
repo.markCooldown('account-1', {
vmId: 'vm-01',
until: Date.now() + 60_000,
reason: 'quota_5h_header',
})
pool = new PoolScheduler({
projectRoot: root,
runtimeRepo: repo,
accountQuota: { canAccept: () => ({ ok: true }) },
workerHealth: async () => ({ ok: true, credential: { generation: 1, has_access: true } }),
})
let dispatched = false
const runner = new FailoverRunner({ scheduler: pool })
const result = await runner.run({
requestId: 'quota-cooldown-pin',
model: 'claude-test',
pinVmId: 'vm-01',
callAttempt: async () => {
dispatched = true
throw new Error('must not dispatch')
},
})
assert.equal(dispatched, false)
assert.equal(result.status, 429)
assert.equal(result.body.error.code, 'pool_rate_limited')
assert.ok(result.retryAfterSec > 0)
assert.deepEqual(result.body.error.details.wait_reasons, ['account_cooldown'])
})

test('long cooldown beyond wait budget fails immediately', async (t) => {
const root = project()
t.after(() => fs.rmSync(root, { recursive: true, force: true }))
Expand Down
31 changes: 28 additions & 3 deletions test/unit/quota-window.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -253,11 +253,36 @@ test('isAccountQuotaExhausted ignores 429s the pool synthesized itself', () => {
assert.equal(isAccountQuotaExhausted(poolRateLimited, 'rate limited'), false)
})

test('isAccountQuotaExhausted keeps provider 429s and plan-limit text', () => {
assert.equal(isAccountQuotaExhausted({ ok: false, status: 429, via: 'go-worker-stream' }, ''), true)
assert.equal(isAccountQuotaExhausted({ ok: false, status: 429 }, ''), true)
test('isAccountQuotaExhausted requires quota evidence rather than a bare provider 429', () => {
assert.equal(isAccountQuotaExhausted({ ok: false, status: 429, via: 'go-worker-stream' }, ''), false)
assert.equal(isAccountQuotaExhausted({ ok: false, status: 429 }, ''), false)
const planText = "You've hit your limit · resets 3pm"
assert.equal(isAccountQuotaExhausted({ ok: false, status: 400, via: 'pool-failover' }, planText), true)
assert.equal(isAccountQuotaExhausted({ ok: true, status: 200 }, planText), false)
assert.equal(isAccountQuotaExhausted({ ok: false, status: 503, via: 'go-worker' }, ''), false)
})

test('account-window rejection headers prove quota while Fable-only rejection does not', () => {
for (const window of ['5h', '7d']) {
assert.equal(
isAccountQuotaExhausted({
ok: false,
status: 429,
headers: {
[`Anthropic-Ratelimit-Unified-${window}-Status`]: 'rejected',
},
}),
true,
)
}
assert.equal(
isAccountQuotaExhausted({
ok: false,
status: 429,
headers: {
'anthropic-ratelimit-unified-7d_oi-status': 'rejected',
},
}),
false,
)
})
Loading