Skip to content

Update VS Code LM model metadata and regression tests #2255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
taltas merged 1 commit into from
Oct 30, 2026
Merged

Update VS Code LM model metadata and regression tests #2255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 86 additions & 6 deletions packages/types/src/__tests__/vscode-llm.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ import { vscodeLlmModels, vscodeLlmDefaultModelId } from "../providers/vscode-ll

describe("vscodeLlmModels", () => {
it("exposes the opus-4.8 row with its measured maxInputTokens and contextWindow", () => {
// claude-opus-4.8 intentionally diverges: maxInputTokens (197897) is the enforced ceiling the
// UI reads, contextWindow (679560) the advertised window. Assert the on-disk literals as a tripwire.
// The two values differ for this row, so pin the exact stored values rather than forcing equality.
expect(vscodeLlmModels).toHaveProperty("claude-opus-4.8")
expect(vscodeLlmModels["claude-opus-4.8"].contextWindow).toBe(679560)
expect(vscodeLlmModels["claude-opus-4.8"].maxInputTokens).toBe(197897)
Expand All @@ -16,13 +15,16 @@ describe("vscodeLlmModels", () => {
expect(vscodeLlmModels["gemini-2.5-pro"].maxInputTokens).toBe(108594)
})
it.each([
["claude-opus-5", "Claude Opus 5", 935793, 197897],
["claude-opus-5.5", "Claude Opus 5.5", 871793, 677108],
["claude-opus-5", "Claude Opus 5", 935793, 680456],
["claude-sonnet-5", "Claude Sonnet 5", 925449, 925449],
["gpt-5.6-luna", "GPT-5.6 Luna", 199753, 199753],
["gpt-5.6-sol", "GPT-5.6 Sol", 271785, 271785],
["gpt-5.6-terra", "GPT-5.6 Terra", 271785, 271785],
["gemini-3.6-flash", "Gemini 3.6 Flash", 935793, 197897],
] as const)("preserves PR #13 row %s (%s) with PR #14 ceilings", (family, name, contextWindow, maxInputTokens) => {
["gemini-3.8-flash", "Gemini 3.8 Flash", 982833, 955113],
["gemini-3.7-flash", "Gemini 3.7 Flash", 935793, 935783],
["gemini-3.6-flash", "Gemini 3.6 Flash", 935793, 935785],
] as const)("preserves measured row %s (%s)", (family, name, contextWindow, maxInputTokens) => {
expect(vscodeLlmModels[family]).toEqual({
contextWindow,
supportsImages: true,
Expand All @@ -38,7 +40,85 @@ describe("vscodeLlmModels", () => {
})

it.each([
["claude-opus-5", "claude-opus-4.8"],
["grok-4.5", "Grok 4.5", 199783],
["grok-4.6", "Grok 4.6", 199784],
] as const)("keeps %s image input disabled pending evidence", (family, name, maxInputTokens) => {
expect(vscodeLlmModels[family]).toEqual({
contextWindow: 424794,
supportsImages: false,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
family,
version: family,
name,
supportsToolCalling: true,
maxInputTokens,
})
})

it("records Astra's measured budget minus a 5K safety margin without unmeasured prices", () => {
expect(vscodeLlmModels["gpt-6-astra"]).toEqual({
contextWindow: 921793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-6-astra",
version: "gpt-6-astra",
name: "GPT-6 Astra",
supportsToolCalling: true,
maxInputTokens: 266789,
})
expect(vscodeLlmModels["gpt-6-astra"]).not.toHaveProperty("inputPrice")
expect(vscodeLlmModels["gpt-6-astra"]).not.toHaveProperty("outputPrice")
})

it.each([
["claude-sonnet-5.5", "Claude Sonnet 5.5", 871793, 680552, true],
["claude-haiku-5.5", "Claude Haiku 5.5", 871793, 99784, true],
["grok-4.7", "Grok 4.7", 424794, 199751, false],
["gemini-4-argon", "Gemini 4 Argon", 982833, 953064, true],
["vscModelF", "MAI-Code-2-Flash (Preview)", 921790, 921736, true],
["oswe-vscode-modelD", "MAI-Code-1.1-Flash", 127790, 127783, true],
["gpt-6.1-sol", "GPT-6.1 Sol", 921793, 269903, true],
["gpt-5.6-sol-fast", "GPT-5.6 Sol Fast (Internal only)", 921793, 269903, true],
["gpt-6.1-terra", "GPT-6.1 Terra", 871793, 269123, true],
["gpt-6-luna", "GPT-6 Luna", 871793, 269122, true],
["gpt-6-sol", "GPT-6 Sol", 871793, 269122, true],
["claude-fable-5.1", "Claude Fable 5.1", 935793, 660686, true],
["claude-fable-5", "Claude Fable 5", 935793, 658785, true],
] as const)(
"records the October capture for %s without unmeasured prices",
(family, name, contextWindow, maxInputTokens, supportsImages) => {
expect(vscodeLlmModels[family]).toEqual({
contextWindow,
maxInputTokens,
supportsImages,
supportsPromptCache: false,
family,
version: family,
name,
supportsToolCalling: true,
})
expect(vscodeLlmModels[family]).not.toHaveProperty("inputPrice")
expect(vscodeLlmModels[family]).not.toHaveProperty("outputPrice")
},
)

it("assigns Fable 5.5 the requested Fable 5.1 values rather than claiming a measurement", () => {
expect(vscodeLlmModels["claude-fable-5.5"]).toEqual({
...vscodeLlmModels["claude-fable-5.1"],
family: "claude-fable-5.5",
version: "claude-fable-5.5",
name: "Claude Fable 5.5",
})
expect(vscodeLlmModels["claude-fable-5.5"].contextWindow).toBe(935793)
expect(vscodeLlmModels["claude-fable-5.5"].maxInputTokens).toBe(660686)
})

it.each([
["claude-opus-5.5", "claude-opus-5", "claude-opus-4.8"],
["gpt-6-astra", "gpt-5.6-luna"],
["grok-4.6", "grok-4.5", "gemini-3.8-flash", "gemini-3.7-flash", "gemini-3.6-flash"],
["claude-sonnet-5", "claude-sonnet-4.6"],
["gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.5"],
["gemini-3.6-flash", "gemini-3.1-pro-preview"],
Expand Down
239 changes: 232 additions & 7 deletions packages/types/src/providers/vscode-llm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,24 @@ export type VscodeLlmModelId = keyof typeof vscodeLlmModels

export const vscodeLlmDefaultModelId: VscodeLlmModelId = "claude-sonnet-4.5"

// The UI and condense gate use maxInputTokens; contextWindow preserves the advertised window.
// https://github.com/simurg79/Roo-Code/pull/13: sonnet-5 and gpt-5.6 were binary-searched.
// https://github.com/simurg79/Roo-Code/pull/14: opus-5 and gemini-3.6-flash retain advertised
// 935793 windows but conservatively reuse the 197897 ceiling measured on other models to avoid
// request overflow; neither model's actual input ceiling has been measured.
// The UI and condense gate use assigned maxInputTokens budgets; contextWindow keeps advertised windows.
// Use measured acceptances, optionally lowered by documented safety margins; Fable 5.5 alone borrows 5.1's values.
// Measurement details: https://github.com/simurg79/Roo-Code/pull/26.
export const vscodeLlmModels = {
// Refusals left the 2026-09-23 bracket open (677108 accepted, 695778 rejected); use the lower bound.
// Image/tool inputs succeeded; caching false and zero prices remain unverified defaults.
"claude-opus-5.5": {
contextWindow: 871793,
supportsImages: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
family: "claude-opus-5.5",
version: "claude-opus-5.5",
name: "Claude Opus 5.5",
supportsToolCalling: true,
maxInputTokens: 677108,
},
"claude-opus-5": {
contextWindow: 935793,
supportsImages: true,
Expand All @@ -65,7 +77,7 @@ export const vscodeLlmModels = {
version: "claude-opus-5",
name: "Claude Opus 5",
supportsToolCalling: true,
maxInputTokens: 197897,
maxInputTokens: 680456,
},
"claude-opus-4.8": {
contextWindow: 679560,
Expand Down Expand Up @@ -115,6 +127,156 @@ export const vscodeLlmModels = {
supportsToolCalling: true,
maxInputTokens: 167790,
},
// Rows below through oswe-vscode-modelD: measured 2026-10-28, VS Code 1.142.0/copilot; maxInputTokens is
// an explicitly accepted size (the smaller when repeated). Prices and caching are unmeasured.
"claude-sonnet-5.5": {
contextWindow: 871793,
supportsImages: true,
supportsPromptCache: false,
family: "claude-sonnet-5.5",
version: "claude-sonnet-5.5",
name: "Claude Sonnet 5.5",
supportsToolCalling: true,
maxInputTokens: 680552,
},
"claude-haiku-5.5": {
contextWindow: 871793,
supportsImages: true,
supportsPromptCache: false,
family: "claude-haiku-5.5",
version: "claude-haiku-5.5",
name: "Claude Haiku 5.5",
supportsToolCalling: true,
maxInputTokens: 99784,
},
"grok-4.7": {
contextWindow: 424794,
// Image request failed with a nondiagnostic HTTP 400: disabled pending evidence.
supportsImages: false,
supportsPromptCache: false,
family: "grok-4.7",
version: "grok-4.7",
name: "Grok 4.7",
supportsToolCalling: true,
maxInputTokens: 199751,
},
"gemini-4-argon": {
contextWindow: 982833,
supportsImages: true,
supportsPromptCache: false,
family: "gemini-4-argon",
version: "gemini-4-argon",
name: "Gemini 4 Argon",
supportsToolCalling: true,
// Accepted at the advertised target; no observed rejection, so only a lower bound.
maxInputTokens: 953064,
},
// Provider lookups need the live family key, not the user-facing id mai-code-2-flash.
vscModelF: {
contextWindow: 921790,
supportsImages: true,
supportsPromptCache: false,
family: "vscModelF",
version: "vscModelF",
name: "MAI-Code-2-Flash (Preview)",
supportsToolCalling: true,
maxInputTokens: 921736,
},
// Live family string; the user-facing id is mai-code-1.1-flash.
"oswe-vscode-modelD": {
contextWindow: 127790,
supportsImages: true,
supportsPromptCache: false,
family: "oswe-vscode-modelD",
version: "oswe-vscode-modelD",
name: "MAI-Code-1.1-Flash",
supportsToolCalling: true,
maxInputTokens: 127783,
},
// Captured 2026-10-28 UTC (Fable 5: 2026-10-29 UTC), VS Code 1.142.0/copilot.
// Image/tool requests were accepted, not verified for understanding or invocation.
// Prices and caching are unmeasured; caching false is a schema default.
"gpt-6.1-sol": {
contextWindow: 921793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-6.1-sol",
version: "gpt-6.1-sol",
name: "GPT-6.1 Sol",
supportsToolCalling: true,
maxInputTokens: 269903,
},
"gpt-5.6-sol-fast": {
contextWindow: 921793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-5.6-sol-fast",
version: "gpt-5.6-sol-fast",
name: "GPT-5.6 Sol Fast (Internal only)",
supportsToolCalling: true,
maxInputTokens: 269903,
},
"gpt-6.1-terra": {
contextWindow: 871793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-6.1-terra",
version: "gpt-6.1-terra",
name: "GPT-6.1 Terra",
supportsToolCalling: true,
maxInputTokens: 269123,
},
"gpt-6-luna": {
contextWindow: 871793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-6-luna",
version: "gpt-6-luna",
name: "GPT-6 Luna",
supportsToolCalling: true,
maxInputTokens: 269122,
},
"gpt-6-sol": {
contextWindow: 871793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-6-sol",
version: "gpt-6-sol",
name: "GPT-6 Sol",
supportsToolCalling: true,
maxInputTokens: 269122,
},
"claude-fable-5.1": {
contextWindow: 935793,
supportsImages: true,
supportsPromptCache: false,
family: "claude-fable-5.1",
version: "claude-fable-5.1",
name: "Claude Fable 5.1",
supportsToolCalling: true,
maxInputTokens: 660686,
},
"claude-fable-5": {
contextWindow: 935793,
supportsImages: true,
supportsPromptCache: false,
family: "claude-fable-5",
version: "claude-fable-5",
name: "Claude Fable 5",
supportsToolCalling: true,
maxInputTokens: 658785,
},
// Values copied from claude-fable-5.1 at the user's request; not measured for claude-fable-5.5.
"claude-fable-5.5": {
contextWindow: 935793,
supportsImages: true,
supportsPromptCache: false,
family: "claude-fable-5.5",
version: "claude-fable-5.5",
name: "Claude Fable 5.5",
supportsToolCalling: true,
maxInputTokens: 660686,
},
"claude-sonnet-5": {
contextWindow: 925449,
supportsImages: true,
Expand Down Expand Up @@ -163,6 +325,18 @@ export const vscodeLlmModels = {
supportsToolCalling: true,
maxInputTokens: 135790,
},
// The 2026-09-23 trials accepted 271789 and rejected 271790; subtract a 5K safety margin.
// Image/tool inputs succeeded; caching false is unverified, and optional prices are unmeasured.
"gpt-6-astra": {
contextWindow: 921793,
supportsImages: true,
supportsPromptCache: false,
family: "gpt-6-astra",
version: "gpt-6-astra",
name: "GPT-6 Astra",
supportsToolCalling: true,
maxInputTokens: 266789,
},
"gpt-5.6-luna": {
contextWindow: 199753,
supportsImages: true,
Expand Down Expand Up @@ -271,6 +445,57 @@ export const vscodeLlmModels = {
supportsToolCalling: true,
maxInputTokens: 12078,
},
"grok-4.6": {
contextWindow: 424794,
// Image trials failed with nondiagnostic HTTP 400s; disable pending evidence.
supportsImages: false,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
family: "grok-4.6",
version: "grok-4.6",
name: "Grok 4.6",
supportsToolCalling: true,
maxInputTokens: 199784,
},
"grok-4.5": {
contextWindow: 424794,
// As with Grok 4.6, image capability remains unverified.
supportsImages: false,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
family: "grok-4.5",
version: "grok-4.5",
name: "Grok 4.5",
supportsToolCalling: true,
maxInputTokens: 199783,
},
"gemini-3.8-flash": {
contextWindow: 982833,
supportsImages: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
family: "gemini-3.8-flash",
version: "gemini-3.8-flash",
name: "Gemini 3.8 Flash",
supportsToolCalling: true,
// No rejection was observed; this is an accepted lower bound, not an exact ceiling.
maxInputTokens: 955113,
},
"gemini-3.7-flash": {
contextWindow: 935793,
supportsImages: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
family: "gemini-3.7-flash",
version: "gemini-3.7-flash",
name: "Gemini 3.7 Flash",
supportsToolCalling: true,
maxInputTokens: 935783,
},
"gemini-3.6-flash": {
contextWindow: 935793,
supportsImages: true,
Expand All @@ -281,7 +506,7 @@ export const vscodeLlmModels = {
version: "gemini-3.6-flash",
name: "Gemini 3.6 Flash",
supportsToolCalling: true,
maxInputTokens: 197897,
maxInputTokens: 935785,
},
"gemini-3.1-pro-preview": {
contextWindow: 197897,
Expand Down
Loading
Loading