From 0f1085b8b88674ce0b07b8579922c059485860d8 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Tue, 20 Oct 2026 21:53:13 +0200 Subject: [PATCH 1/2] fix(task): stop retrying a persistently failing stream forever A mid-stream failure (e.g. OpenRouter 'Upstream idle timeout exceeded') re-queued the same request with no limit. Without auto-approval it did so immediately and silently, and with auto-approval it backed off indefinitely. Retry automatically at most 5 times, then ask the user via api_req_failed. Fixes #2107 Co-Authored-By: Claude Sonnet 5.5 --- src/core/task/Task.ts | 51 ++++++++++++++++++++-------- src/core/task/__tests__/Task.spec.ts | 51 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index e573a7f107..d14a1d2e2e 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -153,6 +153,8 @@ import { } from "./run-state/runStateDivergence" const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes +// Consecutive mid-stream failures retried automatically before the user is asked. +const MAX_MID_STREAM_AUTO_RETRIES = 5 const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds // Upper bound on awaiting lazily loaded model metadata. Some model-catalog // fetchers (e.g. OpenRouter's bare axios GET) have no request timeout, so a @@ -4671,26 +4673,47 @@ export class Task extends EventEmitter implements TaskLike { }) continue } else { + const stateForBackoff = await this.providerRef.deref()?.getState() + const midStreamRetryAttempt = currentItem.retryAttempt ?? 0 + + // Without auto-approval, or once automatic retries are exhausted, a + // persistently failing stream (e.g. an upstream idle timeout) would + // otherwise be re-requested forever. Let the user decide instead. + if ( + !stateForBackoff?.autoApprovalEnabled || + midStreamRetryAttempt >= MAX_MID_STREAM_AUTO_RETRIES + ) { + const { response } = await this.ask("api_req_failed", rawErrorMessage) + + if (response !== "yesButtonClicked") { + throw new Error("API request failed") + } + + await this.say("api_req_retried") + stack.push({ + userContent: currentUserContent, + includeFileDetails: false, + retryAttempt: 0, + }) + continue + } + // Stream failed - log the error and retry with the same content - // The existing rate limiting will prevent rapid retries console.error( `[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`, ) - // Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled - const stateForBackoff = await this.providerRef.deref()?.getState() - if (stateForBackoff?.autoApprovalEnabled) { - await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) + // Apply exponential backoff similar to first-chunk errors + await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) - // Check if task was aborted during the backoff - if (this.abort) { - console.log( - `[Task#${this.taskId}.${this.instanceId}] Task aborted during mid-stream retry backoff`, - ) - this.setAbortReason("user_cancelled") - await this.abortTask() - break - } + // Check if task was aborted during the backoff + if (this.abort) { + console.log( + `[Task#${this.taskId}.${this.instanceId}] Task aborted during mid-stream retry backoff`, + ) + this.setAbortReason("user_cancelled") + await this.abortTask() + break } // Push the same content back onto the stack to retry, incrementing the retry attempt counter diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index ad07230204..a172a9528b 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -475,6 +475,8 @@ describe("Cline", () => { apiConfiguration: mockApiConfig, autoApprovalEnabled: false, }) + // A persistent mid-stream failure asks the user; these tests confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) return task } @@ -621,6 +623,49 @@ describe("Cline", () => { }) }) + describe("persistent mid-stream failure", () => { + async function* idleTimeoutStream(): AsyncGenerator { + yield { type: "text", text: "partial answer" } + throw new Error("Provider ended the request: Upstream idle timeout exceeded.") + } + + // 5 is MAX_MID_STREAM_AUTO_RETRIES: the first attempt plus 5 automatic retries. + it.each([ + { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffs: 5 }, + { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffs: 0 }, + ])( + "asks the user instead of retrying forever (autoApprovalEnabled=$autoApprovalEnabled)", + async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffs }) => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const backoffSpy = vi.spyOn(getTaskTestAccess(task), "backoffAndAnnounce").mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }]).catch(() => {}) + + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(expectedAttempts) + expect(backoffSpy).toHaveBeenCalledTimes(expectedBackoffs) + expect(askSpy).toHaveBeenCalledOnce() + expect(askSpy).toHaveBeenCalledWith("api_req_failed", expect.stringContaining("idle timeout")) + }, + ) + }) + describe("native tool-call request isolation", () => { it("derives readiness only from a closed, idle, fully paired tool turn", () => { const task = new Task({ @@ -1163,6 +1208,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -1211,6 +1258,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -1303,6 +1352,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) From ba7c3d9dfe90f617201f38a6b10222cb86bc4d03 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sun, 25 Oct 2026 11:01:53 +0100 Subject: [PATCH 2/2] fix(task): keep history attempt separate from the mid-stream auto-retry count Co-Authored-By: Claude Sonnet 5.5 --- src/core/task/Task.ts | 13 ++++++-- src/core/task/__tests__/Task.spec.ts | 50 +++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index d14a1d2e2e..dc5e62e9b3 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3866,6 +3866,9 @@ export class Task extends EventEmitter implements TaskLike { userContent: Anthropic.Messages.ContentBlockParam[] includeFileDetails: boolean retryAttempt?: number + // Automatic retries of a mid-stream failure. Kept apart from `retryAttempt`, which also + // tells shouldAddUserMessageToHistory that the user message is already in history. + midStreamAutoRetries?: number userMessageWasRemoved?: boolean // Track if user message was removed due to empty response } @@ -4674,7 +4677,7 @@ export class Task extends EventEmitter implements TaskLike { continue } else { const stateForBackoff = await this.providerRef.deref()?.getState() - const midStreamRetryAttempt = currentItem.retryAttempt ?? 0 + const midStreamRetryAttempt = currentItem.midStreamAutoRetries ?? 0 // Without auto-approval, or once automatic retries are exhausted, a // persistently failing stream (e.g. an upstream idle timeout) would @@ -4690,10 +4693,13 @@ export class Task extends EventEmitter implements TaskLike { } await this.say("api_req_retried") + // The user message is already in history, so keep the history attempt positive + // and only reset the automatic-retry count. stack.push({ userContent: currentUserContent, includeFileDetails: false, - retryAttempt: 0, + retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + midStreamAutoRetries: 0, }) continue } @@ -4704,7 +4710,7 @@ export class Task extends EventEmitter implements TaskLike { ) // Apply exponential backoff similar to first-chunk errors - await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) + await this.backoffAndAnnounce(midStreamRetryAttempt, error) // Check if task was aborted during the backoff if (this.abort) { @@ -4721,6 +4727,7 @@ export class Task extends EventEmitter implements TaskLike { userContent: currentUserContent, includeFileDetails: false, retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + midStreamAutoRetries: midStreamRetryAttempt + 1, }) // Continue to retry the request diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index a172a9528b..a3923a4656 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -631,11 +631,11 @@ describe("Cline", () => { // 5 is MAX_MID_STREAM_AUTO_RETRIES: the first attempt plus 5 automatic retries. it.each([ - { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffs: 5 }, - { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffs: 0 }, + { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffAttempts: [0, 1, 2, 3, 4] }, + { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffAttempts: [] }, ])( "asks the user instead of retrying forever (autoApprovalEnabled=$autoApprovalEnabled)", - async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffs }) => { + async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffAttempts }) => { vi.spyOn(mockProvider, "getState").mockResolvedValue( providerStateWith({ autoApprovalEnabled, requestDelaySeconds: 0 }), ) @@ -656,14 +656,54 @@ describe("Cline", () => { .spyOn(task, "attemptApiRequest") .mockImplementation(() => idleTimeoutStream()) - await task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }]).catch(() => {}) + // Declining ends the turn: the "API request failed" error is handled inside the loop, + // so the call resolves rather than rejecting. + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) expect(attemptApiRequestSpy).toHaveBeenCalledTimes(expectedAttempts) - expect(backoffSpy).toHaveBeenCalledTimes(expectedBackoffs) + expect(backoffSpy.mock.calls.map(([attempt]) => attempt)).toEqual(expectedBackoffAttempts) expect(askSpy).toHaveBeenCalledOnce() expect(askSpy).toHaveBeenCalledWith("api_req_failed", expect.stringContaining("idle timeout")) }, ) + + it("does not add the user message to history again when the user confirms a retry", async () => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled: false, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const historySpy = vi + .spyOn(getTaskTestAccess(task), "addToApiConversationHistory") + .mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValueOnce({ response: "yesButtonClicked" } satisfies TaskAskResult) + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) + + expect(askSpy).toHaveBeenCalledTimes(2) + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(2) + const userMessages = historySpy.mock.calls.filter( + ([message]) => (message as { role: string }).role === "user", + ) + expect(userMessages).toHaveLength(1) + }) }) describe("native tool-call request isolation", () => {