diff --git a/packages/acp-executor-plugin/src/__tests__/acp-process.test.ts b/packages/acp-executor-plugin/src/__tests__/acp-process.test.ts index e1c2c963295..a2c2eb05822 100644 --- a/packages/acp-executor-plugin/src/__tests__/acp-process.test.ts +++ b/packages/acp-executor-plugin/src/__tests__/acp-process.test.ts @@ -313,12 +313,15 @@ for (const prompt of ['wait', 'refusal', 'ignore-cancel', 'crash']) { } assert.equal(values.get('task')?.phase, 'committed'); const terminalEvents: string[] = []; + let stopping: Promise | undefined; for await (const event of backend.send({ turnId: 'stopped', text: prompt })) { terminalEvents.push(event.type); - if (event.type === 'text_delta') await backend.stop('user_stop'); + // Keep draining while Stop waits for the final updates to be consumed. + if (event.type === 'text_delta' && !stopping) stopping = backend.stop('user_stop'); if (event.type === 'complete') assert.equal(values.get('task')?.phase, 'prompt_pending', 'delivery alone cannot commit'); } + await stopping; assert.equal(terminalEvents.at(-1), 'complete'); assert.ok(terminalEvents.includes(['crash', 'refusal'].includes(prompt) ? 'error' : 'abort')); const settled = prompt === 'wait' || prompt === 'refusal'; diff --git a/packages/core/src/__tests__/runtime-event.test.ts b/packages/core/src/__tests__/runtime-event.test.ts index 48c8e6c9f87..19a7a61c7e4 100644 --- a/packages/core/src/__tests__/runtime-event.test.ts +++ b/packages/core/src/__tests__/runtime-event.test.ts @@ -57,6 +57,16 @@ function baseEvent(overrides: Partial = {}): RuntimeEvent { }; } +test('interrupted assistant text survives recovery decoding without becoming model context', () => { + const event = baseEvent({ + modelVisibility: 'hidden', + content: { kind: 'text', text: 'unfinished answer', interrupted: true }, + }); + const decoded = decodeRuntimeEvent(JSON.parse(JSON.stringify(event))); + assert.deepEqual(decoded, event); + assert.equal(runtimeEventHasModelVisibleContent(decoded), false); +}); + test('Stored assistant reasoning parts survive recovery decoding', () => { const parts = [ { diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index 21d56dcb45d..5f178d97ead 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -1041,6 +1041,7 @@ export function decodeRuntimeEvent(value: unknown): RuntimeEvent { content: { kind: 'text', ...normalizeMessageContent(value.content as unknown as MessageContent), + ...(value.content.interrupted === true ? { interrupted: true as const } : {}), ...(value.content.summary !== undefined ? { summary: value.content.summary } : {}), ...(value.content.origin !== undefined ? { origin: decodeTurnOrigin(value.content.origin) } diff --git a/packages/runtime/src/__tests__/plugin-executor-backend.test.ts b/packages/runtime/src/__tests__/plugin-executor-backend.test.ts index 97d8a2c5dff..822b4b6c96b 100644 --- a/packages/runtime/src/__tests__/plugin-executor-backend.test.ts +++ b/packages/runtime/src/__tests__/plugin-executor-backend.test.ts @@ -194,13 +194,106 @@ test('oversized executor completion fails without emitting a large text_complete const events = await collect(backend.send({ turnId: 'turn-a', text: 'task' })); assert.deepEqual( events.map((event) => event.type), - ['text_delta', 'error', 'complete'], + ['text_delta', 'text_complete', 'error', 'complete'], ); - assert.match(events[1]?.type === 'error' ? events[1].message : '', /completion text exceeds/u); - assert.equal(events[2]?.type === 'complete' ? events[2].stopReason : undefined, 'error'); + assert.equal(events[1]?.type === 'text_complete' ? events[1].text : undefined, 'streamed'); + assert.equal(events[1]?.type === 'text_complete' ? events[1].interrupted : undefined, true); + assert.match(events[2]?.type === 'error' ? events[2].message : '', /completion text exceeds/u); + assert.equal(events[3]?.type === 'complete' ? events[3].stopReason : undefined, 'error'); await root.fiber.dispose(); }); +for (const outcome of ['cancelled', 'failed', 'transport-error', 'oversized-completion'] as const) { + test(`interrupted executor text stays bounded after ${outcome}`, async () => { + const chunk = 'x'.repeat(16 * 1024); + const { root, binding } = fixture(async (_request, context) => { + for (let index = 0; index < 17; index++) context.emit({ type: 'output_delta', text: chunk }); + context.emit({ type: 'output_delta', text: 'late update' }); + if (outcome === 'transport-error') throw new Error('transport ended'); + if (outcome === 'oversized-completion') + return { status: 'completed', text: chunk.repeat(17) }; + return outcome === 'cancelled' + ? { status: 'cancelled' } + : { status: 'failed', message: 'provider failed' }; + }); + const backend = new PluginExecutorBackend({ + sessionId: 'session-a', + cwd: '/workspace', + binding, + }); + try { + const events = await collect(backend.send({ turnId: 'turn-a', text: 'task' })); + const answers = events.filter((event) => event.type === 'text_complete'); + assert.equal(answers.length, 1); + const answer = answers[0]!; + assert.equal(Buffer.byteLength(answer.text), 256 * 1024); + assert.equal(answer.interrupted, true); + const marker = '\n[Response truncated]'; + assert.equal(answer.text, 'x'.repeat(256 * 1024 - Buffer.byteLength(marker)) + marker); + assert.equal(events.filter((event) => event.type === 'text_delta').length, 18); + const terminal = events.find((event) => event.type === 'complete'); + assert.equal(terminal?.stopReason, outcome === 'cancelled' ? 'user_stop' : 'error'); + } finally { + await backend.dispose(); + await root.fiber.dispose(); + } + }); +} + +for (const scenario of [ + { + name: 'one oversized Unicode delta', + chunks: ['δΈ­πŸ™‚'.repeat(100_000)], + expected: 'δΈ­πŸ™‚'.repeat(Math.floor((256 * 1024 - 21) / 7)) + '\n[Response truncated]', + }, + { + name: 'a surrogate pair split across deltas at the exact limit', + chunks: ['x'.repeat(256 * 1024 - 4), '\ud83d', '\ude42'], + expected: 'x'.repeat(256 * 1024 - 4) + 'πŸ™‚', + }, + { + name: 'a Unicode boundary before the truncation marker', + chunks: ['x'.repeat(256 * 1024 - 25), '\ud83d', '\ude42', 'z'.repeat(26)], + expected: 'x'.repeat(256 * 1024 - 25) + 'πŸ™‚\n[Response truncated]', + }, +]) { + test(`interrupted executor text preserves ${scenario.name}`, async () => { + const { root, backend } = recordingFixture(async (_request, context) => { + for (const text of scenario.chunks) context.emit({ type: 'output_delta', text }); + return { status: 'cancelled' }; + }); + try { + const events = await collect(backend.send({ turnId: 'turn-a', text: 'task' })); + const answers = events.filter((event) => event.type === 'text_complete'); + assert.equal(answers.length, 1); + assert.equal(answers[0]?.text, scenario.expected); + assert.ok(Buffer.byteLength(answers[0]!.text) <= 256 * 1024); + assert.equal(Buffer.from(answers[0]!.text, 'utf8').toString('utf8'), answers[0]?.text); + } finally { + await backend.dispose(); + await root.fiber.dispose(); + } + }); +} + +test('a bounded interrupted buffer does not replace a successful provider answer', async () => { + const { root, backend, acknowledged } = recordingFixture(async (_request, context) => { + context.emit({ type: 'output_delta', text: 'x'.repeat(256 * 1024 + 1) }); + return { status: 'completed', text: 'authoritative answer' }; + }); + try { + const events = await collect(backend.send({ turnId: 'turn-a', text: 'task' })); + const answers = events.filter((event) => event.type === 'text_complete'); + assert.equal(answers.length, 1); + assert.equal(answers[0]?.text, 'authoritative answer'); + assert.equal(answers[0]?.interrupted, undefined); + assert.deepEqual(acknowledged, ['session-a/turn-a']); + } finally { + await backend.dispose(); + await root.fiber.dispose(); + } +}); + test('executor backend turns stop into abort and terminal events', async () => { let started!: () => void; const ready = new Promise((resolve) => { @@ -233,6 +326,32 @@ test('executor backend turns stop into abort and terminal events', async () => { await root.fiber.dispose(); }); +test('executor stop releases its drain wait when the consumer detaches', { + timeout: 5_000, +}, async () => { + const { root, backend, acknowledged, abandoned } = recordingFixture(async (_request, context) => { + context.emit({ type: 'output_delta', text: 'before stop' }); + await new Promise((resolve) => + context.signal.addEventListener('abort', () => resolve(), { once: true }), + ); + context.emit({ type: 'output_delta', text: 'after stop' }); + return { status: 'cancelled' }; + }); + const iterator = backend.send({ turnId: 'turn-a', text: 'task' })[Symbol.asyncIterator](); + try { + assert.equal((await iterator.next()).value?.type, 'text_delta'); + const stopping = backend.stop('user_stop'); + await iterator.return?.(); + await stopping; + assert.deepEqual(acknowledged, []); + assert.deepEqual(abandoned, ['session-a/turn-a']); + } finally { + await iterator.return?.(); + await backend.dispose(); + await root.fiber.dispose(); + } +}); + test('executor backend projects optional thinking and external tool activity', async () => { const { root, binding } = fixture( async (_request, context) => { diff --git a/packages/runtime/src/__tests__/plugin-executor-history.test.ts b/packages/runtime/src/__tests__/plugin-executor-history.test.ts new file mode 100644 index 00000000000..9778194612c --- /dev/null +++ b/packages/runtime/src/__tests__/plugin-executor-history.test.ts @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { createSessionStore } from '@maka/storage/session-store'; +import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; +import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; +import { PluginExecutorBackend } from '../plugin-executor-backend.js'; +import { PluginExecutorService } from '../plugin-executor-service.js'; +import { Context } from '../plugin-kernel.js'; +import { BackendRegistry, SessionManager } from '../session-manager.js'; +import { projectRuntimeEventsToStoredMessages } from '../runtime-event-read-model.js'; + +for (const outcome of [ + 'cancelled', + 'cancelled-oversized', + 'abort-error', + 'failed', + 'transport-error', +] as const) { + test(`interrupted executor output survives SQLite reopen: ${outcome}`, { + timeout: 10_000, + }, async () => { + const directory = await mkdtemp(join(tmpdir(), 'maka-executor-history-')); + const sessions = createSessionStore(directory); + const runs = createSqliteAgentRunStore(directory); + const events = createWorkspaceRuntimeStore(directory); + const root = new Context(); + const service = new PluginExecutorService(root); + const cancelling = + outcome === 'cancelled' || outcome === 'cancelled-oversized' || outcome === 'abort-error'; + root + .extend({ + maka: { rootId: 'profile', packageId: 'fixture', entryId: 'provider', generation: 1 }, + }) + .executors.register({ + id: 'remote', + execute: async (_request, context) => { + context.emit({ type: 'output_delta', text: '保留 before stop' }); + if (cancelling) { + await new Promise((resolve) => + context.signal.addEventListener('abort', () => resolve(), { once: true }), + ); + } + // Cancellation still drains updates until the provider settles. + if (outcome === 'cancelled-oversized') { + for (let index = 0; index < 17; index++) + context.emit({ type: 'output_delta', text: 'x'.repeat(16 * 1024) }); + } + context.emit({ type: 'output_delta', text: ' and final update.' }); + if (outcome === 'abort-error' || outcome === 'transport-error') + throw new Error('transport ended'); + return cancelling + ? { status: 'cancelled', providerStopReason: 'cancelled' } + : { status: 'failed', message: 'provider failed' }; + }, + }); + const backends = new BackendRegistry(); + backends.register( + 'plugin-executor', + (context) => + new PluginExecutorBackend({ + sessionId: context.sessionId, + cwd: directory, + binding: service.bind(context.sessionId, 'remote'), + }), + ); + const manager = new SessionManager({ + store: sessions, + runStore: runs, + runtimeEventStore: events, + backends, + newId: randomUUID, + now: Date.now, + }); + try { + const session = await manager.createSession({ + cwd: directory, + executorId: 'remote', + llmConnectionSlug: 'executor:remote', + permissionMode: 'bypass', + }); + let stopping: Promise | undefined; + for await (const event of manager.sendMessage(session.id, { + turnId: 'turn-a', + text: 'write a long answer', + })) { + if (event.type === 'text_delta' && cancelling && !stopping) + stopping = manager.stopSession(session.id); + } + await stopping; + events.close(); + const reopened = createWorkspaceRuntimeStore(directory); + try { + const ledger = await reopened.readSessionRuntimeEvents(session.id); + const invocations = await reopened.listSessionInvocations(session.id); + const answers = projectRuntimeEventsToStoredMessages(ledger, { + invocations, + }).messages.filter((message) => message.type === 'assistant'); + assert.equal(answers.length, 1, 'the visible partial answer survives exactly once'); + const marker = '\n[Response truncated]'; + const expected = + outcome === 'cancelled-oversized' + ? '保留 before stop' + + 'x'.repeat(256 * 1024 - Buffer.byteLength('保留 before stop' + marker)) + + marker + : '保留 before stop and final update.'; + assert.equal(answers[0]?.text, expected); + assert.ok(Buffer.byteLength(answers[0]!.text) <= 256 * 1024); + const saved = ledger.filter( + (event) => event.role === 'model' && event.content?.kind === 'text', + ); + assert.equal(saved.length, 1); + assert.equal(saved[0]?.partial, false); + assert.equal( + saved[0]?.modelVisibility, + 'hidden', + 'interrupted output is display history, not completed model context', + ); + assert.equal(saved[0]?.content?.kind === 'text' && saved[0].content.interrupted, true); + assert.equal(invocations[0]?.terminalEvent?.status, cancelling ? 'aborted' : 'failed'); + } finally { + reopened.close(); + } + } finally { + await root.fiber.dispose(); + events.close(); + runs.close?.(); + await sessions.close?.(); + await rm(directory, { recursive: true, force: true }); + } + }); +} diff --git a/packages/runtime/src/plugin-executor-backend.ts b/packages/runtime/src/plugin-executor-backend.ts index 10e490aaaa5..16083e7080b 100644 --- a/packages/runtime/src/plugin-executor-backend.ts +++ b/packages/runtime/src/plugin-executor-backend.ts @@ -19,6 +19,7 @@ import type { ExecutorConfiguration } from '@maka/core/executor-catalog'; import { randomUUID } from 'node:crypto'; +import { truncateUtf8 } from '@maka/core/diagnostic-log'; import type { SessionEvent } from '@maka/core/events'; import { redactSecrets } from '@maka/core/redaction'; import type { @@ -31,6 +32,7 @@ import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; import type { UserQuestionResponse } from '@maka/core/user-question'; import type { ThinkingLevel } from '@maka/core/model-thinking'; import { AsyncEventQueue } from './async-queue.js'; +import { MAX_EXECUTOR_COMPLETION_TEXT_BYTES } from './plugin-executor-service.js'; import type { PluginExecutorBinding, PluginExecutorOutputEvent, @@ -42,6 +44,7 @@ import type { interface ActiveExecution { readonly abort: AbortController; readonly settled: Promise; + readonly queue: AsyncEventQueue; } /** @@ -97,7 +100,7 @@ export class PluginExecutorBackend implements AgentBackend { const producer = this.#produce(input, messageId, abort.signal, queue).finally(() => queue.close(), ); - const active: ActiveExecution = { abort, settled: producer.then(() => undefined) }; + const active: ActiveExecution = { abort, settled: producer.then(() => undefined), queue }; this.#active.add(active); let acknowledged = false; try { @@ -137,10 +140,23 @@ export class PluginExecutorBackend implements AgentBackend { } } + /** + * Cancel execution and wait for its queued events to be consumed or the consumer + * to detach. Keep consuming send() while stopping: awaiting stop() inside that + * consumer loop prevents progress. Start it without awaiting in the loop, then + * await it after the loop exits. + */ async stop(reason: 'user_stop' | 'redirect'): Promise { const active = [...this.#active]; for (const execution of active) execution.abort.abort(new Error(reason)); - await Promise.allSettled(active.map((execution) => execution.settled)); + await Promise.allSettled( + active.map(async (execution) => { + await execution.settled; + // Runtime may seal the ledger when stop returns. Provider settlement + // alone does not mean its queued final updates have been persisted. + await execution.queue.waitUntilConsumedThroughCurrent(); + }), + ); } async respondToSandboxBoundary(_response: SandboxBoundaryResponse): Promise { @@ -165,6 +181,9 @@ export class PluginExecutorBackend implements AgentBackend { ): Promise { const turnId = input.turnId; let thinkingText = ''; + let outputText = ''; + let outputBytes = 0; + let outputTruncated = false; const toolUseIds = new Map(); const toolOutputSequences = new Map(); let result: PluginExecutorResult | undefined; @@ -195,6 +214,25 @@ export class PluginExecutorBackend implements AgentBackend { }, onEvent: (event) => { if (event.type === 'thinking_delta') thinkingText += event.text; + if (event.type === 'output_delta' && !outputTruncated) { + // Bound even a single oversized delta before retaining it. Live + // delivery and the provider's final result remain authoritative. + const chunk = event.text.slice(0, MAX_EXECUTOR_COMPLETION_TEXT_BYTES + 1); + outputText += chunk; + outputBytes += Buffer.byteLength(chunk, 'utf8'); + if (outputBytes > MAX_EXECUTOR_COMPLETION_TEXT_BYTES) { + // A surrogate pair split across deltas uses fewer bytes joined. + outputBytes = Buffer.byteLength(outputText, 'utf8'); + if (outputBytes > MAX_EXECUTOR_COMPLETION_TEXT_BYTES) { + outputText = truncateUtf8( + outputText, + MAX_EXECUTOR_COMPLETION_TEXT_BYTES, + '\n[Response truncated]', + ); + outputTruncated = true; + } + } + } this.#publishOutputEvent( turnId, messageId, @@ -214,6 +252,22 @@ export class PluginExecutorBackend implements AgentBackend { } this.#closeOptionalOutput(turnId, messageId, thinkingText, toolUseIds, queue); + // Deltas are only live presentation. Close the interrupted text before the + // terminal event seals the ledger, including updates drained during cancel. + // Successful results keep their provider-authoritative completion text. + // Like successful completions, history stores one text block after tool + // results; it does not preserve the live text/tool interleaving. + if (outputText && (failed || result?.status !== 'completed')) { + queue.push({ + type: 'text_complete', + id: this.#newId(), + turnId, + ts: this.#now(), + messageId, + text: outputText, + interrupted: true, + }); + } if (failed) { if (signal.aborted) this.#publishCancellation(turnId, { status: 'cancelled', source: 'caller' }, queue); diff --git a/packages/runtime/src/plugin-executor-service.ts b/packages/runtime/src/plugin-executor-service.ts index a3c0b0b611c..123196b410c 100644 --- a/packages/runtime/src/plugin-executor-service.ts +++ b/packages/runtime/src/plugin-executor-service.ts @@ -44,7 +44,7 @@ import { } from './plugin-runtime.js'; import { PluginScopeRegistry } from './plugin-scope-registry.js'; -const MAX_EXECUTOR_COMPLETION_TEXT_BYTES = 256 * 1024; +export const MAX_EXECUTOR_COMPLETION_TEXT_BYTES = 256 * 1024; declare module './plugin-kernel.js' { interface Context {