From 8db9ac233b984b024d9510a00451990288e6a7bf Mon Sep 17 00:00:00 2001 From: crispinhu <353173575+crispinhu27@users.noreply.github.com> Date: Sat, 21 Nov 2026 17:38:55 +0800 Subject: [PATCH] fix(workspace): show Docker build progress without default timeouts --- chatbot/src/ActivityTrace.jsx | 1 + chatbot/src/activity.js | 2 + chatbot/tests/activity-trace-groups.test.jsx | 19 +++ docs/deployment.md | 6 +- internal/domain/runner_workspace.go | 15 ++ internal/httpapi/assistant_activity.go | 16 +++ internal/httpapi/assistant_process.go | 1 + internal/httpapi/assistant_workspace.go | 1 + internal/httpapi/assistant_workspace_build.go | 29 ++++ internal/httpapi/runner_ws.go | 133 +++++++++++++++--- internal/httpapi/runner_ws_workspace_test.go | 110 ++++++++++++++- internal/procwatch/procwatch.go | 6 + internal/runner/control.go | 52 ++++--- internal/runner/workspace_preparation.go | 13 +- internal/runner/workspace_preparation_test.go | 29 ++++ internal/runtime/docker_build_progress.go | 61 ++++++++ .../runtime/docker_build_progress_test.go | 56 ++++++++ internal/runtime/sandbox_docker.go | 35 ++++- 18 files changed, 530 insertions(+), 55 deletions(-) create mode 100644 internal/httpapi/assistant_workspace_build.go create mode 100644 internal/runtime/docker_build_progress.go diff --git a/chatbot/src/ActivityTrace.jsx b/chatbot/src/ActivityTrace.jsx index 195939679..eb7f5a1e3 100644 --- a/chatbot/src/ActivityTrace.jsx +++ b/chatbot/src/ActivityTrace.jsx @@ -94,6 +94,7 @@ function Step({ step, live, interrupted }) { {target} diff --git a/chatbot/src/activity.js b/chatbot/src/activity.js index 84b27277e..119121020 100644 --- a/chatbot/src/activity.js +++ b/chatbot/src/activity.js @@ -467,6 +467,8 @@ export function activityFace(step) { return { kind: "verify", busy: "正在提交节点结果", done: "已提交节点结果" }; if (/workbench_workspace_open/.test(name)) return { kind: "workflow", busy: "正在打开工作区", done: "已打开工作区" }; + if (name === "workspace_docker_build") + return { kind: "workflow", busy: "正在构建 Docker 镜像", done: "已构建 Docker 镜像" }; if (/workbench_delivery_register/.test(name)) return { kind: "workflow", busy: "正在登记交付", done: "已登记交付" }; if (/workbench_publish_artifact/.test(name)) diff --git a/chatbot/tests/activity-trace-groups.test.jsx b/chatbot/tests/activity-trace-groups.test.jsx index 438540a1d..d0641adb9 100644 --- a/chatbot/tests/activity-trace-groups.test.jsx +++ b/chatbot/tests/activity-trace-groups.test.jsx @@ -3,6 +3,7 @@ import { render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, expect, test } from "vitest"; import ActivityTrace, { ReplyTimeline } from "../src/ActivityTrace.jsx"; +import { mergeActivity } from "../src/activity.js"; const start = "2026-01-01T00:00:00Z"; const finish = "2026-01-01T00:01:05Z"; @@ -17,6 +18,24 @@ const tool = (id, detail, extra = {}) => ({ }); describe("Codex-style activity trace", () => { + test("Docker build progress updates one visible row and settles on completion", () => { + const build = { kind: "tool", tool: "workspace_docker_build", id: "build", at: start }; + const steps = mergeActivity([], [{ data: { ...build, detail: "读取基础镜像信息" } }]); + const view = render(); + expect(screen.getByText("正在构建 Docker 镜像")).toBeVisible(); + expect(screen.getByText("读取基础镜像信息")).toBeVisible(); + + const building = mergeActivity(steps, [{ data: { ...build, detail: "执行 Dockerfile 第 2/3 步" } }]); + view.rerender(); + expect(screen.getAllByText("正在构建 Docker 镜像")).toHaveLength(1); + expect(screen.getByText("执行 Dockerfile 第 2/3 步")).toBeVisible(); + expect(screen.queryByText("读取基础镜像信息")).toBeNull(); + + const done = mergeActivity(building, [{ data: { kind: "toolDone", id: "build", at: finish } }]); + view.rerender(); + expect(screen.getByText("已构建 Docker 镜像")).toBeVisible(); + }); + test("the running round lists every step and folds once it finishes", async () => { const steps = [ { id: "t1", kind: "text", text: "先查澄清面板的前端代码。" }, diff --git a/docs/deployment.md b/docs/deployment.md index e29e802c3..9bdf2e0d8 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -101,13 +101,15 @@ Access to the Docker socket is equivalent to root on the host. Two safeguards fo ### Slow networks: clone and image build limits -Cloning a repository and building a custom sandbox image report progress while they work, and their errors keep the last lines of git or Docker output. On the platform host the progress is recorded on the run. On an execution machine it is written to the runner's log, and only the final error reaches the platform. A step that prints nothing for its stall window is treated as hung and stopped. A step that keeps printing may be slow but is stopped only by a total limit. All settings take Go durations such as `90s` or `20m`; `0` turns a limit off. Each host reads its own environment, so set the clone limits on execution machines too. +Cloning a repository and building a custom sandbox image report progress while they work, and their errors keep the last lines of Git or Docker output. Conversations show the current Docker build step on the platform and on connected runners. A long, quiet Dockerfile step can continue until its caller or a configured limit stops the build. All settings take Go durations such as `90s` or `20m`. `0` turns a limit off. Set the variables on the platform process or execution machine that runs the command. + +Runner conversation workspace preparation follows the conversation request's lifetime. Stopping the turn or losing the runner control connection cancels preparation. | Setting | Default | Applies to | |---|---|---| | `AGENTFLOW_GIT_CLONE_STALL_TIMEOUT` | `5m` | `git clone` with no output. Not applied when the platform clones from a repository's local source path, whose final repack is silent. | | `AGENTFLOW_GIT_CLONE_TIMEOUT` | off on the platform, `10m` on execution machines | Whole `git clone`. Execution machines keep their earlier fixed 10 minute limit as the default. | -| `AGENTFLOW_DOCKER_BUILD_STALL_TIMEOUT` | `10m` | `docker build` with no output. A quiet `RUN` step (a long compile) counts as no output. | +| `AGENTFLOW_DOCKER_BUILD_STALL_TIMEOUT` | off | Optional no-output limit for `docker build`. A quiet `RUN` step counts as no output. | | `AGENTFLOW_DOCKER_BUILD_TIMEOUT` | off | Whole `docker build`. | A clone that is stopped this way is not retried, since a retry would wait on the same network. When the execution host reaches GitHub or Debian mirrors slowly, point git at a proxy (`git config --global http.proxy`) and use a nearby package mirror in the Dockerfile rather than raising these limits. diff --git a/internal/domain/runner_workspace.go b/internal/domain/runner_workspace.go index d23367f98..bfe7ba5e1 100644 --- a/internal/domain/runner_workspace.go +++ b/internal/domain/runner_workspace.go @@ -131,6 +131,15 @@ type RunnerToolResponse struct { Error string `json:"error,omitempty"` } +// WorkspaceImageBuildProgress reports a build step without including Docker +// output, Dockerfile arguments, or environment variables. +type WorkspaceImageBuildProgress struct { + ID string `json:"id"` + Step string `json:"step,omitempty"` + Done bool `json:"done,omitempty"` + Failed bool `json:"failed,omitempty"` +} + type WorkspaceCheckpointTransfer struct { ID string `json:"id,omitempty"` Size int64 `json:"size,omitempty"` @@ -139,6 +148,12 @@ type WorkspaceCheckpointTransfer struct { Data []byte `json:"data,omitempty"` } +// IsCancelableRunnerWorkspaceOperation identifies synchronous requests that +// hold a workspace fence. Conversation processes have their own stop protocol. +func IsCancelableRunnerWorkspaceOperation(op string) bool { + return IsWorkspaceCheckpointTransferOperation(op) || op == RunnerWorkspaceOperationCheckpoint || op == RunnerWorkspaceOperationPrepare +} + func IsWorkspaceCheckpointTransferOperation(op string) bool { switch op { case RunnerWorkspaceOperationCheckpointExport, RunnerWorkspaceOperationCheckpointRead, RunnerWorkspaceOperationCheckpointClose, RunnerWorkspaceOperationCheckpointUpload, RunnerWorkspaceOperationCheckpointAppend, RunnerWorkspaceOperationCheckpointCommit: diff --git a/internal/httpapi/assistant_activity.go b/internal/httpapi/assistant_activity.go index e48e0cda7..4f3d6cd15 100644 --- a/internal/httpapi/assistant_activity.go +++ b/internal/httpapi/assistant_activity.go @@ -67,6 +67,22 @@ func (l *assistantActivityLog) add(event assistantStreamEvent) { } event.Input = truncateRunes(event.Input, 8000) event.Output = truncateRunes(event.Output, 8000) + if event.Kind == "tool" && event.Tool == assistantDockerBuildTool && event.ID != "" { + for i := len(l.events) - 1; i >= 0; i-- { + step := &l.events[i] + if step.Kind == "tool" && step.Tool == event.Tool && step.ID == event.ID { + detail := truncateRunes(event.Detail, 4000) + bytes := l.bytes - len(step.Detail) + len(detail) + if bytes > 512<<10 { + l.truncated = true + return + } + l.bytes = bytes + step.Detail = detail + return + } + } + } size := len(event.Text) + len(event.Detail) + len(event.Input) + len(event.Output) if len(l.events) >= 512 || l.bytes+size > 512<<10 { l.truncated = true diff --git a/internal/httpapi/assistant_process.go b/internal/httpapi/assistant_process.go index ea9221587..6e61a78a0 100644 --- a/internal/httpapi/assistant_process.go +++ b/internal/httpapi/assistant_process.go @@ -73,6 +73,7 @@ func (h *assistantHandlers) conversationExecutor(s *AgentToolSession, root strin request := domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationConversationCLI} workspace := "" if s.RuntimeMode == domain.AssistantModeWorkspace { + ctx = withAssistantDockerBuildProgress(ctx, s) runID := s.workspaceRun() run, ok := h.repo.RunState(runID) if !ok { diff --git a/internal/httpapi/assistant_workspace.go b/internal/httpapi/assistant_workspace.go index 7c42af5b5..529aae649 100644 --- a/internal/httpapi/assistant_workspace.go +++ b/internal/httpapi/assistant_workspace.go @@ -163,6 +163,7 @@ func (h *agentToolGateway) mainBotWorkspaceLease(ctx context.Context, s *AgentTo if h.engine == nil { return ctx, nil, release, nil } + ctx = withAssistantDockerBuildProgress(ctx, s) if s.workspaceTurn != nil { ctx, err := s.workspaceTurn.lease(ctx, runID) if err != nil { diff --git a/internal/httpapi/assistant_workspace_build.go b/internal/httpapi/assistant_workspace_build.go new file mode 100644 index 000000000..e8c8b1978 --- /dev/null +++ b/internal/httpapi/assistant_workspace_build.go @@ -0,0 +1,29 @@ +package httpapi + +import ( + "context" + "time" + + "agentflow/internal/domain" + "agentflow/internal/runtime" +) + +const assistantDockerBuildTool = "workspace_docker_build" + +func withAssistantDockerBuildProgress(ctx context.Context, session *AgentToolSession) context.Context { + if session.Notify == nil || runtime.DockerBuildProgressObserver(ctx) != nil { + return ctx + } + return runtime.WithDockerBuildProgress(ctx, func(update domain.WorkspaceImageBuildProgress) { + if ctx.Err() != nil || update.ID == "" { + return + } + id := "docker-build-" + update.ID + at := time.Now().UTC() + if update.Done { + session.Notify("thought", assistantStreamEvent{Kind: "toolDone", ID: id, At: &at, Failed: update.Failed}) + return + } + session.Notify("thought", assistantStreamEvent{Kind: "tool", Tool: assistantDockerBuildTool, ID: id, At: &at, Detail: update.Step}) + }) +} diff --git a/internal/httpapi/runner_ws.go b/internal/httpapi/runner_ws.go index b39e32d54..e388a164f 100644 --- a/internal/httpapi/runner_ws.go +++ b/internal/httpapi/runner_ws.go @@ -47,15 +47,17 @@ var ( ) type runnerControlMessage struct { - ToolRequest *domain.RunnerToolRequest `json:"toolRequest,omitempty"` - ToolResponse *domain.RunnerToolResponse `json:"toolResponse,omitempty"` - Type string `json:"type"` - TaskID string `json:"taskId,omitempty"` - DesiredState string `json:"desiredState,omitempty"` - ActiveTaskIDs []string `json:"activeTaskIds,omitempty"` - RequestID string `json:"requestId,omitempty"` - WorkspaceRequest *domain.RunnerWorkspaceRequest `json:"workspaceRequest,omitempty"` - WorkspaceResponse *domain.RunnerWorkspaceResponse `json:"workspaceResponse,omitempty"` + ToolRequest *domain.RunnerToolRequest `json:"toolRequest,omitempty"` + ToolResponse *domain.RunnerToolResponse `json:"toolResponse,omitempty"` + Type string `json:"type"` + TaskID string `json:"taskId,omitempty"` + DesiredState string `json:"desiredState,omitempty"` + ActiveTaskIDs []string `json:"activeTaskIds,omitempty"` + RequestID string `json:"requestId,omitempty"` + WorkspaceRequest *domain.RunnerWorkspaceRequest `json:"workspaceRequest,omitempty"` + WorkspaceResponse *domain.RunnerWorkspaceResponse `json:"workspaceResponse,omitempty"` + ObserveImageBuild bool `json:"observeImageBuild,omitempty"` + ImageBuildProgress *domain.WorkspaceImageBuildProgress `json:"imageBuildProgress,omitempty"` } type runnerWorkspaceForwarder func(context.Context, string, domain.RunnerWorkspaceRequest) (domain.RunnerWorkspaceResponse, error) @@ -83,8 +85,9 @@ type runnerControlHub struct { } type runnerWorkspaceWaiter struct { - conn *runnerControlConn - result chan domain.RunnerWorkspaceResponse + conn *runnerControlConn + result chan domain.RunnerWorkspaceResponse + imageProgress func(domain.WorkspaceImageBuildProgress) } type runnerControlConn struct { @@ -250,11 +253,12 @@ func (h *runnerControlHub) send(ctx context.Context, conn *runnerControlConn, re } // Older runners ignore the optional workspace field when decoding JSON. request.GitSSHKnownHosts = runnerWorkspaceSSHKnownHosts(gitwork.WithSSHKnownHostsLookup(ctx, h.sshHostLookup), request) + observeImageBuild := runtime.DockerBuildProgressObserver(ctx) h.mu.Lock() h.nextID++ requestID := fmt.Sprintf("workspace-%d", h.nextID) result := make(chan domain.RunnerWorkspaceResponse, 1) - h.pending[requestID] = runnerWorkspaceWaiter{conn: conn, result: result} + h.pending[requestID] = runnerWorkspaceWaiter{conn: conn, result: result, imageProgress: observeImageBuild} h.mu.Unlock() defer func() { h.mu.Lock() @@ -268,7 +272,7 @@ func (h *runnerControlHub) send(ctx context.Context, conn *runnerControlConn, re default: } select { - case conn.send <- runnerControlMessage{Type: "workspace.request", RequestID: requestID, WorkspaceRequest: &request}: + case conn.send <- runnerControlMessage{Type: "workspace.request", RequestID: requestID, WorkspaceRequest: &request, ObserveImageBuild: observeImageBuild != nil}: case <-conn.done: return domain.RunnerWorkspaceResponse{}, false, errRunnerWorkspaceUnavailable case <-ctx.Done(): @@ -280,21 +284,37 @@ func (h *runnerControlHub) send(ctx context.Context, conn *runnerControlConn, re case <-conn.done: return domain.RunnerWorkspaceResponse{}, true, errRunnerWorkspaceUnavailable case <-ctx.Done(): - // Chunked exports hold the runner's workspace fence. End the remote - // request too; cancelling only our waiter would strand the next turn. - if domain.IsWorkspaceCheckpointTransferOperation(request.Operation) || request.Operation == domain.RunnerWorkspaceOperationCheckpoint { - timer := time.NewTimer(time.Second) + // Checkpoints and preparation hold the runner's workspace fence. + // End the remote work too so the next turn can use the workspace. + if domain.IsCancelableRunnerWorkspaceOperation(request.Operation) { + message := runnerControlMessage{Type: "workspace.cancel", RequestID: requestID} select { - case conn.send <- runnerControlMessage{Type: "workspace.cancel", RequestID: requestID}: + case conn.send <- message: case <-conn.done: - case <-timer.C: + default: + // Keep the cancellation queued behind its request even when + // the socket is busy. Disconnect also cancels the runner work. + go func() { + select { + case conn.send <- message: + case <-conn.done: + } + }() } - timer.Stop() } return domain.RunnerWorkspaceResponse{}, true, ctx.Err() } } +func (h *runnerControlHub) reportImageBuild(conn *runnerControlConn, requestID string, progress domain.WorkspaceImageBuildProgress) { + h.mu.Lock() + waiter, ok := h.pending[requestID] + h.mu.Unlock() + if ok && waiter.conn == conn && waiter.imageProgress != nil { + waiter.imageProgress(progress) + } +} + func (h *runnerControlHub) resolveWorkspace(conn *runnerControlConn, requestID string, response domain.RunnerWorkspaceResponse) { h.mu.Lock() waiter, ok := h.pending[requestID] @@ -375,6 +395,7 @@ func (h *runnerControlHub) closeWhenIdle(limit time.Duration) { } const runnerWorkspaceRelayPath = "/internal/runner-workspace" +const runnerWorkspaceRelayStream = "application/x-ndjson" // runnerWorkspaceRelayLimit bounds relayed bodies; the runner socket itself // accepts 20 MiB messages, the envelope adds a little JSON around them. @@ -385,6 +406,12 @@ type runnerWorkspaceRelayRequest struct { Request domain.RunnerWorkspaceRequest `json:"request"` } +type runnerWorkspaceRelayUpdate struct { + Progress *domain.WorkspaceImageBuildProgress `json:"progress,omitempty"` + Response *domain.RunnerWorkspaceResponse `json:"response,omitempty"` + Error string `json:"error,omitempty"` +} + // handleRunnerWorkspaceRelay serves workspace RPCs that a handed-over // predecessor relays to the runner sockets this process now holds. The // predecessor already authorized and bound the request; this endpoint only @@ -400,6 +427,10 @@ func handleRunnerWorkspaceRelay(e *engine.Engine, hub *runnerControlHub) gin.Han c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) return } + if c.GetHeader("Accept") == runnerWorkspaceRelayStream { + streamRunnerWorkspaceRelay(c, hub, input) + return + } response, err := hub.requestRelayed(c.Request.Context(), input.RunnerID, input.Request) if err != nil { status := http.StatusServiceUnavailable @@ -413,6 +444,37 @@ func handleRunnerWorkspaceRelay(e *engine.Engine, hub *runnerControlHub) gin.Han } } +func streamRunnerWorkspaceRelay(c *gin.Context, hub *runnerControlHub, input runnerWorkspaceRelayRequest) { + ctx := c.Request.Context() + c.Header("Content-Type", runnerWorkspaceRelayStream) + c.Header("X-Accel-Buffering", "no") + c.Status(http.StatusOK) + encoder := json.NewEncoder(c.Writer) + send := func(update runnerWorkspaceRelayUpdate) { + if ctx.Err() == nil && encoder.Encode(update) == nil { + c.Writer.Flush() + } + } + var mu sync.Mutex + finished := false + progressCtx := runtime.WithDockerBuildProgress(ctx, func(progress domain.WorkspaceImageBuildProgress) { + mu.Lock() + defer mu.Unlock() + if !finished { + send(runnerWorkspaceRelayUpdate{Progress: &progress}) + } + }) + response, err := hub.requestRelayed(progressCtx, input.RunnerID, input.Request) + mu.Lock() + defer mu.Unlock() + finished = true + if err != nil { + send(runnerWorkspaceRelayUpdate{Error: err.Error()}) + return + } + send(runnerWorkspaceRelayUpdate{Response: &response}) +} + // relayRunnerWorkspaceToSuccessor reaches runners through the process that // took over the listener, over the API address this process was configured // with (never a user-supplied address). @@ -432,6 +494,10 @@ func relayRunnerWorkspaceToSuccessor(e *engine.Engine) runnerWorkspaceForwarder } req.Header.Set(engine.ControlTokenHeader, token) req.Header.Set("Content-Type", "application/json") + observeImageBuild := runtime.DockerBuildProgressObserver(ctx) + if observeImageBuild != nil { + req.Header.Set("Accept", runnerWorkspaceRelayStream) + } res, err := http.DefaultClient.Do(req) if err != nil { if ctx.Err() != nil { @@ -451,14 +517,35 @@ func relayRunnerWorkspaceToSuccessor(e *engine.Engine) runnerWorkspaceForwarder } return domain.RunnerWorkspaceResponse{}, fmt.Errorf("%w: successor returned HTTP %d %s", errRunnerWorkspaceUnavailable, res.StatusCode, strings.TrimSpace(failure.Error)) } + decoder := json.NewDecoder(reader) + if observeImageBuild != nil && res.Header.Get("Content-Type") == runnerWorkspaceRelayStream { + return readRunnerWorkspaceRelayStream(decoder, observeImageBuild) + } var response domain.RunnerWorkspaceResponse - if err := json.NewDecoder(reader).Decode(&response); err != nil { + if err := decoder.Decode(&response); err != nil { return domain.RunnerWorkspaceResponse{}, fmt.Errorf("relay to successor: %w", err) } return response, nil } } +func readRunnerWorkspaceRelayStream(decoder *json.Decoder, observe func(domain.WorkspaceImageBuildProgress)) (domain.RunnerWorkspaceResponse, error) { + for { + var update runnerWorkspaceRelayUpdate + if err := decoder.Decode(&update); err != nil { + return domain.RunnerWorkspaceResponse{}, fmt.Errorf("relay to successor: %w", err) + } + switch { + case update.Progress != nil: + observe(*update.Progress) + case update.Response != nil: + return *update.Response, nil + default: + return domain.RunnerWorkspaceResponse{}, fmt.Errorf("%w: successor: %s", errRunnerWorkspaceUnavailable, update.Error) + } + } +} + func (h *runnerControlHub) sendControl(runnerID, taskID, desiredState string) { if runnerID == "" || taskID == "" || desiredState == "" { return @@ -614,6 +701,10 @@ func (h *runnersHandlers) runnerControlConnect(c *gin.Context) { h.runnerControls.resolveWorkspace(conn, input.RequestID, *input.WorkspaceResponse) continue } + if input.Type == "workspace.image_progress" && input.ImageBuildProgress != nil { + h.runnerControls.reportImageBuild(conn, input.RequestID, *input.ImageBuildProgress) + continue + } if input.Type != "hello" { continue } diff --git a/internal/httpapi/runner_ws_workspace_test.go b/internal/httpapi/runner_ws_workspace_test.go index 55784dd7f..66b472b83 100644 --- a/internal/httpapi/runner_ws_workspace_test.go +++ b/internal/httpapi/runner_ws_workspace_test.go @@ -285,6 +285,96 @@ func TestRunnerWorkspaceRPCIsTransient(t *testing.T) { } } +func TestRunnerWorkspaceImageBuildUpdatesConversationWhilePreparing(t *testing.T) { + hub := quietRunnerHub() + conn := testRunnerConn("runner-a") + registerTestConn(hub, conn) + var activity assistantActivityLog + session := &AgentToolSession{Notify: func(kind string, value any) { + if kind == "thought" { + activity.add(value.(assistantStreamEvent)) + } + }} + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + ctx = withAssistantDockerBuildProgress(ctx, session) + completed := make(chan error, 1) + go func() { + _, err := hub.requestWorkspace(ctx, conn.runnerID, domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationPrepare}) + completed <- err + }() + select { + case request := <-conn.send: + if request.Type != "workspace.request" || !request.ObserveImageBuild { + t.Fatalf("request = %+v", request) + } + hub.reportImageBuild(testRunnerConn("runner-b"), request.RequestID, domain.WorkspaceImageBuildProgress{ID: "foreign", Step: "unrelated"}) + hub.reportImageBuild(conn, request.RequestID, domain.WorkspaceImageBuildProgress{ID: "a", Step: "启动构建"}) + hub.reportImageBuild(conn, request.RequestID, domain.WorkspaceImageBuildProgress{ID: "a", Step: "导出镜像"}) + events := activity.snapshot() + if len(events) != 1 || events[0].Tool != assistantDockerBuildTool || events[0].Detail != "导出镜像" { + t.Fatalf("live activity = %+v", events) + } + hub.reportImageBuild(conn, request.RequestID, domain.WorkspaceImageBuildProgress{ID: "a", Done: true}) + hub.resolveWorkspace(conn, request.RequestID, domain.RunnerWorkspaceResponse{Status: 200}) + if err := <-completed; err != nil { + t.Fatal(err) + } + events = activity.snapshot() + if len(events) != 2 || events[1].ID != events[0].ID || events[1].Kind != "toolDone" || events[1].Failed { + t.Fatalf("completed activity = %+v", events) + } + case <-ctx.Done(): + t.Fatal("runner did not receive the workspace request") + } +} + +func TestRunnerWorkspaceRelaysImageBuildProgressBeforeReturning(t *testing.T) { + gin.SetMode(gin.TestMode) + hub := quietRunnerHub() + conn := testRunnerConn("runner-a") + registerTestConn(hub, conn) + en := relayTestEngine(t) + en.ConfigureControl(engine.ControlConfig{Token: "service-secret"}) + router := gin.New() + router.POST(runnerWorkspaceRelayPath, handleRunnerWorkspaceRelay(en, hub)) + server := httptest.NewServer(router) + defer server.Close() + en.ConfigureControl(engine.ControlConfig{Token: "service-secret", APIURL: server.URL}) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + updates := make(chan domain.WorkspaceImageBuildProgress, 2) + ctx = runtime.WithDockerBuildProgress(ctx, func(update domain.WorkspaceImageBuildProgress) { updates <- update }) + completed := make(chan workspaceResult, 1) + go func() { + response, err := relayRunnerWorkspaceToSuccessor(en)(ctx, conn.runnerID, domain.RunnerWorkspaceRequest{Operation: domain.RunnerWorkspaceOperationPrepare}) + completed <- workspaceResult{response, err} + }() + select { + case request := <-conn.send: + if !request.ObserveImageBuild { + t.Fatalf("relayed request did not subscribe: %+v", request) + } + want := domain.WorkspaceImageBuildProgress{ID: "relayed-build", Step: "读取 Dockerfile"} + hub.reportImageBuild(conn, request.RequestID, want) + select { + case got := <-updates: + if got != want { + t.Fatalf("relayed progress = %+v, want %+v", got, want) + } + case <-ctx.Done(): + t.Fatal("relayed progress was not streamed during the request") + } + hub.resolveWorkspace(conn, request.RequestID, domain.RunnerWorkspaceResponse{Status: 200}) + if got := <-completed; got.err != nil || got.response.Status != 200 { + t.Fatalf("relay completion = %+v", got) + } + case <-ctx.Done(): + t.Fatal("runner did not receive the relayed request") + } +} + func TestRunnerWorkspaceSendsScopedPlatformSSHKnownHosts(t *testing.T) { store, err := repo.OpenTest() if err != nil { @@ -429,19 +519,28 @@ func TestRunnerWorkspaceHandoverDrainIsBounded(t *testing.T) { } } -func TestRunnerCheckpointCancellationReachesExecutorOnlyForOneShotRequests(t *testing.T) { - for _, operation := range []string{domain.RunnerWorkspaceOperationCheckpointExport, "conversation_process"} { +func TestRunnerWorkspaceCancellationReachesExecutorOnlyForOneShotRequests(t *testing.T) { + for _, operation := range []string{domain.RunnerWorkspaceOperationCheckpointExport, domain.RunnerWorkspaceOperationPrepare, domain.RunnerWorkspaceOperationConversationCLI} { t.Run(operation, func(t *testing.T) { hub := quietRunnerHub() conn := testRunnerConn("runner-cancel") + defer conn.close() + conn.send = make(chan runnerControlMessage, 1) registerTestConn(hub, conn) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() done := make(chan error, 1) go func() { _, err := hub.requestWorkspace(ctx, conn.runnerID, domain.RunnerWorkspaceRequest{Operation: operation}) done <- err }() request := <-conn.send + wantCancel := operation != domain.RunnerWorkspaceOperationConversationCLI + if operation == domain.RunnerWorkspaceOperationPrepare { + // The API must release its waiter without losing the cancel + // message when the connection's write queue is full. + conn.send <- runnerControlMessage{Type: "busy"} + } cancel() select { case err := <-done: @@ -451,7 +550,10 @@ func TestRunnerCheckpointCancellationReachesExecutorOnlyForOneShotRequests(t *te case <-time.After(time.Second): t.Fatal("request cancellation blocked") } - if operation == "conversation_process" { + if operation == domain.RunnerWorkspaceOperationPrepare { + <-conn.send + } + if !wantCancel { select { case unexpected := <-conn.send: t.Fatalf("persistent CLI lifecycle cancelled by RPC: %+v", unexpected) @@ -464,7 +566,7 @@ func TestRunnerCheckpointCancellationReachesExecutorOnlyForOneShotRequests(t *te t.Fatalf("wrong cancellation: %+v", stop) } case <-time.After(time.Second): - t.Fatal("export cancellation not sent") + t.Fatal("workspace cancellation not sent") } } }) diff --git a/internal/procwatch/procwatch.go b/internal/procwatch/procwatch.go index 44cc6004d..0db42168c 100644 --- a/internal/procwatch/procwatch.go +++ b/internal/procwatch/procwatch.go @@ -66,6 +66,9 @@ type Options struct { TimeoutEnv string // Redact scrubs secrets from every line before it is kept or reported. Redact func(string) string + // OnLine receives each nonempty line after redaction, independently of + // the throttled progress events. The callback should return promptly. + OnLine func(string) } const ( @@ -95,6 +98,9 @@ func Run(ctx context.Context, build func(context.Context) *exec.Cmd, opts Option started := time.Now() out := &lineWriter{redact: opts.Redact, last: started} out.onLine = func(line string, now time.Time) { + if opts.OnLine != nil { + opts.OnLine(line) + } if now.Sub(out.lastEmit) < progressEvery { return } diff --git a/internal/runner/control.go b/internal/runner/control.go index 28565b78a..4579bd7da 100644 --- a/internal/runner/control.go +++ b/internal/runner/control.go @@ -11,6 +11,7 @@ import ( "time" "agentflow/internal/domain" + "agentflow/internal/runtime" "github.com/gorilla/websocket" ) @@ -29,15 +30,17 @@ var controlReadTimeout = 75 * time.Second var controlWriteTimeout = 10 * time.Second type controlMessage struct { - ToolRequest *domain.RunnerToolRequest `json:"toolRequest,omitempty"` - ToolResponse *domain.RunnerToolResponse `json:"toolResponse,omitempty"` - Type string `json:"type"` - TaskID string `json:"taskId,omitempty"` - DesiredState string `json:"desiredState,omitempty"` - ActiveTaskIDs []string `json:"activeTaskIds,omitempty"` - RequestID string `json:"requestId,omitempty"` - WorkspaceRequest *domain.RunnerWorkspaceRequest `json:"workspaceRequest,omitempty"` - WorkspaceResponse *domain.RunnerWorkspaceResponse `json:"workspaceResponse,omitempty"` + ToolRequest *domain.RunnerToolRequest `json:"toolRequest,omitempty"` + ToolResponse *domain.RunnerToolResponse `json:"toolResponse,omitempty"` + Type string `json:"type"` + TaskID string `json:"taskId,omitempty"` + DesiredState string `json:"desiredState,omitempty"` + ActiveTaskIDs []string `json:"activeTaskIds,omitempty"` + RequestID string `json:"requestId,omitempty"` + WorkspaceRequest *domain.RunnerWorkspaceRequest `json:"workspaceRequest,omitempty"` + WorkspaceResponse *domain.RunnerWorkspaceResponse `json:"workspaceResponse,omitempty"` + ObserveImageBuild bool `json:"observeImageBuild,omitempty"` + ImageBuildProgress *domain.WorkspaceImageBuildProgress `json:"imageBuildProgress,omitempty"` } // controlLoop maintains one outbound WebSocket for immediate server-to-runner @@ -84,8 +87,8 @@ func (d *Daemon) controlLoop(ctx context.Context) { func (d *Daemon) readControls(daemonCtx context.Context, conn *websocket.Conn) error { ctx, cancel := context.WithCancel(daemonCtx) var requests sync.WaitGroup - var checkpointMu sync.Mutex - checkpointCancels := map[string]context.CancelFunc{} + var workspaceCancelMu sync.Mutex + workspaceCancels := map[string]context.CancelFunc{} var writeMu sync.Mutex writeMessage := func(message controlMessage) error { writeMu.Lock() @@ -139,9 +142,9 @@ func (d *Daemon) readControls(daemonCtx context.Context, conn *websocket.Conn) e continue } if message.Type == "workspace.cancel" { - checkpointMu.Lock() - stop := checkpointCancels[message.RequestID] - checkpointMu.Unlock() + workspaceCancelMu.Lock() + stop := workspaceCancels[message.RequestID] + workspaceCancelMu.Unlock() if stop != nil { stop() } @@ -164,11 +167,11 @@ func (d *Daemon) readControls(daemonCtx context.Context, conn *websocket.Conn) e } requestCtx := ctx stopRequest := func() {} - if domain.IsWorkspaceCheckpointTransferOperation(message.WorkspaceRequest.Operation) || message.WorkspaceRequest.Operation == domain.RunnerWorkspaceOperationCheckpoint { + if domain.IsCancelableRunnerWorkspaceOperation(message.WorkspaceRequest.Operation) { requestCtx, stopRequest = context.WithCancel(ctx) - checkpointMu.Lock() - checkpointCancels[message.RequestID] = stopRequest - checkpointMu.Unlock() + workspaceCancelMu.Lock() + workspaceCancels[message.RequestID] = stopRequest + workspaceCancelMu.Unlock() } requests.Add(1) go func(message controlMessage, requestCtx context.Context, stopRequest context.CancelFunc) { @@ -176,15 +179,22 @@ func (d *Daemon) readControls(daemonCtx context.Context, conn *websocket.Conn) e defer func() { <-slots }() defer func() { stopRequest() - checkpointMu.Lock() - delete(checkpointCancels, message.RequestID) - checkpointMu.Unlock() + workspaceCancelMu.Lock() + delete(workspaceCancels, message.RequestID) + workspaceCancelMu.Unlock() }() var response domain.RunnerWorkspaceResponse var startUpdate func() if message.WorkspaceRequest.Operation == domain.RunnerWorkspaceOperationMaintenance { response, startUpdate = d.maintenanceResponse(requestCtx, daemonCtx, message.WorkspaceRequest.Maintenance) } else { + if message.ObserveImageBuild { + requestCtx = runtime.WithDockerBuildProgress(requestCtx, func(progress domain.WorkspaceImageBuildProgress) { + if requestCtx.Err() == nil { + _ = writeMessage(controlMessage{Type: "workspace.image_progress", RequestID: message.RequestID, ImageBuildProgress: &progress}) + } + }) + } response = d.readRunnerWorkspace(requestCtx, *message.WorkspaceRequest) } err := writeMessage(controlMessage{Type: "workspace.response", RequestID: message.RequestID, WorkspaceResponse: &response}) diff --git a/internal/runner/workspace_preparation.go b/internal/runner/workspace_preparation.go index 999880344..526ad354f 100644 --- a/internal/runner/workspace_preparation.go +++ b/internal/runner/workspace_preparation.go @@ -47,8 +47,17 @@ func onceRelease(callbacks ...func()) func() { const runnerWorkspacePreparationTimeout = 15 * time.Minute +func runnerWorkspacePreparationContext(ctx context.Context, request domain.RunnerWorkspaceRequest) (context.Context, context.CancelFunc) { + if request.Operation == domain.RunnerWorkspaceOperationPrepare && request.RunKind == domain.RunKindWorkspace { + // The control plane stops preparation with the conversation. The + // control connection also cancels this work if it goes away. + return context.WithCancel(ctx) + } + return context.WithTimeout(ctx, runnerWorkspacePreparationTimeout) +} + func (d *Daemon) prepareRunnerWorkspace(ctx context.Context, root string, request domain.RunnerWorkspaceRequest) domain.RunnerWorkspaceResponse { - ctx, cancel := context.WithTimeout(ctx, runnerWorkspacePreparationTimeout) + ctx, cancel := runnerWorkspacePreparationContext(ctx, request) defer cancel() task, err := d.workspaceSandboxTask(root, request) if err != nil { @@ -105,7 +114,7 @@ func (d *Daemon) handleRunnerWorkspaceTerminal(ctx context.Context, root, scope if !request.Terminal.ExpiresAt.IsZero() && !request.Terminal.ExpiresAt.After(time.Now()) { return runtime.TerminalResult{}, fmt.Errorf("terminal authorization has expired") } - ctx, cancel := context.WithTimeout(ctx, runnerWorkspacePreparationTimeout) + ctx, cancel := runnerWorkspacePreparationContext(ctx, request) defer cancel() task, err := d.workspaceSandboxTask(root, request) if err != nil { diff --git a/internal/runner/workspace_preparation_test.go b/internal/runner/workspace_preparation_test.go index dda2563b3..00e14d28e 100644 --- a/internal/runner/workspace_preparation_test.go +++ b/internal/runner/workspace_preparation_test.go @@ -28,6 +28,35 @@ func runnerPreparationFixture(t *testing.T) (*Daemon, domain.RunnerWorkspaceRequ }, root } +func TestRunnerWorkspacePreparationDeadline(t *testing.T) { + for _, tc := range []struct { + name, operation, runKind string + followsCaller bool + }{ + {"conversation", domain.RunnerWorkspaceOperationPrepare, domain.RunKindWorkspace, true}, + {"SOP", domain.RunnerWorkspaceOperationPrepare, "", false}, + {"terminal", domain.RunnerWorkspaceOperationTerminal, domain.RunKindWorkspace, false}, + } { + t.Run(tc.name, func(t *testing.T) { + caller, stopCaller := context.WithTimeout(context.Background(), time.Hour) + defer stopCaller() + callerDeadline, _ := caller.Deadline() + ctx, done := runnerWorkspacePreparationContext(caller, domain.RunnerWorkspaceRequest{ + Operation: tc.operation, RunKind: tc.runKind, + }) + defer done() + deadline, ok := ctx.Deadline() + if !ok || deadline.Equal(callerDeadline) != tc.followsCaller { + t.Fatalf("preparation deadline = %v, caller deadline = %v, follows caller = %t", deadline, callerDeadline, tc.followsCaller) + } + stopCaller() + if ctx.Err() != context.Canceled { + t.Fatalf("preparation context error after caller cancellation: %v", ctx.Err()) + } + }) + } +} + func TestRunnerWorkspacePreparationAndTerminalShareFrozenEnvironment(t *testing.T) { d, req, root := runnerPreparationFixture(t) defer d.closeAllRunSandboxes(context.Background()) diff --git a/internal/runtime/docker_build_progress.go b/internal/runtime/docker_build_progress.go new file mode 100644 index 000000000..ff4970843 --- /dev/null +++ b/internal/runtime/docker_build_progress.go @@ -0,0 +1,61 @@ +package runtime + +import ( + "context" + "fmt" + "regexp" + "strings" + + "agentflow/internal/domain" +) + +type dockerBuildProgressKey struct{} + +func WithDockerBuildProgress(ctx context.Context, report func(domain.WorkspaceImageBuildProgress)) context.Context { + return context.WithValue(ctx, dockerBuildProgressKey{}, report) +} + +func DockerBuildProgressObserver(ctx context.Context) func(domain.WorkspaceImageBuildProgress) { + report, _ := ctx.Value(dockerBuildProgressKey{}).(func(domain.WorkspaceImageBuildProgress)) + return report +} + +var ( + buildKitLine = regexp.MustCompile(`^#[0-9]+ `) + buildKitStep = regexp.MustCompile(`^\[(?:[^\[\]]+ )?([0-9]{1,6})/([0-9]{1,6})\] ([A-Z]+)(?: |$)`) + legacyStep = regexp.MustCompile(`^Step ([0-9]{1,6})/([0-9]{1,6}) : ([A-Z]+)(?: |$)`) +) + +// Only recognize Docker's own step headers. RUN output and instruction +// arguments can contain credentials and are never copied into chat. +func dockerBuildStep(line string) string { + if step := legacyStep.FindStringSubmatch(line); step != nil { + return dockerInstructionStep(step) + } + header := buildKitLine.FindStringIndex(line) + if header == nil { + return "" + } + line = line[header[1]:] + if step := buildKitStep.FindStringSubmatch(line); step != nil { + return dockerInstructionStep(step) + } + switch { + case strings.HasPrefix(line, "[internal] load build definition "): + return "读取 Dockerfile" + case strings.HasPrefix(line, "[internal] load metadata for "): + return "读取基础镜像信息" + case strings.HasPrefix(line, "[internal] load build context"): + return "传输构建上下文" + case strings.HasPrefix(line, "exporting to "): + return "导出镜像" + } + return "" +} + +func dockerInstructionStep(step []string) string { + if step[3] == "FROM" { + return "准备基础镜像" + } + return fmt.Sprintf("执行 Dockerfile 第 %s/%s 步", step[1], step[2]) +} diff --git a/internal/runtime/docker_build_progress_test.go b/internal/runtime/docker_build_progress_test.go index d2fce64d1..b76968a7a 100644 --- a/internal/runtime/docker_build_progress_test.go +++ b/internal/runtime/docker_build_progress_test.go @@ -115,3 +115,59 @@ func TestDockerBuildStopsWhenTheBuildStallsOrRunsTooLong(t *testing.T) { } }) } + +func TestDockerBuildReportsStepsBeforeQuietBuildCompletes(t *testing.T) { + t.Setenv(dockerBuildStallEnv, "") + t.Setenv(dockerBuildTimeoutEnv, "") + if watch := dockerBuildWatch(slowDockerfile()); watch.Stall != 0 || watch.Timeout != 0 { + t.Fatalf("default build limits: %+v", watch) + } + gate := filepath.Join(t.TempDir(), "continue") + t.Setenv("TEST_DOCKER_BUILD_CONTINUE", gate) + installFakeDockerBuild(t, ` + printf '%s\n' '#1 [internal] load metadata for private-registry' '#2 [build 2/3] RUN hidden-arguments' '#2 0.1 hidden-output' + until [ -f "$TEST_DOCKER_BUILD_CONTINUE" ]; do sleep 0.05; done + printf '%s\n' '#3 exporting to image' + `) + progress := make(chan domain.WorkspaceImageBuildProgress, 8) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = WithDockerBuildProgress(ctx, func(update domain.WorkspaceImageBuildProgress) { progress <- update }) + done := make(chan error, 1) + workspace := t.TempDir() + go func() { + _, err := ensureDockerImage(ctx, workspace, slowDockerfile()) + done <- err + }() + next := func() domain.WorkspaceImageBuildProgress { + t.Helper() + select { + case update := <-progress: + return update + case <-ctx.Done(): + t.Fatal("build progress was not reported") + return domain.WorkspaceImageBuildProgress{} + } + } + started := next() + if started.ID == "" || started.Step != "启动构建" || started.Done { + t.Fatalf("start = %+v", started) + } + for _, step := range []string{"读取基础镜像信息", "执行 Dockerfile 第 2/3 步"} { + if update := next(); update.ID != started.ID || update.Step != step || update.Done { + t.Fatalf("progress = %+v, want %q", update, step) + } + } + if err := os.WriteFile(gate, nil, 0o600); err != nil { + t.Fatal(err) + } + if update := next(); update.Step != "导出镜像" || update.Done { + t.Fatalf("export = %+v", update) + } + if update := next(); update.ID != started.ID || !update.Done || update.Failed { + t.Fatalf("completion = %+v", update) + } + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/runtime/sandbox_docker.go b/internal/runtime/sandbox_docker.go index c4fb30fa6..2289d50e7 100644 --- a/internal/runtime/sandbox_docker.go +++ b/internal/runtime/sandbox_docker.go @@ -4,6 +4,7 @@ import ( "agentflow/internal/domain" "agentflow/internal/procwatch" "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" @@ -914,8 +915,32 @@ func ensureDockerImage(ctx context.Context, workspace string, dockerfile domain. return "", err } args := dockerBuildArgs(image, contextDir) - build := func(ctx context.Context) *exec.Cmd { return exec.CommandContext(ctx, "docker", args...) } - if tail, err := procwatch.Run(ctx, build, dockerBuildWatch(dockerfile)); err != nil { + watch := dockerBuildWatch(dockerfile) + report := DockerBuildProgressObserver(ctx) + build := func(ctx context.Context) *exec.Cmd { + cmd := exec.CommandContext(ctx, "docker", args...) + if report != nil { + cmd.Env = append(cmd.Environ(), "BUILDKIT_PROGRESS=plain") + } + return cmd + } + var buildID string + if report != nil { + buildID = rand.Text() + step := "启动构建" + report(domain.WorkspaceImageBuildProgress{ID: buildID, Step: step}) + watch.OnLine = func(line string) { + if next := dockerBuildStep(line); next != "" && next != step { + step = next + report(domain.WorkspaceImageBuildProgress{ID: buildID, Step: step}) + } + } + } + tail, err := procwatch.Run(ctx, build, watch) + if report != nil { + report(domain.WorkspaceImageBuildProgress{ID: buildID, Done: true, Failed: err != nil}) + } + if err != nil { // Only the tail: a build killed after minutes of apt output used to // carry all of it in the error. return "", fmt.Errorf("docker build: %w: %s", err, tail) @@ -923,8 +948,8 @@ func ensureDockerImage(ctx context.Context, workspace string, dockerfile domain. return image, nil } -// Build limits. A RUN step can be silent for a while (a compile, a large -// download without a progress bar), so the stall window is wider than for git. +// A Dockerfile step can work quietly for an arbitrary amount of time. +// Apply an output-based limit only when explicitly configured. const ( dockerBuildStallEnv = "AGENTFLOW_DOCKER_BUILD_STALL_TIMEOUT" dockerBuildTimeoutEnv = "AGENTFLOW_DOCKER_BUILD_TIMEOUT" @@ -941,7 +966,7 @@ func dockerBuildWatch(dockerfile domain.DockerfileDefinition) procwatch.Options } return procwatch.Options{ Stage: "image", Label: label, - Stall: procwatch.DurationEnv(dockerBuildStallEnv, 10*time.Minute), StallEnv: dockerBuildStallEnv, + Stall: procwatch.DurationEnv(dockerBuildStallEnv, 0), StallEnv: dockerBuildStallEnv, Timeout: procwatch.DurationEnv(dockerBuildTimeoutEnv, 0), TimeoutEnv: dockerBuildTimeoutEnv, } }