diff --git a/CHANGELOG.md b/CHANGELOG.md index aef1c05e..a7f58b19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Platform releases use `vX.Y.Z` tags and publish `ghcr.io/tencent/loopforge`. Loo - The orchestrator panel embeds the chatbot (`/chatbot/?surface=sop-designer`) instead of its own hand-written chat. Replies render Markdown with the execution trace, a running turn can be stopped, the turn runs as a background generation, so closing the panel or reloading the page no longer loses it, and reopening a SOP shows its earlier conversation. The CLI and model choice keeps its own memory as before. Each reply's draft shows its warnings and can be put on the canvas again. The canvas and the chat talk only through `agentflow-sop-designer-*` window messages; the canvas still animates nodes and wires as the draft streams in. - Editing a SOP in the orchestrator no longer rewrites the whole graph. On a canvas with steps the model submits `operations` (`add_node`, `update_node`, `remove_node`, `add_edge`, `remove_edge`, `update_workflow`) and the server applies them to the canvas it received, so steps, prompts, scripts, layout and the 「构建静态站点」 node the user did not mention stay exactly as they were. An operation that does not fit the canvas (an unknown node, a duplicate edge, removing a node another step rejects to) goes back to the model with the reason instead of reaching the canvas. The canvas no longer guesses intent from the wording: it no longer matches 「重新…」 or 「去掉预览」 phrases or SOP names in the message; the server lists the project's SOPs and the model reads the one the user means. A draft with a different SOP id replaces the canvas only after the user confirms. - The orchestrator offers the model only roles a SOP save accepts: roles of this project and roles another project shares. A built-in role that belongs to no project used to be offered, and the draft then failed on save with 「角色 … 不属于当前项目且尚未共享」. `workflows_save` drafts from any conversation now warn about such a role before saving. A script longer than 400 characters reaches the model as a marker that says it is omitted, not as an empty string; the model used to report the script as empty and offer to write one, and a marker copied back into a draft is replaced by the canvas's script. +- The side browser no longer stays on a blurry frame after a scroll or drag on a page that stops changing. Scrolling switches the screencast to half-size frames and it restarts at full size once input settles, but Chromium does not always re-send a page that is not repainting; the panel now captures the page when no frame arrives within 300 ms. - Scheduled tasks keep an execution history with a session mode, keep conversations per job and gain row actions (#958). AI jobs run with the key of the user who saved them and fall back to the shared key (#966). - Requirements can be created directly and clarified in a chat (#965). Settled execution history can be force-deleted (#210). - Conversation workspaces open by sandbox with agent-owned Git (#947), run native agent CLIs on bound execution machines (#494), can be bound to project SOP workspaces through `workspaceRunId` (#974), and install a missing CLI from the failure message (#183). diff --git a/internal/runtime/browser_motion_test.go b/internal/runtime/browser_motion_test.go index 3b0f13ed..1ef9fd26 100644 --- a/internal/runtime/browser_motion_test.go +++ b/internal/runtime/browser_motion_test.go @@ -14,8 +14,8 @@ import ( "agentflow/internal/domain" ) -// Chromium re-sends the current page when the screencast restarts, so a -// static page still ends on a sharp frame after a scroll. +// A static page still ends on a sharp frame after a scroll, whether or not +// Chromium re-sends it when the screencast restarts. func TestBrowserScrollEndsOnASharpFrame(t *testing.T) { if testing.Short() { t.Skip("short mode") @@ -68,26 +68,16 @@ func TestBrowserScrollEndsOnASharpFrame(t *testing.T) { for width < 700 { seq, width = nextWidth(seq) } + scrolled := time.Now() if _, err := HandleBrowser(ctx, runID, domain.BrowserRequest{Operation: "input", Events: []domain.BrowserInputEvent{{Type: "wheel", X: 0.5, Y: 0.5, DeltaY: 100}}}); err != nil { t.Fatal(err) } - // A full-width frame already in flight can arrive before the screencast - // restarts at the lighter size. Keep reading until a light frame shows up. - sawLight := false + // Light frames may arrive while the scroll is in motion, and a full-width + // frame already in flight can arrive right after the wheel; neither is + // guaranteed on a page that does not repaint. Once input settles a + // full-width frame arrives although nothing changed. for deadline := time.Now().Add(8 * time.Second); time.Now().Before(deadline); { - seq, width = nextWidth(seq) - if width <= 400 { - sawLight = true - break - } - } - if !sawLight { - t.Fatalf("no light frame while scrolling (last %dpx)", width) - } - // More light frames may follow while the scroll animates; once input - // settles a full-width frame arrives although nothing changed. - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - if seq, width = nextWidth(seq); width >= 700 { + if seq, width = nextWidth(seq); width >= 700 && time.Since(scrolled) >= browserMotionSettle { return } } diff --git a/internal/runtime/browser_settled_frame_test.go b/internal/runtime/browser_settled_frame_test.go new file mode 100644 index 00000000..895cd847 --- /dev/null +++ b/internal/runtime/browser_settled_frame_test.go @@ -0,0 +1,204 @@ +package runtime + +import ( + "context" + "encoding/json" + "testing" + "time" +) + +// settledFrameTransport answers like fakeCDPTransport, except that +// Page.captureScreenshot can fail, or be preceded by a screencast frame that +// Chromium sends while the screenshot is being taken. +type settledFrameTransport struct { + *fakeCDPTransport + shotError string + frameDuringShot bool +} + +func (f *settledFrameTransport) write(body []byte) error { + var msg cdpMessage + if err := json.Unmarshal(body, &msg); err != nil { + return err + } + if msg.Method != "Page.captureScreenshot" { + return f.fakeCDPTransport.write(body) + } + f.mu.Lock() + f.sent = append(f.sent, msg) + f.mu.Unlock() + if f.frameDuringShot { + params, _ := json.Marshal(map[string]any{"data": "RESENT", "sessionId": 1, "metadata": map[string]any{"deviceWidth": 1000, "deviceHeight": 500}}) + event, _ := json.Marshal(cdpMessage{Method: "Page.screencastFrame", SessionID: msg.SessionID, Params: params}) + f.replies <- event + } + reply := cdpMessage{ID: msg.ID, Result: json.RawMessage(`{"data":"SHARP"}`)} + if f.shotError != "" { + reply.Result = nil + reply.Error = &struct { + Message string `json:"message"` + }{Message: f.shotError} + } + raw, _ := json.Marshal(reply) + f.replies <- raw + return nil +} + +func cdpCalls(f *fakeCDPTransport, method string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, msg := range f.sent { + if msg.Method == method { + n++ + } + } + return n +} + +// newSettledFrameView is a view showing a light frame at seq 5 on page-1. +// contentBusy keeps screencast frames from starting a layout metrics read. +func newSettledFrameView(t *testing.T, transport cdpTransport) *browserView { + t.Helper() + view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1", + width: 1000, height: 500, frame: "LIGHT", seq: 5, contentBusy: true} + view.client = newCDPClient(transport, view.onEvent) + t.Cleanup(func() { view.client.close() }) + return view +} + +func TestEnsureSettledFramePublishesTheScreenshot(t *testing.T) { + fake := newFakeCDPTransport() + fake.results = map[string]string{"Page.captureScreenshot": `{"data":"SHARP"}`} + view := newSettledFrameView(t, fake) + view.mu.Lock() + changed := view.changed + view.mu.Unlock() + + view.ensureSettledFrame(context.Background(), "page-1", time.Millisecond) + + if state := view.state(0); state.Frame != "SHARP" || state.Seq != 6 { + t.Fatalf("frame %q seq %d, want SHARP at 6", state.Frame, state.Seq) + } + select { + case <-changed: + default: + t.Fatal("watchers were not notified of the captured frame") + } + fake.mu.Lock() + defer fake.mu.Unlock() + if len(fake.sent) != 1 || fake.sent[0].SessionID != "page-1" { + t.Fatalf("sent %+v", fake.sent) + } + var params struct { + Format string `json:"format"` + Quality int `json:"quality"` + } + if err := json.Unmarshal(fake.sent[0].Params, ¶ms); err != nil || params.Format != "jpeg" || params.Quality != 80 { + t.Fatalf("screenshot params %s (%v)", fake.sent[0].Params, err) + } +} + +func TestEnsureSettledFrameKeepsTheViewWhenItShouldNotCapture(t *testing.T) { + cases := []struct { + name string + setup func(*browserView) + }{ + {"input moves again before the wait ends", func(v *browserView) { v.motion = true }}, + {"the panel switched to another tab", func(v *browserView) { v.session = "page-2" }}, + {"a frame already arrived", func(v *browserView) { v.seq = 7 }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := newFakeCDPTransport() + fake.results = map[string]string{"Page.captureScreenshot": `{"data":"SHARP"}`} + view := newSettledFrameView(t, fake) + // Change the view during the wait, right after seq was sampled. + view.settledFrameSampled = func() { + view.mu.Lock() + tc.setup(view) + view.mu.Unlock() + } + + view.ensureSettledFrame(context.Background(), "page-1", time.Millisecond) + + if n := cdpCalls(fake, "Page.captureScreenshot"); n != 0 { + t.Fatalf("screenshots: %d", n) + } + if frame := view.state(0).Frame; frame != "LIGHT" { + t.Fatalf("frame %q", frame) + } + }) + } +} + +func TestEnsureSettledFrameStopsWhenTheContextEnds(t *testing.T) { + fake := newFakeCDPTransport() + view := newSettledFrameView(t, fake) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + done := make(chan struct{}) + go func() { + view.ensureSettledFrame(ctx, "page-1", time.Hour) + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("ensureSettledFrame kept waiting after its context ended") + } + if n := cdpCalls(fake, "Page.captureScreenshot"); n != 0 { + t.Fatalf("screenshots: %d", n) + } + if state := view.state(0); state.Frame != "LIGHT" || state.Seq != 5 { + t.Fatalf("frame %q seq %d", state.Frame, state.Seq) + } +} + +func TestEnsureSettledFrameKeepsTheLightFrameWhenTheCaptureFails(t *testing.T) { + cases := []struct { + name string + transport func() (cdpTransport, *fakeCDPTransport) + }{ + {"cdp error", func() (cdpTransport, *fakeCDPTransport) { + f := &settledFrameTransport{fakeCDPTransport: newFakeCDPTransport(), shotError: "Unable to capture screenshot"} + return f, f.fakeCDPTransport + }}, + {"empty data", func() (cdpTransport, *fakeCDPTransport) { + f := newFakeCDPTransport() + f.results = map[string]string{"Page.captureScreenshot": `{"data":""}`} + return f, f + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + transport, fake := tc.transport() + view := newSettledFrameView(t, transport) + + view.ensureSettledFrame(context.Background(), "page-1", time.Millisecond) + + if n := cdpCalls(fake, "Page.captureScreenshot"); n != 1 { + t.Fatalf("screenshots: %d", n) + } + if state := view.state(0); state.Frame != "LIGHT" || state.Seq != 5 { + t.Fatalf("frame %q seq %d, want LIGHT at 5", state.Frame, state.Seq) + } + }) + } +} + +// A frame Chromium sends while the screenshot is taken is newer than the +// screenshot request, so the screenshot is dropped. +func TestEnsureSettledFrameDropsTheScreenshotWhenAFrameArrivesMeanwhile(t *testing.T) { + transport := &settledFrameTransport{fakeCDPTransport: newFakeCDPTransport(), frameDuringShot: true} + view := newSettledFrameView(t, transport) + + view.ensureSettledFrame(context.Background(), "page-1", time.Millisecond) + + if n := cdpCalls(transport.fakeCDPTransport, "Page.captureScreenshot"); n != 1 { + t.Fatalf("screenshots: %d", n) + } + if state := view.state(0); state.Frame != "RESENT" || state.Seq != 6 { + t.Fatalf("frame %q seq %d, want RESENT at 6", state.Frame, state.Seq) + } +} diff --git a/internal/runtime/browser_test.go b/internal/runtime/browser_test.go index c954bf8e..fbe61418 100644 --- a/internal/runtime/browser_test.go +++ b/internal/runtime/browser_test.go @@ -324,3 +324,68 @@ func TestBrowserViewCapturesLighterFramesWhileScrolling(t *testing.T) { t.Fatalf("drag frames: %v", got) } } + +func TestBrowserViewCapturesASharpFrameWhenTheSettledScreencastSendsNone(t *testing.T) { + settle, wait := browserMotionSettle, browserSettledFrameWait + browserMotionSettle, browserSettledFrameWait = 30*time.Millisecond, 60*time.Millisecond + defer func() { browserMotionSettle, browserSettledFrameWait = settle, wait }() + wheel := []domain.BrowserInputEvent{{Type: "wheel", X: 0.5, Y: 0.5, DeltaY: 100}} + screenshots := func(transport *fakeCDPTransport) int { + transport.mu.Lock() + defer transport.mu.Unlock() + n := 0 + for _, msg := range transport.sent { + if msg.Method == "Page.captureScreenshot" { + n++ + } + } + return n + } + newView := func() (*browserView, *fakeCDPTransport) { + transport := newFakeCDPTransport() + transport.results = map[string]string{"Page.captureScreenshot": `{"data":"SHARP"}`} + view := &browserView{targets: map[string]*browserTarget{}, changed: make(chan struct{}), session: "page-1", active: "t1", width: 1000, height: 500, frameWidth: 1600, frame: "LIGHT", seq: 5} + view.client = newCDPClient(transport, view.onEvent) + return view, transport + } + + // Chromium sends nothing after the restart: the view captures the page. + view, transport := newView() + defer view.client.close() + if err := view.input(context.Background(), wheel); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for view.state(5).Frame != "SHARP" { + if time.Now().After(deadline) { + t.Fatalf("no sharp frame after the scroll settled: %+v", view.state(0)) + } + time.Sleep(10 * time.Millisecond) + } + if n := screenshots(transport); n != 1 { + t.Fatalf("screenshots: %d", n) + } + + // A frame that arrives in time is kept and no screenshot is taken. + view, transport = newView() + defer view.client.close() + // Deliver the frame during the settle wait, right after seq was sampled. + resent := make(chan struct{}) + view.settledFrameSampled = func() { + frame, _ := json.Marshal(map[string]any{"data": "RESENT", "sessionId": 1, "metadata": map[string]any{"deviceWidth": 1000, "deviceHeight": 500}}) + view.onEvent(cdpMessage{Method: "Page.screencastFrame", SessionID: "page-1", Params: frame}) + close(resent) + } + if err := view.input(context.Background(), wheel); err != nil { + t.Fatal(err) + } + select { + case <-resent: + case <-time.After(2 * time.Second): + t.Fatal("the screencast never returned to sharp frames") + } + time.Sleep(browserSettledFrameWait * 2) + if got := view.state(0).Frame; got != "RESENT" || screenshots(transport) != 0 { + t.Fatalf("frame %q, screenshots %d", got, screenshots(transport)) + } +} diff --git a/internal/runtime/browser_view.go b/internal/runtime/browser_view.go index 82896957..55820b3e 100644 --- a/internal/runtime/browser_view.go +++ b/internal/runtime/browser_view.go @@ -52,6 +52,9 @@ type browserView struct { // lighter until the input settles (see markMotion). motion bool motionGen int64 + // settledFrameSampled, when set (tests only), runs once ensureSettledFrame + // has recorded the frame seq it waits on. + settledFrameSampled func() // History flags are cached per active page URL: re-reading them on every // frame would add a CDP round trip to each one. historyKey string @@ -485,8 +488,9 @@ var browserMotionSettle = 400 * time.Millisecond // markMotion switches the screencast to light frames for scrolling and // dragging, and back once input has been quiet for browserMotionSettle. -// Restarting the screencast makes Chromium send the current page again, so -// the view ends on a sharp frame even when nothing moves afterwards. +// Restarting the screencast usually makes Chromium send the current page +// again; ensureSettledFrame covers the times it does not, so the view ends +// on a sharp frame even when nothing moves afterwards. func (v *browserView) markMotion(ctx context.Context) { v.mu.Lock() v.motionGen++ @@ -496,6 +500,7 @@ func (v *browserView) markMotion(ctx context.Context) { if start && session != "" { v.restartScreencast(ctx, session) } + frameWait := browserSettledFrameWait time.AfterFunc(browserMotionSettle, func() { v.mu.Lock() if gen != v.motionGen || !v.motion { @@ -511,9 +516,54 @@ func (v *browserView) markMotion(ctx context.Context) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() v.restartScreencast(ctx, session) + v.ensureSettledFrame(ctx, session, frameWait) }) } +// browserSettledFrameWait is how long a restarted screencast gets to send the +// current page before ensureSettledFrame captures it instead. +var browserSettledFrameWait = 300 * time.Millisecond + +// ensureSettledFrame publishes a sharp screenshot when Chromium sends no frame +// after the screencast returns to sharp. It captures only on repaint and does +// not always re-send a page that stopped changing, which would leave the panel +// on its last light frame. +func (v *browserView) ensureSettledFrame(ctx context.Context, session string, wait time.Duration) { + v.mu.Lock() + seq, sampled := v.seq, v.settledFrameSampled + v.mu.Unlock() + if sampled != nil { + sampled() + } + select { + case <-time.After(wait): + case <-ctx.Done(): + return + } + stale := func() bool { + return v.session == session && v.seq == seq && !v.motion + } + v.mu.Lock() + ok := stale() + v.mu.Unlock() + if !ok { + return + } + var shot struct { + Data string `json:"data"` + } + if v.client.call(ctx, session, "Page.captureScreenshot", map[string]any{"format": "jpeg", "quality": 80}, &shot) != nil || shot.Data == "" { + return + } + v.mu.Lock() + if stale() { + v.frame = shot.Data + v.seq++ + v.notify() + } + v.mu.Unlock() +} + func (v *browserView) restartScreencast(ctx context.Context, session string) { _ = v.client.call(ctx, session, "Page.stopScreencast", nil, nil) _ = v.client.call(ctx, session, "Page.startScreencast", v.screencastParams(), nil) diff --git a/internal/runtime/core_cases_test.go b/internal/runtime/core_cases_test.go index 2e8b7004..c813f203 100644 --- a/internal/runtime/core_cases_test.go +++ b/internal/runtime/core_cases_test.go @@ -153,7 +153,9 @@ func TestBrowserViewOperationsWithFakeCDP(t *testing.T) { ctxDone, cancel := context.WithCancel(ctx) cancel() view.waitFrame(ctxDone, 99, 10) + view.mu.Lock() view.session = "" + view.mu.Unlock() if err := view.pageCall(ctx, "x", nil); err == nil { t.Fatal("page call without session") }