diff --git a/computer-use/internal/contract/ablation.go b/computer-use/internal/contract/ablation.go new file mode 100644 index 0000000..5118a7f --- /dev/null +++ b/computer-use/internal/contract/ablation.go @@ -0,0 +1,139 @@ +package contract + +import ( + "fmt" + "os" + "strings" +) + +// Mechanism ablations remove one HCU mechanism each, for experiments that +// measure what the mechanism contributes. COMPUTER_USE_ABLATE is a +// comma-separated list of the names below; by default nothing is removed. +// +// A removed mechanism is absent from everything the model sees: schemas, +// tool descriptions, server instructions and error texts. Like a disabled +// channel, it is not merely rejected at call time. +// +// - batching: an AX act holds at most one input step; wait steps may +// still surround it. GUI batches are unchanged. +// - wait: AX acts take no wait steps. The tools' own stabilization after +// input is unchanged. +// - diff: an AX act returns its execution evidence and the whole current +// tree of its window, as observe "tree" does, without the effect and +// windows report, the diff, or notes that compare with an earlier read. +// observe always returns the tree, never "unchanged". The tree is read +// with the same post-action stabilization as a diff. Two things stay: +// effect_unknown in the evidence (whether input had a verified effect +// is execution evidence, not a diff) and a settle wait step's message +// (it belongs to wait). +// - guard: the target and layout guards still decide, but only log the +// input they would have stopped (COMPUTER_USE_GUARD_LOG) and let it +// through. Only the Linux provider implements this. +const ( + AblationEnv = "COMPUTER_USE_ABLATE" + GuardLogEnv = "COMPUTER_USE_GUARD_LOG" + AblateBatching = "batching" + AblateWait = "wait" + AblateDiff = "diff" + AblateGuard = "guard" +) + +// AblationNames lists the ablations in their canonical order. +var AblationNames = []string{AblateBatching, AblateWait, AblateDiff, AblateGuard} + +// ParseAblations returns the ablations named in raw. An unknown or repeated +// name is an error: an experiment with a misspelled switch would otherwise +// run the complete HCU under an ablation label. +func ParseAblations(raw string) (map[string]bool, error) { + ablated := map[string]bool{} + if strings.TrimSpace(raw) == "" { + return ablated, nil + } + for _, part := range strings.Split(raw, ",") { + name := strings.TrimSpace(part) + if !ContainsString(AblationNames, name) { + return nil, fmt.Errorf("%s: unknown ablation %q; use a comma-separated subset of %s", + AblationEnv, name, strings.Join(AblationNames, ", ")) + } + if ablated[name] { + return nil, fmt.Errorf("%s: ablation %q is listed twice", AblationEnv, name) + } + ablated[name] = true + } + return ablated, nil +} + +// ValidateAblations checks COMPUTER_USE_ABLATE for a server, daemon or +// command started on goos. Entry points call it before serving anything. +func ValidateAblations(goos string, getenv func(string) string) error { + ablated, err := ParseAblations(getenv(AblationEnv)) + if err != nil { + return err + } + if ablated[AblateGuard] { + if goos != "linux" { + return fmt.Errorf("%s: the guard ablation is implemented only by the Linux provider, not on %s", + AblationEnv, goos) + } + if strings.TrimSpace(getenv(GuardLogEnv)) == "" { + return fmt.Errorf("%s: the guard ablation needs %s, the file that records the input the guards would have stopped", + AblationEnv, GuardLogEnv) + } + } + return nil +} + +// Ablated reports whether the named mechanism is removed. Entry points +// validate the variable at startup, so a value that does not parse here +// means the process skipped that check. +func Ablated(name string) bool { + ablated, err := ParseAblations(os.Getenv(AblationEnv)) + if err != nil { + panic(err) + } + return ablated[name] +} + +// SortedAblations returns the active ablations in canonical order. +func SortedAblations() []string { + names := []string{} + for _, name := range AblationNames { + if Ablated(name) { + names = append(names, name) + } + } + return names +} + +// PublishedAXActionTypes is the AX action list the schema publishes and the +// decoder accepts. +func PublishedAXActionTypes() []string { + if !Ablated(AblateWait) { + return AXActionTypes + } + types := []string{} + for _, actionType := range AXActionTypes { + if actionType != "wait" { + types = append(types, actionType) + } + } + return types +} + +// PublishedObserveValues is the observe list the schema publishes and the +// decoder accepts. Without diff, act takes no observe field (it always +// returns the tree), so diff and none, which only act uses, are gone too. +func PublishedObserveValues() []string { + if !Ablated(AblateDiff) { + return ObserveValues + } + return []string{ObserveTree, ObserveFull, ObserveScreenshot} +} + +// AXActMaxActions is the published maximum length of an AX act. +func AXActMaxActions() int { + if Ablated(AblateBatching) && Ablated(AblateWait) { + return 1 + } + return AXBatchMaxActions +} diff --git a/computer-use/internal/contract/ablation_test.go b/computer-use/internal/contract/ablation_test.go new file mode 100644 index 0000000..18b6fdf --- /dev/null +++ b/computer-use/internal/contract/ablation_test.go @@ -0,0 +1,109 @@ +package contract + +import ( + "strings" + "testing" +) + +func TestParseAblations(t *testing.T) { + for raw, want := range map[string]string{ + "": "", + " ": "", + "wait": "wait", + " guard , batching": "batching,guard", + "batching,wait,diff,guard": "batching,wait,diff,guard", + "diff,wait": "wait,diff", + } { + ablated, err := ParseAblations(raw) + if err != nil { + t.Fatalf("%q: %v", raw, err) + } + got := []string{} + for _, name := range AblationNames { + if ablated[name] { + got = append(got, name) + } + } + if strings.Join(got, ",") != want { + t.Fatalf("%q: got %v want %s", raw, got, want) + } + } + for _, raw := range []string{"waits", "wait,", "Wait", "wait,wait", "compact", ","} { + if _, err := ParseAblations(raw); err == nil { + t.Fatalf("%q must be rejected", raw) + } + } +} + +func TestValidateAblationsAtStartup(t *testing.T) { + env := func(values map[string]string) func(string) string { + return func(key string) string { return values[key] } + } + for _, check := range []struct { + goos string + values map[string]string + fails string + }{ + {"linux", map[string]string{}, ""}, + {"linux", map[string]string{AblationEnv: "batching,wait,diff"}, ""}, + {"darwin", map[string]string{AblationEnv: "wait"}, ""}, + {"linux", map[string]string{AblationEnv: "diffs"}, "unknown ablation"}, + {"linux", map[string]string{AblationEnv: "guard"}, GuardLogEnv}, + {"linux", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, ""}, + {"darwin", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, "only by the Linux provider"}, + {"windows", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, "only by the Linux provider"}, + } { + err := ValidateAblations(check.goos, env(check.values)) + switch { + case check.fails == "" && err != nil: + t.Fatalf("%s %v: %v", check.goos, check.values, err) + case check.fails != "" && (err == nil || !strings.Contains(err.Error(), check.fails)): + t.Fatalf("%s %v: got %v, want an error with %q", check.goos, check.values, err, check.fails) + } + } +} + +func TestPublishedTextWithoutAblationIsTheReleaseText(t *testing.T) { + t.Setenv(AblationEnv, "") + if PublishedServerInstructions() != ServerInstructions { + t.Fatal("server instructions changed without an ablation") + } + if publishedAXToolDescription() != axToolDescription { + t.Fatal("AX description changed without an ablation") + } + t.Setenv(AblationEnv, "wait,diff") + if !strings.Contains(PublishedServerInstructions(), "returns controls as rN refs, and acts on them.") || + !strings.Contains(PublishedServerInstructions(), "before input, and outside input makes their refs stale.") { + t.Fatalf("instructions: %s", PublishedServerInstructions()) + } +} + +func TestBatchWindowDescriptionFollowsBatchingAblation(t *testing.T) { + for mask := 0; mask < 1< 2 { + last := len(parts) - 1 + sentence = strings.Join(parts[:last], ", ") + ", and " + parts[last] + "." + } + if diff { + // The tools state how the ablated act behaves, as the guide does. + sentence += " Its result shows the whole current tree of its window after its " + + "actions instead of a report of what changed." + } + text := strings.Replace(axToolDescription, axActFullSentence, sentence, 1) + text = strings.Replace(text, axActFullExample, example, 1) + if batching { + text = strings.Replace(text, axBatchWindowDescription, "", 1) + } + if diff { + text = strings.Replace(text, " and reports the selection read back", "", 1) + } + if wait { + text = strings.Replace(text, axWaitDescription, + "act cannot pause for a later state; to check one, read again with observe or find. ", 1) + } + if wait || diff { + text = strings.Replace(text, "Skip waits the report answers. ", "", 1) + } + return text +} + var guiToolDescription = fmt.Sprintf("Full-desktop screenshot and pixel input. "+ `{"actions":[{"type":"screenshot"}]} alone observes; {"type":"zoom","region":[x0,y0,x1,y1]} alone `+ "enlarges a frame region to read small text and icons. Any other batch is checked before "+ @@ -480,7 +594,7 @@ func AllToolDefinitions() []ToolDefinition { { Name: PublicToolAX, Title: "Accessibility", - Description: axToolDescription, + Description: publishedAXToolDescription(), Annotations: annotations("Accessibility"), InputSchema: axInputSchema(), }, diff --git a/computer-use/internal/core/ax_ablation.go b/computer-use/internal/core/ax_ablation.go new file mode 100644 index 0000000..4b1acd0 --- /dev/null +++ b/computer-use/internal/core/ax_ablation.go @@ -0,0 +1,107 @@ +package core + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +// Behavior of the mechanism ablations in the AX service; contract/ablation.go +// says what each one removes. + +// axActObservation is the observation an act returns. Under the diff +// ablation an act takes no observe field and always returns the whole +// current working tree, as observe "tree" does; withActReport and +// modelFacingAXNotes leave out what changed. +func axActObservation(obs observation) observation { + if contract.Ablated(contract.AblateDiff) { + obs.mode = contract.ObserveTree + } + return obs +} + +// postActionReadStabilizes reports whether the read after an action waits +// for late content and a replaced top-level window. A diff act does; under +// the diff ablation the act's whole tree takes its place and is read the +// same way, so the ablation changes what is shown, not when it is read. +func postActionReadStabilizes(mode string) bool { + return mode == contract.ObserveDiff || + (mode == contract.ObserveTree && contract.Ablated(contract.AblateDiff)) +} + +// axChangeNote reports a model-facing note that compares the state with an +// earlier read: what the diff ablation removes from AX results. +func axChangeNote(note string) bool { + for _, prefix := range []string{ + "[state] semantic_state=unchanged", "[state] visual_change=", "[state] ax_change=", + "[text-attrs]", "[select] ", "[transition] ", + "The accessibility tree structure is unchanged", "The accessibility tree is unchanged", + "[identity] The new top-level window", "The set of top-level windows changed", + } { + if strings.HasPrefix(note, prefix) { + return true + } + } + return false +} + +// axCheckOneInputStep enforces the batching ablation: an act holds at most +// one input step. Wait steps send no input and may surround it. +func axCheckOneInputStep(earlier []axBatchAction, action axBatchAction, items []any, path string) *axArgError { + if !contract.Ablated(contract.AblateBatching) || action.actionType == "wait" { + return nil + } + for _, step := range earlier { + if step.actionType != "wait" { + return axInvalid(path, fmt.Sprintf( + "act takes one input step per call, and %s is the second; send it in a later call", path), ""). + withRule(contract.ArgumentRuleOneInputStep).withFix(axSet("actions", items[:len(earlier)])) + } + } + return nil +} + +// shadowGuard reports whether the guard ablation lets input through that a +// guard decided to stop. It first appends the decision to +// COMPUTER_USE_GUARD_LOG, in the format of the Linux provider's +// guard_shadow.py, so both sides of one run share one log. +func shadowGuard(channel, guard, reason string, detail map[string]any) bool { + if !contract.Ablated(contract.AblateGuard) { + return false + } + record := map[string]any{ + "at": time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), "channel": channel, + "guard": guard, "reason": reason, "would_block": true, + } + for key, value := range detail { + record[key] = value + } + path := strings.TrimSpace(os.Getenv(contract.GuardLogEnv)) + line, err := json.Marshal(record) + if path == "" || err != nil { + return true + } + // A lost log line must not fail the input it describes. + if dir := filepath.Dir(path); dir != "." { + _ = os.MkdirAll(dir, 0o755) + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return true + } + defer file.Close() + _, _ = file.Write(append(line, '\n')) + return true +} + +// axAblatedActionType reports an AX action type that COMPUTER_USE_ABLATE +// removed from the published contract. +func axAblatedActionType(verb string) bool { + return contract.ContainsString(contract.AXActionTypes, verb) && + !contract.ContainsString(contract.PublishedAXActionTypes(), verb) +} diff --git a/computer-use/internal/core/ax_ablation_test.go b/computer-use/internal/core/ax_ablation_test.go new file mode 100644 index 0000000..8e29a2f --- /dev/null +++ b/computer-use/internal/core/ax_ablation_test.go @@ -0,0 +1,527 @@ +package core + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +// ablationSets returns every subset of the ablations as a +// COMPUTER_USE_ABLATE value, the empty set first. +func ablationSets() []string { + sets := []string{} + for mask := 0; mask < 1< len(fullDescription) { + t.Errorf("AX description grew to %d bytes from %d", len(description), len(fullDescription)) + } + actions := properties["actions"].(map[string]any) + wantMax := contract.AXBatchMaxActions + if strings.Contains(set, "batching") && strings.Contains(set, "wait") { + wantMax = 1 + } + if actions["maxItems"] != wantMax { + t.Errorf("maxItems %v want %d", actions["maxItems"], wantMax) + } + }) + } +} + +// Each text-changing ablation states how the ablated act behaves, so an +// agent does not have to learn it from rejected calls. +func TestAblatedDescriptionStatesTheChangedBehavior(t *testing.T) { + statements := map[string]string{ + contract.AblateBatching: "act runs one input step with optional wait steps before or after it", + contract.AblateWait: "act cannot pause for a later state; to check one, read again with observe or find.", + contract.AblateDiff: "Its result shows the whole current tree of its window after its actions instead of a report of what changed.", + } + for _, set := range ablationSets() { + t.Run("ablate="+set, func(t *testing.T) { + t.Setenv(contract.AblationEnv, set) + description := findToolDefinition(t, contract.PublicToolAX).Description + for name, statement := range statements { + ablated := strings.Contains(","+set+",", ","+name+",") + if name == contract.AblateBatching && strings.Contains(set, contract.AblateWait) { + statement = "act runs one step" + } + if strings.Contains(description, statement) != ablated { + t.Errorf("%s ablated=%v, description:\n%s", name, ablated, description) + } + } + }) + } +} + +func TestAblatedDescriptionsKeepTheirExamplesValid(t *testing.T) { + example := regexp.MustCompile(`\{"mode":"act".*\}`) + for _, set := range ablationSets() { + t.Run("ablate="+set, func(t *testing.T) { + t.Setenv(contract.AblationEnv, set) + t.Setenv(contract.GuardLogEnv, filepath.Join(t.TempDir(), "guard.jsonl")) + description := findToolDefinition(t, contract.PublicToolAX).Description + call := example.FindString(description) + if call == "" { + t.Fatalf("no act example in %s", description) + } + if _, err := DecodeAXRequest(decodeTestArguments(t, call)); err != nil { + t.Fatalf("example %s: %s", call, err.Message) + } + if argErr := contract.ValidateToolArguments(contract.PublicToolAX, decodeTestArguments(t, call)); argErr != nil { + t.Fatalf("example %s: %s", call, argErr.Message) + } + }) + } +} + +func TestBatchingAblationAllowsOneInputStep(t *testing.T) { + t.Setenv(contract.AblationEnv, contract.AblateBatching) + for _, raw := range []string{ + `{"mode":"act","actions":[{"type":"click","ref":"r1"}]}`, + `{"mode":"act","actions":[{"type":"wait","window":"A"},{"type":"click","ref":"r1"},{"type":"wait","settle":true}]}`, + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"wait","window":"A","gone":true}]}`, + } { + if _, err := DecodeAXRequest(decodeTestArguments(t, raw)); err != nil { + t.Fatalf("%s: %s", raw, err.Message) + } + } + _, err := DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"click","ref":"r1"},{"type":"wait","window":"A"},{"type":"select","ref":"r2"}]}`)) + if err == nil || err.rule != contract.ArgumentRuleOneInputStep || err.path != "actions[2]" { + t.Fatalf("second input step: %#v", err) + } + if !strings.Contains(err.Hint, `{"type":"wait","window":"A"}`) || strings.Contains(err.Hint, "select") { + t.Fatalf("hint should keep the steps before the second input: %s", err.Hint) + } + // Input after close is refused as close-not-last, without suggesting a + // wait that would put a second input step in the call. + _, err = DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"click","ref":"r1"}]}`)) + if err == nil || err.rule != contract.ArgumentRuleCloseNotLast || + !strings.HasPrefix(err.Message, "close must be the last step") || strings.Contains(err.Hint, `"gone"`) { + t.Fatalf("input after close: %#v", err) + } + if focus := noFocusMessage(t); strings.Contains(focus, "batch") { + t.Fatalf("no-focus message: %s", focus) + } + + t.Setenv(contract.AblationEnv, "batching,wait") + _, err = DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"click","ref":"r1"},{"type":"click","ref":"r2"}]}`)) + if err == nil || err.Message != "act takes one action per call; send the others in later calls" { + t.Fatalf("two actions without waits: %#v", err) + } +} + +// noFocusMessage returns the no_focused_window message for a desktop where +// no window has focus. +func noFocusMessage(t *testing.T) string { + t.Helper() + backend := &recordingPlatformBackend{accessibilityResponse: &ProviderAccessibilityResult{ + OK: true, Apps: editorApps(testWindow{testHandleReport, "Report - Editor", false}), + }} + _, err := NewServiceWithBackend(backend).resolveFocusedWindow() + if err == nil || err.Code != contract.AXCodeNoFocusedWindow { + t.Fatalf("want no_focused_window, got %#v", err) + } + return err.Message +} + +func TestBatchingAblationKeepsGUIBatches(t *testing.T) { + t.Setenv(contract.AblationEnv, "batching,wait,diff") + args := decodeTestArguments(t, `{"actions":[{"type":"click","x":1,"y":2},{"type":"type","text":"a"},{"type":"keypress","keys":["ENTER"]}]}`) + if err := contract.ValidateToolArguments(contract.PublicToolGUI, args); err != nil { + t.Fatalf("GUI batch: %s", err.Message) + } +} + +func TestWaitAblationRemovesWaitSteps(t *testing.T) { + t.Setenv(contract.AblationEnv, contract.AblateWait) + _, err := DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"type":"wait","window":"A"}]}`)) + if err == nil || !strings.HasPrefix(err.Message, "type must be one of click, type, keypress, set_value, select, scroll, activate, close;") || + strings.Contains(err.Hint, "wait") || err.path != "actions[0].type" { + t.Fatalf("wait step: %#v", err) + } + // A rejected wait points to a fresh read, never to an input action + // guessed from its fields. + for _, step := range []string{ + `{"type":"wait","text":"Saved"}`, + `{"type":"wait","ref":"r1","gone":true}`, + `{"type":"wait","ref":"w1"}`, + `{"type":"wait","keys":["ENTER"]}`, + } { + _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[`+step+`]}`)) + if err == nil || err.Hint != axMinimalCalls[contract.AXModeObserve] { + t.Fatalf("%s: rejected wait got %#v", step, err) + } + } + _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"action":"wait","text":"Saved"}]}`)) + if err == nil || err.Hint != axMinimalCalls[contract.AXModeObserve] { + t.Fatalf("action=wait: %#v", err) + } + if text := (&axNoTreeWindowError{ref: "w2"}).Error(); strings.Contains(text, "wait") { + t.Fatalf("no-tree window text: %s", text) + } + _, err = DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"keypress","keys":["ENTER"]}]}`)) + if err == nil || err.rule != contract.ArgumentRuleCloseNotLast || strings.Contains(err.Message+err.Hint, "wait") { + t.Fatalf("input after close: %#v", err) + } + _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"type":"click","ref":"r1","until":{"type":"exists","ref":"r2"}}]}`)) + if err == nil || strings.Contains(err.Message+err.Hint, "wait") { + t.Fatalf("removed until field: %#v", err) + } + if strings.Contains(strings.Join(continuityNotesForUnchanged(t), "\n"), "wait step") { + t.Fatal("the unchanged-snapshot note still suggests a wait step") + } +} + +// continuityNotesForUnchanged returns the notes for a snapshot identical to +// the delivered one. +func continuityNotesForUnchanged(t *testing.T) []string { + t.Helper() + previous := &AppSnapshot{SnapshotID: "s1", WindowHandle: "h1", WindowTitle: "T"} + current := &AppSnapshot{SnapshotID: "s1", WindowHandle: "h1", WindowTitle: "T"} + return continuityNotes(previous, current) +} + +// modelText is the text a model reads of an AX result. +func modelText(result ToolCallResult) string { + texts := []string{} + for _, item := range FinalizeForModel(contract.PublicToolAX, result, false).Content { + if item.Type == "text" { + texts = append(texts, item.Text) + } + } + return strings.Join(texts, "\n") +} + +// reportActUnder observes the report fixture, sets COMPUTER_USE_ABLATE to +// set, and runs one act whose input applies change. +func reportActUnder(t *testing.T, set string, change func(*reportBackend), act func(map[string]string) map[string]any, names ...string) (*Service, map[string]string, ToolCallResult) { + t.Helper() + s, backend := reportService(t, reportMain()) + _, refs := reportObserve(t, s, names...) + t.Setenv(contract.AblationEnv, set) + backend.onAction = change + return s, refs, s.callTool(contract.PublicToolAX, act(refs)) +} + +func clickOn(name string, extra map[string]any) func(map[string]string) map[string]any { + return func(refs map[string]string) map[string]any { + args := map[string]any{"mode": "act", "actions": []any{map[string]any{"type": "click", "ref": refs[name]}}} + for key, value := range extra { + args[key] = value + } + return args + } +} + +// The full HCU must stay as it was: the act reports the effect, the windows +// and a diff, and observe answers an unchanged tree by reference. +func TestFullHCUActOutputIsUnchanged(t *testing.T) { + s, _, result := reportActUnder(t, "", func(b *reportBackend) { b.checked = true }, + clickOn("Interface", nil), "Interface") + want := `{"ok":true,"input_emitted":true,"completed":1,"requested":1,"final_observation_kind":"diff",` + + `"final_observation_returned":true,"next_identical_observation_required":false,"observation_required":false}` + "\n" + + `effect: r1 checkbox "Interface" checked=false→true (only state flags changed in this window)` + "\n" + + "windows: no change\n\n" + + "app: \"Editor\"\nwindow: \"Report\" [ref=w1]\nax_diff:\n" + + ` - checkbox "Interface" [checked=true] [ref=r1]` + "\n" + if got := modelText(result); got != want { + t.Fatalf("full act:\n%s\nwant:\n%s", got, want) + } + observed := modelText(s.callTool(contract.PublicToolAX, map[string]any{"mode": "observe", "app": "Editor"})) + if !strings.Contains(observed, "Note: [state] semantic_state=unchanged") || + !strings.Contains(observed, "Tree unchanged: identical to the state of this window already "+ + "delivered by an earlier result (last full tree plus later diffs)") { + t.Fatalf("full observe:\n%s", observed) + } +} + +// Under diff an act returns its evidence and the whole current tree of its +// window, as observe "tree" does, without the report of what changed. +func TestDiffAblationActReturnsTheWholeTree(t *testing.T) { + check := func(change func(*reportBackend)) (string, string) { + _, _, full := reportActUnder(t, "", change, clickOn("Interface", map[string]any{"observe": "tree"}), "Interface") + s, _, ablated := reportActUnder(t, contract.AblateDiff, change, clickOn("Interface", nil), "Interface") + text := modelText(ablated) + for _, removed := range []string{"effect:", "windows: no change", "ax_diff", "semantic_state", "visual_change", "ax_change"} { + if strings.Contains(text, removed) { + t.Fatalf("act under diff still has %q:\n%s", removed, text) + } + } + // The internally decided evidence is that of the full HCU. + for _, key := range []string{"target_effect_observed", "effect_unknown", "final_observation_kind"} { + if full.StructuredContent[key] != ablated.StructuredContent[key] { + t.Fatalf("%s: full %v, diff %v", key, full.StructuredContent[key], ablated.StructuredContent[key]) + } + } + // The tree was delivered, so a later observe returns it again, + // never "unchanged". + observed := modelText(s.callTool(contract.PublicToolAX, map[string]any{"mode": "observe", "app": "Editor"})) + if strings.Contains(observed, "Tree unchanged") || strings.Contains(observed, "semantic_state") || + !strings.Contains(observed, `checkbox "Interface"`) { + t.Fatalf("observe under diff:\n%s", observed) + } + return text, modelText(full) + } + text, full := check(func(b *reportBackend) { b.checked = true }) + tree := text[strings.Index(text, "app: "):] + if !strings.HasPrefix(text, `{"ok":true,"input_emitted":true,"completed":1,"requested":1,"final_observation_kind":"tree"`) || + !strings.Contains(tree, `checkbox "Interface" [checked=true]`) || !strings.HasSuffix(full, tree) { + t.Fatalf("act under diff:\n%s\nfull with observe tree:\n%s", text, full) + } + check(func(*reportBackend) {}) + + _, err := DecodeAXRequest(map[string]any{"mode": "act", "observe": "tree", + "actions": []any{map[string]any{"type": "click", "ref": "r1"}}}) + if err == nil || err.path != "observe" { + t.Fatalf("act with observe: %#v", err) + } +} + +// A window the act opened appears with its wN in the tree's window list. +func TestDiffAblationTreeListsAnOpenedWindow(t *testing.T) { + _, _, result := reportActUnder(t, contract.AblateDiff, func(b *reportBackend) { + b.windows[0].active = false + b.windows = append(b.windows, reportWindow{ + handle: reportDialogHandle, role: "dialog", title: "Insert Sheet", path: "/window/2", active: true, + }) + }, clickOn("OK", nil), "OK") + text := modelText(result) + if !regexp.MustCompile(`(?m)^ - dialog "Insert Sheet" \[ref=w[0-9]+\]$`).MatchString(text) || + strings.Contains(text, "opened") { + t.Fatalf("act under diff:\n%s", text) + } +} + +// Error texts belong to other mechanisms (waits, postconditions, recovery) +// and stay as in the full HCU; only the report of what changed goes. +func TestDiffAblationErrorsKeepTheirMessages(t *testing.T) { + s, _ := reportService(t, reportMain()) + reportObserve(t, s, "Interface") + context := &AXBatchContext{AppToken: "Editor"} + message, hint, current := s.axBatchContinuityRecovery(context) + t.Setenv(contract.AblationEnv, contract.AblateDiff) + if m, h, c := s.axBatchContinuityRecovery(context); m != message || h != hint || c != current { + t.Fatalf("recovery under diff: %q %q %q, full: %q %q %q", m, h, c, message, hint, current) + } + t.Setenv(contract.AblationEnv, "") + + evidence := func(result ToolCallResult) map[string]any { + t.Helper() + var line map[string]any + if err := json.Unmarshal([]byte(strings.SplitN(modelText(result), "\n", 2)[0]), &line); err != nil { + t.Fatal(err) + } + return line + } + waitAct := func(refs map[string]string) map[string]any { + return map[string]any{"mode": "act", "actions": []any{ + map[string]any{"type": "click", "ref": refs["Interface"]}, + map[string]any{"type": "wait", "ref": refs["Size"], "value": "99", "timeout_ms": 200}, + }} + } + withTree := func(act func(map[string]string) map[string]any) func(map[string]string) map[string]any { + return func(refs map[string]string) map[string]any { + args := act(refs) + args["observe"] = "tree" + return args + } + } + _, _, full := reportActUnder(t, "", nil, withTree(waitAct), "Interface", "Size") + _, _, ablated := reportActUnder(t, contract.AblateDiff, nil, waitAct, "Interface", "Size") + if evidence(ablated)["code"] != contract.AXCodeWaitTimeout || + fmt.Sprint(evidence(ablated)) != fmt.Sprint(evidence(full)) || + !strings.Contains(fmt.Sprint(evidence(ablated)["message"]), `value is "12"`) || + !strings.Contains(modelText(ablated), `textbox "Size"`) || strings.Contains(modelText(ablated), "effect:") { + t.Fatalf("wait timeout under diff:\n%s\nfull:\n%s", modelText(ablated), modelText(full)) + } + + lose := func(b *reportBackend) { b.loseReadback = true } + _, _, full = reportActUnder(t, "", lose, clickOn("Interface", map[string]any{"observe": "tree"}), "Interface") + _, _, ablated = reportActUnder(t, contract.AblateDiff, lose, clickOn("Interface", nil), "Interface") + if line := evidence(ablated); line["code"] != "effect_unknown" || + fmt.Sprint(line) != fmt.Sprint(evidence(full)) || + !strings.Contains(fmt.Sprint(line["message"]), "post_action_observation_failed") || + !strings.Contains(modelText(ablated), "A fresh current-state observation follows") || + !strings.Contains(modelText(ablated), `checkbox "Interface"`) || strings.Contains(modelText(ablated), "effect:") { + t.Fatalf("lost readback under diff:\n%s\nfull:\n%s", modelText(ablated), modelText(full)) + } +} + +func TestDiffAblationKeepsTheErrorOfAFailedAct(t *testing.T) { + stale := func(map[string]string) map[string]any { + return map[string]any{"mode": "act", "actions": []any{map[string]any{"type": "click", "ref": "r999"}}} + } + _, _, full := reportActUnder(t, "", nil, stale, "Interface") + _, _, ablated := reportActUnder(t, contract.AblateDiff, nil, stale, "Interface") + if !ablated.IsError || modelText(ablated) != modelText(full) { + t.Fatalf("rejected act under diff:\n%s\nfull:\n%s", modelText(ablated), modelText(full)) + } +} + +// Notes that compare with an earlier read go; notes on the current state stay. +func TestDiffAblationDropsChangeNotes(t *testing.T) { + notes := []string{ + "[diagnostic snapshot_unchanged] same tree", "[pixels] persistent_change=false", + "No accessibility change observed: window title, tree, focus and selection match.", + "[text-attrs] Document text formatting changed", `[select] selected 3 chars in 1 elements, from "a" to "c"`, + "[diagnostic snapshot_busy] busy", "[expect] r2 value is \"12\"", + "The accessibility tree structure is unchanged, while accessibility text attributes changed (see the [text-attrs] note above).", + "[transition] A bounded no-input probe observed the transient window finish and returned the current application window.", + "[identity] The new top-level window was refreshed once without input; the current wN/rN refs below are actionable.", + "The set of top-level windows changed but no window took focus within 0.8s. The snapshot below may show a different window.", + } + full := modelFacingAXNotes(notes) + t.Setenv(contract.AblationEnv, contract.AblateDiff) + got := modelFacingAXNotes(notes) + want := []string{"[state] busy=true recovery=wait_then_observe", "[expect] r2 value is \"12\""} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("notes under diff %q, full %q", got, full) + } + if len(full) <= len(got) { + t.Fatalf("full notes %q", full) + } +} + +// The whole tree an act returns under diff is read with the same post-action +// stabilization as a diff: a value that arrives late is in it. +func TestDiffAblationReadsTheTreeAfterLateContent(t *testing.T) { + for _, set := range []string{"", contract.AblateDiff} { + backend := &recordingPlatformBackend{ + accessibilityResponses: []ProviderAccessibilityResult{ + {OK: true, Snapshot: contentStabilizationSnapshot("v1", "Ready")}, + {OK: true, Snapshot: contentStabilizationSnapshot("v1", "Ready")}, + {OK: true, Snapshot: contentStabilizationSnapshot("v2", "Saved")}, + }, + } + service := NewServiceWithBackend(backend) + off := false + service.getAppState("Editor", nil, nil, nil, nil, &off, nil, contract.TreeViewWorking, "", nil) + t.Setenv(contract.AblationEnv, set) + obs := axActObservation(observation{mode: contract.ObserveDiff}) + result := service.actionResult("Editor", AXServiceRequest{ + App: "Editor", Action: axKeypressCommand{Key: "Tab"}, + Observe: obs.resolvedMode(), IncludeScreenshot: &off, TreeView: contract.TreeViewWorking, + }) + if result.StructuredContent["content_stabilized"] != true || !strings.Contains(ResultText(result), `"Saved"`) { + t.Fatalf("ablate=%q read the transient state: %v\n%s", set, + result.StructuredContent["content_stabilized"], ResultText(result)) + } + } +} + +// A verified close returns the current state, the absent window, not a diff. +func TestDiffAblationClosedWindowIsCurrentState(t *testing.T) { + context := &AXBatchContext{WindowRef: "w3"} + full := ResultText(closedWindowFinalObservation(context, observation{mode: contract.ObserveTree})) + t.Setenv(contract.AblationEnv, contract.AblateDiff) + ablated := ResultText(closedWindowFinalObservation(context, observation{mode: contract.ObserveTree})) + if !strings.HasPrefix(full, "ax_diff:") || ablated != "window: null\nabsent_refs: [w3]" { + t.Fatalf("closed window under diff %q, full %q", ablated, full) + } +} + +func TestGuardAblationLogsTheDecisionAndLetsInputThrough(t *testing.T) { + log := filepath.Join(t.TempDir(), "logs", "guard.jsonl") + t.Setenv(contract.GuardLogEnv, log) + if shadowGuard("ax", "batch_window", "changed", nil) { + t.Fatal("without the ablation the guard must stop the input") + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Fatal("a guard log was written without the ablation") + } + t.Setenv(contract.AblationEnv, contract.AblateGuard) + if !shadowGuard("ax", "batch_window", "changed", map[string]any{"window_ref": "w1"}) { + t.Fatal("with the ablation the guard must let the input through") + } + data, err := os.ReadFile(log) + if err != nil { + t.Fatal(err) + } + var record map[string]any + if err := json.Unmarshal(data, &record); err != nil { + t.Fatal(err) + } + if record["channel"] != "ax" || record["guard"] != "batch_window" || record["window_ref"] != "w1" || + record["would_block"] != true || record["at"] == nil { + t.Fatalf("record %v", record) + } +} diff --git a/computer-use/internal/core/ax_act_report.go b/computer-use/internal/core/ax_act_report.go index 59da8b6..34db24a 100644 --- a/computer-use/internal/core/ax_act_report.go +++ b/computer-use/internal/core/ax_act_report.go @@ -103,7 +103,9 @@ func (t *actReportTrace) after(s *Service, index int) *AppSnapshot { } // withActReport puts the report first in an act result. A call rejected -// before any input gets no report: nothing happened. +// before any input gets no report: nothing happened. Under the diff +// ablation the report still decides target_effect_observed but is not +// shown. func (s *Service) withActReport( result ToolCallResult, actions []axBatchAction, @@ -159,6 +161,11 @@ func (s *Service) withActReport( if verified && len(lines) > 1 { result.StructuredContent["target_effect_observed"] = true } + if contract.Ablated(contract.AblateDiff) { + // The diff ablation keeps the evidence the report decides, but + // not the report. + return result + } result.Content = append( []ContentItem{{Type: "text", Text: strings.Join(lines, "\n")}}, result.Content..., diff --git a/computer-use/internal/core/ax_action_evidence.go b/computer-use/internal/core/ax_action_evidence.go index 18d635d..e360af0 100644 --- a/computer-use/internal/core/ax_action_evidence.go +++ b/computer-use/internal/core/ax_action_evidence.go @@ -6,6 +6,8 @@ import ( "sort" "strconv" "strings" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) func unchangedNotes(windowFocusConfirmed bool, pixels pixelVerdict, attrsChanged bool) []string { @@ -350,8 +352,13 @@ func observeNoneNotes(notes []string) []string { func modelFacingAXNotes(notes []string) []string { out := make([]string, 0, len(notes)) seen := map[string]bool{} + withoutChanges := contract.Ablated(contract.AblateDiff) appendOnce := func(value string) { - if value == "" || seen[value] { + if withoutChanges && value == "[state] semantic_state=unchanged busy=true recovery=wait_then_observe" { + // The app's busy state is current state, not a change. + value = "[state] busy=true recovery=wait_then_observe" + } + if value == "" || seen[value] || (withoutChanges && axChangeNote(value)) { return } seen[value] = true diff --git a/computer-use/internal/core/ax_action_stabilization.go b/computer-use/internal/core/ax_action_stabilization.go index b66e204..d0dc1fa 100644 --- a/computer-use/internal/core/ax_action_stabilization.go +++ b/computer-use/internal/core/ax_action_stabilization.go @@ -204,7 +204,7 @@ func postActionWindowIdentityStabilizationTrigger( after *AppSnapshot, obs observation, ) string { - if obs.resolvedMode() != contract.ObserveDiff || + if !postActionReadStabilizes(obs.resolvedMode()) || before == nil || after == nil { return "" @@ -327,7 +327,7 @@ func shouldStabilizePostActionContent( if service == nil || identityStabilized || request.WindowAction != nil || - (observation{mode: request.Observe}).resolvedMode() != contract.ObserveDiff || + !postActionReadStabilizes((observation{mode: request.Observe}).resolvedMode()) || after == nil || after.externalDesktopTransition { return false diff --git a/computer-use/internal/core/ax_argument_hint_test.go b/computer-use/internal/core/ax_argument_hint_test.go index 2dac3a5..50f8cf9 100644 --- a/computer-use/internal/core/ax_argument_hint_test.go +++ b/computer-use/internal/core/ax_argument_hint_test.go @@ -142,8 +142,53 @@ var axBadCalls = []string{ // Every invalid_argument from the decoder has a hint that decodes: a whole // call (starting with "mode") decodes as is; a fragment decodes when it // replaces the object at the error path. +// axAblatedBadCalls reach the rejection sites that exist only under an +// ablation (COMPUTER_USE_ABLATE). +var axAblatedBadCalls = map[string][]string{ + contract.AblateWait: { + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"click","ref":"r1"}]}`, + `{"mode":"observe","app":"x","observe":"none"}`, + `{"mode":"assert_ax"}`, + }, + contract.AblateBatching + "," + contract.AblateWait: { + `{"mode":"act","actions":[{"type":"click","ref":"r1"},{"type":"click","ref":"r2"}]}`, + }, + contract.AblateDiff: { + `{"mode":"act","actions":[{"type":"click","ref":"r1"}],"observe":"none"}`, + `{"mode":"observe","app":"x","observe":"diff"}`, + }, +} + func TestEveryAXArgumentErrorHintDecodes(t *testing.T) { reached := map[int]bool{} + checkAXBadCalls(t, axBadCalls, reached) + for ablation, calls := range axAblatedBadCalls { + t.Run(ablation, func(t *testing.T) { + t.Setenv(contract.AblationEnv, ablation) + checkAXBadCalls(t, calls, reached) + }) + } + source, readErr := os.ReadFile("ax_public_request.go") + if readErr != nil { + t.Fatal(readErr) + } + var missed []string + for index, line := range strings.Split(string(source), "\n") { + if strings.Contains(line, "axInvalid(") && !strings.Contains(line, "func axInvalid(") && + !strings.HasPrefix(strings.TrimSpace(line), "//") && !reached[index+1] { + missed = append(missed, strings.TrimSpace(line)) + } + } + sort.Strings(missed) + if len(missed) > 0 { + t.Fatalf("no bad call reaches these rejection sites:\n%s", strings.Join(missed, "\n")) + } +} + +// checkAXBadCalls decodes each call, checks that its hint decodes, and +// records the decoder lines that rejected it. +func checkAXBadCalls(t *testing.T, calls []string, reached map[int]bool) { + t.Helper() first := false // Only the first rejection of a call is its error site; later ones come // from the decodes that derive the hint. @@ -154,7 +199,7 @@ func TestEveryAXArgumentErrorHintDecodes(t *testing.T) { first = false } defer func() { axInvalidHook = nil }() - for _, raw := range axBadCalls { + for _, raw := range calls { args := decodeTestArguments(t, raw) first = true _, err := DecodeAXRequest(args) @@ -172,21 +217,6 @@ func TestEveryAXArgumentErrorHintDecodes(t *testing.T) { t.Fatalf("%s: hint %s for %s does not decode: %s (%s)", label, err.Hint, err.path, hintErr.Message, hintErr.path) } } - source, readErr := os.ReadFile("ax_public_request.go") - if readErr != nil { - t.Fatal(readErr) - } - var missed []string - for index, line := range strings.Split(string(source), "\n") { - if strings.Contains(line, "axInvalid(") && !strings.Contains(line, "func axInvalid(") && - !strings.HasPrefix(strings.TrimSpace(line), "//") && !reached[index+1] { - missed = append(missed, strings.TrimSpace(line)) - } - } - sort.Strings(missed) - if len(missed) > 0 { - t.Fatalf("no bad call reaches these rejection sites:\n%s", strings.Join(missed, "\n")) - } } func axHintDecodeError(args map[string]any, err *axArgError) *axArgError { diff --git a/computer-use/internal/core/ax_batch_context.go b/computer-use/internal/core/ax_batch_context.go index e40c1bb..f886f8c 100644 --- a/computer-use/internal/core/ax_batch_context.go +++ b/computer-use/internal/core/ax_batch_context.go @@ -271,6 +271,11 @@ func (s *Service) axBatchFocusChanged( if argErr == nil && focus.handle == context.WindowHandle { return ToolCallResult{}, false } + if shadowGuard("ax", "batch_window", "focus_left_bound_window", map[string]any{ + "window_ref": context.WindowRef, "next_action_index": index, + }) { + return ToolCallResult{}, false + } now := "no window has focus" if argErr == nil { now = fmt.Sprintf("focus is now on %s %s (%s)", focus.ref, strconv.Quote(focus.title), focus.appName) diff --git a/computer-use/internal/core/ax_close_observation.go b/computer-use/internal/core/ax_close_observation.go index 62a29c8..4088a0f 100644 --- a/computer-use/internal/core/ax_close_observation.go +++ b/computer-use/internal/core/ax_close_observation.go @@ -17,10 +17,12 @@ func closedWindowFinalObservation( if windowRef != "" { refLine = "\n absent_refs: [" + windowRef + "]" } - result := TextResult( - "ax_diff:\n effect: target_window_absent"+refLine, - false, - ) + text := "ax_diff:\n effect: target_window_absent" + refLine + if contract.Ablated(contract.AblateDiff) { + // The current state, without the diff format. + text = "window: null" + strings.Replace(refLine, "\n ", "\n", 1) + } + result := TextResult(text, false) result.StructuredContent = map[string]any{ "full_tree_returned": false, "tree_reference_only": false, diff --git a/computer-use/internal/core/ax_native_window.go b/computer-use/internal/core/ax_native_window.go index fe62feb..4a85f78 100644 --- a/computer-use/internal/core/ax_native_window.go +++ b/computer-use/internal/core/ax_native_window.go @@ -22,6 +22,10 @@ type axNoTreeWindowError struct { } func (err *axNoTreeWindowError) Error() string { + if contract.Ablated(contract.AblateWait) { + return err.ref + " has no accessibility tree. " + + "Read and operate it with GUI; AX can only activate or close this window." + } return err.ref + " has no accessibility tree. " + "Read and operate it with GUI; AX can only activate, " + "close or wait for this window." diff --git a/computer-use/internal/core/ax_partial_inventory_test.go b/computer-use/internal/core/ax_partial_inventory_test.go index 47083cd..7963926 100644 --- a/computer-use/internal/core/ax_partial_inventory_test.go +++ b/computer-use/internal/core/ax_partial_inventory_test.go @@ -80,6 +80,27 @@ func TestPartialInventoryCannotEstablishUniqueTitleFocusOrAbsence(t *testing.T) } } +// A text wait without a window reads the focused window. When the window +// list cannot be read, the call fails with that reason instead of +// crashing the server on a missing window listing. +func TestFocusWaitOnUnreadableInventoryFails(t *testing.T) { + s, backend := partialInventoryService() + for _, broken := range []bool{false, true} { + if broken { + backend.accessibilityResponse.OK = false + backend.accessibilityResponse.Error = "provider timed out" + } + result := PrepareAXEvidence(s.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{map[string]any{"type": "wait", "text": "Saved", "timeout_ms": 300}}, + })) + text, _ := result.StructuredContent["message"].(string) + if !result.IsError || result.StructuredContent["input_emitted"] != false || + !strings.Contains(text, "give window_ref") || strings.Contains(text, "no window has focus") { + t.Fatalf("broken=%v: %#v", broken, result.StructuredContent) + } + } +} + func TestInventoryCompletenessRecoveryPreservesRefs(t *testing.T) { s, backend := partialInventoryService() s.listApps() diff --git a/computer-use/internal/core/ax_public_request.go b/computer-use/internal/core/ax_public_request.go index 3b82fda..bf8474c 100644 --- a/computer-use/internal/core/ax_public_request.go +++ b/computer-use/internal/core/ax_public_request.go @@ -109,6 +109,15 @@ var axModeFields = map[string][]string{ contract.AXModeAct: {"mode", "window_ref", "actions", "observe"}, } +// axModeAllowedFields lists the fields a mode accepts. Without diff, act +// always returns the tree, so it takes no observe field. +func axModeAllowedFields(mode string) []string { + if mode == contract.AXModeAct && contract.Ablated(contract.AblateDiff) { + return []string{"mode", "window_ref", "actions"} + } + return axModeFields[mode] +} + // axActionFields lists the fields each action type accepts besides type. var axActionFields = map[string][]string{ "click": {"ref", "allow_global_input"}, @@ -159,7 +168,7 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } mode, ok := rawMode.(string) if !ok || !contract.ContainsString([]string{contract.AXModeApps, contract.AXModeObserve, contract.AXModeFind, contract.AXModeAct}, mode) { - if mode == "assert_ax" { + if mode == "assert_ax" && !contract.Ablated(contract.AblateWait) { return nil, axInvalid("mode", `mode "assert_ax" was removed; wait with an act call that has only wait steps`, `{"mode":"act","actions":[{"type":"wait","ref":"r1"}]}`) @@ -173,10 +182,12 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } return nil, err } - if err := axRejectRemovedUntil(args); err != nil { - return nil, err + if !contract.Ablated(contract.AblateWait) { + if err := axRejectRemovedUntil(args); err != nil { + return nil, err + } } - if err := axCheckFields(args, axModeFields[mode], "", "mode "+strconv.Quote(mode)); err != nil { + if err := axCheckFields(args, axModeAllowedFields(mode), "", "mode "+strconv.Quote(mode)); err != nil { return nil, err.withRule(axRuleFieldForMode) } request := &axRequest{mode: mode} @@ -194,7 +205,7 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { if request.scope, err = axOptionalEnum(args, "scope", "", contract.AXScopeValues); err != nil { return nil, err } - if request.observe, err = axOptionalEnum(args, "observe", "", contract.ObserveValues); err != nil { + if request.observe, err = axOptionalEnum(args, "observe", "", contract.PublishedObserveValues()); err != nil { return nil, err } if request.root, err = axOptionalPattern(args, "root", "", axRootPattern, `"focused" or rN`); err != nil { @@ -223,8 +234,11 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } } if request.observe == contract.ObserveNone { - return nil, axInvalid("observe", `observe "none" returns nothing; to wait without input, send act with only wait steps`, - "").withRule(axRuleObserveNone).withFix(axRemove("observe")) + message := `observe "none" returns nothing; to wait without input, send act with only wait steps` + if contract.Ablated(contract.AblateWait) { + message = `observe "none" returns nothing; it is only for act` + } + return nil, axInvalid("observe", message, "").withRule(axRuleObserveNone).withFix(axRemove("observe")) } if request.app == "" && request.windowRef == "" && request.readText == nil { return nil, axInvalid("app", "observe needs app or window_ref", @@ -332,9 +346,12 @@ func decodeAXActions(raw any) ([]axBatchAction, *axArgError) { return nil, axInvalid("actions", `act needs at least one action; use mode "observe" to read`, `[{"type":"click","ref":"r1"}]`) } - if len(items) > contract.AXBatchMaxActions { - return nil, axInvalid("actions", fmt.Sprintf("act accepts at most %d actions", contract.AXBatchMaxActions), ""). - withFix(axSet("actions", items[:contract.AXBatchMaxActions])) + if limit := contract.AXActMaxActions(); len(items) > limit { + message := fmt.Sprintf("act accepts at most %d actions", limit) + if limit == 1 { + message = "act takes one action per call; send the others in later calls" + } + return nil, axInvalid("actions", message, "").withFix(axSet("actions", items[:limit])) } actions := make([]axBatchAction, 0, len(items)) totalText, totalWait := 0, 0 @@ -349,9 +366,18 @@ func decodeAXActions(raw any) ([]axBatchAction, *axArgError) { return nil, err } if index > 0 && actions[index-1].actionType == "close" && action.actionType != "wait" { + // Without wait steps, or with one input step per act, nothing + // can follow close in the same call. + if contract.Ablated(contract.AblateWait) || contract.Ablated(contract.AblateBatching) { + return nil, axInvalid(path, "close must be the last step: a closed window cannot take later input", + "").withRule(contract.ArgumentRuleCloseNotLast).withFix(axSet("actions", items[:index])) + } return nil, axInvalid(path, "a closed window cannot take later input; after close, wait for the window to be gone first", `{"type":"wait","window":"window title","gone":true}`).withRule(contract.ArgumentRuleCloseNotLast) } + if err := axCheckOneInputStep(actions, action, items, path); err != nil { + return nil, err + } totalText += len([]rune(action.text)) + len([]rune(action.value)) if totalText > contract.AXBatchMaxText { return nil, axInvalid(path, fmt.Sprintf("text and value together are limited to %d characters per call", contract.AXBatchMaxText), "").withRule(contract.ArgumentRuleTextBudget) @@ -379,7 +405,7 @@ func decodeAXAction(object map[string]any, path string) (axBatchAction, *axArgEr action := axBatchAction{pages: 1} rawType, hasType := object["type"] actionType, _ := rawType.(string) - if !hasType || !contract.ContainsString(contract.AXActionTypes, actionType) { + if !hasType || !contract.ContainsString(contract.PublishedAXActionTypes(), actionType) { if hint := axActionShapeHint(object); hint != "" && !hasType { field := "type" for name := range object { @@ -389,15 +415,24 @@ func decodeAXAction(object map[string]any, path string) (axBatchAction, *axArgEr } return action, axInvalid(path+"."+field, `actions use "type", not "`+field+`"`, hint) } - message := "type must be one of " + strings.Join(contract.AXActionTypes, ", ") + message := "type must be one of " + strings.Join(contract.PublishedAXActionTypes(), ", ") if hasType { message += "; got " + contract.CompactJSON(rawType) } else { - message = "action needs type: " + strings.Join(contract.AXActionTypes, ", ") + message = "action needs type: " + strings.Join(contract.PublishedAXActionTypes(), ", ") } - err := axInvalid(path+".type", message, axActionShapeHint(object)) + hint := axActionShapeHint(object) + verb := actionType + if !hasType { + verb, _ = object["action"].(string) + } + if axAblatedActionType(verb) { + // The tool description sends a removed wait to a fresh read. + hint = axMinimalCalls[contract.AXModeObserve] + } + err := axInvalid(path+".type", message, hint) switch { - case actionType == "settle": + case actionType == "settle" && !axAblatedActionType("wait"): err.Message += `; use type "wait" with settle:true after an input step in the same call` err.Hint = "" err.fix = []axEdit{axSet(path+".type", "wait"), axSet(path+".settle", true)} @@ -592,7 +627,12 @@ func axActionShapeHint(object map[string]any) string { if synonym, known := axActionVerbSynonyms[verb]; known { verb = synonym } - if !contract.ContainsString(contract.AXActionTypes, verb) { + // An action type removed by an ablation has no AX equivalent; guessing + // one from its fields would turn a wait for "Saved" into typing it. + if axAblatedActionType(verb) { + return "" + } + if !contract.ContainsString(contract.PublishedAXActionTypes(), verb) { switch { case AXWindowRefPattern.MatchString(ref) && strings.Contains(strings.ToLower(verb), "close"): verb = "close" diff --git a/computer-use/internal/core/ax_request.go b/computer-use/internal/core/ax_request.go index d627c07..bd4a263 100644 --- a/computer-use/internal/core/ax_request.go +++ b/computer-use/internal/core/ax_request.go @@ -252,7 +252,7 @@ func (s *Service) axAct( ) ToolCallResult { result := s.axBatchDecoded( request.actions, - obs, + axActObservation(obs), request.windowRef, ).withEvidenceScope("delivery_or_local_ui") return WithAXModeEvidence(result, contract.AXModeAct) diff --git a/computer-use/internal/core/ax_snapshot_notes.go b/computer-use/internal/core/ax_snapshot_notes.go index c2b2fec..1c2fa6e 100644 --- a/computer-use/internal/core/ax_snapshot_notes.go +++ b/computer-use/internal/core/ax_snapshot_notes.go @@ -202,11 +202,10 @@ func continuityNotes(previous, current *AppSnapshot) []string { "window instance, same tree, same focus, same selection.%s "+ "Nothing in the tree reports work in progress, but that is not proof your "+ "last action failed: effects may not reach the accessibility tree. "+ - "Check the screenshot, or "+ - "wait for the specific element with an AX wait step before deciding to retry. "+ + "Check the screenshot%s before deciding to retry. "+ "Note also what this line rules OUT: the content is not missing because a "+ "budget trimmed it — this tree is the same one you already have.", - gap)) + gap, map[bool]string{false: ", or wait for the specific element with an AX wait step", true: ""}[contract.Ablated(contract.AblateWait)])) return notes } diff --git a/computer-use/internal/core/ax_snapshot_store.go b/computer-use/internal/core/ax_snapshot_store.go index 80f2d83..a9f4534 100644 --- a/computer-use/internal/core/ax_snapshot_store.go +++ b/computer-use/internal/core/ax_snapshot_store.go @@ -261,8 +261,10 @@ func (s *Service) getAppStateResult( providerCoverageSummary(snapshot.Annotations).AcquisitionIncomplete == 0 // A stable provider snapshot can receive new public refs after invalidation. // Only reuse the model's tree when its delivered lines still match. + // The diff ablation always returns the tree. previousDelivery := s.deliveredSnapshotFor(snapshot) if acquisitionComplete && !forceFull && previousDelivery != nil && + !contract.Ablated(contract.AblateDiff) && slices.Equal(previousDelivery.TreeLines, snapshot.TreeLines) && ((snapshot.identicalToPrevious && delivered) || s.deliveredStateMatches(snapshot)) { diff --git a/computer-use/internal/core/ax_text_read_test.go b/computer-use/internal/core/ax_text_read_test.go index 9ed62c0..508da1a 100644 --- a/computer-use/internal/core/ax_text_read_test.go +++ b/computer-use/internal/core/ax_text_read_test.go @@ -141,6 +141,18 @@ func TestTextReadBudgetDoesNotCutDeliveredPage(t *testing.T) { } } +func TestTextReadSurvivesMechanismAblations(t *testing.T) { + for _, ablation := range []string{"", contract.AblateDiff, contract.AblateBatching, contract.AblateWait} { + t.Run(ablation, func(t *testing.T) { + t.Setenv(contract.AblationEnv, ablation) + request, err := DecodeAXRequest(axTextReadCall("", "r1", "value", 0, 4000)) + if err != nil || request.readText.Limit != 4000 { + t.Fatalf("read_text was lost in ablation: %v", err) + } + }) + } +} + func TestProviderTruncationIsMarkedEvenWhenEllipsisFillsTheGap(t *testing.T) { for _, total := range []int{501, 502, 503} { record := &ElementRecord{Name: strings.Repeat("x", 500) + "...", TextLengths: map[string]int{"name": total}, TextTruncated: []string{"name"}} diff --git a/computer-use/internal/core/ax_wait_batch.go b/computer-use/internal/core/ax_wait_batch.go index 2398ef4..d997c62 100644 --- a/computer-use/internal/core/ax_wait_batch.go +++ b/computer-use/internal/core/ax_wait_batch.go @@ -207,6 +207,11 @@ func (s *Service) axBatchContinuityStop( if continuity == axWindowSame { return ToolCallResult{}, false } + if shadowGuard("ax", "batch_window", "logical_window_"+string(continuity)+"_between_actions", map[string]any{ + "window_ref": context.WindowRef, "next_action_index": index + 1, + }) { + return ToolCallResult{}, false + } anyInputEmitted, anyPartialPossible, anyMutationUnknown := batchMutationSummary(actionEvidence) failureCode := "stale_target" messageCode := "batch_window_changed" @@ -291,8 +296,14 @@ func (s *Service) axWaitOnly( var argErr *axArgError focus, argErr = s.resolveFocusedWindow() if argErr != nil { - argErr.Message = "a wait without ref or window_ref reads the focused window, and no window has focus; " + - "give window_ref, or wait for a window by title. Windows:" + strings.TrimPrefix(argErr.Message[strings.Index(argErr.Message, "Windows:"):], "Windows:") + // Only a missing focus lists the windows; an unreadable window + // list keeps its own code and message. + if index := strings.Index(argErr.Message, "Windows:"); index >= 0 { + argErr.Message = "a wait without ref or window_ref reads the focused window, and no window has focus; " + + "give window_ref, or wait for a window by title. " + argErr.Message[index:] + } else { + argErr.Message += "; a wait without ref or window_ref reads the focused window, so give window_ref" + } argErr.Hint = `{"mode":"act","window_ref":"w1","actions":[{"type":"wait","text":"Saved"}]}` return axArgErrorResult(argErr, len(actions)) } diff --git a/computer-use/internal/core/ax_window_inventory.go b/computer-use/internal/core/ax_window_inventory.go index 6eff2dd..966f208 100644 --- a/computer-use/internal/core/ax_window_inventory.go +++ b/computer-use/internal/core/ax_window_inventory.go @@ -268,10 +268,14 @@ func (s *Service) resolveFocusedWindow() (*axDesktopWindow, *axArgError) { return &inventory.windows[index], nil } } + start := "Start the batch with activate. " + if contract.Ablated(contract.AblateBatching) { + start = "Activate a window in its own call first. " + } return nil, &axArgError{ Code: contract.AXCodeNoFocusedWindow, Message: "no application window has keyboard focus, so keys or text without ref have no target. " + - "Start the batch with activate. Windows:\n" + inventory.text, + start + "Windows:\n" + inventory.text, Hint: `{"type":"activate","title":"window title"}`, } } diff --git a/computer-use/internal/core/ax_window_inventory_test.go b/computer-use/internal/core/ax_window_inventory_test.go index ba3ff91..fe5b6d4 100644 --- a/computer-use/internal/core/ax_window_inventory_test.go +++ b/computer-use/internal/core/ax_window_inventory_test.go @@ -1,6 +1,8 @@ package core import ( + "os" + "path/filepath" "strings" "testing" @@ -300,6 +302,26 @@ func TestFocusChangeBeforeFirstActionStopsWithoutInput(t *testing.T) { } func TestFocusChangeAfterActionKStopsAtKPlusOne(t *testing.T) { + focusChangeBatch(t, false) +} + +// With guard ablated the focus check only logs, and every key is sent. +func TestGuardAblationLetsFocusBoundBatchContinue(t *testing.T) { + log := filepath.Join(t.TempDir(), "guard.jsonl") + t.Setenv(contract.AblationEnv, contract.AblateGuard) + t.Setenv(contract.GuardLogEnv, log) + focusChangeBatch(t, true) + data, err := os.ReadFile(log) + if err != nil || !strings.Contains(string(data), `"reason":"focus_left_bound_window"`) || + !strings.Contains(string(data), `"next_action_index":2`) { + t.Fatalf("guard log %q err=%v", data, err) + } +} + +// focusChangeBatch sends three keys to the focused window; focus moves to +// another window before the third. +func focusChangeBatch(t *testing.T, guardAblated bool) { + t.Helper() focused := editorApps( testWindow{testHandleReport, "Report - Editor", true}, ) @@ -334,6 +356,12 @@ func TestFocusChangeAfterActionKStopsAtKPlusOne(t *testing.T) { map[string]any{"type": "keypress", "keys": []any{"C"}}, }) result := service.axBatchDecoded(actions, observation{mode: contract.ObserveNone}, "") + if guardAblated { + if result.IsError || len(backend.actionRequests) != 3 { + t.Fatalf("guard ablation stopped the batch after %d keys: %#v", len(backend.actionRequests), result.StructuredContent) + } + return + } if !result.IsError || result.StructuredContent[contract.AXPublicErrorCodeKey] != contract.AXCodeWindowChanged { t.Fatalf("focus change after action 2: %#v", result.StructuredContent) } diff --git a/computer-use/internal/core/schema_consistency_test.go b/computer-use/internal/core/schema_consistency_test.go index 9c9f016..c7306af 100644 --- a/computer-use/internal/core/schema_consistency_test.go +++ b/computer-use/internal/core/schema_consistency_test.go @@ -9,6 +9,7 @@ import ( "fmt" "math/big" "os" + "path/filepath" "sort" "strings" "testing" @@ -33,6 +34,7 @@ var crossFieldRuleCodes = map[string]bool{ contract.ArgumentRuleTextBudget: true, contract.ArgumentRuleRefOwnsIdentity: true, contract.ArgumentRuleWaitBudget: true, + contract.ArgumentRuleOneInputStep: true, // The flat AX schema has no oneOf or if/then, so rules that depend on // the value of mode or of an action type are declared rules of // the AX decoder: which fields a mode or type accepts or requires, @@ -217,6 +219,22 @@ func checkConsistency(t testing.TB, sample consistencySample) *consistencyMismat } func TestPublishedSchemaMatchesRuntimeAcceptance(t *testing.T) { + checkPublishedSchemaMatchesRuntime(t) +} + +// Each ablation changes the schema and the decoder together; they must +// still accept the same calls. +func TestAblatedSchemaMatchesRuntimeAcceptance(t *testing.T) { + for _, ablation := range append(append([]string{}, contract.AblationNames...), "batching,wait,diff,guard") { + t.Run(ablation, func(t *testing.T) { + t.Setenv(contract.AblationEnv, ablation) + t.Setenv(contract.GuardLogEnv, filepath.Join(t.TempDir(), "guard.jsonl")) + checkPublishedSchemaMatchesRuntime(t) + }) + } +} + +func checkPublishedSchemaMatchesRuntime(t *testing.T) { samples := consistencySamples(t) if len(samples) < 1000 { t.Fatalf("only %d consistency samples were generated", len(samples)) diff --git a/computer-use/internal/provider/linux_runtime_bundle.go b/computer-use/internal/provider/linux_runtime_bundle.go index 52ebead..e6840ac 100644 --- a/computer-use/internal/provider/linux_runtime_bundle.go +++ b/computer-use/internal/provider/linux_runtime_bundle.go @@ -6,6 +6,7 @@ import runtimebundle "github.com/microsoft/Hybrid-Computer-Use/computer-use" var linuxProviderPaths = []string{ "native/linux/provider/foundation.py", + "native/linux/provider/guard_shadow.py", "native/linux/provider/node_geometry.py", "native/linux/provider/node_visibility.py", "native/linux/provider/a11y_enable.py", diff --git a/computer-use/internal/provider/linux_runtime_bundle_test.go b/computer-use/internal/provider/linux_runtime_bundle_test.go index 9b434da..f3753b1 100644 --- a/computer-use/internal/provider/linux_runtime_bundle_test.go +++ b/computer-use/internal/provider/linux_runtime_bundle_test.go @@ -39,6 +39,7 @@ func TestLinuxRuntimeBundleIsComplete(t *testing.T) { "native/linux/provider/find_walk.py", "native/linux/provider/focus_probe.py", "native/linux/provider/foundation.py", + "native/linux/provider/guard_shadow.py", "native/linux/provider/gui_capture.py", "native/linux/provider/gui_dispatch.py", "native/linux/provider/gui_input.py", diff --git a/computer-use/internal/transport/cli_main.go b/computer-use/internal/transport/cli_main.go index 662858d..fbb3f4b 100644 --- a/computer-use/internal/transport/cli_main.go +++ b/computer-use/internal/transport/cli_main.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "runtime" "strings" "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" @@ -18,6 +19,16 @@ func RunCLI(args []string, stdout io.Writer) error { return nil } + switch args[0] { + case "-h", "--help", "help", "-v", "--version", "version": + default: + // Fail before serving anything: a misspelled ablation must not run + // the complete HCU under an ablation label. + if err := contract.ValidateAblations(runtime.GOOS, os.Getenv); err != nil { + return err + } + } + switch args[0] { case "-h", "--help", "help": topic := "" diff --git a/computer-use/internal/transport/cli_main_test.go b/computer-use/internal/transport/cli_main_test.go new file mode 100644 index 0000000..0381bcb --- /dev/null +++ b/computer-use/internal/transport/cli_main_test.go @@ -0,0 +1,50 @@ +package transport + +import ( + "bytes" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/core" +) + +// A misspelled ablation stops every entry point before it serves anything; +// help and version still answer. +func TestInvalidAblationStopsStartup(t *testing.T) { + t.Setenv(contract.AblationEnv, "wait,diffs") + for _, args := range [][]string{ + {"mcp"}, {"serve"}, {"daemon", "status"}, {"schema"}, {"call", "AX", "--args", `{"mode":"apps"}`}, {"apps"}, + } { + var stdout bytes.Buffer + err := RunCLI(args, &stdout) + if err == nil || !strings.Contains(err.Error(), `unknown ablation "diffs"`) { + t.Fatalf("%v: got %v", args, err) + } + if stdout.Len() != 0 { + t.Fatalf("%v wrote %q before failing", args, stdout.String()) + } + } + for _, args := range [][]string{{"help"}, {"version"}} { + if err := RunCLI(args, &bytes.Buffer{}); err != nil { + t.Fatalf("%v: %v", args, err) + } + } + t.Setenv(contract.AblationEnv, "guard") + t.Setenv(contract.GuardLogEnv, "") + if err := RunCLI([]string{"mcp"}, &bytes.Buffer{}); err == nil || + !strings.Contains(err.Error(), contract.GuardLogEnv) { + t.Fatalf("guard without a log: %v", err) + } +} + +func TestMCPInitializeServesTheAblatedInstructions(t *testing.T) { + t.Setenv(contract.AblationEnv, "wait") + response := handleMCPRequest(map[string]any{ + "jsonrpc": "2.0", "id": float64(1), "method": "initialize", "params": map[string]any{}, + }, core.NewService()) + instructions := response["result"].(map[string]any)["instructions"].(string) + if instructions == contract.ServerInstructions || strings.Contains(instructions, "wait step") { + t.Fatalf("instructions: %s", instructions) + } +} diff --git a/computer-use/internal/transport/mcp_transport.go b/computer-use/internal/transport/mcp_transport.go index 8cccf0c..1f3ce46 100644 --- a/computer-use/internal/transport/mcp_transport.go +++ b/computer-use/internal/transport/mcp_transport.go @@ -215,7 +215,7 @@ func handleMCPRequest(request map[string]any, svc *core.Service) map[string]any "version": contract.ServerVersion, }, "capabilities": map[string]any{"tools": map[string]any{"listChanged": false}}, - "instructions": contract.ServerInstructions, + "instructions": contract.PublishedServerInstructions(), }) case "ping": return jsonRPCResult(id, map[string]any{}) diff --git a/computer-use/native/linux/provider/actionability.py b/computer-use/native/linux/provider/actionability.py index 2709d02..1045c4a 100644 --- a/computer-use/native/linux/provider/actionability.py +++ b/computer-use/native/linux/provider/actionability.py @@ -30,6 +30,7 @@ ) from . import action_evidence from .input_primitives import require_window_focus +from .guard_shadow import shadow_guard from .element_resolver import ( same_frame, @@ -277,6 +278,10 @@ def require_stable_coordinate_geometry(element, element_record, window_bounds): input_events_emitted=False, failure_subject="element", ) + if not settled and shadow_guard( + "ax", "target_geometry", "target_unstable", samples=samples, + ): + settled = True if not settled: action_evidence.update(geometry_stable=False) raise OperationFailure( @@ -399,6 +404,10 @@ def require_coordinate_delivery(window, x, y, what): return action_evidence.update(point_owner_verified=owner) reason = "unknown" if owner is None else "different_window" + if shadow_guard( + "ax", "point_owner", reason, x=x, y=y, action=what, + ): + return raise OperationFailure( "coordinate_target_unverified: point owner is {}; no input was " "emitted.".format(reason), diff --git a/computer-use/native/linux/provider/ax_actions.py b/computer-use/native/linux/provider/ax_actions.py index 9a72951..084f1e2 100644 --- a/computer-use/native/linux/provider/ax_actions.py +++ b/computer-use/native/linux/provider/ax_actions.py @@ -25,6 +25,7 @@ from .tree_walk import iter_all from .ax_records import combo_edit_target, combo_edit_identity +from .guard_shadow import shadow_guard from .input_method import InputMethodText @@ -46,6 +47,10 @@ def resolve_combo_editor(element, record): return element, False editor = combo_edit_target(element) identity = combo_edit_identity(editor) + if (expected is not None and identity is not None and identity != expected + and shadow_guard("ax", "target_identity", "composite_edit_target_replaced", + expected=expected, live=identity)): + return editor, True if expected is None or identity != expected: raise OperationFailure( "The combobox has no unique unchanged editable partner; no text " diff --git a/computer-use/native/linux/provider/ax_dispatch.py b/computer-use/native/linux/provider/ax_dispatch.py index f53fff8..51ff65d 100644 --- a/computer-use/native/linux/provider/ax_dispatch.py +++ b/computer-use/native/linux/provider/ax_dispatch.py @@ -35,9 +35,11 @@ from .app_resolver import app_window_inventory, list_apps_inventory, windowless_snapshot from .native_windows import native_window_for_operation, perform_native_window_action +from .guard_shadow import shadow_guard from .element_resolver import ( find_element_at_snapshot_path, resolve_semantic_target, + shadow_semantic_refusal, ) from .window_resolver import ( perform_ax_window_action, @@ -99,6 +101,11 @@ def _verify_expected_window(window, expectation): and current_role == expected_role ) if not common_matches or (transient and not transient_matches): + if shadow_guard( + "ax", "batch_window", "logical_window_changed_before_input", + expected_role=expected_role, current_role=current_role, + ): + return raise OperationFailure( "stale_target: the batch logical window changed before keyboard " "input; no input was emitted.", @@ -499,21 +506,28 @@ def _resolve_action_target(app, operation, operation_kind, action_input): bounds = operation.get("window_bounds") or window_bounds( app, window_index, window) if semantic_target is not None and operation_kind != "window_action": - ( - element, - element_record, - semantic_note, - window, - bounds, - ) = resolve_semantic_target( - app, - window_index, - window, - bounds, - semantic_target, - operation.get("known_refs"), - operation, - ) + try: + ( + element, + element_record, + semantic_note, + window, + bounds, + ) = resolve_semantic_target( + app, + window_index, + window, + bounds, + semantic_target, + operation.get("known_refs"), + operation, + ) + except OperationFailure as refused: + shadowed = shadow_semantic_refusal( + app, window_index, window, bounds, semantic_target, refused) + if shadowed is None: + raise + element, element_record, window, bounds = shadowed elif semantic_target is None and action_input.get("window_centre"): element = None elif semantic_target is None: @@ -550,10 +564,17 @@ def _resolve_select_end(app, operation, action_input, window, bounds): failure_phase="precondition", input_events_emitted=False, ) - to_element, _record, _note, to_window, _bounds = resolve_semantic_target( - app, window_index, window, bounds, to_target, - operation.get("known_refs"), operation, - ) + try: + to_element, _record, _note, to_window, _bounds = resolve_semantic_target( + app, window_index, window, bounds, to_target, + operation.get("known_refs"), operation, + ) + except OperationFailure as refused: + shadowed = shadow_semantic_refusal( + app, window_index, window, bounds, to_target, refused) + if shadowed is None: + raise + to_element, _record, to_window, _bounds = shadowed if provider_object_path(to_window) != window_path: raise OperationFailure( "select_invalid_range: to_ref is in a different window than ref; " diff --git a/computer-use/native/linux/provider/ax_input_delivery.py b/computer-use/native/linux/provider/ax_input_delivery.py index 882175a..92eba64 100644 --- a/computer-use/native/linux/provider/ax_input_delivery.py +++ b/computer-use/native/linux/provider/ax_input_delivery.py @@ -3,6 +3,7 @@ import math import time +from .guard_shadow import guards_enforced from .foundation import ( Atspi, OperationFailure, @@ -408,10 +409,13 @@ def _pointer_click_for_state_only_action( return None, "the window is not in the foreground" evidence_before = dict(action_evidence) try: - record, _ = require_stable_coordinate_geometry( - element, element_record, bounds) - x, y = screen_point(bounds, record, None, None) - require_coordinate_delivery(window, x, y, "click") + # These checks only choose the route: a refusal keeps the semantic + # action, so the guard ablation has nothing to let through. + with guards_enforced(): + record, _ = require_stable_coordinate_geometry( + element, element_record, bounds) + x, y = screen_point(bounds, record, None, None) + require_coordinate_delivery(window, x, y, "click") hits = point_hits_element(window, element, x, y) except (OperationFailure, RuntimeError) as refused: action_evidence.clear() diff --git a/computer-use/native/linux/provider/element_resolver.py b/computer-use/native/linux/provider/element_resolver.py index 27dd487..f86af60 100644 --- a/computer-use/native/linux/provider/element_resolver.py +++ b/computer-use/native/linux/provider/element_resolver.py @@ -7,24 +7,31 @@ DEFAULT_TEXT_LIMIT, OperationFailure, child_at, + node_name, node_role, safe, + state_value, ) from .node_geometry import extents from .primitives import name_fingerprint +from .guard_shadow import guards_shadowed, shadow_guard from .ax_node_properties import ( accessible_id, parse_text_limit, ) from .ax_records import ( + combo_edit_identity, + combo_edit_target, effective_name, effective_role, + gtk_combo_partner, ) from .x11_windows import ( main_window, relative_frame, + window_bounds, ) from .primitives import semantic_text_key from .semantic_window_candidates import ( @@ -132,6 +139,7 @@ def find_element_at_snapshot_path_with_bounds(app, record, window_bounds): paths = [record.get("runtimeId") or []] paths.extend(record.get("runtimeAliases") or []) seen_paths = set() + mismatched = None for path in paths: path_key = tuple(path) if path_key in seen_paths: @@ -142,6 +150,8 @@ def find_element_at_snapshot_path_with_bounds(app, record, window_bounds): node, record, window_bounds ): return node + if node is not None and mismatched is None: + mismatched = (node, path) # Self-managed tables expose visible cells through Table.get_accessible_at # instead of child enumeration. render_visible_cells gives those records a # stable virtual suffix [row, col], so the action resolver must understand @@ -170,6 +180,13 @@ def find_element_at_snapshot_path_with_bounds(app, record, window_bounds): and record_still_matches(node, record, window_bounds) ): return node + if mismatched is not None and shadow_guard( + "ax", "target_identity", "path_resolves_to_a_different_element", + path=list(mismatched[1]), + expected={"role": record.get("controlType"), "name": record.get("name")}, + live={"role": effective_role(mismatched[0]), "name": effective_name(mismatched[0])}, + ): + return mismatched[0] return None @@ -528,3 +545,61 @@ def _semantic_resolution_note( ), ) return note + + +# Identity refusals of a semantic target. The guard ablation acts on the +# snapshot path instead when a node is still there. +_SHADOWED_TARGET_FAILURES = { + "stale_element_ref", "semantic_target_not_found", "semantic_target_ambiguous", +} + + +def shadow_semantic_refusal(app, window_index, window, bounds, target, refused): + """Return (node, record, window, bounds) at the target's snapshot path. + + The window and bounds are those of the top-level window that holds the + node, as a resolved target has. None keeps the refusal: the guard + ablation is off, the refusal is not an identity check, or no live node is + left on the path. + """ + # The full arm must not pay for the AT-SPI reads below. + if not guards_shadowed(): + return None + if getattr(refused, "failure_code", None) not in _SHADOWED_TARGET_FAILURES: + return None + paths = [target.get("runtimeId") or []] + paths.extend(target.get("runtimeAliases") or []) + for path in paths: + if not path: + continue + node = resolve_path(app, path) + if node is None or state_value(node, Atspi.StateType.DEFUNCT) is True: + continue + role = node_role(node) + if not shadow_guard( + "ax", "target_identity", refused.failure_code, + message=str(refused), selector=target.get("selector"), path=list(path), + live={"role": role, "name": node_name(node)}, + ): + return None + node_window, node_bounds = window, bounds + if path[0] != window_index: + top = safe(lambda: app.get_child_at_index(path[0])) + if top is not None: + node_window = top + node_bounds = window_bounds(app, path[0], top) + record = { + "runtimeId": list(path), + "controlType": role, + "name": node_name(node), + "frame": safe(lambda: relative_frame(node, node_bounds)), + } + # A projected combo edits through its partner, as a scanned record + # says (ax_records.py). + if role == "combo box" or gtk_combo_partner(node, role) is not None: + record["controlType"] = "combo box" + identity = combo_edit_identity(combo_edit_target(node)) + if identity is not None: + record["editTarget"] = identity + return node, record, node_window, node_bounds + return None diff --git a/computer-use/native/linux/provider/guard_shadow.py b/computer-use/native/linux/provider/guard_shadow.py new file mode 100644 index 0000000..66c6cb7 --- /dev/null +++ b/computer-use/native/linux/provider/guard_shadow.py @@ -0,0 +1,81 @@ +"""Shadow mode of the input guards for the guard ablation. + +With ``guard`` in COMPUTER_USE_ABLATE, the AX target guard and the GUI +layout guard still make their decision, but a decision to stop input is +only recorded: one JSON line per decision in COMPUTER_USE_GUARD_LOG, which +the model never sees. The input then goes ahead. Checks that find nothing +to act on (no node on the snapshot path, no captured frame) are not guards +and still fail. +""" + +import contextlib +import json +import os +import time + +_enforced = 0 + + +@contextlib.contextmanager +def guards_enforced(): + """Keep guards deciding normally, for checks that only choose a route.""" + global _enforced + _enforced += 1 + try: + yield + finally: + _enforced -= 1 + + +def guard_ablated(): + names = os.environ.get("COMPUTER_USE_ABLATE", "").split(",") + return "guard" in {name.strip() for name in names} + + +def guards_shadowed(): + """Report whether a stopped input would be logged and let through.""" + return not _enforced and guard_ablated() + + +def shadow_guard(channel, guard, reason, **detail): + """Return True when the guard ablation lets the stopped input through. + + The decision is appended to the guard log first. Without the ablation + nothing is written and the caller stops the input as before. + """ + if not guards_shadowed(): + return False + record = { + "at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + + ".%03dZ" % (int(time.time() * 1000) % 1000), + "channel": channel, + "guard": guard, + "reason": reason, + "would_block": True, + } + record.update(detail) + path = os.environ.get("COMPUTER_USE_GUARD_LOG", "").strip() + if path: + # A lost log line must not fail the input it describes. + try: + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + with open(path, "a", encoding="utf-8") as log: + log.write(json.dumps(record, sort_keys=True, default=str) + "\n") + except OSError: + pass + return True + + +def shadow_layout_guard(reason, frame_id, actions=(), region=None): + """shadow_guard for the GUI layout guard; logs each action's point.""" + points = [] + for action in actions: + point = {"type": action.get("type")} + point.update({key: action[key] for key in ("x", "y", "path") if key in action}) + points.append(point) + detail = {"frame_id": frame_id, "actions": points} + if region is not None: + detail["region"] = region + return shadow_guard("gui", "zoom_layout" if region is not None else "layout", reason, **detail) diff --git a/computer-use/native/linux/provider/gui_dispatch.py b/computer-use/native/linux/provider/gui_dispatch.py index 7d274a5..70a36dd 100644 --- a/computer-use/native/linux/provider/gui_dispatch.py +++ b/computer-use/native/linux/provider/gui_dispatch.py @@ -33,6 +33,7 @@ validate_gui_actions, ) from .input_primitives import InputEmissionError +from .guard_shadow import shadow_layout_guard from .snapshot import capture_gui_snapshot from .desktop_layout import ( desktop_bounds, @@ -394,7 +395,8 @@ def stale(message, capture): left, top, width, height = geometry["native"] if not region_still_valid( _published_layouts.get(frame_id), capture_layout(current), - (left, top, width, height)): + (left, top, width, height)) and not shadow_layout_guard( + "zoom_region_changed", frame_id, region=[left, top, width, height]): return stale( "The windows over the zoom region changed since the frame its " "coordinates came from.", current) @@ -573,7 +575,8 @@ def failure_evidence(fresh=True): if frame_id and frame_guarded_actions: if (current_capture.frame_id != frame_id and not coordinates_still_valid( - reference_layout, capture_layout(current_capture), validated)): + reference_layout, capture_layout(current_capture), validated) + and not shadow_layout_guard("changed_before_batch", frame_id, validated)): return { "ok": False, "error": "The desktop layout changed since the screenshot used " @@ -614,7 +617,8 @@ def failure_evidence(fresh=True): live_frame_id = desktop_layout_frame_id(live_layout) if (live_frame_id != frame_id and not coordinates_still_valid( - reference_layout, live_layout, [action])): + reference_layout, live_layout, [action]) + and not shadow_layout_guard("changed_during_batch", frame_id, [action])): error_snapshot, evidence_note = failure_evidence() return { "ok": False, diff --git a/computer-use/runtime_test.py b/computer-use/runtime_test.py index 2167b7a..294a150 100644 --- a/computer-use/runtime_test.py +++ b/computer-use/runtime_test.py @@ -36,6 +36,7 @@ from runtime_tests.test_native_windows import * # noqa: F401,F403 from runtime_tests.test_x11_client_identity import * # noqa: F401,F403 from runtime_tests.test_window_geometry import * # noqa: F401,F403 +from runtime_tests.test_guard_shadow import * # noqa: F401,F403 if __name__ == "__main__": diff --git a/computer-use/runtime_test_support.py b/computer-use/runtime_test_support.py index ae058f3..f168976 100644 --- a/computer-use/runtime_test_support.py +++ b/computer-use/runtime_test_support.py @@ -47,6 +47,7 @@ semantic_window_candidates, semantic_scan, foundation, + guard_shadow, node_geometry, node_visibility, node_facts, @@ -76,6 +77,7 @@ PROVIDER_MODULES = ( foundation, + guard_shadow, node_geometry, node_visibility, a11y_enable, diff --git a/computer-use/runtime_tests/test_guard_shadow.py b/computer-use/runtime_tests/test_guard_shadow.py new file mode 100644 index 0000000..b32229d --- /dev/null +++ b/computer-use/runtime_tests/test_guard_shadow.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Linux provider tests for the guard ablation's shadow mode.""" + +import json +import os +import tempfile + +from runtime_test_support import * # noqa: F401,F403 +from runtime_tests import test_gui_capture + + +class GuardLogMixin: + """Run with or without the guard ablation and read the guard log.""" + + def setUp(self): + super().setUp() + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.log_path = os.path.join(directory.name, "guard.jsonl") + patcher = mock.patch.dict(os.environ, { + "COMPUTER_USE_ABLATE": "", "COMPUTER_USE_GUARD_LOG": self.log_path, + }) + patcher.start() + self.addCleanup(patcher.stop) + + def ablate_guard(self): + os.environ["COMPUTER_USE_ABLATE"] = "wait,guard" + + def guard_log(self): + if not os.path.exists(self.log_path): + return [] + with open(self.log_path, encoding="utf-8") as log: + return [json.loads(line) for line in log] + + +class AXGuardShadowTests(GuardLogMixin, unittest.TestCase): + record = { + "runtimeId": [0, 2], "controlType": "push button", "name": "Rename…", + } + + def resolve_mismatch(self): + node = object() + with mock.patch.object(element_resolver, "resolve_path", return_value=node), \ + mock.patch.object(element_resolver, "record_still_matches", return_value=False), \ + mock.patch.object(element_resolver, "effective_role", return_value="toggle button"), \ + mock.patch.object(element_resolver, "effective_name", return_value="Bold"): + return node, element_resolver.find_element_at_snapshot_path_with_bounds( + object(), dict(self.record), None) + + def test_identity_mismatch_stops_by_default(self): + _node, resolved = self.resolve_mismatch() + self.assertIsNone(resolved) + self.assertEqual(self.guard_log(), []) + + def test_identity_mismatch_acts_on_the_path_node_and_logs_it(self): + self.ablate_guard() + node, resolved = self.resolve_mismatch() + self.assertIs(resolved, node) + [entry] = self.guard_log() + self.assertEqual(entry["channel"], "ax") + self.assertEqual(entry["guard"], "target_identity") + self.assertEqual(entry["path"], [0, 2]) + self.assertEqual(entry["expected"]["name"], "Rename…") + self.assertEqual(entry["live"], {"role": "toggle button", "name": "Bold"}) + self.assertTrue(entry["would_block"]) + + def test_a_path_without_a_node_stays_stale(self): + self.ablate_guard() + with mock.patch.object(element_resolver, "resolve_path", return_value=None): + resolved = element_resolver.find_element_at_snapshot_path_with_bounds( + object(), dict(self.record), None) + self.assertIsNone(resolved) + self.assertEqual(self.guard_log(), []) + + def semantic_refusal(self, code, node, path=(0, 4), role="push button", app=None, + combo=None, edit_identity=None): + refused = runtime.OperationFailure( + code + " selector='button:Save'; no input was emitted.", + failure_code=code, failure_phase="precondition", input_events_emitted=False) + target = {"selector": "button:Save", "runtimeId": list(path)} + with mock.patch.object(element_resolver, "resolve_path", return_value=node) as resolve, \ + mock.patch.object(element_resolver, "state_value", return_value=None), \ + mock.patch.object(element_resolver, "node_role", return_value=role), \ + mock.patch.object(element_resolver, "node_name", return_value="Cancel"), \ + mock.patch.object(element_resolver, "relative_frame", return_value=None), \ + mock.patch.object(element_resolver, "window_bounds", return_value=(1, 2, 3, 4)), \ + mock.patch.object(element_resolver, "gtk_combo_partner", return_value=combo), \ + mock.patch.object(element_resolver, "combo_edit_target", return_value=object()), \ + mock.patch.object(element_resolver, "combo_edit_identity", return_value=edit_identity): + self.resolve_path = resolve + return element_resolver.shadow_semantic_refusal( + app or object(), 0, "main", (0, 0, 9, 9), target, refused) + + def test_semantic_identity_refusal_falls_back_to_the_snapshot_path(self): + node = object() + self.assertIsNone(self.semantic_refusal("semantic_target_not_found", node)) + # The full arm does not read the tree for a shadow it will not use. + self.resolve_path.assert_not_called() + self.ablate_guard() + for code in ("semantic_target_not_found", "semantic_target_ambiguous", "stale_element_ref"): + resolved, record, window, bounds = self.semantic_refusal(code, node) + self.assertIs(resolved, node) + self.assertEqual(record["runtimeId"], [0, 4]) + self.assertEqual((window, bounds), ("main", (0, 0, 9, 9))) + self.assertEqual([entry["reason"] for entry in self.guard_log()], + ["semantic_target_not_found", "semantic_target_ambiguous", "stale_element_ref"]) + # Refusals that are not identity checks, and paths without a node, stay. + self.assertIsNone(self.semantic_refusal("observation_incomplete", node)) + self.assertIsNone(self.semantic_refusal("stale_element_ref", None)) + self.assertEqual(len(self.guard_log()), 3) + + def test_shadowed_target_keeps_its_own_window(self): + self.ablate_guard() + dialog = object() + app = mock.Mock() + app.get_child_at_index.return_value = dialog + _node, _record, window, bounds = self.semantic_refusal( + "stale_element_ref", object(), path=(2, 1), app=app) + app.get_child_at_index.assert_called_once_with(2) + self.assertEqual((window, bounds), (dialog, (1, 2, 3, 4))) + + def test_shadowed_combo_keeps_its_edit_partner(self): + self.ablate_guard() + identity = {"providerBusName": ":1.4", "providerObjectPath": "/e", "processGeneration": "g"} + _node, record, _window, _bounds = self.semantic_refusal( + "stale_element_ref", object(), role="toggle button", combo=object(), + edit_identity=identity) + self.assertEqual(record["controlType"], "combo box") + self.assertEqual(record["editTarget"], identity) + + def test_select_end_identity_refusal_is_logged_and_let_through(self): + refused = runtime.OperationFailure( + "stale_element_ref; no input was emitted.", failure_code="stale_element_ref", + failure_phase="precondition", input_events_emitted=False) + window, end = object(), object() + app = mock.Mock() + app.get_child_count.return_value = 1 + app.get_child_at_index.return_value = window + operation = {} + action_input = {"to_target": {"runtimeId": [0, 7]}} + with mock.patch.object(ax_dispatch, "provider_object_path", return_value="/w"), \ + mock.patch.object(ax_dispatch, "resolve_semantic_target", side_effect=refused), \ + mock.patch.object(ax_dispatch, "shadow_semantic_refusal", + wraps=lambda *args: None if not guard_shadow.guards_shadowed() + else (end, {}, window, None)): + with self.assertRaises(runtime.OperationFailure): + ax_dispatch._resolve_select_end(app, operation, action_input, window, None) + self.ablate_guard() + self.assertIs(ax_dispatch._resolve_select_end(app, operation, action_input, window, None), end) + + def expected_window_mismatch(self): + with mock.patch.object(ax_dispatch, "provider_bus_name", return_value=":1.9"), \ + mock.patch.object(ax_dispatch, "process_generation", return_value="g2"), \ + mock.patch.object(ax_dispatch, "node_pid", return_value=7), \ + mock.patch.object(ax_dispatch, "node_role", return_value="frame"): + ax_dispatch._verify_expected_window(object(), { + "role": "frame", "providerBusName": ":1.4", "processGeneration": "g1", + }) + + def test_batch_window_change_is_logged_and_let_through(self): + with self.assertRaises(runtime.OperationFailure): + self.expected_window_mismatch() + self.ablate_guard() + self.expected_window_mismatch() + [entry] = self.guard_log() + self.assertEqual(entry["guard"], "batch_window") + + def test_point_owner_is_logged_and_let_through(self): + with mock.patch.object(actionability, "require_window_focus"), \ + mock.patch.object(actionability, "coordinate_point_owner", return_value=False): + with self.assertRaises(runtime.OperationFailure): + actionability.require_coordinate_delivery(object(), 5, 6, "click") + self.ablate_guard() + actionability.require_coordinate_delivery(object(), 5, 6, "click") + # A check that only chooses a route keeps deciding normally. + with guard_shadow.guards_enforced(), self.assertRaises(runtime.OperationFailure): + actionability.require_coordinate_delivery(object(), 5, 6, "click") + [entry] = self.guard_log() + self.assertEqual((entry["guard"], entry["reason"], entry["x"]), ("point_owner", "different_window", 5)) + + +class GUIGuardShadowTests(GuardLogMixin, unittest.TestCase): + # The capture pipeline fixture, without its tests. + patch = test_gui_capture.GUICapturePipelineTests.patch + sample_layout = test_gui_capture.GUICapturePipelineTests.sample_layout + capture_pixels = test_gui_capture.GUICapturePipelineTests.capture_pixels + sample_pixels = test_gui_capture.GUICapturePipelineTests.sample_pixels + encode_pixels = test_gui_capture.GUICapturePipelineTests.encode_pixels + layouts = test_gui_capture.GUICapturePipelineTests.layouts + publish = test_gui_capture.GUICapturePipelineTests.publish + click_after = test_gui_capture.GUICapturePipelineTests.click_after + zoom_after = test_gui_capture.GUICapturePipelineTests.zoom_after + + def setUp(self): + super().setUp() + test_gui_capture.GUICapturePipelineTests.setUp(self) + + def covered(self): + return self.layouts([("0xa", 0, 0, 50, 50)], + [("0xa", 0, 0, 50, 50), ("0xb", 5, 5, 20, 20)]) + + def test_layout_change_stops_the_batch_by_default(self): + response = self.click_after(*self.covered(), (10, 10)) + self.assertEqual(response["failureCode"], "stale_gui_frame") + self.execute.assert_not_called() + self.assertEqual(self.guard_log(), []) + + def test_layout_change_is_logged_and_the_click_is_sent(self): + self.ablate_guard() + response = self.click_after(*self.covered(), (10, 10)) + self.assertTrue(response["ok"], response.get("error")) + self.execute.assert_called_once() + [entry] = self.guard_log() + self.assertEqual(entry["channel"], "gui") + self.assertEqual((entry["guard"], entry["reason"]), ("layout", "changed_before_batch")) + self.assertEqual(entry["actions"], [{"type": "click", "x": 10, "y": 10}]) + + def test_zoom_over_a_changed_region_is_logged_and_returned(self): + self.ablate_guard() + response = self.zoom_after(*self.covered(), [0, 0, 30, 30]) + self.assertTrue(response["ok"], response.get("error")) + self.assertEqual(len(self.zoomed), 1) + [entry] = self.guard_log() + self.assertEqual(entry["guard"], "zoom_layout") + + +if __name__ == "__main__": + unittest.main() diff --git a/configs/ablation/osworld-v1-codex-full.example.json b/configs/ablation/osworld-v1-codex-full.example.json new file mode 100644 index 0000000..05728ba --- /dev/null +++ b/configs/ablation/osworld-v1-codex-full.example.json @@ -0,0 +1,98 @@ +{ + "run_id": "example-osworld-v1-codex-full", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-all4.example.json b/configs/ablation/osworld-v1-codex-no-all4.example.json new file mode 100644 index 0000000..17a5c53 --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-all4.example.json @@ -0,0 +1,103 @@ +{ + "run_id": "example-osworld-v1-codex-no-all4", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "batching", + "wait", + "diff", + "guard" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-batching.example.json b/configs/ablation/osworld-v1-codex-no-batching.example.json new file mode 100644 index 0000000..814a8e4 --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-batching.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-batching", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "batching" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-diff.example.json b/configs/ablation/osworld-v1-codex-no-diff.example.json new file mode 100644 index 0000000..b52861d --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-diff.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-diff", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "diff" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-guard.example.json b/configs/ablation/osworld-v1-codex-no-guard.example.json new file mode 100644 index 0000000..3b3a7a9 --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-guard.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-guard", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "guard" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-wait.example.json b/configs/ablation/osworld-v1-codex-no-wait.example.json new file mode 100644 index 0000000..130aebc --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-wait.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-wait", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "wait" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/docs/OSWORLD.md b/docs/OSWORLD.md index b3fe64d..5c2ec1b 100644 --- a/docs/OSWORLD.md +++ b/docs/OSWORLD.md @@ -220,13 +220,42 @@ The gateway defaults to the Docker bridge address at the agent's port; `--result-dir`. `--channels a11y,keyboard` or `gui,keyboard` exposes one channel only; `--instructions-file ""` runs without a profile. +`--ablate` removes HCU mechanisms from the computer-use server: a +comma-separated subset of `batching`, `wait`, `diff` and `guard`. With +`diff` removed, an `act` returns the whole current tree of its window in +place of the change report and diff, and `observe` always repeats the tree. +`effect_unknown` in the act's evidence and the message of a settle wait +step are kept: the first is execution evidence and the second belongs to +`wait`. +The runner +sets `COMPUTER_USE_ABLATE` in the MCP server environment of every harness and +gives the agent copies of `instructions/base.md` and `instructions/hybrid.md` +without the passages that describe the removed mechanisms. For each of +`batching`, `wait` and `diff` the guide instead gets one sentence that +states how the ablated `act` behaves, as the AX tool description does, so +the agent does not learn the change from rejected calls. It writes the +copies to `ablated-instructions/` in the task's result directory. Other +guides, `--instructions-file ""` and `--code-mode` are refused with +`--ablate`, because they can still describe a removed mechanism. With `guard` +removed, the server logs the guard decisions in the guest, and the runner +copies the log to `guard.jsonl` in the result directory. Without `--ablate` +the harness configurations do not change. The preflight probe also checks +that the server applies the ablation: it sends one call per removed +mechanism and requires each to be refused, and it requires the server to +refuse an unknown `COMPUTER_USE_ABLATE` value. A build without the switches +therefore fails preparation instead of running full HCU under an ablation +label. `meta.json` records the SHA-256 of the MCP binary. + ## Run a panel A run is one JSON config. Relative paths in it are relative to the config file; omitted paths default to the repository's `artifacts/`, `runs/`, `assets/` and `instructions/`. See [`configs/osworld-v1-codex-queue.example.json`](../configs/osworld-v1-codex-queue.example.json) -for the 20-task V1 panel. +for the 20-task V1 panel. [`configs/ablation/`](../configs/ablation/) holds +the same panel for each mechanism-ablation group. Groups that share one +gateway must share its `--out` file: set the same `token_log` in each config. +A task fails before its agent starts when the gateway writes another file. | Key | Meaning | | --- | --- | @@ -238,6 +267,7 @@ for the 20-task V1 panel. | `max_active_tasks`, `launch_stagger_seconds` | Concurrency | | `gateway`, `token_log`, `results_dir`, `artifact_dir`, `asset_cache`, `offline_assets` | Optional overrides | | `agent_binary`, `mcp_binary`, `instructions_file`, `base_instructions_file`, `channels` | Optional agent overrides | +| `ablate` | Removed HCU mechanisms, as a list or a comma-separated string (see `--ablate`) | ```bash $PY -m hcu.osworld.run_queue configs/osworld-v1-codex-queue.example.json --dry-run @@ -258,10 +288,11 @@ runs// config.json the config this directory belongs to gateway-tokens.jsonl gateway token log (default location) tasks// - result.json the outcome: status, score, termination_reason, usage + result.json the outcome: status, score, termination_reason, usage, ablate meta.json guest and runtime diagnostics agent.jsonl, phase-*-turn-*-* agent traces per turn final-*.png screenshots before and after evaluation + guard.jsonl guard decisions, only with guard ablated logs/.log runner output retries/-/ attempts interrupted by infrastructure, not scored QUEUE_STATUS.md diff --git a/hcu/osworld/agents.py b/hcu/osworld/agents.py index 7b9080f..142719b 100644 --- a/hcu/osworld/agents.py +++ b/hcu/osworld/agents.py @@ -16,8 +16,15 @@ from hcu.osworld.guest import ( GUEST_CLAUDE_MCP_ADAPTER, + GUEST_GUARD_LOG, GUEST_MCP, ) +from hcu.osworld.guide_ablation import ( + ABLATION_ENV, + GUARD_LOG_ENV, + ablation_value, + parse_ablation, +) # Per-agent facts the runner needs: the file under artifacts/, the gateway @@ -116,6 +123,33 @@ def public_tools(channels: str) -> tuple[str, ...]: return CHANNEL_TOOLS[channels] +def mcp_server_env(channels: str = "", ablate: tuple[str, ...] = ()) -> dict[str, str]: + """Environment of the computer-use MCP server. + + A variable is set only when it changes the server, so the client config + of a run without a channel restriction or ablation has no MCP env. With + ``guard`` ablated the server requires a log for the guard decisions; the + runner collects it from the guest. + """ + public_tools(channels) + if tuple(ablate) != parse_ablation(ablate): + raise ValueError(f"ablate must be in canonical order, got {ablate!r}") + env = {"COMPUTER_USE_CHANNELS": channels} if channels else {} + if ablate: + env[ABLATION_ENV] = ablation_value(ablate) + if "guard" in ablate: + env[GUARD_LOG_ENV] = GUEST_GUARD_LOG + return env + + +def _toml_env_table(env: dict[str, str]) -> str: + if not env: + return "" + return "\n[mcp_servers.computer_use.env]\n" + "".join( + f"{key} = {_toml_string(value)}\n" for key, value in env.items() + ) + + def _toml_string(value: str) -> str: # JSON's UTF-16 surrogate escapes are not valid TOML Unicode escapes. return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") @@ -130,6 +164,7 @@ def codex_config( base_instructions: str, mcp_command: str, channels: str = "", + ablate: tuple[str, ...] = (), developer_instructions: str | None = None, execution_context: Literal["standalone", "isolated-guest"] = "standalone", code_mode: bool = False, @@ -155,12 +190,7 @@ def codex_config( "execution_context must be standalone or isolated-guest, " f"got {execution_context!r}" ) - public_tools(channels) - mcp_env = ( - f'\n[mcp_servers.computer_use.env]\nCOMPUTER_USE_CHANNELS = {_toml_string(channels)}\n' - if channels - else "" - ) + mcp_env = _toml_env_table(mcp_server_env(channels, ablate)) disabled_system_skills = "\n\n".join( ( "[[skills.config]]\n" @@ -281,17 +311,18 @@ def claude_mcp_config( adapter: str = GUEST_CLAUDE_MCP_ADAPTER, mcp_command: str = GUEST_MCP, channels: str = "", + ablate: tuple[str, ...] = (), python_executable: str = "/usr/bin/python3", ) -> str: """Claude Code MCP configuration that runs the MCP behind the adapter.""" - public_tools(channels) + env = mcp_server_env(channels, ablate) server: dict = { "type": "stdio", "command": python_executable, "args": [adapter, mcp_command, "mcp"], } - if channels: - server["env"] = {"COMPUTER_USE_CHANNELS": channels} + if env: + server["env"] = env return json.dumps({"mcpServers": {"computer_use": server}}, indent=2) + "\n" @@ -343,6 +374,7 @@ def gemini_settings( effort: str, mcp_command: str = GUEST_MCP, channels: str = "", + ablate: tuple[str, ...] = (), ) -> str: """Gemini CLI ``settings.json`` with ambient features off and the MCP allowed. @@ -351,7 +383,7 @@ def gemini_settings( loaded as context, and it carries the profile; directory discovery is off. ``effort`` is a Gemini thinking level, or ``native`` for the model default. """ - public_tools(channels) + env = mcp_server_env(channels, ablate) if effort != "native" and effort not in GEMINI_THINKING_LEVELS: raise ValueError( "Gemini CLI effort must be native or a thinking level (" @@ -363,8 +395,8 @@ def gemini_settings( "timeout": 180000, "trust": False, } - if channels: - server["env"] = {"COMPUTER_USE_CHANNELS": channels} + if env: + server["env"] = env settings = { # The gateway enforces the model-call budget; the CLI has no turn cap. "model": {"name": model, "maxSessionTurns": -1}, @@ -388,7 +420,9 @@ def gemini_settings( "mcp": {"allowed": [GEMINI_MCP_SERVER]}, "security": { "auth": {"selectedType": "gemini-api-key", "enforcedType": "gemini-api-key"}, - "environmentVariableRedaction": {"allowed": ["COMPUTER_USE_CHANNELS"]}, + "environmentVariableRedaction": {"allowed": [ + "COMPUTER_USE_CHANNELS", *(key for key in env if key != "COMPUTER_USE_CHANNELS"), + ]}, }, "context": { "fileName": "GEMINI.md", @@ -424,19 +458,15 @@ def grok_config( effort: str, mcp_command: str = GUEST_MCP, channels: str = "", + ablate: tuple[str, ...] = (), ) -> str: """Grok Build ``config.toml`` bound to the task gateway with ambient features off. The model is registered under the key ``gateway`` so the CLI never contacts x.ai directly; the gateway URL already carries the task path. """ - public_tools(channels) q = _toml_string - mcp_env = ( - f"\n[mcp_servers.computer_use.env]\nCOMPUTER_USE_CHANNELS = {q(channels)}\n" - if channels - else "" - ) + mcp_env = _toml_env_table(mcp_server_env(channels, ablate)) off = lambda *keys: "\n".join(f"{key} = false" for key in keys) # noqa: E731 return f"""\ [cli] diff --git a/hcu/osworld/config.py b/hcu/osworld/config.py index a7142c3..4c7f488 100644 --- a/hcu/osworld/config.py +++ b/hcu/osworld/config.py @@ -12,7 +12,13 @@ from pathlib import Path import sys -from hcu.paths import DEFAULT_ASSET_CACHE, DEFAULT_RESULTS_DIR +from hcu.osworld.guide_ablation import ablation_value, parse_ablation +from hcu.paths import ( + DEFAULT_ASSET_CACHE, + DEFAULT_BASE_INSTRUCTIONS_FILE, + DEFAULT_INSTRUCTIONS_FILE, + DEFAULT_RESULTS_DIR, +) # Config key -> run_task flag, for options passed through unchanged. VALUE_OPTIONS = { @@ -51,8 +57,14 @@ "token_log", "guest_image", "artifact_dir", "agent_binary", "mcp_binary", "instructions_file", "base_instructions_file", "asset_cache", "results_dir", ) +# ``ablate`` is a comma-separated string or a list; the loader stores it as a +# list in canonical order and passes it on as one --ablate value. +ABLATION_KEY = "ablate" QUEUE_KEYS = ("run_id", "tasks", "results_dir", "max_active_tasks", "launch_stagger_seconds") -CONFIG_KEYS = frozenset(VALUE_OPTIONS) | set(SWITCHES) | set(TASK_OPTIONS) | set(QUEUE_KEYS) +CONFIG_KEYS = ( + frozenset(VALUE_OPTIONS) | set(SWITCHES) | set(TASK_OPTIONS) | set(QUEUE_KEYS) + | {ABLATION_KEY} +) TASK_KEYS = frozenset(TASK_OPTIONS) | {"id", "stratum"} @@ -65,6 +77,37 @@ def single_path_component(value: object, label: str) -> str: return value +def ablation_input_error( + ablate: tuple[str, ...], + *, + instructions_file: str | Path | None, + base_instructions_file: str | Path | None, + code_mode: bool, +) -> str | None: + """Why the selected guides cannot run under ``ablate``, or None. + + An ablation run uses copies of the default guides without the passages + that describe the removed mechanisms. Other guides, no guide, and the + code-mode add-on may still describe them, so they are refused. + """ + if not ablate: + return None + if code_mode: + return "--ablate cannot be combined with --code-mode" + for label, value, default in ( + ("instructions file", instructions_file, DEFAULT_INSTRUCTIONS_FILE), + ("base instructions file", base_instructions_file, DEFAULT_BASE_INSTRUCTIONS_FILE), + ): + if value is None: + continue + if not value or Path(value).expanduser().resolve() != default.resolve(): + return ( + f"--ablate uses ablated copies of the default guides; " + f"the {label} must be {default}, got {str(value)!r}" + ) + return None + + def load_config(path: Path) -> dict: """Read, validate and normalize a run configuration file.""" path = path.expanduser().resolve() @@ -93,6 +136,8 @@ def load_config(path: Path) -> dict: if len({task["id"] for task in tasks}) != len(tasks): raise ValueError("task ids must be unique") config["tasks"] = tasks + if ABLATION_KEY in config: + config[ABLATION_KEY] = list(parse_ablation(config[ABLATION_KEY])) if int(config["max_active_tasks"]) < 1: raise ValueError("max_active_tasks must be at least 1") config.setdefault("results_dir", str(DEFAULT_RESULTS_DIR)) @@ -101,6 +146,14 @@ def load_config(path: Path) -> dict: # An empty instructions_file disables the profile; keep it empty. if config.get(key): config[key] = str((path.parent / Path(config[key]).expanduser()).resolve()) + error = ablation_input_error( + tuple(config.get(ABLATION_KEY) or ()), + instructions_file=config.get("instructions_file"), + base_instructions_file=config.get("base_instructions_file"), + code_mode=bool(config.get("code_mode")), + ) + if error: + raise ValueError(error) # The gateway appends here; the runner reads budget and usage from it. config.setdefault( "token_log", str(Path(config["results_dir"]) / config["run_id"] / "gateway-tokens.jsonl") @@ -127,5 +180,7 @@ def runner_command(config: dict, task: dict, result_dir: Path, run_started_at: s value = task.get(key, config.get(key)) if value is not None: command += [flag, str(value)] + if config.get(ABLATION_KEY): + command += ["--ablate", ablation_value(tuple(config[ABLATION_KEY]))] command += [flag for key, flag in SWITCHES.items() if config.get(key)] return command diff --git a/hcu/osworld/guest.py b/hcu/osworld/guest.py index d85b8e5..db8c96b 100644 --- a/hcu/osworld/guest.py +++ b/hcu/osworld/guest.py @@ -51,6 +51,9 @@ GUEST_FINAL = f"{GUEST_TASK_DIR}/final.txt" GUEST_DONE = f"{GUEST_TASK_DIR}/done.json" GUEST_LAUNCHER_STATUS = f"{GUEST_TASK_DIR}/launcher-status.json" +# Guard decisions the MCP server appends when the guard is ablated; it +# spans all turns of the task, so turn cleanup leaves it in place. +GUEST_GUARD_LOG = f"{GUEST_TASK_DIR}/guard.jsonl" GUEST_GEMINI_ARCHIVE = f"{GUEST_TASK_DIR}/gemini-cli.tgz" diff --git a/hcu/osworld/guest_lifecycle.py b/hcu/osworld/guest_lifecycle.py index 57b53a5..e31b1e5 100644 --- a/hcu/osworld/guest_lifecycle.py +++ b/hcu/osworld/guest_lifecycle.py @@ -13,8 +13,10 @@ GUEST_GEMINI_CLI, GUEST_GEMINI_NODE, GUEST_GEMINI_PACKAGE, + GUEST_GUARD_LOG, GUEST_MCP, request_execute, + save_guest_artifact, transient_workload_command, ) from hcu.osworld.runtime import ( @@ -27,7 +29,7 @@ ) from hcu.osworld.staging import GUEST_AGENT_BINARIES, TaskRun, runtime_uploads from hcu.osworld.task import task_value -from hcu.osworld.usage import check_gateway_budget +from hcu.osworld.usage import check_gateway_budget, check_gateway_token_log from desktop_env.cache import run_scoped_cache_dir from desktop_env.desktop_env import DesktopEnv @@ -160,6 +162,7 @@ def prepare_guest(run: TaskRun, env: DesktopEnv) -> str: ) meta["gateway_health"] = health.get("output", "").strip() check_gateway_budget(meta["gateway_health"], args.max_model_calls) + check_gateway_token_log(meta["gateway_health"], run.token_log) agent_version = ( f"{shlex.quote(GUEST_GEMINI_NODE)} {shlex.quote(GUEST_GEMINI_CLI)}" @@ -193,6 +196,9 @@ def prepare_guest(run: TaskRun, env: DesktopEnv) -> str: timeout=180, ) meta["mcp_probe"] = mcp_probe.get("output", "").strip() + if "guard" in run.ablate: + # The guard log records the agent's run, not the preflight probe. + request_execute(base_url, ["rm", "-f", GUEST_GUARD_LOG], timeout=30) initial_wait_remaining = max( 0.0, 60.0 - (time.monotonic() - reset_finished_at), @@ -206,8 +212,33 @@ def prepare_guest(run: TaskRun, env: DesktopEnv) -> str: return base_url +GUARD_LOG_FILE = "guard.jsonl" + + +def save_guard_log(run: TaskRun, env: DesktopEnv) -> None: + """Copy the guest's guard log into the result directory. + + The MCP server writes it only with ``guard`` ablated. ``meta.json`` + names the copy, or holds None when the guest has no log. + """ + try: + data = save_guest_artifact( + env.setup_controller.http_server, + GUEST_GUARD_LOG, + run.result_dir / GUARD_LOG_FILE, + ) + run.meta["guard_log"] = GUARD_LOG_FILE if data is not None else None + except Exception as exc: + logging.exception("Failed to copy the guard log") + run.meta["guard_log"] = None + run.meta["guard_log_error"] = f"{type(exc).__name__}: {exc}" + run.write_meta(ensure_ascii=False) + + def close_guest(run: TaskRun, env: DesktopEnv) -> None: - """Stop recording, then remove or retain the task container.""" + """Copy the guard log, stop recording, then remove or retain the container.""" + if "guard" in run.ablate: + save_guard_log(run, env) if run.recording_started: try: env.controller.end_recording( diff --git a/hcu/osworld/guide_ablation.py b/hcu/osworld/guide_ablation.py new file mode 100644 index 0000000..e9ac1a9 --- /dev/null +++ b/hcu/osworld/guide_ablation.py @@ -0,0 +1,256 @@ +"""Interaction guides for HCU mechanism ablations. + +``COMPUTER_USE_ABLATE`` removes HCU mechanisms from the computer-use server +(see ``computer-use/internal/contract/ablation.go``). The guides given to the +agent must not describe a removed mechanism either, so this module deletes +the passages that describe it. It also adds one sentence per ablation that +states how the ablated tools behave, the same fact the tool description +states, and no new advice: otherwise an ablation run would also measure +the agent learning the changed rules from rejected calls. Nothing else +changes. + +The passages are listed here as exact text, not marked inside the guides: +the guides are model input, and markers in them would change the text of +every run, including runs without an ablation. Each passage must occur +exactly once in its file. Without ablations the guides are returned +unchanged. + +Usage: python -m hcu.osworld.guide_ablation ABLATE OUTPUT_DIR [INSTRUCTIONS_DIR] +writes the ablated ``base.md`` and ``hybrid.md`` to OUTPUT_DIR. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ABLATION_ENV = "COMPUTER_USE_ABLATE" +GUARD_LOG_ENV = "COMPUTER_USE_GUARD_LOG" +# Canonical order; the server accepts the same names. +ABLATION_NAMES = ("batching", "wait", "diff", "guard") + +_PLAN_FROM_ONE_READ = """\ +- Plan from one read. After `observe` or `find`, put every action you can + already determine into one `act`, while later refs and focus remain valid. + Split when the next action needs a new observation. +- Use `activate` with a `wN` or title to switch the batch target. + Later refs and ref-less steps use that window. AX rejects refs from another + window without explicit `activate`; activate back before reusing the + previous window's refs. Each step reports its window; the final observation + uses the last executed window. +""" + +_ACT_REPORT = """\ +- Every `act` result starts with what happened, then a diff: + +```text +effect: r58 checkbox "Interface" checked=false→true +windows: opened w5 "Insert Sheet" (dialog, focused) +``` + + `effect` gives each target's changed fields, `no visible change`, or + `unknown` with the reason. `(only state flags changed in this window)` means + no page, content, or window changed; if the control should have switched + something, the application did not respond. `windows` names windows that opened, closed, or + took focus, with a `wN` you can use at once. When these lines already show + the result, do not read or wait again. Continue from the diff for other + changes; read again only when the next decision needs more. +""" + +_WAIT_STEPS = """\ +- One `act` can mix actions and `wait` steps; the first failed step stops + the batch. Add a `wait` only when a later step needs a state that may + arrive after the action returns: a slow dialog, a menu item, a value + computed later. A checkbox that shows `checked=false→true`, or a dialog + listed as `opened`, needs no wait. +- Wait for a known state with `window`, `query`, `ref`, or `text`. When you + know the UI will change but not what it becomes, such as the exact title of + the next dialog, use `{"type":"wait","settle":true}`. It waits until the UI + changes and then stops changing, names what changed, and fails with "no + change" only when nothing changed. Keys after it go to the window that took + focus. Give a slow operation a longer `timeout_ms`. +""" + +# The examples batch several actions with wait steps between them. +_ACT_EXAMPLES = """ +```json +{"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"wait","window":"Export"},{"type":"type","text":"q3.csv"},{"type":"keypress","keys":["ENTER"]},{"type":"wait","window":"Export","gone":true}]} +{"mode":"act","actions":[{"type":"click","ref":"r7"},{"type":"wait","query":{"role":"menu item","name_contains":"Paragraph"}},{"type":"click","ref":"r9"}]} +``` + +""" + +_SELECT_READBACK = """ The result reports what is now + selected; check it before applying the command.""" + +_WAIT_TIMEOUT = """\ +- `wait_timeout`: the batch stopped at that `wait`; the steps before it ran + and none after it. The message says what was seen instead, or that the + state could not be read. Read the returned state and revise the plan; do + not resend the same batch. +""" + +# The wait advice names report lines that the diff ablation removes. +_REPORT_NEEDS_NO_WAIT = """ A checkbox that shows `checked=false→true`, or a dialog + listed as `opened`, needs no wait.""" + +_APP_STARTING_WAIT = """if the application is still starting, wait once for + its window with `{"mode":"act","actions":[{"type":"wait","window":"..."}]}`; + otherwise """ + +# Removed passages per ablation, as (file name, exact text). A passage may +# belong to several ablations; overlapping passages are removed once. +PASSAGES: dict[str, tuple[tuple[str, str], ...]] = { + "batching": ( + ("hybrid.md", _PLAN_FROM_ONE_READ), + ("hybrid.md", _ACT_EXAMPLES), + ("hybrid.md", "- `no_focused_window`: start the batch with `activate`.\n"), + ), + "wait": ( + ("hybrid.md", " or wait again"), + ("hybrid.md", _WAIT_STEPS), + ("hybrid.md", _ACT_EXAMPLES), + ("hybrid.md", _WAIT_TIMEOUT), + ("hybrid.md", _APP_STARTING_WAIT), + ("hybrid.md", " or a timed-out wait"), + ), + "diff": ( + ("hybrid.md", _ACT_REPORT), + ("hybrid.md", _SELECT_READBACK), + ("hybrid.md", _REPORT_NEEDS_NO_WAIT), + ), + "guard": (), +} + +# The sentence each ablation adds, after this bullet, which no ablation +# removes. guard adds none: the agent cannot notice the shadow mode. +_STATEMENT_ANCHOR = """\ +- Once the source, target, and operation are known, act. Further checks must + answer a concrete blocker. +""" +STATEMENTS = { + "batching": """\ +- Each `act` sends one input step, with optional `wait` steps before or after + it. Send further input in later calls. +""", + "wait": """\ +- An AX `act` cannot pause for a later state. To check one, read again with + `observe` or `find`. +""", + "diff": """\ +- An `act` result shows the whole current tree of its window after its + actions instead of a report of what changed. +""", +} +# With both batching and wait ablated, an act holds one step and no waits. +_ONE_STEP = """\ +- Each `act` sends one step. Send further steps in later calls. +""" + + +def parse_ablation(value: str | list[str] | tuple[str, ...] | None) -> tuple[str, ...]: + """Return the ablations in canonical order. + + ``value`` is a comma-separated string, as in ``COMPUTER_USE_ABLATE``, or + a list of names. Unknown and repeated names raise ValueError: a typo must + not run the complete HCU under an ablation label. + """ + if value is None: + return () + parts = value.split(",") if isinstance(value, str) else list(value) + names: list[str] = [] + for part in parts: + if not isinstance(part, str): + raise ValueError(f"ablation names must be strings, got {part!r}") + name = part.strip() + if not name: + if isinstance(value, str) and not value.strip(): + continue + raise ValueError(f"empty ablation name in {value!r}") + if name not in ABLATION_NAMES: + raise ValueError( + f"unknown ablation {name!r}; use {', '.join(ABLATION_NAMES)}" + ) + if name in names: + raise ValueError(f"ablation {name!r} is listed twice") + names.append(name) + return tuple(name for name in ABLATION_NAMES if name in names) + + +def ablation_value(ablations: tuple[str, ...]) -> str: + """The COMPUTER_USE_ABLATE value for parsed ablations; empty for none.""" + return ",".join(ablations) + + +def ablation_label(ablations: tuple[str, ...]) -> str: + """A file-name-safe label: ``full`` or ``no-batching-wait``.""" + return "no-" + "-".join(ablations) if ablations else "full" + + +def ablated_guide(text: str, file_name: str, ablations: tuple[str, ...]) -> str: + """Return ``text`` without the passages of ``ablations`` for ``file_name``.""" + ranges: list[tuple[int, int]] = [] + for name in ablations: + for owner, passage in PASSAGES[name]: + if owner != file_name: + continue + start = text.find(passage) + if start < 0 or text.find(passage, start + 1) >= 0: + raise ValueError( + f"{file_name}: the {name} passage must occur exactly once: " + f"{passage[:60]!r}" + ) + ranges.append((start, start + len(passage))) + kept, position = [], 0 + for start, end in sorted(ranges): + if start > position: + kept.append(text[position:start]) + position = max(position, end) + kept.append(text[position:]) + text = "".join(kept) + if file_name != "hybrid.md": + return text + added = [STATEMENTS[name] for name in ablations if name in STATEMENTS] + if "batching" in ablations and "wait" in ablations: + added[0] = _ONE_STEP + if not added: + return text + if text.count(_STATEMENT_ANCHOR) != 1: + raise ValueError("hybrid.md: the ablation statements' anchor must occur exactly once") + return text.replace(_STATEMENT_ANCHOR, _STATEMENT_ANCHOR + "".join(added)) + + +def write_ablated_guides( + instructions_dir: Path, output_dir: Path, ablations: tuple[str, ...] +) -> dict[str, Path]: + """Write the ablated base.md and hybrid.md; return their paths by name.""" + output_dir.mkdir(parents=True, exist_ok=True) + written = {} + for file_name in ("base.md", "hybrid.md"): + source = (instructions_dir / file_name).read_text(encoding="utf-8") + target = output_dir / file_name + target.write_text(ablated_guide(source, file_name, ablations), encoding="utf-8") + written[file_name] = target + return written + + +def main(argv: list[str]) -> int: + if len(argv) not in (2, 3): + print(__doc__, file=sys.stderr) + return 2 + instructions_dir = ( + Path(argv[2]) if len(argv) == 3 + else Path(__file__).resolve().parents[2] / "instructions" + ) + try: + ablations = parse_ablation(argv[0]) + except ValueError as error: + print(error, file=sys.stderr) + return 2 + for path in write_ablated_guides(instructions_dir, Path(argv[1]), ablations).values(): + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/hcu/osworld/phases.py b/hcu/osworld/phases.py index 4d881dd..785d70f 100644 --- a/hcu/osworld/phases.py +++ b/hcu/osworld/phases.py @@ -259,6 +259,7 @@ def write_result(run: TaskRun, status: str, **fields) -> dict: "agent": args.agent, "model": args.model, "effort": args.effort, + "ablate": list(getattr(args, "ablate", ())), "status": status, "score": None, **fields, diff --git a/hcu/osworld/run_task.py b/hcu/osworld/run_task.py index 5163dbc..724a010 100644 --- a/hcu/osworld/run_task.py +++ b/hcu/osworld/run_task.py @@ -21,6 +21,8 @@ anchor_path_args, ) from hcu.osworld.agents import AGENT_PROFILES, AGENTS +from hcu.osworld.config import ablation_input_error +from hcu.osworld.guide_ablation import ABLATION_NAMES, parse_ablation from hcu.osworld.guest_lifecycle import close_guest, prepare_guest, start_guest from hcu.osworld.phases import finish_run, record_failure, run_phases from hcu.osworld.preflight import docker_runtime @@ -115,6 +117,15 @@ def parse_args() -> argparse.Namespace: "channel (gui,keyboard); empty exposes both." ), ) + parser.add_argument( + "--ablate", + default="", + help=( + "Comma-separated HCU mechanisms to remove (" + + ", ".join(ABLATION_NAMES) + + "); the agent gets the default guides without their passages." + ), + ) parser.add_argument( "--code-mode", action="store_true", @@ -210,6 +221,18 @@ def parse_args() -> argparse.Namespace: args = parser.parse_args() if args.code_mode and args.agent != "codex": parser.error("--code-mode applies only to --agent codex") + try: + args.ablate = parse_ablation(args.ablate) + except ValueError as error: + parser.error(f"--ablate: {error}") + error = ablation_input_error( + args.ablate, + instructions_file=args.instructions_file, + base_instructions_file=args.base_instructions_file, + code_mode=args.code_mode, + ) + if error: + parser.error(error) profile = AGENT_PROFILES[args.agent] args.model = args.model or profile["default_model"] args.effort = args.effort or profile["default_effort"] diff --git a/hcu/osworld/runtime.py b/hcu/osworld/runtime.py index a8ce988..5599d01 100644 --- a/hcu/osworld/runtime.py +++ b/hcu/osworld/runtime.py @@ -363,6 +363,46 @@ def call(name, arguments): raise RuntimeError("unexpected public MCP tools: " + repr(names)) report = {"public_tools": names, "requested_protocol": config["protocol"], "negotiated_protocol": negotiated, "child_env_verified": True, "checks": {}} + ablated = [name.strip() for name in config.get("env", {}).get("COMPUTER_USE_ABLATE", "").split(",") if name.strip()] + if ablated: + # The env reached the server; check that the server applies it. A + # build without the switches would run full HCU under an ablation + # label. Each call is refused before input only when its mechanism is + # removed. + uses = { + "wait": ({"mode": "act", "actions": [{"type": "wait", "window": "__hcu_ablation_probe__", "timeout_ms": 100}]}, + {"actions[0].type"}), + "batching": ({"mode": "act", "actions": [{"type": "click", "ref": "r1"}, {"type": "click", "ref": "r2"}]}, + {"actions[1]", "actions"}), + "diff": ({"mode": "observe", "app": "__hcu_ablation_probe__", "observe": "diff"}, {"observe"}), + } + for name in ablated: + if name not in uses or "AX" not in names: + continue + arguments, paths = uses[name] + result = request("tools/call", {"name": "AX", "arguments": arguments}) + evidence = {} + for item in result.get("content") or []: + try: + evidence = json.loads(item.get("text", "")) + break + except (ValueError, TypeError): + continue + if not result.get("isError") or evidence.get("code") != "invalid_argument" or evidence.get("path") not in paths: + raise RuntimeError("MCP server ignores the %s ablation: %s" % (name, json.dumps(evidence)[:300])) + # guard changes no schema; a build that parses the switches refuses + # an unknown one before it answers initialize. + bogus = subprocess.run( + [config["command"], *config.get("args", [])], + input=json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": config["protocol"], "capabilities": {}, + "clientInfo": {"name": "osworld-v2-harness-preflight", "version": "1"}}}) + "\n", + env=dict(child_env, COMPUTER_USE_ABLATE="__hcu_ablation_probe__"), cwd=config.get("cwd"), + capture_output=True, text=True, timeout=30, + ) + if '"result"' in bogus.stdout: + raise RuntimeError("MCP server accepts an unknown COMPUTER_USE_ABLATE value; it cannot apply ablations") + report["checks"]["ablation"] = "passed: " + ",".join(ablated) if "AX" in names: apps, _ = call("AX", {"mode": "apps"}) windows = re.findall(r"^(w[1-9][0-9]*)\s+.*$", apps, re.M) diff --git a/hcu/osworld/staging.py b/hcu/osworld/staging.py index 33259c2..f19158d 100644 --- a/hcu/osworld/staging.py +++ b/hcu/osworld/staging.py @@ -5,6 +5,7 @@ import argparse from dataclasses import dataclass, field import datetime +import hashlib import json from pathlib import Path import time @@ -13,6 +14,7 @@ from hcu.paths import ( CODE_MODE_INSTRUCTIONS_FILE, + DEFAULT_INSTRUCTIONS_FILE, DEFAULT_RESULTS_DIR, OSWORLD_ROOT, ) @@ -29,6 +31,7 @@ public_tools, task_gateway_base, ) +from hcu.osworld.guide_ablation import write_ablated_guides from hcu.osworld.launcher import launcher_script from hcu.gateway.common import TASK_AFFINITY_PREFIX from hcu.osworld.guest import ( @@ -114,6 +117,11 @@ class TaskRun: def profile(self) -> dict: return AGENT_PROFILES[self.args.agent] + @property + def ablate(self) -> tuple[str, ...]: + """Removed HCU mechanisms in canonical order; empty for full HCU.""" + return tuple(getattr(self.args, "ablate", ())) + @property def config_path(self) -> Path: return self.result_dir / { @@ -196,6 +204,14 @@ def prepare_run(args: argparse.Namespace) -> TaskRun: base_instructions_file = Path( args.base_instructions_file ).resolve() + ablate = tuple(getattr(args, "ablate", ())) + if ablate: + # run_task has checked that the default guides are selected; the + # agent gets copies without the passages of the removed mechanisms. + guides = write_ablated_guides( + DEFAULT_INSTRUCTIONS_FILE.parent, result_dir / "ablated-instructions", ablate, + ) + instructions_file, base_instructions_file = guides["hybrid.md"], guides["base.md"] required_files = [agent_binary, mcp_binary, base_instructions_file] if instructions_file is not None: required_files.append(instructions_file) @@ -278,14 +294,16 @@ def stage_local_files(run: TaskRun) -> None: base_instructions=GUEST_BASE_INSTRUCTIONS, mcp_command=GUEST_MCP, channels=args.channels, + ablate=run.ablate, execution_context="isolated-guest", code_mode=getattr(args, "code_mode", False), ) elif args.agent == "claude-code": - config = claude_mcp_config(channels=args.channels) + config = claude_mcp_config(channels=args.channels, ablate=run.ablate) elif args.agent == "gemini-cli": config = gemini_settings( - model=args.model, effort=args.effort, channels=args.channels + model=args.model, effort=args.effort, channels=args.channels, + ablate=run.ablate, ) else: config = grok_config( @@ -293,6 +311,7 @@ def stage_local_files(run: TaskRun) -> None: base_url=run.agent_endpoint + "/v1", effort=args.effort, channels=args.channels, + ablate=run.ablate, ) write_text(run.config_path, config) write_text( @@ -378,7 +397,13 @@ def initial_meta(run: TaskRun, docker_runtime: dict) -> dict: else None ), "mcp_binary": str(run.mcp_binary), + # Identifies the build: every build reports the same version. + "mcp_binary_sha256": ( + hashlib.sha256(run.mcp_binary.read_bytes()).hexdigest() + if run.mcp_binary.is_file() else None + ), "mcp_channels": args.channels or None, + "ablate": list(run.ablate), "code_mode": bool(getattr(args, "code_mode", False)), "mcp_public_tools": run.mcp_public_tools, "instructions_file": str(run.instructions_file) if run.instructions_file else None, diff --git a/hcu/osworld/usage.py b/hcu/osworld/usage.py index bfd51be..f0941af 100644 --- a/hcu/osworld/usage.py +++ b/hcu/osworld/usage.py @@ -233,3 +233,26 @@ def check_gateway_budget(health_output: str, max_model_calls: int) -> None: f"runner={max_model_calls}, gateway={limit!r}. " "Start the gateway with --max-requests-per-task equal to max_model_calls." ) + + +def check_gateway_token_log(health_output: str, token_log: Path) -> None: + """Fail before the agent starts if the gateway logs to another file. + + The runner reads its usage and retry exhaustion from ``token_log``. Runs + that share a gateway with a different ``--out`` would otherwise report + zero usage and never see exhausted retries. + """ + try: + out = json.loads(health_output).get("out") + except (ValueError, AttributeError): + return + # Only the OpenAI gateway reports its log; a relative path is relative + # to the gateway's working directory, which the runner cannot know. + if not isinstance(out, str) or not Path(out).is_absolute(): + return + if Path(out).resolve() != Path(token_log).resolve(): + raise RuntimeError( + f"Gateway writes its token log to {out}, but this run reads {token_log}. " + "Start the gateway with --out set to the run's token_log, or give every " + "run that shares the gateway the same token_log." + ) diff --git a/scripts/maint/check_ablation_text.py b/scripts/maint/check_ablation_text.py new file mode 100644 index 0000000..2bc9a27 --- /dev/null +++ b/scripts/maint/check_ablation_text.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Check that an ablated HCU no longer tells the model about what it removed. + +Usage: check_ablation_text.py BINARY + +For every subset of COMPUTER_USE_ABLATE (batching, wait, diff, guard) the +script starts BINARY as an MCP server, sends initialize, tools/list and AX +calls that use each removed mechanism, and generates the guides with +hcu.osworld.guide_ablation. It searches the server instructions, tool +definitions, error results and guides for the words of each removed +mechanism, and checks that every act example obeys the ablation. It also +checks that the server refuses to start on a misspelled ablation. No +desktop is needed: every call is rejected before input. + +Exit status 1 lists each finding. +""" +from __future__ import annotations + +import itertools +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO)) + +from hcu.osworld.guide_ablation import ABLATION_NAMES, ablated_guide # noqa: E402 + +# Words of each mechanism. GUI keeps its own wait action and settle_ms, so +# the wait words name only the AX forms. +WORDS = { + "wait": [r'"type":\s*"wait"', r"(?i)wait step", r'"settle"', r"settle needs", + r"\bsettle\b(?!_ms)", r"timeout_ms", r"wait_timeout", r"waits with", + r"`wait`"], + "diff": [r"\bdiffs?\b", r"(?m)^effect:", r"(?m)^windows:", r"report what changed", + r"reports the selection", r"reports what is now", r"reports each target", + r"checked=false→true", r"listed as `opened`"], + "batching": [r"ordered steps", r"into one `act`", r"(?i)start the batch with"], + "guard": [], +} + +# A call that uses each mechanism; the ablated server must reject it. +USES = { + "wait": {"mode": "act", "actions": [{"type": "wait", "window": "Export"}]}, + "diff": {"mode": "act", "observe": "diff", "actions": [{"type": "click", "ref": "r1"}]}, + "batching": {"mode": "act", "actions": [{"type": "click", "ref": "r1"}, {"type": "click", "ref": "r2"}]}, +} +BAD_CALL = {"mode": "act", "actions": []} + + +def mcp_session(binary: str, env: dict[str, str], calls: list[dict]) -> list[dict]: + process = subprocess.Popen( + [binary, "mcp"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, text=True, + ) + assert process.stdin is not None and process.stdout is not None + requests = [ + ("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "hcu-ablation-check", "version": "1"}}), + ("tools/list", {}), + ] + [("tools/call", {"name": "AX", "arguments": call}) for call in calls] + responses = [] + try: + for request_id, (method, params) in enumerate(requests, 1): + process.stdin.write(json.dumps( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + "\n") + process.stdin.flush() + while True: + line = process.stdout.readline() + if not line: + raise RuntimeError(f"EOF before {method}: {process.stderr.read()}") + response = json.loads(line) + if response.get("id") == request_id: + if "error" in response: + raise RuntimeError(f"{method}: {response['error']}") + responses.append(response["result"]) + break + finally: + process.terminate() + process.wait(timeout=5) + return responses + + +def act_examples(text: str) -> list[list[dict]]: + """The action lists of every act call written in text.""" + found = [] + decoder = json.JSONDecoder() + for match in re.finditer(r'\{\\?"mode\\?":\s*\\?"act\\?"', text): + chunk = text[match.start():] + if chunk.startswith('{\\"'): + continue # escaped copy inside JSON; the raw copy is checked + try: + call, _ = decoder.raw_decode(chunk) + except json.JSONDecodeError: + continue + found.append(call.get("actions", [])) + return found + + +def check(binary: str, ablations: tuple[str, ...], guard_log: str) -> list[str]: + env = {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", "/tmp")} + if ablations: + env["COMPUTER_USE_ABLATE"] = ",".join(ablations) + if "guard" in ablations: + env["COMPUTER_USE_GUARD_LOG"] = guard_log + calls = [BAD_CALL] + [USES[name] for name in ablations if name in USES] + initialize, tools, *results = mcp_session(binary, env, calls) + names = [tool["name"] for tool in tools["tools"]] + findings = [] if names == ["AX", "GUI"] else [f"public tools {names}"] + texts = {"instructions": initialize.get("instructions", "")} + for tool in tools["tools"]: + # Raw description plus the schema, so quoted examples are searchable. + texts[f"{tool['name']} description"] = tool["description"] + schema = tool["inputSchema"] + if tool["name"] == "GUI": + # Only settle_ms of GUI speaks about AX mechanisms. + schema = schema["oneOf"][1]["properties"]["settle_ms"] + texts[f"{tool['name']} schema"] = json.dumps(schema) + for call, result in zip(calls, results): + text = "\n".join(item.get("text", "") for item in result.get("content", [])) + texts[f"result of {json.dumps(call)}"] = text + if not result.get("isError") or '"code":"invalid_argument"' not in text: + findings.append(f"{json.dumps(call)} was not rejected as invalid_argument: {text[:200]}") + for file_name in ("base.md", "hybrid.md"): + source = (REPO / "instructions" / file_name).read_text(encoding="utf-8") + texts[file_name] = ablated_guide(source, file_name, ablations) + for name in ablations: + for label, text in texts.items(): + if label.startswith("result of") and name in USES and label.endswith(json.dumps(USES[name])): + continue # the rejected call may echo what it sent + for word in WORDS[name]: + match = re.search(word, text) + if match: + start = max(0, match.start() - 50) + findings.append(f"{name}: {label} still has {match.group(0)!r}: ...{text[start:match.end() + 50]!r}") + for label, text in texts.items(): + if label.startswith("result of"): + continue + for actions in act_examples(text): + inputs = [step for step in actions if step.get("type") != "wait"] + if "batching" in ablations and len(inputs) > 1: + findings.append(f"batching: {label} shows an act with {len(inputs)} input steps") + if "wait" in ablations and len(inputs) != len(actions): + findings.append(f"wait: {label} shows an act with a wait step") + return findings + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + return 2 + binary = sys.argv[1] + failed = False + with tempfile.TemporaryDirectory() as scratch: + for count in range(len(ABLATION_NAMES) + 1): + for ablations in itertools.combinations(ABLATION_NAMES, count): + findings = check(binary, ablations, os.path.join(scratch, "guard.jsonl")) + label = ",".join(ablations) or "(none)" + print(f"{'FAIL' if findings else 'ok '} {label}") + for finding in findings: + print(f" {finding}") + failed = failed or bool(findings) + for bad in ("waits", "wait,wait", "guard"): + env = {"PATH": "/usr/bin:/bin", "COMPUTER_USE_ABLATE": bad} + started = subprocess.run([binary, "mcp"], input="", capture_output=True, + text=True, env=env, timeout=30) + refused = started.returncode != 0 and "COMPUTER_USE_ABLATE" in started.stderr + print(f"{'ok ' if refused else 'FAIL'} refuses COMPUTER_USE_ABLATE={bad!r}" + + ("" if refused else f": exit {started.returncode}")) + failed = failed or not refused + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/maint/check_all.sh b/scripts/maint/check_all.sh index 13a7fb0..e9a37ca 100755 --- a/scripts/maint/check_all.sh +++ b/scripts/maint/check_all.sh @@ -3,8 +3,9 @@ # # Usage: scripts/maint/check_all.sh [BASELINE_BINARY] # -# With BASELINE_BINARY, the script also builds the current tree and compares -# MCP initialize/tools/list, help and schema bytes with that binary. +# With BASELINE_BINARY, the script also builds the current tree, compares +# MCP initialize/tools/list, help and schema bytes with that binary, and +# checks the model-visible text of every COMPUTER_USE_ABLATE setting. # Logs go to $LOG_DIR (default /tmp/hcu-maint-check); each step prints one # summary line, and the log tail on failure. The script exits non-zero at the # first failing step. @@ -63,4 +64,5 @@ if [ -n "$baseline_binary" ]; then (cd "$repo/computer-use" && CGO_ENABLED=0 go build -trimpath \ -ldflags "-X main.version=$(cat "$repo/VERSION")" -o "$candidate" ./cmd/computer-use) step mcp-bytes python3 "$repo/scripts/maint/compare_mcp_bytes.py" "$baseline_binary" "$candidate" + step ablation-text python3 "$repo/scripts/maint/check_ablation_text.py" "$candidate" fi diff --git a/tests/test_ablation_switches.py b/tests/test_ablation_switches.py new file mode 100644 index 0000000..0e32ea0 --- /dev/null +++ b/tests/test_ablation_switches.py @@ -0,0 +1,336 @@ +"""The runner side of the HCU mechanism ablations, without a guest or model calls.""" + +import argparse +import importlib.util +import hashlib +import json +from pathlib import Path +import sys +from types import ModuleType +from unittest.mock import Mock + +import pytest + +from hcu.osworld import agents +from hcu.osworld.config import load_config, runner_command +from hcu.osworld.guest import GUEST_GUARD_LOG +from hcu.osworld.guide_ablation import ABLATION_NAMES, ablated_guide +from hcu.osworld.runtime import public_mcp_probe_config + +ROOT = Path(__file__).resolve().parents[1] +GROUPS = { + "full": [], + "no-batching": ["batching"], + "no-wait": ["wait"], + "no-diff": ["diff"], + "no-guard": ["guard"], + "no-all4": list(ABLATION_NAMES), +} + + +def write_config(path: Path, **values) -> Path: + config = {"tasks": ["a"], "results_dir": str(path.parent / "runs"), **values} + path.write_text(json.dumps(config), encoding="utf-8") + return path + + +# --- configuration and command line ------------------------------------------------- + + +def test_config_ablate_is_canonical_and_reaches_the_runner(tmp_path): + config = load_config(write_config(tmp_path / "run.json", ablate="guard, batching")) + assert config["ablate"] == ["batching", "guard"] + command = runner_command(config, config["tasks"][0], tmp_path / "a") + assert command[command.index("--ablate") + 1] == "batching,guard" + listed = load_config(write_config(tmp_path / "list.json", ablate=["diff", "wait"])) + assert listed["ablate"] == ["wait", "diff"] + + +@pytest.mark.parametrize("values", [{}, {"ablate": []}, {"ablate": ""}]) +def test_config_without_ablation_passes_no_flag(tmp_path, values): + config = load_config(write_config(tmp_path / "run.json", **values)) + assert "--ablate" not in runner_command(config, config["tasks"][0], tmp_path / "a") + # A config without the key keeps its saved form, so existing runs resume. + assert ("ablate" in config) == ("ablate" in values) + + +@pytest.mark.parametrize("value", ["gaurd", "wait,wait", "wait,,diff", ["wait", 1]]) +def test_config_rejects_invalid_ablation(tmp_path, value): + with pytest.raises(ValueError): + load_config(write_config(tmp_path / "run.json", ablate=value)) + + +@pytest.mark.parametrize( + "values, message", + [ + ({"instructions_file": "custom.md"}, "instructions file"), + ({"instructions_file": ""}, "instructions file"), + ({"base_instructions_file": "custom-base.md"}, "base instructions file"), + ({"code_mode": True}, "code-mode"), + ], +) +def test_config_refuses_guides_that_may_describe_removed_mechanisms(tmp_path, values, message): + with pytest.raises(ValueError, match=message): + load_config(write_config(tmp_path / "run.json", ablate="wait", **values)) + # Without an ablation the same guide choice stays valid. + load_config(write_config(tmp_path / "plain.json", **values)) + + +def test_config_accepts_the_default_guides_named_explicitly(tmp_path): + config = load_config(write_config( + tmp_path / "run.json", ablate="diff", + instructions_file=str(ROOT / "instructions/hybrid.md"), + base_instructions_file=str(ROOT / "instructions/base.md"), + )) + assert config["ablate"] == ["diff"] + + +def test_ablation_example_configs_load_with_their_group(): + paths = sorted((ROOT / "configs/ablation").glob("*.example.json")) + assert {path.name for path in paths} == { + f"osworld-v1-codex-{label}.example.json" for label in GROUPS + } + reference = load_config(ROOT / "configs/osworld-v1-codex-queue.example.json") + for path in paths: + label = path.name.removeprefix("osworld-v1-codex-").removesuffix(".example.json") + config = load_config(path) + assert config["ablate"] == GROUPS[label] + for key in ("benchmark_version", "agent", "model", "effort", "max_model_calls", "tasks"): + assert config[key] == reference[key], (path, key) + command = runner_command(config, config["tasks"][0], Path("/unused")) + assert ("--ablate" in command) == bool(GROUPS[label]) + + +# --- harness MCP environments -------------------------------------------------------- + + +def render(agent: str, **kwargs) -> str: + if agent == "codex": + return agents.codex_config( + model="m", effort="xhigh", base_url="http://gateway/v1", codex_home="/codex", + base_instructions="/base.md", mcp_command="/computer-use", **kwargs, + ) + if agent == "claude-code": + return agents.claude_mcp_config(**kwargs) + if agent == "gemini-cli": + return agents.gemini_settings(model="m", effort="high", **kwargs) + return agents.grok_config(model="m", base_url="http://gateway/v1", effort="xhigh", **kwargs) + + +HARNESSES = ("codex", "claude-code", "gemini-cli", "grok-build") + + +@pytest.mark.parametrize("agent", HARNESSES) +@pytest.mark.parametrize("channels", ["", "a11y,keyboard", "gui,keyboard"]) +def test_full_hcu_renders_the_same_harness_config(agent, channels): + plain = render(agent, channels=channels) + assert render(agent, channels=channels, ablate=()) == plain + assert "COMPUTER_USE_ABLATE" not in plain and "COMPUTER_USE_GUARD_LOG" not in plain + if agent == "gemini-cli": + redaction = json.loads(plain)["security"]["environmentVariableRedaction"] + assert redaction == {"allowed": ["COMPUTER_USE_CHANNELS"]} + + +@pytest.mark.parametrize("agent", HARNESSES) +@pytest.mark.parametrize( + "ablate, expected", + [ + (("wait",), {"COMPUTER_USE_ABLATE": "wait"}), + (("batching", "diff"), {"COMPUTER_USE_ABLATE": "batching,diff"}), + (ABLATION_NAMES, { + "COMPUTER_USE_ABLATE": "batching,wait,diff,guard", + "COMPUTER_USE_GUARD_LOG": GUEST_GUARD_LOG, + }), + (("guard",), { + "COMPUTER_USE_ABLATE": "guard", "COMPUTER_USE_GUARD_LOG": GUEST_GUARD_LOG, + }), + ], +) +@pytest.mark.parametrize("channels", ["", "gui,keyboard"]) +def test_ablation_reaches_every_harness_mcp_env(agent, ablate, expected, channels): + pytest.importorskip("tomllib") + config = render(agent, channels=channels, ablate=ablate) + want = {**({"COMPUTER_USE_CHANNELS": channels} if channels else {}), **expected} + # The preflight probe starts the server with the env it reads from the config. + spec = public_mcp_probe_config(config, agent, list(agents.public_tools(channels))) + assert spec["env"] == want + if agent == "gemini-cli": + allowed = json.loads(config)["security"]["environmentVariableRedaction"]["allowed"] + assert set(want) <= set(allowed) + + +def test_harness_rejects_ablations_out_of_canonical_order(): + with pytest.raises(ValueError, match="canonical"): + agents.mcp_server_env("", ("guard", "wait")) + with pytest.raises(ValueError): + agents.mcp_server_env("", ("typo",)) + + +# --- runner modules loaded without OSWorld ------------------------------------------- + + +@pytest.fixture +def runner(monkeypatch): + """Fresh runner modules with the OSWorld imports replaced by test doubles.""" + stubs = { + "desktop_env": {}, + "desktop_env.cache": {"run_scoped_cache_dir": None}, + "desktop_env.desktop_env": { + "DesktopEnv": Mock(side_effect=AssertionError("No guest may be constructed")), + }, + "desktop_env.file_source": {"configure": None}, + "hcu.osworld.preflight": {"docker_runtime": None}, + "hcu.osworld.task_loader": {"load_task_config": None}, + } + for name, attributes in stubs.items(): + module = ModuleType(name) + vars(module).update(attributes) + monkeypatch.setitem(sys.modules, name, module) + modules = {} + for name in ("staging", "guest_lifecycle", "phases", "run_task"): + spec = importlib.util.spec_from_file_location( + f"hcu.osworld.{name}", ROOT / f"hcu/osworld/{name}.py", + ) + modules[name] = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, modules[name]) + spec.loader.exec_module(modules[name]) + staging = modules["staging"] + monkeypatch.setattr(staging, "load_task_config", lambda *a, **kw: {"instruction": "offline"}) + monkeypatch.setattr(staging, "resolve_guest_image", lambda *a: "/unused/image.qcow2") + return modules + + +def parse(runner, monkeypatch, *flags): + monkeypatch.setattr(sys, "argv", ["run_task", "--task-id", "t", *flags]) + return runner["run_task"].parse_args() + + +def test_run_task_parses_ablate_in_canonical_order(runner, monkeypatch): + assert parse(runner, monkeypatch).ablate == () + assert parse(runner, monkeypatch, "--ablate", "guard,wait").ablate == ("wait", "guard") + + +@pytest.mark.parametrize( + "flags", + [ + ["--ablate", "batching,typo"], + ["--ablate", "diff,diff"], + ["--ablate", "wait", "--instructions-file", ""], + ["--ablate", "wait", "--instructions-file", "instructions/ax-only.md"], + ["--ablate", "wait", "--code-mode"], + ], +) +def test_run_task_rejects_invalid_ablation_before_any_work(runner, monkeypatch, flags): + with pytest.raises(SystemExit) as exit_info: + parse(runner, monkeypatch, *flags) + assert exit_info.value.code == 2 + + +def prepared_run(runner, tmp_path, agent="codex", ablate=()): + artifact = tmp_path / "artifact" + artifact.touch() + profile = agents.AGENT_PROFILES[agent] + args = argparse.Namespace( + task_id="t", agent=agent, model=profile["default_model"], + effort=profile["default_effort"], gateway="http://gateway:8611", + result_dir=str(tmp_path / "task"), token_log=str(tmp_path / "tokens.jsonl"), + max_model_calls=0, artifact_dir=str(tmp_path), agent_binary=str(artifact), + mcp_binary=str(artifact), + instructions_file=str(ROOT / "instructions/hybrid.md"), + base_instructions_file=str(ROOT / "instructions/base.md"), + channels="", code_mode=False, task_domain="", benchmark_version="v1", + run_started_at="2000-01-01T00:00:00+00:00", timezone="UTC", + asset_cache="/unused", offline_assets=False, retain_container=False, ablate=ablate, + ) + if agent == "gemini-cli": + (tmp_path / "node").touch() + staging = runner["staging"] + run = staging.prepare_run(args) + staging.stage_local_files(run) + run.meta = staging.initial_meta(run, {}) + return run + + +def test_full_hcu_uses_the_default_guides_unchanged(runner, tmp_path): + run = prepared_run(runner, tmp_path) + assert run.instructions_file == ROOT / "instructions/hybrid.md" + assert run.base_instructions_file == ROOT / "instructions/base.md" + assert not (run.result_dir / "ablated-instructions").exists() + assert run.meta["ablate"] == [] + + +@pytest.mark.parametrize("agent", HARNESSES) +def test_ablation_run_gives_the_agent_ablated_guides(runner, tmp_path, agent): + ablate = ("wait", "diff") + run = prepared_run(runner, tmp_path, agent, ablate) + guides = run.result_dir / "ablated-instructions" + assert run.instructions_file == guides / "hybrid.md" + assert run.base_instructions_file == guides / "base.md" + hybrid = ablated_guide( + (ROOT / "instructions/hybrid.md").read_text(encoding="utf-8"), "hybrid.md", ablate, + ) + assert hybrid != (ROOT / "instructions/hybrid.md").read_text(encoding="utf-8") + assert run.instructions_file.read_text(encoding="utf-8") == hybrid + # Each harness receives the ablated guide through its own channel. + if agent in {"codex", "gemini-cli"}: + assert run.instructions_copy.read_text(encoding="utf-8") == hybrid + else: + assert run.base_instructions_copy.read_text(encoding="utf-8").endswith(hybrid) + assert "COMPUTER_USE_ABLATE" in run.config_path.read_text(encoding="utf-8") + assert run.meta["ablate"] == ["wait", "diff"] + assert run.meta["mcp_binary_sha256"] == hashlib.sha256(run.mcp_binary.read_bytes()).hexdigest() + assert run.meta["instructions_file"] == str(guides / "hybrid.md") + + +@pytest.mark.parametrize("ablate", [(), ("batching", "guard")]) +def test_result_records_the_ablation(runner, tmp_path, ablate): + run = prepared_run(runner, tmp_path, ablate=ablate) + result = runner["phases"].write_result(run, "error", error="test") + assert result["ablate"] == list(ablate) + saved = json.loads((run.result_dir / "result.json").read_text(encoding="utf-8")) + assert saved["ablate"] == list(ablate) + assert json.loads((run.result_dir / "meta.json").read_text())["ablate"] == list(ablate) + + +class FakeEnv: + def __init__(self): + self.setup_controller = type("Controller", (), {"http_server": "http://guest:5000"})() + self.closed = False + + def close(self): + self.closed = True + + +@pytest.mark.parametrize("guest_log", [b'{"decision":"skipped"}\n', None]) +def test_guard_ablation_copies_the_guest_guard_log(runner, tmp_path, monkeypatch, guest_log): + lifecycle = runner["guest_lifecycle"] + copied = [] + + def save(base_url, guest_path, local_path): + copied.append((base_url, guest_path)) + if guest_log is not None: + local_path.write_bytes(guest_log) + return guest_log + + monkeypatch.setattr(lifecycle, "save_guest_artifact", save) + run = prepared_run(runner, tmp_path, ablate=("guard",)) + env = FakeEnv() + lifecycle.close_guest(run, env) + assert copied == [("http://guest:5000", GUEST_GUARD_LOG)] + assert env.closed + meta = json.loads((run.result_dir / "meta.json").read_text()) + if guest_log is None: + assert meta["guard_log"] is None + assert not (run.result_dir / "guard.jsonl").exists() + else: + assert meta["guard_log"] == "guard.jsonl" + assert (run.result_dir / "guard.jsonl").read_bytes() == guest_log + + +@pytest.mark.parametrize("ablate", [(), ("wait", "diff")]) +def test_runs_with_the_guard_do_not_read_a_guard_log(runner, tmp_path, monkeypatch, ablate): + lifecycle = runner["guest_lifecycle"] + monkeypatch.setattr(lifecycle, "save_guest_artifact", Mock(side_effect=AssertionError)) + run = prepared_run(runner, tmp_path, ablate=ablate) + lifecycle.close_guest(run, FakeEnv()) + assert "guard_log" not in run.meta diff --git a/tests/test_agent_budget.py b/tests/test_agent_budget.py index 8589bec..0259b08 100644 --- a/tests/test_agent_budget.py +++ b/tests/test_agent_budget.py @@ -123,6 +123,19 @@ def test_gateway_cap_is_checked_once_before_the_agent_starts(health, limit, ok): check_gateway_budget(health, limit) +def test_gateway_token_log_must_be_the_runs(tmp_path): + from hcu.osworld.usage import check_gateway_token_log + + log = tmp_path / "run" / "gateway-tokens.jsonl" + check_gateway_token_log(json.dumps({"out": str(log)}), log) + check_gateway_token_log(json.dumps({"out": str(tmp_path / "run" / ".." / "run" / log.name)}), log) + # The Anthropic gateway reports no log; a relative path cannot be checked. + for health in ("{}", "not json", json.dumps({"out": "gateway-tokens.jsonl"})): + check_gateway_token_log(health, log) + with pytest.raises(RuntimeError, match="token log"): + check_gateway_token_log(json.dumps({"out": str(tmp_path / "other.jsonl")}), log) + + def test_rate_limit_waits_extend_the_deadline_and_count_as_progress(polling, monkeypatch): monkeypatch.setattr(agent_loop, "MODEL_PROGRESS_TIMEOUT_SEC", 5) waits = [] diff --git a/tests/test_guide_ablation.py b/tests/test_guide_ablation.py new file mode 100644 index 0000000..88d65bf --- /dev/null +++ b/tests/test_guide_ablation.py @@ -0,0 +1,81 @@ +"""Ablated interaction guides: removed passages and the added statements.""" + +from __future__ import annotations + +import itertools + +import pytest + +from hcu.osworld.guide_ablation import ABLATION_NAMES, STATEMENTS, ablated_guide +from hcu.paths import DEFAULT_INSTRUCTIONS_FILE + +GUIDES = DEFAULT_INSTRUCTIONS_FILE.parent +ALL_SETS = [ + combo + for count in range(len(ABLATION_NAMES) + 1) + for combo in itertools.combinations(ABLATION_NAMES, count) +] + + +def source(file_name: str) -> str: + return (GUIDES / file_name).read_text(encoding="utf-8") + + +@pytest.mark.parametrize("file_name", ["base.md", "hybrid.md"]) +def test_guides_without_ablation_are_unchanged(file_name): + assert ablated_guide(source(file_name), file_name, ()) == source(file_name) + + +def test_guard_changes_no_guide(): + for file_name in ("base.md", "hybrid.md"): + assert ablated_guide(source(file_name), file_name, ("guard",)) == source(file_name) + + +@pytest.mark.parametrize("ablations", ALL_SETS, ids=lambda combo: ",".join(combo) or "full") +def test_each_ablation_states_the_changed_behavior_once(ablations): + guide = ablated_guide(source("hybrid.md"), "hybrid.md", ablations) + one_step = "batching" in ablations and "wait" in ablations + for name, statement in STATEMENTS.items(): + expected = 1 if name in ablations and not (name == "batching" and one_step) else 0 + assert guide.count(statement) == expected, name + assert guide.count("- Each `act` sends one step.") == (1 if one_step else 0) + # base.md is common policy; no ablation adds to it. + assert ablated_guide(source("base.md"), "base.md", ablations) == source("base.md") + + +@pytest.mark.parametrize("ablations", ALL_SETS, ids=lambda combo: ",".join(combo) or "full") +def test_ablations_only_remove_text_besides_the_statements(ablations): + original = source("hybrid.md") + guide = ablated_guide(original, "hybrid.md", ablations) + for statement in [*STATEMENTS.values(), "- Each `act` sends one step. Send further steps in later calls.\n"]: + guide = guide.replace(statement, "", 1) + # What remains is the original with characters removed, nothing added. + remaining = iter(original) + assert all(character in remaining for character in guide) + + +@pytest.mark.parametrize("ablations", ALL_SETS, ids=lambda combo: ",".join(combo) or "full") +def test_batch_window_guidance_follows_batching_ablation(ablations): + guide = " ".join(ablated_guide(source("hybrid.md"), "hybrid.md", ablations).split()) + expected = 0 if "batching" in ablations else 1 + for fragment in ( + "Use `activate` with a `wN` or title to switch the batch target.", + "Later refs and ref-less steps use that window.", + "AX rejects refs from another window without explicit `activate`", + "activate back before reusing the previous window's refs.", + "Each step reports its window; the final observation uses the last executed window.", + ): + assert guide.count(fragment) == expected, fragment + + +def test_diff_guide_keeps_the_returned_state_and_drops_the_report(): + guide = ablated_guide(source("hybrid.md"), "hybrid.md", ("diff",)) + # An act returns the whole tree, so reading the returned state stays. + assert "Read the\n returned state and send only what remains." in guide + assert "Read the returned state and revise the plan" in guide + assert "The message says what was seen instead" in guide + # The report of what changed, and advice that relies on it, goes. + for removed in ("then a diff", "effect: r58", "windows: opened", "checked=false→true", + "listed as `opened`", "reports what is now"): + assert removed not in guide, removed + assert "shows the whole current tree of its window" in STATEMENTS["diff"] diff --git a/tests/test_public_mcp_probe.py b/tests/test_public_mcp_probe.py index ef10d30..58f0198 100644 --- a/tests/test_public_mcp_probe.py +++ b/tests/test_public_mcp_probe.py @@ -14,6 +14,12 @@ SERVER = r''' import json, os, sys scenario = os.environ.get("PROBE_SCENARIO", "ok") +ablated = [n for n in os.environ.get("COMPUTER_USE_ABLATE", "").split(",") if n] +# The "ablated" server applies the switches and refuses unknown ones; the +# "stale" server ignores them, as a build without the switches does. +if scenario == "ablated" and any(n not in ("batching", "wait", "diff", "guard") for n in ablated): + sys.exit("unknown COMPUTER_USE_ABLATE") +REFUSED = {"wait": "actions[0].type", "batching": "actions[1]", "diff": "observe"} for line in sys.stdin: req = json.loads(line) if "id" not in req: @@ -24,7 +30,16 @@ elif method == "tools/list": result = {"tools": [{"name": x} for x in json.loads(sys.argv[1])]} else: - mode = params.get("arguments", {}).get("mode", "screenshot") + arguments = params.get("arguments", {}) + mode = arguments.get("mode", "screenshot") + uses = ("wait" if "wait" in json.dumps(arguments) else + "batching" if len(arguments.get("actions", [])) > 1 else + "diff" if arguments.get("observe") == "diff" else None) + if uses and scenario == "ablated" and uses in ablated: + evidence = {"ok": False, "code": "invalid_argument", "path": REFUSED[uses]} + print(json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": { + "isError": True, "content": [{"type": "text", "text": json.dumps(evidence)}]}}), flush=True) + continue if mode == "apps": text = 'w1 "Fixture" app=fixture active\n' if scenario == "empty": @@ -55,14 +70,17 @@ ''' -def probe(tmp_path, scenario="ok", names=None): +def probe(tmp_path, scenario="ok", names=None, ablate=""): names = ["AX", "GUI"] if names is None else names server = tmp_path / "server.py" server.write_text(SERVER) + env = {"PROBE_SCENARIO": scenario} + if ablate: + env["COMPUTER_USE_ABLATE"] = ablate config = { "command": sys.executable, "args": [str(server), json.dumps(names)], - "env": {"PROBE_SCENARIO": scenario}, + "env": env, "expected_tools": names, "protocol": "2026-07-28", } @@ -96,6 +114,17 @@ def test_probe_rejects_errors_or_missing_evidence(tmp_path, scenario): assert "RuntimeError" in result.stderr +@pytest.mark.parametrize("ablate", ["guard", "wait", "batching,wait,diff,guard"]) +def test_probe_confirms_the_server_applies_the_ablation(tmp_path, ablate): + result = probe(tmp_path, "ablated", ablate=ablate) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["checks"]["ablation"] == "passed: " + ablate + # A build without the switches runs full HCU; the probe must stop it. + stale = probe(tmp_path, "stale", ablate=ablate) + assert stale.returncode != 0 + assert "ignores the" in stale.stderr or "unknown COMPUTER_USE_ABLATE" in stale.stderr + + def test_empty_desktop_does_not_require_creating_a_task_window(tmp_path): result = probe(tmp_path, "empty", ["AX"]) assert result.returncode == 0, result.stderr