From f8360e10ceebfe3e9fee676b94b6af42c20710bc Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:44:16 +0800 Subject: [PATCH 01/13] contract: add COMPUTER_USE_ABLATE mechanism ablation switches Parse and validate batching, wait, diff and guard; refuse to start on an unknown value, on guard outside Linux, and on guard without COMPUTER_USE_GUARD_LOG. The schema, tool descriptions and server instructions drop the fields and sentences of ablated mechanisms; with no ablation they are unchanged. Co-Authored-By: Claude Opus 5.5 --- computer-use/internal/contract/ablation.go | 133 ++++++++++++++++++ .../internal/contract/schema_validator.go | 33 +++-- .../internal/contract/tool_argument_rules.go | 3 + .../internal/contract/tool_contract.go | 111 ++++++++++++++- .../internal/provider/linux_runtime_bundle.go | 1 + .../provider/linux_runtime_bundle_test.go | 1 + computer-use/internal/transport/cli_main.go | 11 ++ .../internal/transport/mcp_transport.go | 2 +- .../native/linux/provider/actionability.py | 9 ++ .../native/linux/provider/ax_dispatch.py | 85 +++++++++-- .../linux/provider/ax_input_delivery.py | 12 +- .../native/linux/provider/element_resolver.py | 11 ++ .../native/linux/provider/guard_shadow.py | 63 +++++++++ .../native/linux/provider/gui_dispatch.py | 28 +++- computer-use/runtime_test_support.py | 2 + 15 files changed, 470 insertions(+), 35 deletions(-) create mode 100644 computer-use/internal/contract/ablation.go create mode 100644 computer-use/native/linux/provider/guard_shadow.py diff --git a/computer-use/internal/contract/ablation.go b/computer-use/internal/contract/ablation.go new file mode 100644 index 0000000..b5a4650 --- /dev/null +++ b/computer-use/internal/contract/ablation.go @@ -0,0 +1,133 @@ +package contract + +import ( + "fmt" + "os" + "strings" +) + +// Mechanism ablations remove one HCU mechanism each, for experiments that +// measure what the mechanism contributes. COMPUTER_USE_ABLATE is a +// comma-separated list of the names below; by default nothing is removed. +// +// A removed mechanism is absent from everything the model sees: schemas, +// tool descriptions, server instructions and error texts. Like a disabled +// channel, it is not merely rejected at call time. +// +// - batching: an AX act holds at most one input step; wait steps may +// still surround it. GUI batches are unchanged. +// - wait: AX acts take no wait steps. The tools' own stabilization after +// input is unchanged. +// - diff: an AX act returns only its execution evidence, not the effect +// and windows report, the diff or any other observation. +// - guard: the target and layout guards still decide, but only log the +// input they would have stopped (COMPUTER_USE_GUARD_LOG) and let it +// through. Only the Linux provider implements this. +const ( + AblationEnv = "COMPUTER_USE_ABLATE" + GuardLogEnv = "COMPUTER_USE_GUARD_LOG" + AblateBatching = "batching" + AblateWait = "wait" + AblateDiff = "diff" + AblateGuard = "guard" +) + +// AblationNames lists the ablations in their canonical order. +var AblationNames = []string{AblateBatching, AblateWait, AblateDiff, AblateGuard} + +// ParseAblations returns the ablations named in raw. An unknown or repeated +// name is an error: an experiment with a misspelled switch would otherwise +// run the complete HCU under an ablation label. +func ParseAblations(raw string) (map[string]bool, error) { + ablated := map[string]bool{} + if strings.TrimSpace(raw) == "" { + return ablated, nil + } + for _, part := range strings.Split(raw, ",") { + name := strings.TrimSpace(part) + if !ContainsString(AblationNames, name) { + return nil, fmt.Errorf("%s: unknown ablation %q; use a comma-separated subset of %s", + AblationEnv, name, strings.Join(AblationNames, ", ")) + } + if ablated[name] { + return nil, fmt.Errorf("%s: ablation %q is listed twice", AblationEnv, name) + } + ablated[name] = true + } + return ablated, nil +} + +// ValidateAblations checks COMPUTER_USE_ABLATE for a server, daemon or +// command started on goos. Entry points call it before serving anything. +func ValidateAblations(goos string, getenv func(string) string) error { + ablated, err := ParseAblations(getenv(AblationEnv)) + if err != nil { + return err + } + if ablated[AblateGuard] { + if goos != "linux" { + return fmt.Errorf("%s: the guard ablation is implemented only by the Linux provider, not on %s", + AblationEnv, goos) + } + if strings.TrimSpace(getenv(GuardLogEnv)) == "" { + return fmt.Errorf("%s: the guard ablation needs %s, the file that records the input the guards would have stopped", + AblationEnv, GuardLogEnv) + } + } + return nil +} + +// Ablated reports whether the named mechanism is removed. Entry points +// validate the variable at startup, so a value that does not parse here +// means the process skipped that check. +func Ablated(name string) bool { + ablated, err := ParseAblations(os.Getenv(AblationEnv)) + if err != nil { + panic(err) + } + return ablated[name] +} + +// SortedAblations returns the active ablations in canonical order. +func SortedAblations() []string { + names := []string{} + for _, name := range AblationNames { + if Ablated(name) { + names = append(names, name) + } + } + return names +} + +// PublishedAXActionTypes is the AX action list the schema publishes and the +// decoder accepts. +func PublishedAXActionTypes() []string { + if !Ablated(AblateWait) { + return AXActionTypes + } + types := []string{} + for _, actionType := range AXActionTypes { + if actionType != "wait" { + types = append(types, actionType) + } + } + return types +} + +// PublishedObserveValues is the observe list the schema publishes and the +// decoder accepts. Without diff, act takes no observe field, so diff and +// none, which only act uses, are gone too. +func PublishedObserveValues() []string { + if !Ablated(AblateDiff) { + return ObserveValues + } + return []string{ObserveTree, ObserveFull, ObserveScreenshot} +} + +// AXActMaxActions is the published maximum length of an AX act. +func AXActMaxActions() int { + if Ablated(AblateBatching) && Ablated(AblateWait) { + return 1 + } + return AXBatchMaxActions +} diff --git a/computer-use/internal/contract/schema_validator.go b/computer-use/internal/contract/schema_validator.go index d8ad6cf..670cfcb 100644 --- a/computer-use/internal/contract/schema_validator.go +++ b/computer-use/internal/contract/schema_validator.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "os" "sync" "github.com/santhosh-tekuri/jsonschema/v6" @@ -36,24 +37,38 @@ type ArgumentError struct { func (e *ArgumentError) Error() string { return e.Message } -var publicInputSchemas struct { - once sync.Once +type compiledInputSchemas struct { schemas map[string]*jsonschema.Schema err error } +// publicInputSchemas holds the compiled schemas per COMPUTER_USE_ABLATE +// value, since ablations change the published schemas. +var publicInputSchemas struct { + mu sync.Mutex + byAblation map[string]*compiledInputSchemas +} + // CompiledInputSchema returns the compiled published input schema of a // public tool, or nil for a name without one. All schemas are compiled once, // on first use. func CompiledInputSchema(tool string) (*jsonschema.Schema, error) { - publicInputSchemas.once.Do(func() { - publicInputSchemas.schemas, publicInputSchemas.err = - CompileInputSchemas(AllToolDefinitions()) - }) - if publicInputSchemas.err != nil { - return nil, publicInputSchemas.err + key := os.Getenv(AblationEnv) + publicInputSchemas.mu.Lock() + compiled := publicInputSchemas.byAblation[key] + if compiled == nil { + compiled = &compiledInputSchemas{} + compiled.schemas, compiled.err = CompileInputSchemas(AllToolDefinitions()) + if publicInputSchemas.byAblation == nil { + publicInputSchemas.byAblation = map[string]*compiledInputSchemas{} + } + publicInputSchemas.byAblation[key] = compiled + } + publicInputSchemas.mu.Unlock() + if compiled.err != nil { + return nil, compiled.err } - return publicInputSchemas.schemas[tool], nil + return compiled.schemas[tool], nil } func CompileInputSchemas(tools []ToolDefinition) (map[string]*jsonschema.Schema, error) { diff --git a/computer-use/internal/contract/tool_argument_rules.go b/computer-use/internal/contract/tool_argument_rules.go index 73d6057..110d6bc 100644 --- a/computer-use/internal/contract/tool_argument_rules.go +++ b/computer-use/internal/contract/tool_argument_rules.go @@ -83,6 +83,9 @@ const ( ArgumentRuleTextBudget = "text_budget" ArgumentRuleRefOwnsIdentity = "ref_owns_identity" ArgumentRuleWaitBudget = "wait_budget" + // ArgumentRuleOneInputStep is the batching ablation's limit of one + // input step per act. + ArgumentRuleOneInputStep = "one_input_step" ) // ArgumentRuleError is a cross-field rule violation. diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index fcacda9..9f4295d 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -55,6 +55,22 @@ const ServerInstructions = "" + "before input and report what changed, and outside input makes their refs and " + "diffs stale. After GUI input, read AX again before reusing refs." +// PublishedServerInstructions is ServerInstructions without the sentences +// about ablated mechanisms. +func PublishedServerInstructions() string { + text := ServerInstructions + if Ablated(AblateWait) { + text = strings.Replace(text, "returns controls as rN refs, acts on "+ + "them, and waits with wait steps.", "returns controls as rN refs, and acts on them.", 1) + } + if Ablated(AblateDiff) { + text = strings.Replace(text, "check the target window before input and report what changed, "+ + "and outside input makes their refs and diffs stale.", + "check the target window before input, and outside input makes their refs stale.", 1) + } + return text +} + type ToolDefinition struct { Name string `json:"name"` Title string `json:"title,omitempty"` @@ -175,7 +191,7 @@ func axInputSchema() map[string]any { for _, field := range AXFindFilters { query[field] = nonBlankString("") } - return map[string]any{ + schema := map[string]any{ "type": "object", "additionalProperties": false, "required": []string{"mode"}, @@ -265,6 +281,42 @@ func axInputSchema() map[string]any { }, }, } + ablateAXInputSchema(schema) + return schema +} + +// ablateAXInputSchema removes the fields and words of ablated mechanisms +// from the AX schema. +func ablateAXInputSchema(schema map[string]any) { + properties := schema["properties"].(map[string]any) + actions := properties["actions"].(map[string]any) + item := actions["items"].(map[string]any)["properties"].(map[string]any) + wait, batching := Ablated(AblateWait), Ablated(AblateBatching) + if wait { + for _, field := range []string{"window", "query", "state", "gone", "settle", "timeout_ms"} { + delete(item, field) + } + item["type"].(map[string]any)["enum"] = PublishedAXActionTypes() + item["text"].(map[string]any)["description"] = "select: substring of ref to select." + } + actions["maxItems"] = AXActMaxActions() + switch { + case batching && wait: + actions["description"] = "act: one step." + case batching: + actions["description"] = "act: one input step, with optional wait steps before or after it; " + + "a failed step stops the batch. Wait timeouts total at most 60000 ms; an omitted timeout counts as 5000 ms." + case wait: + actions["description"] = "act: ordered steps; a failed step stops the batch." + } + if Ablated(AblateDiff) { + properties["mode"].(map[string]any)["description"] = "apps lists confirmed top-level windows " + + "(wN, title, app, active) and reports incomplete inventories; observe returns a tree with rN; " + + "find returns matching rN; act runs actions." + observe := properties["observe"].(map[string]any) + observe["enum"] = PublishedObserveValues() + observe["description"] = "observe: returned evidence; default tree, full adds the full-desktop GUI screenshot." + } } func guiActionProperty(includeScreenshot bool) map[string]any { @@ -415,7 +467,8 @@ func GUIInputSchema() map[string]any { fmt.Sprintf( "Visual stillness: default %dms, max %dms. "+ "Final screenshot stillness, not semantic success and not extra sleep. "+ - "After wait, usually omit it. At the cap inspect the screenshot or use an AX wait step.", + "After wait, usually omit it. At the cap inspect the screenshot"+ + map[bool]string{false: " or use an AX wait step.", true: "."}[Ablated(AblateWait)], DefaultSettleMS, GUIMaxSettleMS, ), @@ -458,6 +511,58 @@ const axToolDescription = "Accessibility for application windows. Linux: observe `{"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"wait","window":"Export"},{"type":"type","text":"q3.csv"},{"type":"keypress","keys":["ENTER"]}]}` + "\n" + `{"mode":"find","window_ref":"w2","query":{"role":"button","name_contains":"Save"}}` +const ( + axActFullSentence = "act runs ordered steps, checks them before input, " + + "stops at the first failed step, and reports each target's change, windows " + + "opened/closed/focused, and a diff." + axActFullExample = `{"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"wait","window":"Export"},{"type":"type","text":"q3.csv"},{"type":"keypress","keys":["ENTER"]}]}` + axWaitDescription = "A wait step sends no input and fails the batch on timeout: window (title, any app), " + + "query (as find), ref with value/state/text, or text; gone inverts. settle needs no " + + "prediction: it waits until the UI changes after the step before it and stops changing; " + + "it fails only if nothing changed. After a window wait, keys go to that window, after " + + "settle to the focused one. " +) + +// publishedAXToolDescription is axToolDescription without the sentences +// and example steps of ablated mechanisms. +func publishedAXToolDescription() string { + wait, batching, diff := Ablated(AblateWait), Ablated(AblateBatching), Ablated(AblateDiff) + subject := "act runs ordered steps" + clauses := []string{"checks them before input", "stops at the first failed step"} + example := axActFullExample + switch { + case batching && wait: + subject, clauses = "act runs one step", []string{"checks it before input"} + example = `{"mode":"act","actions":[{"type":"click","ref":"r12"}]}` + case batching: + subject = "act runs one input step with optional wait steps before or after it" + example = `{"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"wait","window":"Export"}]}` + case wait: + example = `{"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"type","text":"q3.csv"},{"type":"keypress","keys":["ENTER"]}]}` + } + if !diff { + clauses = append(clauses, "reports each target's change, windows opened/closed/focused, and a diff") + } + parts := append([]string{subject}, clauses...) + sentence := parts[0] + " and " + parts[1] + "." + if len(parts) > 2 { + last := len(parts) - 1 + sentence = strings.Join(parts[:last], ", ") + ", and " + parts[last] + "." + } + text := strings.Replace(axToolDescription, axActFullSentence, sentence, 1) + text = strings.Replace(text, axActFullExample, example, 1) + if diff { + text = strings.Replace(text, " and reports the selection read back", "", 1) + } + if wait { + text = strings.Replace(text, axWaitDescription, "", 1) + } + if wait || diff { + text = strings.Replace(text, "Skip waits the report answers. ", "", 1) + } + return text +} + var guiToolDescription = fmt.Sprintf("Full-desktop screenshot and pixel input. "+ `{"actions":[{"type":"screenshot"}]} alone observes; {"type":"zoom","region":[x0,y0,x1,y1]} alone `+ "enlarges a frame region to read small text and icons. Any other batch is checked before "+ @@ -480,7 +585,7 @@ func AllToolDefinitions() []ToolDefinition { { Name: PublicToolAX, Title: "Accessibility", - Description: axToolDescription, + Description: publishedAXToolDescription(), Annotations: annotations("Accessibility"), InputSchema: axInputSchema(), }, diff --git a/computer-use/internal/provider/linux_runtime_bundle.go b/computer-use/internal/provider/linux_runtime_bundle.go index 52ebead..e6840ac 100644 --- a/computer-use/internal/provider/linux_runtime_bundle.go +++ b/computer-use/internal/provider/linux_runtime_bundle.go @@ -6,6 +6,7 @@ import runtimebundle "github.com/microsoft/Hybrid-Computer-Use/computer-use" var linuxProviderPaths = []string{ "native/linux/provider/foundation.py", + "native/linux/provider/guard_shadow.py", "native/linux/provider/node_geometry.py", "native/linux/provider/node_visibility.py", "native/linux/provider/a11y_enable.py", diff --git a/computer-use/internal/provider/linux_runtime_bundle_test.go b/computer-use/internal/provider/linux_runtime_bundle_test.go index 9b434da..f3753b1 100644 --- a/computer-use/internal/provider/linux_runtime_bundle_test.go +++ b/computer-use/internal/provider/linux_runtime_bundle_test.go @@ -39,6 +39,7 @@ func TestLinuxRuntimeBundleIsComplete(t *testing.T) { "native/linux/provider/find_walk.py", "native/linux/provider/focus_probe.py", "native/linux/provider/foundation.py", + "native/linux/provider/guard_shadow.py", "native/linux/provider/gui_capture.py", "native/linux/provider/gui_dispatch.py", "native/linux/provider/gui_input.py", diff --git a/computer-use/internal/transport/cli_main.go b/computer-use/internal/transport/cli_main.go index 662858d..fbb3f4b 100644 --- a/computer-use/internal/transport/cli_main.go +++ b/computer-use/internal/transport/cli_main.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "runtime" "strings" "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" @@ -18,6 +19,16 @@ func RunCLI(args []string, stdout io.Writer) error { return nil } + switch args[0] { + case "-h", "--help", "help", "-v", "--version", "version": + default: + // Fail before serving anything: a misspelled ablation must not run + // the complete HCU under an ablation label. + if err := contract.ValidateAblations(runtime.GOOS, os.Getenv); err != nil { + return err + } + } + switch args[0] { case "-h", "--help", "help": topic := "" diff --git a/computer-use/internal/transport/mcp_transport.go b/computer-use/internal/transport/mcp_transport.go index 8cccf0c..1f3ce46 100644 --- a/computer-use/internal/transport/mcp_transport.go +++ b/computer-use/internal/transport/mcp_transport.go @@ -215,7 +215,7 @@ func handleMCPRequest(request map[string]any, svc *core.Service) map[string]any "version": contract.ServerVersion, }, "capabilities": map[string]any{"tools": map[string]any{"listChanged": false}}, - "instructions": contract.ServerInstructions, + "instructions": contract.PublishedServerInstructions(), }) case "ping": return jsonRPCResult(id, map[string]any{}) diff --git a/computer-use/native/linux/provider/actionability.py b/computer-use/native/linux/provider/actionability.py index 2709d02..1045c4a 100644 --- a/computer-use/native/linux/provider/actionability.py +++ b/computer-use/native/linux/provider/actionability.py @@ -30,6 +30,7 @@ ) from . import action_evidence from .input_primitives import require_window_focus +from .guard_shadow import shadow_guard from .element_resolver import ( same_frame, @@ -277,6 +278,10 @@ def require_stable_coordinate_geometry(element, element_record, window_bounds): input_events_emitted=False, failure_subject="element", ) + if not settled and shadow_guard( + "ax", "target_geometry", "target_unstable", samples=samples, + ): + settled = True if not settled: action_evidence.update(geometry_stable=False) raise OperationFailure( @@ -399,6 +404,10 @@ def require_coordinate_delivery(window, x, y, what): return action_evidence.update(point_owner_verified=owner) reason = "unknown" if owner is None else "different_window" + if shadow_guard( + "ax", "point_owner", reason, x=x, y=y, action=what, + ): + return raise OperationFailure( "coordinate_target_unverified: point owner is {}; no input was " "emitted.".format(reason), diff --git a/computer-use/native/linux/provider/ax_dispatch.py b/computer-use/native/linux/provider/ax_dispatch.py index f53fff8..524c271 100644 --- a/computer-use/native/linux/provider/ax_dispatch.py +++ b/computer-use/native/linux/provider/ax_dispatch.py @@ -6,6 +6,7 @@ MAX_DEPTH, MAX_ELEMENTS, OperationFailure, + node_name, node_pid, node_role, process_generation, @@ -27,6 +28,7 @@ from .x11_windows import ( main_window, select_window_by_handle, + relative_frame, window_identity_set, window_bounds, ) @@ -35,8 +37,10 @@ from .app_resolver import app_window_inventory, list_apps_inventory, windowless_snapshot from .native_windows import native_window_for_operation, perform_native_window_action +from .guard_shadow import shadow_guard from .element_resolver import ( find_element_at_snapshot_path, + resolve_path, resolve_semantic_target, ) from .window_resolver import ( @@ -99,6 +103,11 @@ def _verify_expected_window(window, expectation): and current_role == expected_role ) if not common_matches or (transient and not transient_matches): + if shadow_guard( + "ax", "batch_window", "logical_window_changed_before_input", + expected_role=expected_role, current_role=current_role, + ): + return raise OperationFailure( "stale_target: the batch logical window changed before keyboard " "input; no input was emitted.", @@ -499,21 +508,28 @@ def _resolve_action_target(app, operation, operation_kind, action_input): bounds = operation.get("window_bounds") or window_bounds( app, window_index, window) if semantic_target is not None and operation_kind != "window_action": - ( - element, - element_record, - semantic_note, - window, - bounds, - ) = resolve_semantic_target( - app, - window_index, - window, - bounds, - semantic_target, - operation.get("known_refs"), - operation, - ) + try: + ( + element, + element_record, + semantic_note, + window, + bounds, + ) = resolve_semantic_target( + app, + window_index, + window, + bounds, + semantic_target, + operation.get("known_refs"), + operation, + ) + except OperationFailure as refused: + shadowed = _shadow_semantic_refusal( + app, window_index, bounds, semantic_target, refused) + if shadowed is None: + raise + element, element_record = shadowed elif semantic_target is None and action_input.get("window_centre"): element = None elif semantic_target is None: @@ -530,6 +546,45 @@ def _resolve_action_target(app, operation, operation_kind, action_input): return element, element_record, semantic_note, window, bounds +# Identity refusals of a semantic target. The guard ablation acts on the +# snapshot path instead when a node is still there. +_SHADOWED_TARGET_FAILURES = { + "stale_element_ref", "semantic_target_not_found", "semantic_target_ambiguous", +} + + +def _shadow_semantic_refusal(app, window_index, bounds, target, refused): + """Return (node, record) at the target's snapshot path, or None. + + None keeps the refusal: the guard ablation is off, the refusal is not an + identity check, or no live node is left on the path. + """ + if getattr(refused, "failure_code", None) not in _SHADOWED_TARGET_FAILURES: + return None + paths = [target.get("runtimeId") or []] + paths.extend(target.get("runtimeAliases") or []) + for path in paths: + if not path: + continue + node = resolve_path(app, path) + if node is None or state_value(node, Atspi.StateType.DEFUNCT) is True: + continue + if not shadow_guard( + "ax", "target_identity", refused.failure_code, + message=str(refused), selector=target.get("selector"), path=list(path), + live={"role": node_role(node), "name": node_name(node)}, + ): + return None + record = { + "runtimeId": list(path), + "controlType": node_role(node), + "name": node_name(node), + "frame": safe(lambda: relative_frame(node, bounds)), + } + return node, record + return None + + def _resolve_select_end(app, operation, action_input, window, bounds): """Resolve the select range end (to_ref) in the target's window, or None.""" to_target = action_input.get("to_target") diff --git a/computer-use/native/linux/provider/ax_input_delivery.py b/computer-use/native/linux/provider/ax_input_delivery.py index 882175a..92eba64 100644 --- a/computer-use/native/linux/provider/ax_input_delivery.py +++ b/computer-use/native/linux/provider/ax_input_delivery.py @@ -3,6 +3,7 @@ import math import time +from .guard_shadow import guards_enforced from .foundation import ( Atspi, OperationFailure, @@ -408,10 +409,13 @@ def _pointer_click_for_state_only_action( return None, "the window is not in the foreground" evidence_before = dict(action_evidence) try: - record, _ = require_stable_coordinate_geometry( - element, element_record, bounds) - x, y = screen_point(bounds, record, None, None) - require_coordinate_delivery(window, x, y, "click") + # These checks only choose the route: a refusal keeps the semantic + # action, so the guard ablation has nothing to let through. + with guards_enforced(): + record, _ = require_stable_coordinate_geometry( + element, element_record, bounds) + x, y = screen_point(bounds, record, None, None) + require_coordinate_delivery(window, x, y, "click") hits = point_hits_element(window, element, x, y) except (OperationFailure, RuntimeError) as refused: action_evidence.clear() diff --git a/computer-use/native/linux/provider/element_resolver.py b/computer-use/native/linux/provider/element_resolver.py index 27dd487..73bf262 100644 --- a/computer-use/native/linux/provider/element_resolver.py +++ b/computer-use/native/linux/provider/element_resolver.py @@ -12,6 +12,7 @@ ) from .node_geometry import extents from .primitives import name_fingerprint +from .guard_shadow import shadow_guard from .ax_node_properties import ( accessible_id, @@ -132,6 +133,7 @@ def find_element_at_snapshot_path_with_bounds(app, record, window_bounds): paths = [record.get("runtimeId") or []] paths.extend(record.get("runtimeAliases") or []) seen_paths = set() + mismatched = None for path in paths: path_key = tuple(path) if path_key in seen_paths: @@ -142,6 +144,8 @@ def find_element_at_snapshot_path_with_bounds(app, record, window_bounds): node, record, window_bounds ): return node + if node is not None and mismatched is None: + mismatched = (node, path) # Self-managed tables expose visible cells through Table.get_accessible_at # instead of child enumeration. render_visible_cells gives those records a # stable virtual suffix [row, col], so the action resolver must understand @@ -170,6 +174,13 @@ def find_element_at_snapshot_path_with_bounds(app, record, window_bounds): and record_still_matches(node, record, window_bounds) ): return node + if mismatched is not None and shadow_guard( + "ax", "target_identity", "path_resolves_to_a_different_element", + path=list(mismatched[1]), + expected={"role": record.get("controlType"), "name": record.get("name")}, + live={"role": effective_role(mismatched[0]), "name": effective_name(mismatched[0])}, + ): + return mismatched[0] return None diff --git a/computer-use/native/linux/provider/guard_shadow.py b/computer-use/native/linux/provider/guard_shadow.py new file mode 100644 index 0000000..d9fe867 --- /dev/null +++ b/computer-use/native/linux/provider/guard_shadow.py @@ -0,0 +1,63 @@ +"""Shadow mode of the input guards for the guard ablation. + +With ``guard`` in COMPUTER_USE_ABLATE, the AX target guard and the GUI +layout guard still make their decision, but a decision to stop input is +only recorded: one JSON line per decision in COMPUTER_USE_GUARD_LOG, which +the model never sees. The input then goes ahead. Checks that find nothing +to act on (no node on the snapshot path, no captured frame) are not guards +and still fail. +""" + +import contextlib +import json +import os +import time + +_enforced = 0 + + +@contextlib.contextmanager +def guards_enforced(): + """Keep guards deciding normally, for checks that only choose a route.""" + global _enforced + _enforced += 1 + try: + yield + finally: + _enforced -= 1 + + +def guard_ablated(): + names = os.environ.get("COMPUTER_USE_ABLATE", "").split(",") + return "guard" in {name.strip() for name in names} + + +def shadow_guard(channel, guard, reason, **detail): + """Return True when the guard ablation lets the stopped input through. + + The decision is appended to the guard log first. Without the ablation + nothing is written and the caller stops the input as before. + """ + if _enforced or not guard_ablated(): + return False + record = { + "at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + + ".%03dZ" % (int(time.time() * 1000) % 1000), + "channel": channel, + "guard": guard, + "reason": reason, + "would_block": True, + } + record.update(detail) + path = os.environ.get("COMPUTER_USE_GUARD_LOG", "").strip() + if path: + # A lost log line must not fail the input it describes. + try: + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + with open(path, "a", encoding="utf-8") as log: + log.write(json.dumps(record, sort_keys=True, default=str) + "\n") + except OSError: + pass + return True diff --git a/computer-use/native/linux/provider/gui_dispatch.py b/computer-use/native/linux/provider/gui_dispatch.py index 7d274a5..ee4f588 100644 --- a/computer-use/native/linux/provider/gui_dispatch.py +++ b/computer-use/native/linux/provider/gui_dispatch.py @@ -33,6 +33,7 @@ validate_gui_actions, ) from .input_primitives import InputEmissionError +from .guard_shadow import shadow_guard from .snapshot import capture_gui_snapshot from .desktop_layout import ( desktop_bounds, @@ -286,6 +287,18 @@ def layout_hit(layout, x, y): return None +def _guard_points(actions): + """The action types and coordinates a guard log line records.""" + points = [] + for action in actions: + point = {"type": action.get("type")} + for key in ("x", "y", "path"): + if key in action: + point[key] = action[key] + points.append(point) + return points + + def coordinates_still_valid(reference, current, actions): """Whether every coordinate of actions still lands where it did. @@ -394,7 +407,9 @@ def stale(message, capture): left, top, width, height = geometry["native"] if not region_still_valid( _published_layouts.get(frame_id), capture_layout(current), - (left, top, width, height)): + (left, top, width, height)) and not shadow_guard( + "gui", "zoom_layout", "windows_over_region_changed", + frame_id=frame_id, region=[left, top, width, height]): return stale( "The windows over the zoom region changed since the frame its " "coordinates came from.", current) @@ -573,7 +588,10 @@ def failure_evidence(fresh=True): if frame_id and frame_guarded_actions: if (current_capture.frame_id != frame_id and not coordinates_still_valid( - reference_layout, capture_layout(current_capture), validated)): + reference_layout, capture_layout(current_capture), validated) + and not shadow_guard( + "gui", "layout", "layout_changed_before_batch", + frame_id=frame_id, actions=_guard_points(validated))): return { "ok": False, "error": "The desktop layout changed since the screenshot used " @@ -614,7 +632,11 @@ def failure_evidence(fresh=True): live_frame_id = desktop_layout_frame_id(live_layout) if (live_frame_id != frame_id and not coordinates_still_valid( - reference_layout, live_layout, [action])): + reference_layout, live_layout, [action]) + and not shadow_guard( + "gui", "layout", "layout_changed_during_batch", + frame_id=frame_id, index=index, + actions=_guard_points([action]))): error_snapshot, evidence_note = failure_evidence() return { "ok": False, diff --git a/computer-use/runtime_test_support.py b/computer-use/runtime_test_support.py index ae058f3..f168976 100644 --- a/computer-use/runtime_test_support.py +++ b/computer-use/runtime_test_support.py @@ -47,6 +47,7 @@ semantic_window_candidates, semantic_scan, foundation, + guard_shadow, node_geometry, node_visibility, node_facts, @@ -76,6 +77,7 @@ PROVIDER_MODULES = ( foundation, + guard_shadow, node_geometry, node_visibility, a11y_enable, From ed07bc9bd8a86967dced01c25c4f2bc5d3f2c2d4 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:44:16 +0800 Subject: [PATCH 02/13] core: apply the batching, wait and diff ablations batching: an AX act takes one input step; wait steps may surround it (rule one_input_step). wait: wait steps are no longer an action type, and error texts no longer suggest them. diff: act runs as before but returns only its execution evidence; its reads are not recorded as delivered, so a later observe compares with what the model actually saw. Co-Authored-By: Claude Opus 5.5 --- computer-use/internal/core/ax_ablation.go | 71 +++++++++++++++++++ .../internal/core/ax_argument_hint_test.go | 62 +++++++++++----- .../internal/core/ax_public_request.go | 52 ++++++++++---- computer-use/internal/core/ax_request.go | 3 + .../internal/core/ax_snapshot_notes.go | 5 +- .../internal/core/ax_snapshot_store.go | 4 +- computer-use/internal/core/service.go | 3 + computer-use/internal/core/tool_result.go | 5 ++ 8 files changed, 170 insertions(+), 35 deletions(-) create mode 100644 computer-use/internal/core/ax_ablation.go diff --git a/computer-use/internal/core/ax_ablation.go b/computer-use/internal/core/ax_ablation.go new file mode 100644 index 0000000..3b2e969 --- /dev/null +++ b/computer-use/internal/core/ax_ablation.go @@ -0,0 +1,71 @@ +package core + +import ( + "fmt" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +// Behavior of the mechanism ablations in the AX service; contract/ablation.go +// says what each one removes. + +// axActWithoutFeedback runs an act for the diff ablation. The batch runs +// exactly as with diff, including the post-action reads that time its +// return, but the model receives none of what those reads found: +// withoutAXActFeedback keeps only the execution evidence. The reads are not +// recorded as delivered, so a later observe compares with what the model +// actually saw and never answers "unchanged" from a hidden read. +func (s *Service) axActWithoutFeedback(request *axRequest, obs observation) ToolCallResult { + s.observations.hidden = true + defer func() { s.observations.hidden = false }() + result := s.axBatchDecoded(request.actions, obs, request.windowRef). + withEvidenceScope("delivery_or_local_ui") + // Drop images here, before callTool registers a delivered image as the + // GUI frame; the text goes in withoutAXActFeedback, after the evidence + // line has taken the error message from it. + kept := []ContentItem{} + for _, item := range result.Content { + if item.Type == "text" { + kept = append(kept, item) + } + } + result.Content = kept + return WithAXModeEvidence(result, contract.AXModeAct) +} + +// axActEvidenceFields are the public evidence fields an act keeps under the +// diff ablation: whether it succeeded, the error, and how far input went. +var axActEvidenceFields = append(append([]string{}, axPublicFields...), + "failure_phase", "partial_input_possible", "retry_same_request") + +// withoutAXActFeedback reduces a prepared act result to its execution +// evidence: no effect or windows lines, diff, tree, notes or selection +// readback. +func withoutAXActFeedback(result ToolCallResult) ToolCallResult { + kept := map[string]any{} + for _, key := range axActEvidenceFields { + if value, present := result.StructuredContent[key]; present { + kept[key] = value + } + } + result.StructuredContent = kept + result.Content = []ContentItem{} + return result +} + +// axCheckOneInputStep enforces the batching ablation: an act holds at most +// one input step. Wait steps send no input and may surround it. +func axCheckOneInputStep(earlier []axBatchAction, action axBatchAction, items []any, path string) *axArgError { + if !contract.Ablated(contract.AblateBatching) || action.actionType == "wait" { + return nil + } + for _, step := range earlier { + if step.actionType != "wait" { + return axInvalid(path, fmt.Sprintf( + "act takes one input step per call, and %s is the second; send it in a later call", path), ""). + withRule(contract.ArgumentRuleOneInputStep).withFix(axSet("actions", items[:len(earlier)])) + } + } + return nil +} + diff --git a/computer-use/internal/core/ax_argument_hint_test.go b/computer-use/internal/core/ax_argument_hint_test.go index 2dac3a5..50f8cf9 100644 --- a/computer-use/internal/core/ax_argument_hint_test.go +++ b/computer-use/internal/core/ax_argument_hint_test.go @@ -142,8 +142,53 @@ var axBadCalls = []string{ // Every invalid_argument from the decoder has a hint that decodes: a whole // call (starting with "mode") decodes as is; a fragment decodes when it // replaces the object at the error path. +// axAblatedBadCalls reach the rejection sites that exist only under an +// ablation (COMPUTER_USE_ABLATE). +var axAblatedBadCalls = map[string][]string{ + contract.AblateWait: { + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"click","ref":"r1"}]}`, + `{"mode":"observe","app":"x","observe":"none"}`, + `{"mode":"assert_ax"}`, + }, + contract.AblateBatching + "," + contract.AblateWait: { + `{"mode":"act","actions":[{"type":"click","ref":"r1"},{"type":"click","ref":"r2"}]}`, + }, + contract.AblateDiff: { + `{"mode":"act","actions":[{"type":"click","ref":"r1"}],"observe":"none"}`, + `{"mode":"observe","app":"x","observe":"diff"}`, + }, +} + func TestEveryAXArgumentErrorHintDecodes(t *testing.T) { reached := map[int]bool{} + checkAXBadCalls(t, axBadCalls, reached) + for ablation, calls := range axAblatedBadCalls { + t.Run(ablation, func(t *testing.T) { + t.Setenv(contract.AblationEnv, ablation) + checkAXBadCalls(t, calls, reached) + }) + } + source, readErr := os.ReadFile("ax_public_request.go") + if readErr != nil { + t.Fatal(readErr) + } + var missed []string + for index, line := range strings.Split(string(source), "\n") { + if strings.Contains(line, "axInvalid(") && !strings.Contains(line, "func axInvalid(") && + !strings.HasPrefix(strings.TrimSpace(line), "//") && !reached[index+1] { + missed = append(missed, strings.TrimSpace(line)) + } + } + sort.Strings(missed) + if len(missed) > 0 { + t.Fatalf("no bad call reaches these rejection sites:\n%s", strings.Join(missed, "\n")) + } +} + +// checkAXBadCalls decodes each call, checks that its hint decodes, and +// records the decoder lines that rejected it. +func checkAXBadCalls(t *testing.T, calls []string, reached map[int]bool) { + t.Helper() first := false // Only the first rejection of a call is its error site; later ones come // from the decodes that derive the hint. @@ -154,7 +199,7 @@ func TestEveryAXArgumentErrorHintDecodes(t *testing.T) { first = false } defer func() { axInvalidHook = nil }() - for _, raw := range axBadCalls { + for _, raw := range calls { args := decodeTestArguments(t, raw) first = true _, err := DecodeAXRequest(args) @@ -172,21 +217,6 @@ func TestEveryAXArgumentErrorHintDecodes(t *testing.T) { t.Fatalf("%s: hint %s for %s does not decode: %s (%s)", label, err.Hint, err.path, hintErr.Message, hintErr.path) } } - source, readErr := os.ReadFile("ax_public_request.go") - if readErr != nil { - t.Fatal(readErr) - } - var missed []string - for index, line := range strings.Split(string(source), "\n") { - if strings.Contains(line, "axInvalid(") && !strings.Contains(line, "func axInvalid(") && - !strings.HasPrefix(strings.TrimSpace(line), "//") && !reached[index+1] { - missed = append(missed, strings.TrimSpace(line)) - } - } - sort.Strings(missed) - if len(missed) > 0 { - t.Fatalf("no bad call reaches these rejection sites:\n%s", strings.Join(missed, "\n")) - } } func axHintDecodeError(args map[string]any, err *axArgError) *axArgError { diff --git a/computer-use/internal/core/ax_public_request.go b/computer-use/internal/core/ax_public_request.go index 3b82fda..9e43392 100644 --- a/computer-use/internal/core/ax_public_request.go +++ b/computer-use/internal/core/ax_public_request.go @@ -109,6 +109,15 @@ var axModeFields = map[string][]string{ contract.AXModeAct: {"mode", "window_ref", "actions", "observe"}, } +// axModeAllowedFields lists the fields a mode accepts. Without diff, act +// returns no observation, so it takes no observe field. +func axModeAllowedFields(mode string) []string { + if mode == contract.AXModeAct && contract.Ablated(contract.AblateDiff) { + return []string{"mode", "window_ref", "actions"} + } + return axModeFields[mode] +} + // axActionFields lists the fields each action type accepts besides type. var axActionFields = map[string][]string{ "click": {"ref", "allow_global_input"}, @@ -159,7 +168,7 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } mode, ok := rawMode.(string) if !ok || !contract.ContainsString([]string{contract.AXModeApps, contract.AXModeObserve, contract.AXModeFind, contract.AXModeAct}, mode) { - if mode == "assert_ax" { + if mode == "assert_ax" && !contract.Ablated(contract.AblateWait) { return nil, axInvalid("mode", `mode "assert_ax" was removed; wait with an act call that has only wait steps`, `{"mode":"act","actions":[{"type":"wait","ref":"r1"}]}`) @@ -173,10 +182,12 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } return nil, err } - if err := axRejectRemovedUntil(args); err != nil { - return nil, err + if !contract.Ablated(contract.AblateWait) { + if err := axRejectRemovedUntil(args); err != nil { + return nil, err + } } - if err := axCheckFields(args, axModeFields[mode], "", "mode "+strconv.Quote(mode)); err != nil { + if err := axCheckFields(args, axModeAllowedFields(mode), "", "mode "+strconv.Quote(mode)); err != nil { return nil, err.withRule(axRuleFieldForMode) } request := &axRequest{mode: mode} @@ -194,7 +205,7 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { if request.scope, err = axOptionalEnum(args, "scope", "", contract.AXScopeValues); err != nil { return nil, err } - if request.observe, err = axOptionalEnum(args, "observe", "", contract.ObserveValues); err != nil { + if request.observe, err = axOptionalEnum(args, "observe", "", contract.PublishedObserveValues()); err != nil { return nil, err } if request.root, err = axOptionalPattern(args, "root", "", axRootPattern, `"focused" or rN`); err != nil { @@ -223,8 +234,11 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } } if request.observe == contract.ObserveNone { - return nil, axInvalid("observe", `observe "none" returns nothing; to wait without input, send act with only wait steps`, - "").withRule(axRuleObserveNone).withFix(axRemove("observe")) + message := `observe "none" returns nothing; to wait without input, send act with only wait steps` + if contract.Ablated(contract.AblateWait) { + message = `observe "none" returns nothing; it is only for act` + } + return nil, axInvalid("observe", message, "").withRule(axRuleObserveNone).withFix(axRemove("observe")) } if request.app == "" && request.windowRef == "" && request.readText == nil { return nil, axInvalid("app", "observe needs app or window_ref", @@ -332,9 +346,12 @@ func decodeAXActions(raw any) ([]axBatchAction, *axArgError) { return nil, axInvalid("actions", `act needs at least one action; use mode "observe" to read`, `[{"type":"click","ref":"r1"}]`) } - if len(items) > contract.AXBatchMaxActions { - return nil, axInvalid("actions", fmt.Sprintf("act accepts at most %d actions", contract.AXBatchMaxActions), ""). - withFix(axSet("actions", items[:contract.AXBatchMaxActions])) + if limit := contract.AXActMaxActions(); len(items) > limit { + message := fmt.Sprintf("act accepts at most %d actions", limit) + if limit == 1 { + message = "act takes one action per call; send the others in later calls" + } + return nil, axInvalid("actions", message, "").withFix(axSet("actions", items[:limit])) } actions := make([]axBatchAction, 0, len(items)) totalText, totalWait := 0, 0 @@ -349,9 +366,16 @@ func decodeAXActions(raw any) ([]axBatchAction, *axArgError) { return nil, err } if index > 0 && actions[index-1].actionType == "close" && action.actionType != "wait" { + if contract.Ablated(contract.AblateWait) { + return nil, axInvalid(path, "close must be the last step: a closed window cannot take later input", + "").withRule(contract.ArgumentRuleCloseNotLast).withFix(axSet("actions", items[:index])) + } return nil, axInvalid(path, "a closed window cannot take later input; after close, wait for the window to be gone first", `{"type":"wait","window":"window title","gone":true}`).withRule(contract.ArgumentRuleCloseNotLast) } + if err := axCheckOneInputStep(actions, action, items, path); err != nil { + return nil, err + } totalText += len([]rune(action.text)) + len([]rune(action.value)) if totalText > contract.AXBatchMaxText { return nil, axInvalid(path, fmt.Sprintf("text and value together are limited to %d characters per call", contract.AXBatchMaxText), "").withRule(contract.ArgumentRuleTextBudget) @@ -379,7 +403,7 @@ func decodeAXAction(object map[string]any, path string) (axBatchAction, *axArgEr action := axBatchAction{pages: 1} rawType, hasType := object["type"] actionType, _ := rawType.(string) - if !hasType || !contract.ContainsString(contract.AXActionTypes, actionType) { + if !hasType || !contract.ContainsString(contract.PublishedAXActionTypes(), actionType) { if hint := axActionShapeHint(object); hint != "" && !hasType { field := "type" for name := range object { @@ -389,11 +413,11 @@ func decodeAXAction(object map[string]any, path string) (axBatchAction, *axArgEr } return action, axInvalid(path+"."+field, `actions use "type", not "`+field+`"`, hint) } - message := "type must be one of " + strings.Join(contract.AXActionTypes, ", ") + message := "type must be one of " + strings.Join(contract.PublishedAXActionTypes(), ", ") if hasType { message += "; got " + contract.CompactJSON(rawType) } else { - message = "action needs type: " + strings.Join(contract.AXActionTypes, ", ") + message = "action needs type: " + strings.Join(contract.PublishedAXActionTypes(), ", ") } err := axInvalid(path+".type", message, axActionShapeHint(object)) switch { @@ -592,7 +616,7 @@ func axActionShapeHint(object map[string]any) string { if synonym, known := axActionVerbSynonyms[verb]; known { verb = synonym } - if !contract.ContainsString(contract.AXActionTypes, verb) { + if !contract.ContainsString(contract.PublishedAXActionTypes(), verb) { switch { case AXWindowRefPattern.MatchString(ref) && strings.Contains(strings.ToLower(verb), "close"): verb = "close" diff --git a/computer-use/internal/core/ax_request.go b/computer-use/internal/core/ax_request.go index d627c07..1d21ff0 100644 --- a/computer-use/internal/core/ax_request.go +++ b/computer-use/internal/core/ax_request.go @@ -250,6 +250,9 @@ func (s *Service) axAct( request *axRequest, obs observation, ) ToolCallResult { + if contract.Ablated(contract.AblateDiff) { + return s.axActWithoutFeedback(request, obs) + } result := s.axBatchDecoded( request.actions, obs, diff --git a/computer-use/internal/core/ax_snapshot_notes.go b/computer-use/internal/core/ax_snapshot_notes.go index c2b2fec..1c2fa6e 100644 --- a/computer-use/internal/core/ax_snapshot_notes.go +++ b/computer-use/internal/core/ax_snapshot_notes.go @@ -202,11 +202,10 @@ func continuityNotes(previous, current *AppSnapshot) []string { "window instance, same tree, same focus, same selection.%s "+ "Nothing in the tree reports work in progress, but that is not proof your "+ "last action failed: effects may not reach the accessibility tree. "+ - "Check the screenshot, or "+ - "wait for the specific element with an AX wait step before deciding to retry. "+ + "Check the screenshot%s before deciding to retry. "+ "Note also what this line rules OUT: the content is not missing because a "+ "budget trimmed it — this tree is the same one you already have.", - gap)) + gap, map[bool]string{false: ", or wait for the specific element with an AX wait step", true: ""}[contract.Ablated(contract.AblateWait)])) return notes } diff --git a/computer-use/internal/core/ax_snapshot_store.go b/computer-use/internal/core/ax_snapshot_store.go index 80f2d83..29bc85c 100644 --- a/computer-use/internal/core/ax_snapshot_store.go +++ b/computer-use/internal/core/ax_snapshot_store.go @@ -140,7 +140,7 @@ func DeliveredSnapshotKey(snapshot *AppSnapshot) string { } func (s *Service) rememberDeliveredTree(key string) { - if key == "" { + if key == "" || s.observations.hidden { return } if len(s.observations.lastFullTree) >= contract.MaxDeliveredTrees && @@ -153,7 +153,7 @@ func (s *Service) rememberDeliveredTree(key string) { } func (s *Service) rememberDeliveredSnapshot(snapshot *AppSnapshot) { - if snapshot == nil { + if snapshot == nil || s.observations.hidden { return } if providerCoverageSummary(snapshot.Annotations).AcquisitionIncomplete > 0 { diff --git a/computer-use/internal/core/service.go b/computer-use/internal/core/service.go index c56b560..6e42598 100644 --- a/computer-use/internal/core/service.go +++ b/computer-use/internal/core/service.go @@ -22,6 +22,9 @@ type failedObservationEntry struct { type observationLedger struct { lastComplete map[string]*AppSnapshot lastFullTree map[string]bool + // hidden is set while an act runs whose observations the model never + // receives (the diff ablation); they are then not recorded as delivered. + hidden bool } // DesktopState is shared by every caller of one desktop: the provider, the diff --git a/computer-use/internal/core/tool_result.go b/computer-use/internal/core/tool_result.go index cb13b81..841bae2 100644 --- a/computer-use/internal/core/tool_result.go +++ b/computer-use/internal/core/tool_result.go @@ -268,8 +268,13 @@ func FinalizeForModel(name string, result ToolCallResult, structured bool) ToolC func prepareModelEvidence(name string, result ToolCallResult) ToolCallResult { switch name { case contract.PublicToolAX: + actFeedbackHidden := contract.Ablated(contract.AblateDiff) && + result.StructuredContent["ax_mode"] == contract.AXModeAct result = EnforceAXResponseBudget(result) result = PrepareAXEvidence(result) + if actFeedbackHidden { + result = withoutAXActFeedback(result) + } case contract.PublicToolGUI: result = prepareGUIEvidence(result) } From 4233d2ce29cd5a708bf7acb2efc600dbb49a220c Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:44:16 +0800 Subject: [PATCH 03/13] provider: shadow mode of the input guards for the guard ablation With guard ablated, the AX target identity, batch window, point owner and geometry checks and the GUI layout and zoom checks still decide, but only log the input they would stop to COMPUTER_USE_GUARD_LOG and let it through. A path with no live node still fails as stale. Co-Authored-By: Claude Opus 5.5 --- computer-use/internal/core/ax_ablation.go | 37 +++++++++++++++++++++ computer-use/internal/core/ax_wait_batch.go | 5 +++ 2 files changed, 42 insertions(+) diff --git a/computer-use/internal/core/ax_ablation.go b/computer-use/internal/core/ax_ablation.go index 3b2e969..496216b 100644 --- a/computer-use/internal/core/ax_ablation.go +++ b/computer-use/internal/core/ax_ablation.go @@ -1,7 +1,12 @@ package core import ( + "encoding/json" "fmt" + "os" + "path/filepath" + "strings" + "time" "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) @@ -69,3 +74,35 @@ func axCheckOneInputStep(earlier []axBatchAction, action axBatchAction, items [] return nil } +// shadowGuard reports whether the guard ablation lets input through that a +// guard decided to stop. It first appends the decision to +// COMPUTER_USE_GUARD_LOG, in the format of the Linux provider's +// guard_shadow.py, so both sides of one run share one log. +func shadowGuard(channel, guard, reason string, detail map[string]any) bool { + if !contract.Ablated(contract.AblateGuard) { + return false + } + record := map[string]any{ + "at": time.Now().UTC().Format("2006-01-02T15:04:05.000Z"), "channel": channel, + "guard": guard, "reason": reason, "would_block": true, + } + for key, value := range detail { + record[key] = value + } + path := strings.TrimSpace(os.Getenv(contract.GuardLogEnv)) + line, err := json.Marshal(record) + if path == "" || err != nil { + return true + } + // A lost log line must not fail the input it describes. + if dir := filepath.Dir(path); dir != "." { + _ = os.MkdirAll(dir, 0o755) + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return true + } + defer file.Close() + _, _ = file.Write(append(line, '\n')) + return true +} diff --git a/computer-use/internal/core/ax_wait_batch.go b/computer-use/internal/core/ax_wait_batch.go index 2398ef4..5b06c76 100644 --- a/computer-use/internal/core/ax_wait_batch.go +++ b/computer-use/internal/core/ax_wait_batch.go @@ -207,6 +207,11 @@ func (s *Service) axBatchContinuityStop( if continuity == axWindowSame { return ToolCallResult{}, false } + if shadowGuard("ax", "batch_window", "logical_window_"+string(continuity)+"_between_actions", map[string]any{ + "window_ref": context.WindowRef, "next_action_index": index + 1, + }) { + return ToolCallResult{}, false + } anyInputEmitted, anyPartialPossible, anyMutationUnknown := batchMutationSummary(actionEvidence) failureCode := "stale_target" messageCode := "batch_window_changed" From 22173537207ae83afd3a431afeb1b1933ed0119f Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:45:03 +0800 Subject: [PATCH 04/13] osworld: run HCU mechanism ablations with --ablate --ablate (config key ablate) selects the removed mechanisms, passes them to the MCP server env of all four harnesses, gives the agent guides without the passages of the removed mechanisms, and records the set in result.json and meta.json. With guard ablated the runner collects the guest's guard log. configs/ablation holds the six OSWorld 1.0 groups. Co-Authored-By: Claude Opus 5.5 --- .../osworld-v1-codex-full.example.json | 98 +++++ .../osworld-v1-codex-no-all4.example.json | 103 ++++++ .../osworld-v1-codex-no-batching.example.json | 100 ++++++ .../osworld-v1-codex-no-diff.example.json | 100 ++++++ .../osworld-v1-codex-no-guard.example.json | 100 ++++++ .../osworld-v1-codex-no-wait.example.json | 100 ++++++ docs/OSWORLD.md | 19 +- hcu/osworld/agents.py | 68 +++- hcu/osworld/config.py | 59 +++- hcu/osworld/guest.py | 3 + hcu/osworld/guest_lifecycle.py | 32 +- hcu/osworld/guide_ablation.py | 209 +++++++++++ hcu/osworld/phases.py | 1 + hcu/osworld/run_task.py | 23 ++ hcu/osworld/staging.py | 23 +- tests/test_ablation_switches.py | 334 ++++++++++++++++++ 16 files changed, 1346 insertions(+), 26 deletions(-) create mode 100644 configs/ablation/osworld-v1-codex-full.example.json create mode 100644 configs/ablation/osworld-v1-codex-no-all4.example.json create mode 100644 configs/ablation/osworld-v1-codex-no-batching.example.json create mode 100644 configs/ablation/osworld-v1-codex-no-diff.example.json create mode 100644 configs/ablation/osworld-v1-codex-no-guard.example.json create mode 100644 configs/ablation/osworld-v1-codex-no-wait.example.json create mode 100644 hcu/osworld/guide_ablation.py create mode 100644 tests/test_ablation_switches.py diff --git a/configs/ablation/osworld-v1-codex-full.example.json b/configs/ablation/osworld-v1-codex-full.example.json new file mode 100644 index 0000000..05728ba --- /dev/null +++ b/configs/ablation/osworld-v1-codex-full.example.json @@ -0,0 +1,98 @@ +{ + "run_id": "example-osworld-v1-codex-full", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-all4.example.json b/configs/ablation/osworld-v1-codex-no-all4.example.json new file mode 100644 index 0000000..17a5c53 --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-all4.example.json @@ -0,0 +1,103 @@ +{ + "run_id": "example-osworld-v1-codex-no-all4", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "batching", + "wait", + "diff", + "guard" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-batching.example.json b/configs/ablation/osworld-v1-codex-no-batching.example.json new file mode 100644 index 0000000..814a8e4 --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-batching.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-batching", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "batching" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-diff.example.json b/configs/ablation/osworld-v1-codex-no-diff.example.json new file mode 100644 index 0000000..b52861d --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-diff.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-diff", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "diff" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-guard.example.json b/configs/ablation/osworld-v1-codex-no-guard.example.json new file mode 100644 index 0000000..3b3a7a9 --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-guard.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-guard", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "guard" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/configs/ablation/osworld-v1-codex-no-wait.example.json b/configs/ablation/osworld-v1-codex-no-wait.example.json new file mode 100644 index 0000000..130aebc --- /dev/null +++ b/configs/ablation/osworld-v1-codex-no-wait.example.json @@ -0,0 +1,100 @@ +{ + "run_id": "example-osworld-v1-codex-no-wait", + "benchmark_version": "v1", + "agent": "codex", + "model": "gpt-5.4", + "effort": "xhigh", + "ablate": [ + "wait" + ], + "guest_image": "/absolute/path/to/osworld-v1-workstation-deps-64g.qcow2", + "client_password": "password", + "volume_size": 64, + "ram_size": "4G", + "max_model_calls": 100, + "timeout_seconds": 1800, + "max_active_tasks": 20, + "launch_stagger_seconds": 5, + "tasks": [ + { + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "task_domain": "chrome" + }, + { + "id": "7767eef2-56a3-4cea-8c9f-48c070c7d65b", + "task_domain": "gimp" + }, + { + "id": "7b7617bd-57cc-468e-9c91-40c4ec2bcb3d", + "task_domain": "gimp" + }, + { + "id": "1d17d234-e39d-4ed7-b46f-4417922a4e7c", + "task_domain": "libreoffice_calc" + }, + { + "id": "04d9aeaf-7bed-4024-bedb-e10e6f00eb7f", + "task_domain": "libreoffice_calc" + }, + { + "id": "7ae48c60-f143-4119-b659-15b8f485eb9a", + "task_domain": "libreoffice_impress" + }, + { + "id": "a434992a-89df-4577-925c-0c58b747f0f4", + "task_domain": "libreoffice_impress" + }, + { + "id": "0b17a146-2934-46c7-8727-73ff6b6483e8", + "task_domain": "libreoffice_writer" + }, + { + "id": "0810415c-bde4-4443-9047-d5f70165a697", + "task_domain": "libreoffice_writer" + }, + { + "id": "bc2b57f3-686d-4ec9-87ce-edf850b7e442", + "task_domain": "multi_apps" + }, + { + "id": "00fa164e-2612-4439-992e-157d019a8436", + "task_domain": "multi_apps" + }, + { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "task_domain": "os" + }, + { + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "task_domain": "os" + }, + { + "id": "a10b69e1-6034-4a2b-93e1-571d45194f75", + "task_domain": "thunderbird" + }, + { + "id": "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", + "task_domain": "vlc" + }, + { + "id": "9195653c-f4aa-453d-aa95-787f6ccfaae9", + "task_domain": "vlc" + }, + { + "id": "57242fad-77ca-454f-b71b-f187181a9f23", + "task_domain": "vs_code" + }, + { + "id": "70745df8-f2f5-42bd-8074-fbc10334fcc5", + "task_domain": "vs_code" + }, + { + "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", + "task_domain": "thunderbird" + }, + { + "id": "af630914-714e-4a24-a7bb-f9af687d3b91", + "task_domain": "chrome" + } + ] +} diff --git a/docs/OSWORLD.md b/docs/OSWORLD.md index b3fe64d..08887ea 100644 --- a/docs/OSWORLD.md +++ b/docs/OSWORLD.md @@ -220,13 +220,26 @@ The gateway defaults to the Docker bridge address at the agent's port; `--result-dir`. `--channels a11y,keyboard` or `gui,keyboard` exposes one channel only; `--instructions-file ""` runs without a profile. +`--ablate` removes HCU mechanisms from the computer-use server: a +comma-separated subset of `batching`, `wait`, `diff` and `guard`. The runner +sets `COMPUTER_USE_ABLATE` in the MCP server environment of every harness and +gives the agent copies of `instructions/base.md` and `instructions/hybrid.md` +without the passages that describe the removed mechanisms. It writes the +copies to `ablated-instructions/` in the task's result directory. Other +guides, `--instructions-file ""` and `--code-mode` are refused with +`--ablate`, because they can still describe a removed mechanism. With `guard` +removed, the server logs the guard decisions in the guest, and the runner +copies the log to `guard.jsonl` in the result directory. Without `--ablate` +the harness configurations do not change. + ## Run a panel A run is one JSON config. Relative paths in it are relative to the config file; omitted paths default to the repository's `artifacts/`, `runs/`, `assets/` and `instructions/`. See [`configs/osworld-v1-codex-queue.example.json`](../configs/osworld-v1-codex-queue.example.json) -for the 20-task V1 panel. +for the 20-task V1 panel. [`configs/ablation/`](../configs/ablation/) holds +the same panel for each mechanism-ablation group. | Key | Meaning | | --- | --- | @@ -238,6 +251,7 @@ for the 20-task V1 panel. | `max_active_tasks`, `launch_stagger_seconds` | Concurrency | | `gateway`, `token_log`, `results_dir`, `artifact_dir`, `asset_cache`, `offline_assets` | Optional overrides | | `agent_binary`, `mcp_binary`, `instructions_file`, `base_instructions_file`, `channels` | Optional agent overrides | +| `ablate` | Removed HCU mechanisms, as a list or a comma-separated string (see `--ablate`) | ```bash $PY -m hcu.osworld.run_queue configs/osworld-v1-codex-queue.example.json --dry-run @@ -258,10 +272,11 @@ runs// config.json the config this directory belongs to gateway-tokens.jsonl gateway token log (default location) tasks// - result.json the outcome: status, score, termination_reason, usage + result.json the outcome: status, score, termination_reason, usage, ablate meta.json guest and runtime diagnostics agent.jsonl, phase-*-turn-*-* agent traces per turn final-*.png screenshots before and after evaluation + guard.jsonl guard decisions, only with guard ablated logs/.log runner output retries/-/ attempts interrupted by infrastructure, not scored QUEUE_STATUS.md diff --git a/hcu/osworld/agents.py b/hcu/osworld/agents.py index 7b9080f..142719b 100644 --- a/hcu/osworld/agents.py +++ b/hcu/osworld/agents.py @@ -16,8 +16,15 @@ from hcu.osworld.guest import ( GUEST_CLAUDE_MCP_ADAPTER, + GUEST_GUARD_LOG, GUEST_MCP, ) +from hcu.osworld.guide_ablation import ( + ABLATION_ENV, + GUARD_LOG_ENV, + ablation_value, + parse_ablation, +) # Per-agent facts the runner needs: the file under artifacts/, the gateway @@ -116,6 +123,33 @@ def public_tools(channels: str) -> tuple[str, ...]: return CHANNEL_TOOLS[channels] +def mcp_server_env(channels: str = "", ablate: tuple[str, ...] = ()) -> dict[str, str]: + """Environment of the computer-use MCP server. + + A variable is set only when it changes the server, so the client config + of a run without a channel restriction or ablation has no MCP env. With + ``guard`` ablated the server requires a log for the guard decisions; the + runner collects it from the guest. + """ + public_tools(channels) + if tuple(ablate) != parse_ablation(ablate): + raise ValueError(f"ablate must be in canonical order, got {ablate!r}") + env = {"COMPUTER_USE_CHANNELS": channels} if channels else {} + if ablate: + env[ABLATION_ENV] = ablation_value(ablate) + if "guard" in ablate: + env[GUARD_LOG_ENV] = GUEST_GUARD_LOG + return env + + +def _toml_env_table(env: dict[str, str]) -> str: + if not env: + return "" + return "\n[mcp_servers.computer_use.env]\n" + "".join( + f"{key} = {_toml_string(value)}\n" for key, value in env.items() + ) + + def _toml_string(value: str) -> str: # JSON's UTF-16 surrogate escapes are not valid TOML Unicode escapes. return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") @@ -130,6 +164,7 @@ def codex_config( base_instructions: str, mcp_command: str, channels: str = "", + ablate: tuple[str, ...] = (), developer_instructions: str | None = None, execution_context: Literal["standalone", "isolated-guest"] = "standalone", code_mode: bool = False, @@ -155,12 +190,7 @@ def codex_config( "execution_context must be standalone or isolated-guest, " f"got {execution_context!r}" ) - public_tools(channels) - mcp_env = ( - f'\n[mcp_servers.computer_use.env]\nCOMPUTER_USE_CHANNELS = {_toml_string(channels)}\n' - if channels - else "" - ) + mcp_env = _toml_env_table(mcp_server_env(channels, ablate)) disabled_system_skills = "\n\n".join( ( "[[skills.config]]\n" @@ -281,17 +311,18 @@ def claude_mcp_config( adapter: str = GUEST_CLAUDE_MCP_ADAPTER, mcp_command: str = GUEST_MCP, channels: str = "", + ablate: tuple[str, ...] = (), python_executable: str = "/usr/bin/python3", ) -> str: """Claude Code MCP configuration that runs the MCP behind the adapter.""" - public_tools(channels) + env = mcp_server_env(channels, ablate) server: dict = { "type": "stdio", "command": python_executable, "args": [adapter, mcp_command, "mcp"], } - if channels: - server["env"] = {"COMPUTER_USE_CHANNELS": channels} + if env: + server["env"] = env return json.dumps({"mcpServers": {"computer_use": server}}, indent=2) + "\n" @@ -343,6 +374,7 @@ def gemini_settings( effort: str, mcp_command: str = GUEST_MCP, channels: str = "", + ablate: tuple[str, ...] = (), ) -> str: """Gemini CLI ``settings.json`` with ambient features off and the MCP allowed. @@ -351,7 +383,7 @@ def gemini_settings( loaded as context, and it carries the profile; directory discovery is off. ``effort`` is a Gemini thinking level, or ``native`` for the model default. """ - public_tools(channels) + env = mcp_server_env(channels, ablate) if effort != "native" and effort not in GEMINI_THINKING_LEVELS: raise ValueError( "Gemini CLI effort must be native or a thinking level (" @@ -363,8 +395,8 @@ def gemini_settings( "timeout": 180000, "trust": False, } - if channels: - server["env"] = {"COMPUTER_USE_CHANNELS": channels} + if env: + server["env"] = env settings = { # The gateway enforces the model-call budget; the CLI has no turn cap. "model": {"name": model, "maxSessionTurns": -1}, @@ -388,7 +420,9 @@ def gemini_settings( "mcp": {"allowed": [GEMINI_MCP_SERVER]}, "security": { "auth": {"selectedType": "gemini-api-key", "enforcedType": "gemini-api-key"}, - "environmentVariableRedaction": {"allowed": ["COMPUTER_USE_CHANNELS"]}, + "environmentVariableRedaction": {"allowed": [ + "COMPUTER_USE_CHANNELS", *(key for key in env if key != "COMPUTER_USE_CHANNELS"), + ]}, }, "context": { "fileName": "GEMINI.md", @@ -424,19 +458,15 @@ def grok_config( effort: str, mcp_command: str = GUEST_MCP, channels: str = "", + ablate: tuple[str, ...] = (), ) -> str: """Grok Build ``config.toml`` bound to the task gateway with ambient features off. The model is registered under the key ``gateway`` so the CLI never contacts x.ai directly; the gateway URL already carries the task path. """ - public_tools(channels) q = _toml_string - mcp_env = ( - f"\n[mcp_servers.computer_use.env]\nCOMPUTER_USE_CHANNELS = {q(channels)}\n" - if channels - else "" - ) + mcp_env = _toml_env_table(mcp_server_env(channels, ablate)) off = lambda *keys: "\n".join(f"{key} = false" for key in keys) # noqa: E731 return f"""\ [cli] diff --git a/hcu/osworld/config.py b/hcu/osworld/config.py index a7142c3..4c7f488 100644 --- a/hcu/osworld/config.py +++ b/hcu/osworld/config.py @@ -12,7 +12,13 @@ from pathlib import Path import sys -from hcu.paths import DEFAULT_ASSET_CACHE, DEFAULT_RESULTS_DIR +from hcu.osworld.guide_ablation import ablation_value, parse_ablation +from hcu.paths import ( + DEFAULT_ASSET_CACHE, + DEFAULT_BASE_INSTRUCTIONS_FILE, + DEFAULT_INSTRUCTIONS_FILE, + DEFAULT_RESULTS_DIR, +) # Config key -> run_task flag, for options passed through unchanged. VALUE_OPTIONS = { @@ -51,8 +57,14 @@ "token_log", "guest_image", "artifact_dir", "agent_binary", "mcp_binary", "instructions_file", "base_instructions_file", "asset_cache", "results_dir", ) +# ``ablate`` is a comma-separated string or a list; the loader stores it as a +# list in canonical order and passes it on as one --ablate value. +ABLATION_KEY = "ablate" QUEUE_KEYS = ("run_id", "tasks", "results_dir", "max_active_tasks", "launch_stagger_seconds") -CONFIG_KEYS = frozenset(VALUE_OPTIONS) | set(SWITCHES) | set(TASK_OPTIONS) | set(QUEUE_KEYS) +CONFIG_KEYS = ( + frozenset(VALUE_OPTIONS) | set(SWITCHES) | set(TASK_OPTIONS) | set(QUEUE_KEYS) + | {ABLATION_KEY} +) TASK_KEYS = frozenset(TASK_OPTIONS) | {"id", "stratum"} @@ -65,6 +77,37 @@ def single_path_component(value: object, label: str) -> str: return value +def ablation_input_error( + ablate: tuple[str, ...], + *, + instructions_file: str | Path | None, + base_instructions_file: str | Path | None, + code_mode: bool, +) -> str | None: + """Why the selected guides cannot run under ``ablate``, or None. + + An ablation run uses copies of the default guides without the passages + that describe the removed mechanisms. Other guides, no guide, and the + code-mode add-on may still describe them, so they are refused. + """ + if not ablate: + return None + if code_mode: + return "--ablate cannot be combined with --code-mode" + for label, value, default in ( + ("instructions file", instructions_file, DEFAULT_INSTRUCTIONS_FILE), + ("base instructions file", base_instructions_file, DEFAULT_BASE_INSTRUCTIONS_FILE), + ): + if value is None: + continue + if not value or Path(value).expanduser().resolve() != default.resolve(): + return ( + f"--ablate uses ablated copies of the default guides; " + f"the {label} must be {default}, got {str(value)!r}" + ) + return None + + def load_config(path: Path) -> dict: """Read, validate and normalize a run configuration file.""" path = path.expanduser().resolve() @@ -93,6 +136,8 @@ def load_config(path: Path) -> dict: if len({task["id"] for task in tasks}) != len(tasks): raise ValueError("task ids must be unique") config["tasks"] = tasks + if ABLATION_KEY in config: + config[ABLATION_KEY] = list(parse_ablation(config[ABLATION_KEY])) if int(config["max_active_tasks"]) < 1: raise ValueError("max_active_tasks must be at least 1") config.setdefault("results_dir", str(DEFAULT_RESULTS_DIR)) @@ -101,6 +146,14 @@ def load_config(path: Path) -> dict: # An empty instructions_file disables the profile; keep it empty. if config.get(key): config[key] = str((path.parent / Path(config[key]).expanduser()).resolve()) + error = ablation_input_error( + tuple(config.get(ABLATION_KEY) or ()), + instructions_file=config.get("instructions_file"), + base_instructions_file=config.get("base_instructions_file"), + code_mode=bool(config.get("code_mode")), + ) + if error: + raise ValueError(error) # The gateway appends here; the runner reads budget and usage from it. config.setdefault( "token_log", str(Path(config["results_dir"]) / config["run_id"] / "gateway-tokens.jsonl") @@ -127,5 +180,7 @@ def runner_command(config: dict, task: dict, result_dir: Path, run_started_at: s value = task.get(key, config.get(key)) if value is not None: command += [flag, str(value)] + if config.get(ABLATION_KEY): + command += ["--ablate", ablation_value(tuple(config[ABLATION_KEY]))] command += [flag for key, flag in SWITCHES.items() if config.get(key)] return command diff --git a/hcu/osworld/guest.py b/hcu/osworld/guest.py index d85b8e5..db8c96b 100644 --- a/hcu/osworld/guest.py +++ b/hcu/osworld/guest.py @@ -51,6 +51,9 @@ GUEST_FINAL = f"{GUEST_TASK_DIR}/final.txt" GUEST_DONE = f"{GUEST_TASK_DIR}/done.json" GUEST_LAUNCHER_STATUS = f"{GUEST_TASK_DIR}/launcher-status.json" +# Guard decisions the MCP server appends when the guard is ablated; it +# spans all turns of the task, so turn cleanup leaves it in place. +GUEST_GUARD_LOG = f"{GUEST_TASK_DIR}/guard.jsonl" GUEST_GEMINI_ARCHIVE = f"{GUEST_TASK_DIR}/gemini-cli.tgz" diff --git a/hcu/osworld/guest_lifecycle.py b/hcu/osworld/guest_lifecycle.py index 57b53a5..4be4c0c 100644 --- a/hcu/osworld/guest_lifecycle.py +++ b/hcu/osworld/guest_lifecycle.py @@ -13,8 +13,10 @@ GUEST_GEMINI_CLI, GUEST_GEMINI_NODE, GUEST_GEMINI_PACKAGE, + GUEST_GUARD_LOG, GUEST_MCP, request_execute, + save_guest_artifact, transient_workload_command, ) from hcu.osworld.runtime import ( @@ -193,6 +195,9 @@ def prepare_guest(run: TaskRun, env: DesktopEnv) -> str: timeout=180, ) meta["mcp_probe"] = mcp_probe.get("output", "").strip() + if "guard" in run.ablate: + # The guard log records the agent's run, not the preflight probe. + request_execute(base_url, ["rm", "-f", GUEST_GUARD_LOG], timeout=30) initial_wait_remaining = max( 0.0, 60.0 - (time.monotonic() - reset_finished_at), @@ -206,8 +211,33 @@ def prepare_guest(run: TaskRun, env: DesktopEnv) -> str: return base_url +GUARD_LOG_FILE = "guard.jsonl" + + +def save_guard_log(run: TaskRun, env: DesktopEnv) -> None: + """Copy the guest's guard log into the result directory. + + The MCP server writes it only with ``guard`` ablated. ``meta.json`` + names the copy, or holds None when the guest has no log. + """ + try: + data = save_guest_artifact( + env.setup_controller.http_server, + GUEST_GUARD_LOG, + run.result_dir / GUARD_LOG_FILE, + ) + run.meta["guard_log"] = GUARD_LOG_FILE if data is not None else None + except Exception as exc: + logging.exception("Failed to copy the guard log") + run.meta["guard_log"] = None + run.meta["guard_log_error"] = f"{type(exc).__name__}: {exc}" + run.write_meta(ensure_ascii=False) + + def close_guest(run: TaskRun, env: DesktopEnv) -> None: - """Stop recording, then remove or retain the task container.""" + """Copy the guard log, stop recording, then remove or retain the container.""" + if "guard" in run.ablate: + save_guard_log(run, env) if run.recording_started: try: env.controller.end_recording( diff --git a/hcu/osworld/guide_ablation.py b/hcu/osworld/guide_ablation.py new file mode 100644 index 0000000..b9073cd --- /dev/null +++ b/hcu/osworld/guide_ablation.py @@ -0,0 +1,209 @@ +"""Interaction guides for HCU mechanism ablations. + +``COMPUTER_USE_ABLATE`` removes HCU mechanisms from the computer-use server +(see ``computer-use/internal/contract/ablation.go``). The guides given to the +agent must not describe a removed mechanism either, so this module deletes +the passages that describe it and changes nothing else. + +The passages are listed here as exact text, not marked inside the guides: +the guides are model input, and markers in them would change the text of +every run, including runs without an ablation. Each passage must occur +exactly once in its file. Without ablations the guides are returned +unchanged. + +Usage: python -m hcu.osworld.guide_ablation ABLATE OUTPUT_DIR [INSTRUCTIONS_DIR] +writes the ablated ``base.md`` and ``hybrid.md`` to OUTPUT_DIR. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ABLATION_ENV = "COMPUTER_USE_ABLATE" +GUARD_LOG_ENV = "COMPUTER_USE_GUARD_LOG" +# Canonical order; the server accepts the same names. +ABLATION_NAMES = ("batching", "wait", "diff", "guard") + +_PLAN_FROM_ONE_READ = """\ +- Plan from one read. After `observe` or `find`, put every action you can + already determine for that window into one `act`, while later refs and + focus remain valid. Split when the next action needs a new observation. +""" + +_ACT_REPORT = """\ +- Every `act` result starts with what happened, then a diff: + +```text +effect: r58 checkbox "Interface" checked=false→true +windows: opened w5 "Insert Sheet" (dialog, focused) +``` + + `effect` gives each target's changed fields, `no visible change`, or + `unknown` with the reason. `(only state flags changed in this window)` means + no page, content, or window changed; if the control should have switched + something, the application did not respond. `windows` names windows that opened, closed, or + took focus, with a `wN` you can use at once. When these lines already show + the result, do not read or wait again. Continue from the diff for other + changes; read again only when the next decision needs more. +""" + +_WAIT_STEPS = """\ +- One `act` can mix actions and `wait` steps; the first failed step stops + the batch. Add a `wait` only when a later step needs a state that may + arrive after the action returns: a slow dialog, a menu item, a value + computed later. A checkbox that shows `checked=false→true`, or a dialog + listed as `opened`, needs no wait. +- Wait for a known state with `window`, `query`, `ref`, or `text`. When you + know the UI will change but not what it becomes, such as the exact title of + the next dialog, use `{"type":"wait","settle":true}`. It waits until the UI + changes and then stops changing, names what changed, and fails with "no + change" only when nothing changed. Keys after it go to the window that took + focus. Give a slow operation a longer `timeout_ms`. +""" + +# The examples batch several actions with wait steps between them. +_ACT_EXAMPLES = """ +```json +{"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"wait","window":"Export"},{"type":"type","text":"q3.csv"},{"type":"keypress","keys":["ENTER"]},{"type":"wait","window":"Export","gone":true}]} +{"mode":"act","actions":[{"type":"click","ref":"r7"},{"type":"wait","query":{"role":"menu item","name_contains":"Paragraph"}},{"type":"click","ref":"r9"}]} +``` +""" + +_SELECT_READBACK = """ The result reports what is now + selected; check it before applying the command.""" + +_WAIT_TIMEOUT = """\ +- `wait_timeout`: the batch stopped at that `wait`; the steps before it ran + and none after it. The message says what was seen instead, or that the + state could not be read. Read the returned state and revise the plan; do + not resend the same batch. +""" + +_WINDOW_CHANGED_READ = """ Read the + returned state and send only what remains.""" + +_APP_STARTING_WAIT = """if the application is still starting, wait once for + its window with `{"mode":"act","actions":[{"type":"wait","window":"..."}]}`; + otherwise """ + +# Removed passages per ablation, as (file name, exact text). A passage may +# belong to several ablations; overlapping passages are removed once. +PASSAGES: dict[str, tuple[tuple[str, str], ...]] = { + "batching": ( + ("hybrid.md", _PLAN_FROM_ONE_READ), + ("hybrid.md", _ACT_EXAMPLES), + ), + "wait": ( + ("hybrid.md", " or wait again"), + ("hybrid.md", _WAIT_STEPS), + ("hybrid.md", _ACT_EXAMPLES), + ("hybrid.md", _WAIT_TIMEOUT), + ("hybrid.md", _APP_STARTING_WAIT), + ("hybrid.md", " or a timed-out wait"), + ), + "diff": ( + ("hybrid.md", _ACT_REPORT), + ("hybrid.md", _SELECT_READBACK), + ("hybrid.md", _WINDOW_CHANGED_READ), + ), + "guard": (), +} + + +def parse_ablation(value: str | list[str] | tuple[str, ...] | None) -> tuple[str, ...]: + """Return the ablations in canonical order. + + ``value`` is a comma-separated string, as in ``COMPUTER_USE_ABLATE``, or + a list of names. Unknown and repeated names raise ValueError: a typo must + not run the complete HCU under an ablation label. + """ + if value is None: + return () + parts = value.split(",") if isinstance(value, str) else list(value) + names: list[str] = [] + for part in parts: + if not isinstance(part, str): + raise ValueError(f"ablation names must be strings, got {part!r}") + name = part.strip() + if not name: + if isinstance(value, str) and not value.strip(): + continue + raise ValueError(f"empty ablation name in {value!r}") + if name not in ABLATION_NAMES: + raise ValueError( + f"unknown ablation {name!r}; use {', '.join(ABLATION_NAMES)}" + ) + if name in names: + raise ValueError(f"ablation {name!r} is listed twice") + names.append(name) + return tuple(name for name in ABLATION_NAMES if name in names) + + +def ablation_value(ablations: tuple[str, ...]) -> str: + """The COMPUTER_USE_ABLATE value for parsed ablations; empty for none.""" + return ",".join(ablations) + + +def ablation_label(ablations: tuple[str, ...]) -> str: + """A file-name-safe label: ``full`` or ``no-batching-wait``.""" + return "no-" + "-".join(ablations) if ablations else "full" + + +def ablated_guide(text: str, file_name: str, ablations: tuple[str, ...]) -> str: + """Return ``text`` without the passages of ``ablations`` for ``file_name``.""" + ranges: list[tuple[int, int]] = [] + for name in ablations: + for owner, passage in PASSAGES[name]: + if owner != file_name: + continue + start = text.find(passage) + if start < 0 or text.find(passage, start + 1) >= 0: + raise ValueError( + f"{file_name}: the {name} passage must occur exactly once: " + f"{passage[:60]!r}" + ) + ranges.append((start, start + len(passage))) + kept, position = [], 0 + for start, end in sorted(ranges): + if start > position: + kept.append(text[position:start]) + position = max(position, end) + kept.append(text[position:]) + return "".join(kept) + + +def write_ablated_guides( + instructions_dir: Path, output_dir: Path, ablations: tuple[str, ...] +) -> dict[str, Path]: + """Write the ablated base.md and hybrid.md; return their paths by name.""" + output_dir.mkdir(parents=True, exist_ok=True) + written = {} + for file_name in ("base.md", "hybrid.md"): + source = (instructions_dir / file_name).read_text(encoding="utf-8") + target = output_dir / file_name + target.write_text(ablated_guide(source, file_name, ablations), encoding="utf-8") + written[file_name] = target + return written + + +def main(argv: list[str]) -> int: + if len(argv) not in (2, 3): + print(__doc__, file=sys.stderr) + return 2 + instructions_dir = ( + Path(argv[2]) if len(argv) == 3 + else Path(__file__).resolve().parents[2] / "instructions" + ) + try: + ablations = parse_ablation(argv[0]) + except ValueError as error: + print(error, file=sys.stderr) + return 2 + for path in write_ablated_guides(instructions_dir, Path(argv[1]), ablations).values(): + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/hcu/osworld/phases.py b/hcu/osworld/phases.py index 4d881dd..785d70f 100644 --- a/hcu/osworld/phases.py +++ b/hcu/osworld/phases.py @@ -259,6 +259,7 @@ def write_result(run: TaskRun, status: str, **fields) -> dict: "agent": args.agent, "model": args.model, "effort": args.effort, + "ablate": list(getattr(args, "ablate", ())), "status": status, "score": None, **fields, diff --git a/hcu/osworld/run_task.py b/hcu/osworld/run_task.py index 5163dbc..724a010 100644 --- a/hcu/osworld/run_task.py +++ b/hcu/osworld/run_task.py @@ -21,6 +21,8 @@ anchor_path_args, ) from hcu.osworld.agents import AGENT_PROFILES, AGENTS +from hcu.osworld.config import ablation_input_error +from hcu.osworld.guide_ablation import ABLATION_NAMES, parse_ablation from hcu.osworld.guest_lifecycle import close_guest, prepare_guest, start_guest from hcu.osworld.phases import finish_run, record_failure, run_phases from hcu.osworld.preflight import docker_runtime @@ -115,6 +117,15 @@ def parse_args() -> argparse.Namespace: "channel (gui,keyboard); empty exposes both." ), ) + parser.add_argument( + "--ablate", + default="", + help=( + "Comma-separated HCU mechanisms to remove (" + + ", ".join(ABLATION_NAMES) + + "); the agent gets the default guides without their passages." + ), + ) parser.add_argument( "--code-mode", action="store_true", @@ -210,6 +221,18 @@ def parse_args() -> argparse.Namespace: args = parser.parse_args() if args.code_mode and args.agent != "codex": parser.error("--code-mode applies only to --agent codex") + try: + args.ablate = parse_ablation(args.ablate) + except ValueError as error: + parser.error(f"--ablate: {error}") + error = ablation_input_error( + args.ablate, + instructions_file=args.instructions_file, + base_instructions_file=args.base_instructions_file, + code_mode=args.code_mode, + ) + if error: + parser.error(error) profile = AGENT_PROFILES[args.agent] args.model = args.model or profile["default_model"] args.effort = args.effort or profile["default_effort"] diff --git a/hcu/osworld/staging.py b/hcu/osworld/staging.py index 33259c2..40e0f33 100644 --- a/hcu/osworld/staging.py +++ b/hcu/osworld/staging.py @@ -13,6 +13,7 @@ from hcu.paths import ( CODE_MODE_INSTRUCTIONS_FILE, + DEFAULT_INSTRUCTIONS_FILE, DEFAULT_RESULTS_DIR, OSWORLD_ROOT, ) @@ -29,6 +30,7 @@ public_tools, task_gateway_base, ) +from hcu.osworld.guide_ablation import write_ablated_guides from hcu.osworld.launcher import launcher_script from hcu.gateway.common import TASK_AFFINITY_PREFIX from hcu.osworld.guest import ( @@ -114,6 +116,11 @@ class TaskRun: def profile(self) -> dict: return AGENT_PROFILES[self.args.agent] + @property + def ablate(self) -> tuple[str, ...]: + """Removed HCU mechanisms in canonical order; empty for full HCU.""" + return tuple(getattr(self.args, "ablate", ())) + @property def config_path(self) -> Path: return self.result_dir / { @@ -196,6 +203,14 @@ def prepare_run(args: argparse.Namespace) -> TaskRun: base_instructions_file = Path( args.base_instructions_file ).resolve() + ablate = tuple(getattr(args, "ablate", ())) + if ablate: + # run_task has checked that the default guides are selected; the + # agent gets copies without the passages of the removed mechanisms. + guides = write_ablated_guides( + DEFAULT_INSTRUCTIONS_FILE.parent, result_dir / "ablated-instructions", ablate, + ) + instructions_file, base_instructions_file = guides["hybrid.md"], guides["base.md"] required_files = [agent_binary, mcp_binary, base_instructions_file] if instructions_file is not None: required_files.append(instructions_file) @@ -278,14 +293,16 @@ def stage_local_files(run: TaskRun) -> None: base_instructions=GUEST_BASE_INSTRUCTIONS, mcp_command=GUEST_MCP, channels=args.channels, + ablate=run.ablate, execution_context="isolated-guest", code_mode=getattr(args, "code_mode", False), ) elif args.agent == "claude-code": - config = claude_mcp_config(channels=args.channels) + config = claude_mcp_config(channels=args.channels, ablate=run.ablate) elif args.agent == "gemini-cli": config = gemini_settings( - model=args.model, effort=args.effort, channels=args.channels + model=args.model, effort=args.effort, channels=args.channels, + ablate=run.ablate, ) else: config = grok_config( @@ -293,6 +310,7 @@ def stage_local_files(run: TaskRun) -> None: base_url=run.agent_endpoint + "/v1", effort=args.effort, channels=args.channels, + ablate=run.ablate, ) write_text(run.config_path, config) write_text( @@ -379,6 +397,7 @@ def initial_meta(run: TaskRun, docker_runtime: dict) -> dict: ), "mcp_binary": str(run.mcp_binary), "mcp_channels": args.channels or None, + "ablate": list(run.ablate), "code_mode": bool(getattr(args, "code_mode", False)), "mcp_public_tools": run.mcp_public_tools, "instructions_file": str(run.instructions_file) if run.instructions_file else None, diff --git a/tests/test_ablation_switches.py b/tests/test_ablation_switches.py new file mode 100644 index 0000000..cc7224c --- /dev/null +++ b/tests/test_ablation_switches.py @@ -0,0 +1,334 @@ +"""The runner side of the HCU mechanism ablations, without a guest or model calls.""" + +import argparse +import importlib.util +import json +from pathlib import Path +import sys +from types import ModuleType +from unittest.mock import Mock + +import pytest + +from hcu.osworld import agents +from hcu.osworld.config import load_config, runner_command +from hcu.osworld.guest import GUEST_GUARD_LOG +from hcu.osworld.guide_ablation import ABLATION_NAMES, ablated_guide +from hcu.osworld.runtime import public_mcp_probe_config + +ROOT = Path(__file__).resolve().parents[1] +GROUPS = { + "full": [], + "no-batching": ["batching"], + "no-wait": ["wait"], + "no-diff": ["diff"], + "no-guard": ["guard"], + "no-all4": list(ABLATION_NAMES), +} + + +def write_config(path: Path, **values) -> Path: + config = {"tasks": ["a"], "results_dir": str(path.parent / "runs"), **values} + path.write_text(json.dumps(config), encoding="utf-8") + return path + + +# --- configuration and command line ------------------------------------------------- + + +def test_config_ablate_is_canonical_and_reaches_the_runner(tmp_path): + config = load_config(write_config(tmp_path / "run.json", ablate="guard, batching")) + assert config["ablate"] == ["batching", "guard"] + command = runner_command(config, config["tasks"][0], tmp_path / "a") + assert command[command.index("--ablate") + 1] == "batching,guard" + listed = load_config(write_config(tmp_path / "list.json", ablate=["diff", "wait"])) + assert listed["ablate"] == ["wait", "diff"] + + +@pytest.mark.parametrize("values", [{}, {"ablate": []}, {"ablate": ""}]) +def test_config_without_ablation_passes_no_flag(tmp_path, values): + config = load_config(write_config(tmp_path / "run.json", **values)) + assert "--ablate" not in runner_command(config, config["tasks"][0], tmp_path / "a") + # A config without the key keeps its saved form, so existing runs resume. + assert ("ablate" in config) == ("ablate" in values) + + +@pytest.mark.parametrize("value", ["gaurd", "wait,wait", "wait,,diff", ["wait", 1]]) +def test_config_rejects_invalid_ablation(tmp_path, value): + with pytest.raises(ValueError): + load_config(write_config(tmp_path / "run.json", ablate=value)) + + +@pytest.mark.parametrize( + "values, message", + [ + ({"instructions_file": "custom.md"}, "instructions file"), + ({"instructions_file": ""}, "instructions file"), + ({"base_instructions_file": "custom-base.md"}, "base instructions file"), + ({"code_mode": True}, "code-mode"), + ], +) +def test_config_refuses_guides_that_may_describe_removed_mechanisms(tmp_path, values, message): + with pytest.raises(ValueError, match=message): + load_config(write_config(tmp_path / "run.json", ablate="wait", **values)) + # Without an ablation the same guide choice stays valid. + load_config(write_config(tmp_path / "plain.json", **values)) + + +def test_config_accepts_the_default_guides_named_explicitly(tmp_path): + config = load_config(write_config( + tmp_path / "run.json", ablate="diff", + instructions_file=str(ROOT / "instructions/hybrid.md"), + base_instructions_file=str(ROOT / "instructions/base.md"), + )) + assert config["ablate"] == ["diff"] + + +def test_ablation_example_configs_load_with_their_group(): + paths = sorted((ROOT / "configs/ablation").glob("*.example.json")) + assert {path.name for path in paths} == { + f"osworld-v1-codex-{label}.example.json" for label in GROUPS + } + reference = load_config(ROOT / "configs/osworld-v1-codex-queue.example.json") + for path in paths: + label = path.name.removeprefix("osworld-v1-codex-").removesuffix(".example.json") + config = load_config(path) + assert config["ablate"] == GROUPS[label] + for key in ("benchmark_version", "agent", "model", "effort", "max_model_calls", "tasks"): + assert config[key] == reference[key], (path, key) + command = runner_command(config, config["tasks"][0], Path("/unused")) + assert ("--ablate" in command) == bool(GROUPS[label]) + + +# --- harness MCP environments -------------------------------------------------------- + + +def render(agent: str, **kwargs) -> str: + if agent == "codex": + return agents.codex_config( + model="m", effort="xhigh", base_url="http://gateway/v1", codex_home="/codex", + base_instructions="/base.md", mcp_command="/computer-use", **kwargs, + ) + if agent == "claude-code": + return agents.claude_mcp_config(**kwargs) + if agent == "gemini-cli": + return agents.gemini_settings(model="m", effort="high", **kwargs) + return agents.grok_config(model="m", base_url="http://gateway/v1", effort="xhigh", **kwargs) + + +HARNESSES = ("codex", "claude-code", "gemini-cli", "grok-build") + + +@pytest.mark.parametrize("agent", HARNESSES) +@pytest.mark.parametrize("channels", ["", "a11y,keyboard", "gui,keyboard"]) +def test_full_hcu_renders_the_same_harness_config(agent, channels): + plain = render(agent, channels=channels) + assert render(agent, channels=channels, ablate=()) == plain + assert "COMPUTER_USE_ABLATE" not in plain and "COMPUTER_USE_GUARD_LOG" not in plain + if agent == "gemini-cli": + redaction = json.loads(plain)["security"]["environmentVariableRedaction"] + assert redaction == {"allowed": ["COMPUTER_USE_CHANNELS"]} + + +@pytest.mark.parametrize("agent", HARNESSES) +@pytest.mark.parametrize( + "ablate, expected", + [ + (("wait",), {"COMPUTER_USE_ABLATE": "wait"}), + (("batching", "diff"), {"COMPUTER_USE_ABLATE": "batching,diff"}), + (ABLATION_NAMES, { + "COMPUTER_USE_ABLATE": "batching,wait,diff,guard", + "COMPUTER_USE_GUARD_LOG": GUEST_GUARD_LOG, + }), + (("guard",), { + "COMPUTER_USE_ABLATE": "guard", "COMPUTER_USE_GUARD_LOG": GUEST_GUARD_LOG, + }), + ], +) +@pytest.mark.parametrize("channels", ["", "gui,keyboard"]) +def test_ablation_reaches_every_harness_mcp_env(agent, ablate, expected, channels): + pytest.importorskip("tomllib") + config = render(agent, channels=channels, ablate=ablate) + want = {**({"COMPUTER_USE_CHANNELS": channels} if channels else {}), **expected} + # The preflight probe starts the server with the env it reads from the config. + spec = public_mcp_probe_config(config, agent, list(agents.public_tools(channels))) + assert spec["env"] == want + if agent == "gemini-cli": + allowed = json.loads(config)["security"]["environmentVariableRedaction"]["allowed"] + assert set(want) <= set(allowed) + + +def test_harness_rejects_ablations_out_of_canonical_order(): + with pytest.raises(ValueError, match="canonical"): + agents.mcp_server_env("", ("guard", "wait")) + with pytest.raises(ValueError): + agents.mcp_server_env("", ("typo",)) + + +# --- runner modules loaded without OSWorld ------------------------------------------- + + +@pytest.fixture +def runner(monkeypatch): + """Fresh runner modules with the OSWorld imports replaced by test doubles.""" + stubs = { + "desktop_env": {}, + "desktop_env.cache": {"run_scoped_cache_dir": None}, + "desktop_env.desktop_env": { + "DesktopEnv": Mock(side_effect=AssertionError("No guest may be constructed")), + }, + "desktop_env.file_source": {"configure": None}, + "hcu.osworld.preflight": {"docker_runtime": None}, + "hcu.osworld.task_loader": {"load_task_config": None}, + } + for name, attributes in stubs.items(): + module = ModuleType(name) + vars(module).update(attributes) + monkeypatch.setitem(sys.modules, name, module) + modules = {} + for name in ("staging", "guest_lifecycle", "phases", "run_task"): + spec = importlib.util.spec_from_file_location( + f"hcu.osworld.{name}", ROOT / f"hcu/osworld/{name}.py", + ) + modules[name] = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, modules[name]) + spec.loader.exec_module(modules[name]) + staging = modules["staging"] + monkeypatch.setattr(staging, "load_task_config", lambda *a, **kw: {"instruction": "offline"}) + monkeypatch.setattr(staging, "resolve_guest_image", lambda *a: "/unused/image.qcow2") + return modules + + +def parse(runner, monkeypatch, *flags): + monkeypatch.setattr(sys, "argv", ["run_task", "--task-id", "t", *flags]) + return runner["run_task"].parse_args() + + +def test_run_task_parses_ablate_in_canonical_order(runner, monkeypatch): + assert parse(runner, monkeypatch).ablate == () + assert parse(runner, monkeypatch, "--ablate", "guard,wait").ablate == ("wait", "guard") + + +@pytest.mark.parametrize( + "flags", + [ + ["--ablate", "batching,typo"], + ["--ablate", "diff,diff"], + ["--ablate", "wait", "--instructions-file", ""], + ["--ablate", "wait", "--instructions-file", "instructions/ax-only.md"], + ["--ablate", "wait", "--code-mode"], + ], +) +def test_run_task_rejects_invalid_ablation_before_any_work(runner, monkeypatch, flags): + with pytest.raises(SystemExit) as exit_info: + parse(runner, monkeypatch, *flags) + assert exit_info.value.code == 2 + + +def prepared_run(runner, tmp_path, agent="codex", ablate=()): + artifact = tmp_path / "artifact" + artifact.touch() + profile = agents.AGENT_PROFILES[agent] + args = argparse.Namespace( + task_id="t", agent=agent, model=profile["default_model"], + effort=profile["default_effort"], gateway="http://gateway:8611", + result_dir=str(tmp_path / "task"), token_log=str(tmp_path / "tokens.jsonl"), + max_model_calls=0, artifact_dir=str(tmp_path), agent_binary=str(artifact), + mcp_binary=str(artifact), + instructions_file=str(ROOT / "instructions/hybrid.md"), + base_instructions_file=str(ROOT / "instructions/base.md"), + channels="", code_mode=False, task_domain="", benchmark_version="v1", + run_started_at="2000-01-01T00:00:00+00:00", timezone="UTC", + asset_cache="/unused", offline_assets=False, retain_container=False, ablate=ablate, + ) + if agent == "gemini-cli": + (tmp_path / "node").touch() + staging = runner["staging"] + run = staging.prepare_run(args) + staging.stage_local_files(run) + run.meta = staging.initial_meta(run, {}) + return run + + +def test_full_hcu_uses_the_default_guides_unchanged(runner, tmp_path): + run = prepared_run(runner, tmp_path) + assert run.instructions_file == ROOT / "instructions/hybrid.md" + assert run.base_instructions_file == ROOT / "instructions/base.md" + assert not (run.result_dir / "ablated-instructions").exists() + assert run.meta["ablate"] == [] + + +@pytest.mark.parametrize("agent", HARNESSES) +def test_ablation_run_gives_the_agent_ablated_guides(runner, tmp_path, agent): + ablate = ("wait", "diff") + run = prepared_run(runner, tmp_path, agent, ablate) + guides = run.result_dir / "ablated-instructions" + assert run.instructions_file == guides / "hybrid.md" + assert run.base_instructions_file == guides / "base.md" + hybrid = ablated_guide( + (ROOT / "instructions/hybrid.md").read_text(encoding="utf-8"), "hybrid.md", ablate, + ) + assert hybrid != (ROOT / "instructions/hybrid.md").read_text(encoding="utf-8") + assert run.instructions_file.read_text(encoding="utf-8") == hybrid + # Each harness receives the ablated guide through its own channel. + if agent in {"codex", "gemini-cli"}: + assert run.instructions_copy.read_text(encoding="utf-8") == hybrid + else: + assert run.base_instructions_copy.read_text(encoding="utf-8").endswith(hybrid) + assert "COMPUTER_USE_ABLATE" in run.config_path.read_text(encoding="utf-8") + assert run.meta["ablate"] == ["wait", "diff"] + assert run.meta["instructions_file"] == str(guides / "hybrid.md") + + +@pytest.mark.parametrize("ablate", [(), ("batching", "guard")]) +def test_result_records_the_ablation(runner, tmp_path, ablate): + run = prepared_run(runner, tmp_path, ablate=ablate) + result = runner["phases"].write_result(run, "error", error="test") + assert result["ablate"] == list(ablate) + saved = json.loads((run.result_dir / "result.json").read_text(encoding="utf-8")) + assert saved["ablate"] == list(ablate) + assert json.loads((run.result_dir / "meta.json").read_text())["ablate"] == list(ablate) + + +class FakeEnv: + def __init__(self): + self.setup_controller = type("Controller", (), {"http_server": "http://guest:5000"})() + self.closed = False + + def close(self): + self.closed = True + + +@pytest.mark.parametrize("guest_log", [b'{"decision":"skipped"}\n', None]) +def test_guard_ablation_copies_the_guest_guard_log(runner, tmp_path, monkeypatch, guest_log): + lifecycle = runner["guest_lifecycle"] + copied = [] + + def save(base_url, guest_path, local_path): + copied.append((base_url, guest_path)) + if guest_log is not None: + local_path.write_bytes(guest_log) + return guest_log + + monkeypatch.setattr(lifecycle, "save_guest_artifact", save) + run = prepared_run(runner, tmp_path, ablate=("guard",)) + env = FakeEnv() + lifecycle.close_guest(run, env) + assert copied == [("http://guest:5000", GUEST_GUARD_LOG)] + assert env.closed + meta = json.loads((run.result_dir / "meta.json").read_text()) + if guest_log is None: + assert meta["guard_log"] is None + assert not (run.result_dir / "guard.jsonl").exists() + else: + assert meta["guard_log"] == "guard.jsonl" + assert (run.result_dir / "guard.jsonl").read_bytes() == guest_log + + +@pytest.mark.parametrize("ablate", [(), ("wait", "diff")]) +def test_runs_with_the_guard_do_not_read_a_guard_log(runner, tmp_path, monkeypatch, ablate): + lifecycle = runner["guest_lifecycle"] + monkeypatch.setattr(lifecycle, "save_guest_artifact", Mock(side_effect=AssertionError)) + run = prepared_run(runner, tmp_path, ablate=ablate) + lifecycle.close_guest(run, FakeEnv()) + assert "guard_log" not in run.meta From e247b64c50392aac1620462da60910ad3614d152 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:59:20 +0800 Subject: [PATCH 05/13] provider: move guard shadow helpers out of the dispatch modules The semantic refusal fallback joins the other resolution code in element_resolver.py, and the GUI layout log line is built in guard_shadow.py, so the dispatch modules stay near the size limit. Co-Authored-By: Claude Opus 5.5 --- .../native/linux/provider/ax_dispatch.py | 45 +------------------ .../native/linux/provider/element_resolver.py | 41 +++++++++++++++++ .../native/linux/provider/guard_shadow.py | 13 ++++++ .../native/linux/provider/gui_dispatch.py | 28 +++--------- 4 files changed, 61 insertions(+), 66 deletions(-) diff --git a/computer-use/native/linux/provider/ax_dispatch.py b/computer-use/native/linux/provider/ax_dispatch.py index 524c271..be947a9 100644 --- a/computer-use/native/linux/provider/ax_dispatch.py +++ b/computer-use/native/linux/provider/ax_dispatch.py @@ -6,7 +6,6 @@ MAX_DEPTH, MAX_ELEMENTS, OperationFailure, - node_name, node_pid, node_role, process_generation, @@ -28,7 +27,6 @@ from .x11_windows import ( main_window, select_window_by_handle, - relative_frame, window_identity_set, window_bounds, ) @@ -40,8 +38,8 @@ from .guard_shadow import shadow_guard from .element_resolver import ( find_element_at_snapshot_path, - resolve_path, resolve_semantic_target, + shadow_semantic_refusal, ) from .window_resolver import ( perform_ax_window_action, @@ -525,7 +523,7 @@ def _resolve_action_target(app, operation, operation_kind, action_input): operation, ) except OperationFailure as refused: - shadowed = _shadow_semantic_refusal( + shadowed = shadow_semantic_refusal( app, window_index, bounds, semantic_target, refused) if shadowed is None: raise @@ -546,45 +544,6 @@ def _resolve_action_target(app, operation, operation_kind, action_input): return element, element_record, semantic_note, window, bounds -# Identity refusals of a semantic target. The guard ablation acts on the -# snapshot path instead when a node is still there. -_SHADOWED_TARGET_FAILURES = { - "stale_element_ref", "semantic_target_not_found", "semantic_target_ambiguous", -} - - -def _shadow_semantic_refusal(app, window_index, bounds, target, refused): - """Return (node, record) at the target's snapshot path, or None. - - None keeps the refusal: the guard ablation is off, the refusal is not an - identity check, or no live node is left on the path. - """ - if getattr(refused, "failure_code", None) not in _SHADOWED_TARGET_FAILURES: - return None - paths = [target.get("runtimeId") or []] - paths.extend(target.get("runtimeAliases") or []) - for path in paths: - if not path: - continue - node = resolve_path(app, path) - if node is None or state_value(node, Atspi.StateType.DEFUNCT) is True: - continue - if not shadow_guard( - "ax", "target_identity", refused.failure_code, - message=str(refused), selector=target.get("selector"), path=list(path), - live={"role": node_role(node), "name": node_name(node)}, - ): - return None - record = { - "runtimeId": list(path), - "controlType": node_role(node), - "name": node_name(node), - "frame": safe(lambda: relative_frame(node, bounds)), - } - return node, record - return None - - def _resolve_select_end(app, operation, action_input, window, bounds): """Resolve the select range end (to_ref) in the target's window, or None.""" to_target = action_input.get("to_target") diff --git a/computer-use/native/linux/provider/element_resolver.py b/computer-use/native/linux/provider/element_resolver.py index 73bf262..c616ff0 100644 --- a/computer-use/native/linux/provider/element_resolver.py +++ b/computer-use/native/linux/provider/element_resolver.py @@ -7,8 +7,10 @@ DEFAULT_TEXT_LIMIT, OperationFailure, child_at, + node_name, node_role, safe, + state_value, ) from .node_geometry import extents from .primitives import name_fingerprint @@ -539,3 +541,42 @@ def _semantic_resolution_note( ), ) return note + + +# Identity refusals of a semantic target. The guard ablation acts on the +# snapshot path instead when a node is still there. +_SHADOWED_TARGET_FAILURES = { + "stale_element_ref", "semantic_target_not_found", "semantic_target_ambiguous", +} + + +def shadow_semantic_refusal(app, window_index, bounds, target, refused): + """Return (node, record) at the target's snapshot path, or None. + + None keeps the refusal: the guard ablation is off, the refusal is not an + identity check, or no live node is left on the path. + """ + if getattr(refused, "failure_code", None) not in _SHADOWED_TARGET_FAILURES: + return None + paths = [target.get("runtimeId") or []] + paths.extend(target.get("runtimeAliases") or []) + for path in paths: + if not path: + continue + node = resolve_path(app, path) + if node is None or state_value(node, Atspi.StateType.DEFUNCT) is True: + continue + if not shadow_guard( + "ax", "target_identity", refused.failure_code, + message=str(refused), selector=target.get("selector"), path=list(path), + live={"role": node_role(node), "name": node_name(node)}, + ): + return None + record = { + "runtimeId": list(path), + "controlType": node_role(node), + "name": node_name(node), + "frame": safe(lambda: relative_frame(node, bounds)), + } + return node, record + return None diff --git a/computer-use/native/linux/provider/guard_shadow.py b/computer-use/native/linux/provider/guard_shadow.py index d9fe867..ce0a695 100644 --- a/computer-use/native/linux/provider/guard_shadow.py +++ b/computer-use/native/linux/provider/guard_shadow.py @@ -61,3 +61,16 @@ def shadow_guard(channel, guard, reason, **detail): except OSError: pass return True + + +def shadow_layout_guard(reason, frame_id, actions=(), region=None): + """shadow_guard for the GUI layout guard; logs each action's point.""" + points = [] + for action in actions: + point = {"type": action.get("type")} + point.update({key: action[key] for key in ("x", "y", "path") if key in action}) + points.append(point) + detail = {"frame_id": frame_id, "actions": points} + if region is not None: + detail["region"] = region + return shadow_guard("gui", "zoom_layout" if region is not None else "layout", reason, **detail) diff --git a/computer-use/native/linux/provider/gui_dispatch.py b/computer-use/native/linux/provider/gui_dispatch.py index ee4f588..70a36dd 100644 --- a/computer-use/native/linux/provider/gui_dispatch.py +++ b/computer-use/native/linux/provider/gui_dispatch.py @@ -33,7 +33,7 @@ validate_gui_actions, ) from .input_primitives import InputEmissionError -from .guard_shadow import shadow_guard +from .guard_shadow import shadow_layout_guard from .snapshot import capture_gui_snapshot from .desktop_layout import ( desktop_bounds, @@ -287,18 +287,6 @@ def layout_hit(layout, x, y): return None -def _guard_points(actions): - """The action types and coordinates a guard log line records.""" - points = [] - for action in actions: - point = {"type": action.get("type")} - for key in ("x", "y", "path"): - if key in action: - point[key] = action[key] - points.append(point) - return points - - def coordinates_still_valid(reference, current, actions): """Whether every coordinate of actions still lands where it did. @@ -407,9 +395,8 @@ def stale(message, capture): left, top, width, height = geometry["native"] if not region_still_valid( _published_layouts.get(frame_id), capture_layout(current), - (left, top, width, height)) and not shadow_guard( - "gui", "zoom_layout", "windows_over_region_changed", - frame_id=frame_id, region=[left, top, width, height]): + (left, top, width, height)) and not shadow_layout_guard( + "zoom_region_changed", frame_id, region=[left, top, width, height]): return stale( "The windows over the zoom region changed since the frame its " "coordinates came from.", current) @@ -589,9 +576,7 @@ def failure_evidence(fresh=True): if (current_capture.frame_id != frame_id and not coordinates_still_valid( reference_layout, capture_layout(current_capture), validated) - and not shadow_guard( - "gui", "layout", "layout_changed_before_batch", - frame_id=frame_id, actions=_guard_points(validated))): + and not shadow_layout_guard("changed_before_batch", frame_id, validated)): return { "ok": False, "error": "The desktop layout changed since the screenshot used " @@ -633,10 +618,7 @@ def failure_evidence(fresh=True): if (live_frame_id != frame_id and not coordinates_still_valid( reference_layout, live_layout, [action]) - and not shadow_guard( - "gui", "layout", "layout_changed_during_batch", - frame_id=frame_id, index=index, - actions=_guard_points([action]))): + and not shadow_layout_guard("changed_during_batch", frame_id, [action])): error_snapshot, evidence_note = failure_evidence() return { "ok": False, From 9104c71defd374997c87c60918ef2ef5110cc9f6 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:59:20 +0800 Subject: [PATCH 06/13] test: cover the mechanism ablations Go: parsing and startup refusal, model-visible text of every ablation set, valid description examples, schema and decoder agreement under each ablation, the one-input-step rule, wait removal, act results without feedback, and the guard log. Provider: shadow mode logs and lets input through for target identity, batch window, point owner and GUI layout, and changes nothing without the ablation. Co-Authored-By: Claude Opus 5.5 --- .../internal/contract/ablation_test.go | 79 ++++++ .../internal/core/ax_ablation_test.go | 264 ++++++++++++++++++ .../internal/core/schema_consistency_test.go | 18 ++ .../internal/transport/cli_main_test.go | 50 ++++ computer-use/runtime_test.py | 1 + .../runtime_tests/test_guard_shadow.py | 179 ++++++++++++ 6 files changed, 591 insertions(+) create mode 100644 computer-use/internal/contract/ablation_test.go create mode 100644 computer-use/internal/core/ax_ablation_test.go create mode 100644 computer-use/internal/transport/cli_main_test.go create mode 100644 computer-use/runtime_tests/test_guard_shadow.py diff --git a/computer-use/internal/contract/ablation_test.go b/computer-use/internal/contract/ablation_test.go new file mode 100644 index 0000000..5aff611 --- /dev/null +++ b/computer-use/internal/contract/ablation_test.go @@ -0,0 +1,79 @@ +package contract + +import ( + "strings" + "testing" +) + +func TestParseAblations(t *testing.T) { + for raw, want := range map[string]string{ + "": "", + " ": "", + "wait": "wait", + " guard , batching": "batching,guard", + "batching,wait,diff,guard": "batching,wait,diff,guard", + "diff,wait": "wait,diff", + } { + ablated, err := ParseAblations(raw) + if err != nil { + t.Fatalf("%q: %v", raw, err) + } + got := []string{} + for _, name := range AblationNames { + if ablated[name] { + got = append(got, name) + } + } + if strings.Join(got, ",") != want { + t.Fatalf("%q: got %v want %s", raw, got, want) + } + } + for _, raw := range []string{"waits", "wait,", "Wait", "wait,wait", "compact", ","} { + if _, err := ParseAblations(raw); err == nil { + t.Fatalf("%q must be rejected", raw) + } + } +} + +func TestValidateAblationsAtStartup(t *testing.T) { + env := func(values map[string]string) func(string) string { + return func(key string) string { return values[key] } + } + for _, check := range []struct { + goos string + values map[string]string + fails string + }{ + {"linux", map[string]string{}, ""}, + {"linux", map[string]string{AblationEnv: "batching,wait,diff"}, ""}, + {"darwin", map[string]string{AblationEnv: "wait"}, ""}, + {"linux", map[string]string{AblationEnv: "diffs"}, "unknown ablation"}, + {"linux", map[string]string{AblationEnv: "guard"}, GuardLogEnv}, + {"linux", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, ""}, + {"darwin", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, "only by the Linux provider"}, + {"windows", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, "only by the Linux provider"}, + } { + err := ValidateAblations(check.goos, env(check.values)) + switch { + case check.fails == "" && err != nil: + t.Fatalf("%s %v: %v", check.goos, check.values, err) + case check.fails != "" && (err == nil || !strings.Contains(err.Error(), check.fails)): + t.Fatalf("%s %v: got %v, want an error with %q", check.goos, check.values, err, check.fails) + } + } +} + +func TestPublishedTextWithoutAblationIsTheReleaseText(t *testing.T) { + t.Setenv(AblationEnv, "") + if PublishedServerInstructions() != ServerInstructions { + t.Fatal("server instructions changed without an ablation") + } + if publishedAXToolDescription() != axToolDescription { + t.Fatal("AX description changed without an ablation") + } + t.Setenv(AblationEnv, "wait,diff") + if !strings.Contains(PublishedServerInstructions(), "returns controls as rN refs, and acts on them.") || + !strings.Contains(PublishedServerInstructions(), "before input, and outside input makes their refs stale.") { + t.Fatalf("instructions: %s", PublishedServerInstructions()) + } +} diff --git a/computer-use/internal/core/ax_ablation_test.go b/computer-use/internal/core/ax_ablation_test.go new file mode 100644 index 0000000..19d1397 --- /dev/null +++ b/computer-use/internal/core/ax_ablation_test.go @@ -0,0 +1,264 @@ +package core + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +// ablationSets returns every subset of the ablations as a +// COMPUTER_USE_ABLATE value, the empty set first. +func ablationSets() []string { + sets := []string{} + for mask := 0; mask < 1< len(fullDescription) { + t.Errorf("AX description grew to %d bytes from %d", len(description), len(fullDescription)) + } + actions := properties["actions"].(map[string]any) + wantMax := contract.AXBatchMaxActions + if strings.Contains(set, "batching") && strings.Contains(set, "wait") { + wantMax = 1 + } + if actions["maxItems"] != wantMax { + t.Errorf("maxItems %v want %d", actions["maxItems"], wantMax) + } + }) + } +} + +func TestAblatedDescriptionsKeepTheirExamplesValid(t *testing.T) { + example := regexp.MustCompile(`\{"mode":"act".*\}`) + for _, set := range ablationSets() { + t.Run("ablate="+set, func(t *testing.T) { + t.Setenv(contract.AblationEnv, set) + t.Setenv(contract.GuardLogEnv, filepath.Join(t.TempDir(), "guard.jsonl")) + description := findToolDefinition(t, contract.PublicToolAX).Description + call := example.FindString(description) + if call == "" { + t.Fatalf("no act example in %s", description) + } + if _, err := DecodeAXRequest(decodeTestArguments(t, call)); err != nil { + t.Fatalf("example %s: %s", call, err.Message) + } + if argErr := contract.ValidateToolArguments(contract.PublicToolAX, decodeTestArguments(t, call)); argErr != nil { + t.Fatalf("example %s: %s", call, argErr.Message) + } + }) + } +} + +func TestBatchingAblationAllowsOneInputStep(t *testing.T) { + t.Setenv(contract.AblationEnv, contract.AblateBatching) + for _, raw := range []string{ + `{"mode":"act","actions":[{"type":"click","ref":"r1"}]}`, + `{"mode":"act","actions":[{"type":"wait","window":"A"},{"type":"click","ref":"r1"},{"type":"wait","settle":true}]}`, + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"wait","window":"A","gone":true}]}`, + } { + if _, err := DecodeAXRequest(decodeTestArguments(t, raw)); err != nil { + t.Fatalf("%s: %s", raw, err.Message) + } + } + _, err := DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"click","ref":"r1"},{"type":"wait","window":"A"},{"type":"select","ref":"r2"}]}`)) + if err == nil || err.rule != contract.ArgumentRuleOneInputStep || err.path != "actions[2]" { + t.Fatalf("second input step: %#v", err) + } + if !strings.Contains(err.Hint, `{"type":"wait","window":"A"}`) || strings.Contains(err.Hint, "select") { + t.Fatalf("hint should keep the steps before the second input: %s", err.Hint) + } + + t.Setenv(contract.AblationEnv, "batching,wait") + _, err = DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"click","ref":"r1"},{"type":"click","ref":"r2"}]}`)) + if err == nil || err.Message != "act takes one action per call; send the others in later calls" { + t.Fatalf("two actions without waits: %#v", err) + } +} + +func TestBatchingAblationKeepsGUIBatches(t *testing.T) { + t.Setenv(contract.AblationEnv, "batching,wait,diff") + args := decodeTestArguments(t, `{"actions":[{"type":"click","x":1,"y":2},{"type":"type","text":"a"},{"type":"keypress","keys":["ENTER"]}]}`) + if err := contract.ValidateToolArguments(contract.PublicToolGUI, args); err != nil { + t.Fatalf("GUI batch: %s", err.Message) + } +} + +func TestWaitAblationRemovesWaitSteps(t *testing.T) { + t.Setenv(contract.AblationEnv, contract.AblateWait) + _, err := DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"type":"wait","window":"A"}]}`)) + if err == nil || !strings.HasPrefix(err.Message, "type must be one of click, type, keypress, set_value, select, scroll, activate, close;") || + strings.Contains(err.Hint, "wait") || err.path != "actions[0].type" { + t.Fatalf("wait step: %#v", err) + } + _, err = DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"keypress","keys":["ENTER"]}]}`)) + if err == nil || err.rule != contract.ArgumentRuleCloseNotLast || strings.Contains(err.Message+err.Hint, "wait") { + t.Fatalf("input after close: %#v", err) + } + _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"type":"click","ref":"r1","until":{"type":"exists","ref":"r2"}}]}`)) + if err == nil || strings.Contains(err.Message+err.Hint, "wait") { + t.Fatalf("removed until field: %#v", err) + } + if strings.Contains(strings.Join(continuityNotesForUnchanged(t), "\n"), "wait step") { + t.Fatal("the unchanged-snapshot note still suggests a wait step") + } +} + +// continuityNotesForUnchanged returns the notes for a snapshot identical to +// the delivered one. +func continuityNotesForUnchanged(t *testing.T) []string { + t.Helper() + previous := &AppSnapshot{SnapshotID: "s1", WindowHandle: "h1", WindowTitle: "T"} + current := &AppSnapshot{SnapshotID: "s1", WindowHandle: "h1", WindowTitle: "T"} + return continuityNotes(previous, current) +} + +func TestDiffAblationReturnsOnlyExecutionEvidence(t *testing.T) { + s, backend := reportService(t, reportMain()) + _, refs := reportObserve(t, s, "Interface") + t.Setenv(contract.AblationEnv, contract.AblateDiff) + backend.onAction = func(b *reportBackend) { b.checked = true } + result := FinalizeForModel(contract.PublicToolAX, s.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{map[string]any{"type": "click", "ref": refs["Interface"]}}, + }), false) + if len(result.Content) != 1 { + t.Fatalf("want only the evidence line, got %d items: %s", len(result.Content), ResultText(result)) + } + line := result.Content[0].Text + if line != `{"ok":true,"input_emitted":true,"completed":1,"requested":1}` { + t.Fatalf("evidence line %s", line) + } + // The act's own read was never delivered, so observe returns the tree + // instead of calling it unchanged. + observed := ResultText(s.callTool(contract.PublicToolAX, map[string]any{"mode": "observe", "app": "Editor"})) + if strings.Contains(observed, "Tree unchanged") || !strings.Contains(observed, `"Interface"`) { + t.Fatalf("observe after a hidden act read:\n%s", observed) + } + _, err := DecodeAXRequest(map[string]any{"mode": "act", "observe": "tree", + "actions": []any{map[string]any{"type": "click", "ref": "r1"}}}) + if err == nil || err.path != "observe" { + t.Fatalf("act with observe: %#v", err) + } +} + +func TestDiffAblationKeepsTheErrorOfAFailedAct(t *testing.T) { + s, _ := reportService(t, reportMain()) + reportObserve(t, s, "Interface") + t.Setenv(contract.AblationEnv, contract.AblateDiff) + result := FinalizeForModel(contract.PublicToolAX, s.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{map[string]any{"type": "click", "ref": "r999"}}, + }), false) + if len(result.Content) != 1 || !result.IsError { + t.Fatalf("want one evidence line for the error: %s", ResultText(result)) + } + var evidence map[string]any + if err := json.Unmarshal([]byte(result.Content[0].Text), &evidence); err != nil { + t.Fatal(err) + } + if evidence["ok"] != false || evidence["code"] == nil || evidence["message"] == nil { + t.Fatalf("error evidence %v", evidence) + } +} + +func TestGuardAblationLogsTheDecisionAndLetsInputThrough(t *testing.T) { + log := filepath.Join(t.TempDir(), "logs", "guard.jsonl") + t.Setenv(contract.GuardLogEnv, log) + if shadowGuard("ax", "batch_window", "changed", nil) { + t.Fatal("without the ablation the guard must stop the input") + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Fatal("a guard log was written without the ablation") + } + t.Setenv(contract.AblationEnv, contract.AblateGuard) + if !shadowGuard("ax", "batch_window", "changed", map[string]any{"window_ref": "w1"}) { + t.Fatal("with the ablation the guard must let the input through") + } + data, err := os.ReadFile(log) + if err != nil { + t.Fatal(err) + } + var record map[string]any + if err := json.Unmarshal(data, &record); err != nil { + t.Fatal(err) + } + if record["channel"] != "ax" || record["guard"] != "batch_window" || record["window_ref"] != "w1" || + record["would_block"] != true || record["at"] == nil { + t.Fatalf("record %v", record) + } +} diff --git a/computer-use/internal/core/schema_consistency_test.go b/computer-use/internal/core/schema_consistency_test.go index 9c9f016..c7306af 100644 --- a/computer-use/internal/core/schema_consistency_test.go +++ b/computer-use/internal/core/schema_consistency_test.go @@ -9,6 +9,7 @@ import ( "fmt" "math/big" "os" + "path/filepath" "sort" "strings" "testing" @@ -33,6 +34,7 @@ var crossFieldRuleCodes = map[string]bool{ contract.ArgumentRuleTextBudget: true, contract.ArgumentRuleRefOwnsIdentity: true, contract.ArgumentRuleWaitBudget: true, + contract.ArgumentRuleOneInputStep: true, // The flat AX schema has no oneOf or if/then, so rules that depend on // the value of mode or of an action type are declared rules of // the AX decoder: which fields a mode or type accepts or requires, @@ -217,6 +219,22 @@ func checkConsistency(t testing.TB, sample consistencySample) *consistencyMismat } func TestPublishedSchemaMatchesRuntimeAcceptance(t *testing.T) { + checkPublishedSchemaMatchesRuntime(t) +} + +// Each ablation changes the schema and the decoder together; they must +// still accept the same calls. +func TestAblatedSchemaMatchesRuntimeAcceptance(t *testing.T) { + for _, ablation := range append(append([]string{}, contract.AblationNames...), "batching,wait,diff,guard") { + t.Run(ablation, func(t *testing.T) { + t.Setenv(contract.AblationEnv, ablation) + t.Setenv(contract.GuardLogEnv, filepath.Join(t.TempDir(), "guard.jsonl")) + checkPublishedSchemaMatchesRuntime(t) + }) + } +} + +func checkPublishedSchemaMatchesRuntime(t *testing.T) { samples := consistencySamples(t) if len(samples) < 1000 { t.Fatalf("only %d consistency samples were generated", len(samples)) diff --git a/computer-use/internal/transport/cli_main_test.go b/computer-use/internal/transport/cli_main_test.go new file mode 100644 index 0000000..0381bcb --- /dev/null +++ b/computer-use/internal/transport/cli_main_test.go @@ -0,0 +1,50 @@ +package transport + +import ( + "bytes" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/core" +) + +// A misspelled ablation stops every entry point before it serves anything; +// help and version still answer. +func TestInvalidAblationStopsStartup(t *testing.T) { + t.Setenv(contract.AblationEnv, "wait,diffs") + for _, args := range [][]string{ + {"mcp"}, {"serve"}, {"daemon", "status"}, {"schema"}, {"call", "AX", "--args", `{"mode":"apps"}`}, {"apps"}, + } { + var stdout bytes.Buffer + err := RunCLI(args, &stdout) + if err == nil || !strings.Contains(err.Error(), `unknown ablation "diffs"`) { + t.Fatalf("%v: got %v", args, err) + } + if stdout.Len() != 0 { + t.Fatalf("%v wrote %q before failing", args, stdout.String()) + } + } + for _, args := range [][]string{{"help"}, {"version"}} { + if err := RunCLI(args, &bytes.Buffer{}); err != nil { + t.Fatalf("%v: %v", args, err) + } + } + t.Setenv(contract.AblationEnv, "guard") + t.Setenv(contract.GuardLogEnv, "") + if err := RunCLI([]string{"mcp"}, &bytes.Buffer{}); err == nil || + !strings.Contains(err.Error(), contract.GuardLogEnv) { + t.Fatalf("guard without a log: %v", err) + } +} + +func TestMCPInitializeServesTheAblatedInstructions(t *testing.T) { + t.Setenv(contract.AblationEnv, "wait") + response := handleMCPRequest(map[string]any{ + "jsonrpc": "2.0", "id": float64(1), "method": "initialize", "params": map[string]any{}, + }, core.NewService()) + instructions := response["result"].(map[string]any)["instructions"].(string) + if instructions == contract.ServerInstructions || strings.Contains(instructions, "wait step") { + t.Fatalf("instructions: %s", instructions) + } +} diff --git a/computer-use/runtime_test.py b/computer-use/runtime_test.py index 2167b7a..294a150 100644 --- a/computer-use/runtime_test.py +++ b/computer-use/runtime_test.py @@ -36,6 +36,7 @@ from runtime_tests.test_native_windows import * # noqa: F401,F403 from runtime_tests.test_x11_client_identity import * # noqa: F401,F403 from runtime_tests.test_window_geometry import * # noqa: F401,F403 +from runtime_tests.test_guard_shadow import * # noqa: F401,F403 if __name__ == "__main__": diff --git a/computer-use/runtime_tests/test_guard_shadow.py b/computer-use/runtime_tests/test_guard_shadow.py new file mode 100644 index 0000000..3a4912a --- /dev/null +++ b/computer-use/runtime_tests/test_guard_shadow.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Linux provider tests for the guard ablation's shadow mode.""" + +import json +import os +import tempfile + +from runtime_test_support import * # noqa: F401,F403 +from runtime_tests import test_gui_capture + + +class GuardLogMixin: + """Run with or without the guard ablation and read the guard log.""" + + def setUp(self): + super().setUp() + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.log_path = os.path.join(directory.name, "guard.jsonl") + patcher = mock.patch.dict(os.environ, { + "COMPUTER_USE_ABLATE": "", "COMPUTER_USE_GUARD_LOG": self.log_path, + }) + patcher.start() + self.addCleanup(patcher.stop) + + def ablate_guard(self): + os.environ["COMPUTER_USE_ABLATE"] = "wait,guard" + + def guard_log(self): + if not os.path.exists(self.log_path): + return [] + with open(self.log_path, encoding="utf-8") as log: + return [json.loads(line) for line in log] + + +class AXGuardShadowTests(GuardLogMixin, unittest.TestCase): + record = { + "runtimeId": [0, 2], "controlType": "push button", "name": "Rename…", + } + + def resolve_mismatch(self): + node = object() + with mock.patch.object(element_resolver, "resolve_path", return_value=node), \ + mock.patch.object(element_resolver, "record_still_matches", return_value=False), \ + mock.patch.object(element_resolver, "effective_role", return_value="toggle button"), \ + mock.patch.object(element_resolver, "effective_name", return_value="Bold"): + return node, element_resolver.find_element_at_snapshot_path_with_bounds( + object(), dict(self.record), None) + + def test_identity_mismatch_stops_by_default(self): + _node, resolved = self.resolve_mismatch() + self.assertIsNone(resolved) + self.assertEqual(self.guard_log(), []) + + def test_identity_mismatch_acts_on_the_path_node_and_logs_it(self): + self.ablate_guard() + node, resolved = self.resolve_mismatch() + self.assertIs(resolved, node) + [entry] = self.guard_log() + self.assertEqual(entry["channel"], "ax") + self.assertEqual(entry["guard"], "target_identity") + self.assertEqual(entry["path"], [0, 2]) + self.assertEqual(entry["expected"]["name"], "Rename…") + self.assertEqual(entry["live"], {"role": "toggle button", "name": "Bold"}) + self.assertTrue(entry["would_block"]) + + def test_a_path_without_a_node_stays_stale(self): + self.ablate_guard() + with mock.patch.object(element_resolver, "resolve_path", return_value=None): + resolved = element_resolver.find_element_at_snapshot_path_with_bounds( + object(), dict(self.record), None) + self.assertIsNone(resolved) + self.assertEqual(self.guard_log(), []) + + def semantic_refusal(self, code, node): + refused = runtime.OperationFailure( + code + " selector='button:Save'; no input was emitted.", + failure_code=code, failure_phase="precondition", input_events_emitted=False) + target = {"selector": "button:Save", "runtimeId": [0, 4]} + with mock.patch.object(element_resolver, "resolve_path", return_value=node), \ + mock.patch.object(element_resolver, "state_value", return_value=None), \ + mock.patch.object(element_resolver, "node_role", return_value="push button"), \ + mock.patch.object(element_resolver, "node_name", return_value="Cancel"), \ + mock.patch.object(element_resolver, "relative_frame", return_value=None): + return element_resolver.shadow_semantic_refusal(object(), 0, None, target, refused) + + def test_semantic_identity_refusal_falls_back_to_the_snapshot_path(self): + node = object() + self.assertIsNone(self.semantic_refusal("semantic_target_not_found", node)) + self.ablate_guard() + for code in ("semantic_target_not_found", "semantic_target_ambiguous", "stale_element_ref"): + resolved, record = self.semantic_refusal(code, node) + self.assertIs(resolved, node) + self.assertEqual(record["runtimeId"], [0, 4]) + self.assertEqual([entry["reason"] for entry in self.guard_log()], + ["semantic_target_not_found", "semantic_target_ambiguous", "stale_element_ref"]) + # Refusals that are not identity checks, and paths without a node, stay. + self.assertIsNone(self.semantic_refusal("observation_incomplete", node)) + self.assertIsNone(self.semantic_refusal("stale_element_ref", None)) + self.assertEqual(len(self.guard_log()), 3) + + def expected_window_mismatch(self): + with mock.patch.object(ax_dispatch, "provider_bus_name", return_value=":1.9"), \ + mock.patch.object(ax_dispatch, "process_generation", return_value="g2"), \ + mock.patch.object(ax_dispatch, "node_pid", return_value=7), \ + mock.patch.object(ax_dispatch, "node_role", return_value="frame"): + ax_dispatch._verify_expected_window(object(), { + "role": "frame", "providerBusName": ":1.4", "processGeneration": "g1", + }) + + def test_batch_window_change_is_logged_and_let_through(self): + with self.assertRaises(runtime.OperationFailure): + self.expected_window_mismatch() + self.ablate_guard() + self.expected_window_mismatch() + [entry] = self.guard_log() + self.assertEqual(entry["guard"], "batch_window") + + def test_point_owner_is_logged_and_let_through(self): + with mock.patch.object(actionability, "require_window_focus"), \ + mock.patch.object(actionability, "coordinate_point_owner", return_value=False): + with self.assertRaises(runtime.OperationFailure): + actionability.require_coordinate_delivery(object(), 5, 6, "click") + self.ablate_guard() + actionability.require_coordinate_delivery(object(), 5, 6, "click") + # A check that only chooses a route keeps deciding normally. + with guard_shadow.guards_enforced(), self.assertRaises(runtime.OperationFailure): + actionability.require_coordinate_delivery(object(), 5, 6, "click") + [entry] = self.guard_log() + self.assertEqual((entry["guard"], entry["reason"], entry["x"]), ("point_owner", "different_window", 5)) + + +class GUIGuardShadowTests(GuardLogMixin, unittest.TestCase): + # The capture pipeline fixture, without its tests. + patch = test_gui_capture.GUICapturePipelineTests.patch + sample_layout = test_gui_capture.GUICapturePipelineTests.sample_layout + capture_pixels = test_gui_capture.GUICapturePipelineTests.capture_pixels + sample_pixels = test_gui_capture.GUICapturePipelineTests.sample_pixels + encode_pixels = test_gui_capture.GUICapturePipelineTests.encode_pixels + layouts = test_gui_capture.GUICapturePipelineTests.layouts + publish = test_gui_capture.GUICapturePipelineTests.publish + click_after = test_gui_capture.GUICapturePipelineTests.click_after + zoom_after = test_gui_capture.GUICapturePipelineTests.zoom_after + + def setUp(self): + super().setUp() + test_gui_capture.GUICapturePipelineTests.setUp(self) + + def covered(self): + return self.layouts([("0xa", 0, 0, 50, 50)], + [("0xa", 0, 0, 50, 50), ("0xb", 5, 5, 20, 20)]) + + def test_layout_change_stops_the_batch_by_default(self): + response = self.click_after(*self.covered(), (10, 10)) + self.assertEqual(response["failureCode"], "stale_gui_frame") + self.execute.assert_not_called() + self.assertEqual(self.guard_log(), []) + + def test_layout_change_is_logged_and_the_click_is_sent(self): + self.ablate_guard() + response = self.click_after(*self.covered(), (10, 10)) + self.assertTrue(response["ok"], response.get("error")) + self.execute.assert_called_once() + [entry] = self.guard_log() + self.assertEqual(entry["channel"], "gui") + self.assertEqual((entry["guard"], entry["reason"]), ("layout", "changed_before_batch")) + self.assertEqual(entry["actions"], [{"type": "click", "x": 10, "y": 10}]) + + def test_zoom_over_a_changed_region_is_logged_and_returned(self): + self.ablate_guard() + response = self.zoom_after(*self.covered(), [0, 0, 30, 30]) + self.assertTrue(response["ok"], response.get("error")) + self.assertEqual(len(self.zoomed), 1) + [entry] = self.guard_log() + self.assertEqual(entry["guard"], "zoom_layout") + + +if __name__ == "__main__": + unittest.main() From 5932a797eb0659217d32ca17e76228fc608b7ed6 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 00:59:20 +0800 Subject: [PATCH 07/13] maint: check the model-visible text of every ablation set check_ablation_text.py starts the server for each COMPUTER_USE_ABLATE subset, reads instructions, tools and rejected calls, generates the guides, and fails on words or act examples of a removed mechanism. check_all.sh runs it with the candidate binary. Co-Authored-By: Claude Opus 5.5 --- scripts/maint/check_ablation_text.py | 178 +++++++++++++++++++++++++++ scripts/maint/check_all.sh | 6 +- 2 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 scripts/maint/check_ablation_text.py diff --git a/scripts/maint/check_ablation_text.py b/scripts/maint/check_ablation_text.py new file mode 100644 index 0000000..253b7fc --- /dev/null +++ b/scripts/maint/check_ablation_text.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Check that an ablated HCU no longer tells the model about what it removed. + +Usage: check_ablation_text.py BINARY + +For every subset of COMPUTER_USE_ABLATE (batching, wait, diff, guard) the +script starts BINARY as an MCP server, sends initialize, tools/list and AX +calls that use each removed mechanism, and generates the guides with +hcu.osworld.guide_ablation. It searches the server instructions, tool +definitions, error results and guides for the words of each removed +mechanism, and checks that every act example obeys the ablation. It also +checks that the server refuses to start on a misspelled ablation. No +desktop is needed: every call is rejected before input. + +Exit status 1 lists each finding. +""" +from __future__ import annotations + +import itertools +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO)) + +from hcu.osworld.guide_ablation import ABLATION_NAMES, ablated_guide # noqa: E402 + +# Words of each mechanism. GUI keeps its own wait action and settle_ms, so +# the wait words name only the AX forms. +WORDS = { + "wait": [r'"type":\s*"wait"', r"(?i)wait step", r'"settle"', r"settle needs", + r"\bsettle\b(?!_ms)", r"timeout_ms", r"wait_timeout", r"waits with", + r"`wait`"], + "diff": [r"\bdiffs?\b", r"(?m)^effect:", r"(?m)^windows:", r"report what changed", + r"reports the selection", r"reports what is now", r"reports each target"], + "batching": [r"ordered steps", r"into one `act`"], + "guard": [], +} + +# A call that uses each mechanism; the ablated server must reject it. +USES = { + "wait": {"mode": "act", "actions": [{"type": "wait", "window": "Export"}]}, + "diff": {"mode": "act", "observe": "diff", "actions": [{"type": "click", "ref": "r1"}]}, + "batching": {"mode": "act", "actions": [{"type": "click", "ref": "r1"}, {"type": "click", "ref": "r2"}]}, +} +BAD_CALL = {"mode": "act", "actions": []} + + +def mcp_session(binary: str, env: dict[str, str], calls: list[dict]) -> list[dict]: + process = subprocess.Popen( + [binary, "mcp"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, text=True, + ) + assert process.stdin is not None and process.stdout is not None + requests = [ + ("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "hcu-ablation-check", "version": "1"}}), + ("tools/list", {}), + ] + [("tools/call", {"name": "AX", "arguments": call}) for call in calls] + responses = [] + try: + for request_id, (method, params) in enumerate(requests, 1): + process.stdin.write(json.dumps( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + "\n") + process.stdin.flush() + while True: + line = process.stdout.readline() + if not line: + raise RuntimeError(f"EOF before {method}: {process.stderr.read()}") + response = json.loads(line) + if response.get("id") == request_id: + if "error" in response: + raise RuntimeError(f"{method}: {response['error']}") + responses.append(response["result"]) + break + finally: + process.terminate() + process.wait(timeout=5) + return responses + + +def act_examples(text: str) -> list[list[dict]]: + """The action lists of every act call written in text.""" + found = [] + decoder = json.JSONDecoder() + for match in re.finditer(r'\{\\?"mode\\?":\s*\\?"act\\?"', text): + chunk = text[match.start():] + if chunk.startswith('{\\"'): + continue # escaped copy inside JSON; the raw copy is checked + try: + call, _ = decoder.raw_decode(chunk) + except json.JSONDecodeError: + continue + found.append(call.get("actions", [])) + return found + + +def check(binary: str, ablations: tuple[str, ...], guard_log: str) -> list[str]: + env = {"PATH": "/usr/bin:/bin", "HOME": os.environ.get("HOME", "/tmp")} + if ablations: + env["COMPUTER_USE_ABLATE"] = ",".join(ablations) + if "guard" in ablations: + env["COMPUTER_USE_GUARD_LOG"] = guard_log + calls = [BAD_CALL] + [USES[name] for name in ablations if name in USES] + initialize, tools, *results = mcp_session(binary, env, calls) + names = [tool["name"] for tool in tools["tools"]] + findings = [] if names == ["AX", "GUI"] else [f"public tools {names}"] + texts = {"instructions": initialize.get("instructions", "")} + for tool in tools["tools"]: + # Raw description plus the schema, so quoted examples are searchable. + texts[f"{tool['name']} description"] = tool["description"] + schema = tool["inputSchema"] + if tool["name"] == "GUI": + # Only settle_ms of GUI speaks about AX mechanisms. + schema = schema["oneOf"][1]["properties"]["settle_ms"] + texts[f"{tool['name']} schema"] = json.dumps(schema) + for call, result in zip(calls, results): + text = "\n".join(item.get("text", "") for item in result.get("content", [])) + texts[f"result of {json.dumps(call)}"] = text + if not result.get("isError") or '"code":"invalid_argument"' not in text: + findings.append(f"{json.dumps(call)} was not rejected as invalid_argument: {text[:200]}") + for file_name in ("base.md", "hybrid.md"): + source = (REPO / "instructions" / file_name).read_text(encoding="utf-8") + texts[file_name] = ablated_guide(source, file_name, ablations) + for name in ablations: + for label, text in texts.items(): + if label.startswith("result of") and name in USES and label.endswith(json.dumps(USES[name])): + continue # the rejected call may echo what it sent + for word in WORDS[name]: + match = re.search(word, text) + if match: + start = max(0, match.start() - 50) + findings.append(f"{name}: {label} still has {match.group(0)!r}: ...{text[start:match.end() + 50]!r}") + for label, text in texts.items(): + if label.startswith("result of"): + continue + for actions in act_examples(text): + inputs = [step for step in actions if step.get("type") != "wait"] + if "batching" in ablations and len(inputs) > 1: + findings.append(f"batching: {label} shows an act with {len(inputs)} input steps") + if "wait" in ablations and len(inputs) != len(actions): + findings.append(f"wait: {label} shows an act with a wait step") + return findings + + +def main() -> int: + if len(sys.argv) != 2: + print(__doc__, file=sys.stderr) + return 2 + binary = sys.argv[1] + failed = False + with tempfile.TemporaryDirectory() as scratch: + for count in range(len(ABLATION_NAMES) + 1): + for ablations in itertools.combinations(ABLATION_NAMES, count): + findings = check(binary, ablations, os.path.join(scratch, "guard.jsonl")) + label = ",".join(ablations) or "(none)" + print(f"{'FAIL' if findings else 'ok '} {label}") + for finding in findings: + print(f" {finding}") + failed = failed or bool(findings) + for bad in ("waits", "wait,wait", "guard"): + env = {"PATH": "/usr/bin:/bin", "COMPUTER_USE_ABLATE": bad} + started = subprocess.run([binary, "mcp"], input="", capture_output=True, + text=True, env=env, timeout=30) + refused = started.returncode != 0 and "COMPUTER_USE_ABLATE" in started.stderr + print(f"{'ok ' if refused else 'FAIL'} refuses COMPUTER_USE_ABLATE={bad!r}" + + ("" if refused else f": exit {started.returncode}")) + failed = failed or not refused + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/maint/check_all.sh b/scripts/maint/check_all.sh index 13a7fb0..e9a37ca 100755 --- a/scripts/maint/check_all.sh +++ b/scripts/maint/check_all.sh @@ -3,8 +3,9 @@ # # Usage: scripts/maint/check_all.sh [BASELINE_BINARY] # -# With BASELINE_BINARY, the script also builds the current tree and compares -# MCP initialize/tools/list, help and schema bytes with that binary. +# With BASELINE_BINARY, the script also builds the current tree, compares +# MCP initialize/tools/list, help and schema bytes with that binary, and +# checks the model-visible text of every COMPUTER_USE_ABLATE setting. # Logs go to $LOG_DIR (default /tmp/hcu-maint-check); each step prints one # summary line, and the log tail on failure. The script exits non-zero at the # first failing step. @@ -63,4 +64,5 @@ if [ -n "$baseline_binary" ]; then (cd "$repo/computer-use" && CGO_ENABLED=0 go build -trimpath \ -ldflags "-X main.version=$(cat "$repo/VERSION")" -o "$candidate" ./cmd/computer-use) step mcp-bytes python3 "$repo/scripts/maint/compare_mcp_bytes.py" "$baseline_binary" "$candidate" + step ablation-text python3 "$repo/scripts/maint/check_ablation_text.py" "$candidate" fi From 371bdb2f09ec30017168a6bbfa48e53f9d988d25 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 01:25:41 +0800 Subject: [PATCH 08/13] ablation: state the ablated act behavior in the guide and AX description Deleting the passages of a removed mechanism left the agent to learn the changed rules from rejected calls. The guide now adds one sentence per ablation (batching, wait, diff), matching the AX tool description, and no new advice; guard adds none. Everything else in the guide is still only removed text. Co-Authored-By: Claude Opus 5.5 --- .../internal/contract/tool_contract.go | 8 ++- .../internal/core/ax_ablation_test.go | 25 +++++++++ docs/OSWORLD.md | 5 +- hcu/osworld/guide_ablation.py | 44 ++++++++++++++- tests/test_guide_ablation.py | 54 +++++++++++++++++++ 5 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 tests/test_guide_ablation.py diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index 9f4295d..596b44d 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -549,13 +549,19 @@ func publishedAXToolDescription() string { last := len(parts) - 1 sentence = strings.Join(parts[:last], ", ") + ", and " + parts[last] + "." } + if diff { + // The tools state how the ablated act behaves, as the guide does. + sentence += " Its result reports only whether input was sent and how many steps completed; " + + "see what changed with observe, find, or GUI." + } text := strings.Replace(axToolDescription, axActFullSentence, sentence, 1) text = strings.Replace(text, axActFullExample, example, 1) if diff { text = strings.Replace(text, " and reports the selection read back", "", 1) } if wait { - text = strings.Replace(text, axWaitDescription, "", 1) + text = strings.Replace(text, axWaitDescription, + "act cannot pause for a later state; to check one, read again with observe or find. ", 1) } if wait || diff { text = strings.Replace(text, "Skip waits the report answers. ", "", 1) diff --git a/computer-use/internal/core/ax_ablation_test.go b/computer-use/internal/core/ax_ablation_test.go index 19d1397..137b990 100644 --- a/computer-use/internal/core/ax_ablation_test.go +++ b/computer-use/internal/core/ax_ablation_test.go @@ -102,6 +102,31 @@ func TestAblatedTextOmitsTheRemovedMechanisms(t *testing.T) { } } +// Each text-changing ablation states how the ablated act behaves, so an +// agent does not have to learn it from rejected calls. +func TestAblatedDescriptionStatesTheChangedBehavior(t *testing.T) { + statements := map[string]string{ + contract.AblateBatching: "act runs one input step with optional wait steps before or after it", + contract.AblateWait: "act cannot pause for a later state; to check one, read again with observe or find.", + contract.AblateDiff: "Its result reports only whether input was sent and how many steps completed", + } + for _, set := range ablationSets() { + t.Run("ablate="+set, func(t *testing.T) { + t.Setenv(contract.AblationEnv, set) + description := findToolDefinition(t, contract.PublicToolAX).Description + for name, statement := range statements { + ablated := strings.Contains(","+set+",", ","+name+",") + if name == contract.AblateBatching && strings.Contains(set, contract.AblateWait) { + statement = "act runs one step" + } + if strings.Contains(description, statement) != ablated { + t.Errorf("%s ablated=%v, description:\n%s", name, ablated, description) + } + } + }) + } +} + func TestAblatedDescriptionsKeepTheirExamplesValid(t *testing.T) { example := regexp.MustCompile(`\{"mode":"act".*\}`) for _, set := range ablationSets() { diff --git a/docs/OSWORLD.md b/docs/OSWORLD.md index 08887ea..1527ec4 100644 --- a/docs/OSWORLD.md +++ b/docs/OSWORLD.md @@ -224,7 +224,10 @@ one channel only; `--instructions-file ""` runs without a profile. comma-separated subset of `batching`, `wait`, `diff` and `guard`. The runner sets `COMPUTER_USE_ABLATE` in the MCP server environment of every harness and gives the agent copies of `instructions/base.md` and `instructions/hybrid.md` -without the passages that describe the removed mechanisms. It writes the +without the passages that describe the removed mechanisms. For each of +`batching`, `wait` and `diff` the guide instead gets one sentence that +states how the ablated `act` behaves, as the AX tool description does, so +the agent does not learn the change from rejected calls. It writes the copies to `ablated-instructions/` in the task's result directory. Other guides, `--instructions-file ""` and `--code-mode` are refused with `--ablate`, because they can still describe a removed mechanism. With `guard` diff --git a/hcu/osworld/guide_ablation.py b/hcu/osworld/guide_ablation.py index b9073cd..ba0f59c 100644 --- a/hcu/osworld/guide_ablation.py +++ b/hcu/osworld/guide_ablation.py @@ -3,7 +3,11 @@ ``COMPUTER_USE_ABLATE`` removes HCU mechanisms from the computer-use server (see ``computer-use/internal/contract/ablation.go``). The guides given to the agent must not describe a removed mechanism either, so this module deletes -the passages that describe it and changes nothing else. +the passages that describe it. It also adds one sentence per ablation that +states how the ablated tools behave, the same fact the tool description +states, and no new advice: otherwise an ablation run would also measure +the agent learning the changed rules from rejected calls. Nothing else +changes. The passages are listed here as exact text, not marked inside the guides: the guides are model input, and markers in them would change the text of @@ -68,6 +72,7 @@ {"mode":"act","actions":[{"type":"click","ref":"r12"},{"type":"wait","window":"Export"},{"type":"type","text":"q3.csv"},{"type":"keypress","keys":["ENTER"]},{"type":"wait","window":"Export","gone":true}]} {"mode":"act","actions":[{"type":"click","ref":"r7"},{"type":"wait","query":{"role":"menu item","name_contains":"Paragraph"}},{"type":"click","ref":"r9"}]} ``` + """ _SELECT_READBACK = """ The result reports what is now @@ -110,6 +115,31 @@ "guard": (), } +# The sentence each ablation adds, after this bullet, which no ablation +# removes. guard adds none: the agent cannot notice the shadow mode. +_STATEMENT_ANCHOR = """\ +- Once the source, target, and operation are known, act. Further checks must + answer a concrete blocker. +""" +STATEMENTS = { + "batching": """\ +- Each `act` sends one input step, with optional `wait` steps before or after + it. Send further input in later calls. +""", + "wait": """\ +- An AX `act` cannot pause for a later state. To check one, read again with + `observe` or `find`. +""", + "diff": """\ +- An `act` result reports only whether input was sent and how many steps + completed. To see what changed, use `observe`, `find`, or a GUI screenshot. +""", +} +# With both batching and wait ablated, an act holds one step and no waits. +_ONE_STEP = """\ +- Each `act` sends one step. Send further steps in later calls. +""" + def parse_ablation(value: str | list[str] | tuple[str, ...] | None) -> tuple[str, ...]: """Return the ablations in canonical order. @@ -170,7 +200,17 @@ def ablated_guide(text: str, file_name: str, ablations: tuple[str, ...]) -> str: kept.append(text[position:start]) position = max(position, end) kept.append(text[position:]) - return "".join(kept) + text = "".join(kept) + if file_name != "hybrid.md": + return text + added = [STATEMENTS[name] for name in ablations if name in STATEMENTS] + if "batching" in ablations and "wait" in ablations: + added[0] = _ONE_STEP + if not added: + return text + if text.count(_STATEMENT_ANCHOR) != 1: + raise ValueError("hybrid.md: the ablation statements' anchor must occur exactly once") + return text.replace(_STATEMENT_ANCHOR, _STATEMENT_ANCHOR + "".join(added)) def write_ablated_guides( diff --git a/tests/test_guide_ablation.py b/tests/test_guide_ablation.py new file mode 100644 index 0000000..af648be --- /dev/null +++ b/tests/test_guide_ablation.py @@ -0,0 +1,54 @@ +"""Ablated interaction guides: removed passages and the added statements.""" + +from __future__ import annotations + +import itertools + +import pytest + +from hcu.osworld.guide_ablation import ABLATION_NAMES, STATEMENTS, ablated_guide +from hcu.paths import DEFAULT_INSTRUCTIONS_FILE + +GUIDES = DEFAULT_INSTRUCTIONS_FILE.parent +ALL_SETS = [ + combo + for count in range(len(ABLATION_NAMES) + 1) + for combo in itertools.combinations(ABLATION_NAMES, count) +] + + +def source(file_name: str) -> str: + return (GUIDES / file_name).read_text(encoding="utf-8") + + +@pytest.mark.parametrize("file_name", ["base.md", "hybrid.md"]) +def test_guides_without_ablation_are_unchanged(file_name): + assert ablated_guide(source(file_name), file_name, ()) == source(file_name) + + +def test_guard_changes_no_guide(): + for file_name in ("base.md", "hybrid.md"): + assert ablated_guide(source(file_name), file_name, ("guard",)) == source(file_name) + + +@pytest.mark.parametrize("ablations", ALL_SETS, ids=lambda combo: ",".join(combo) or "full") +def test_each_ablation_states_the_changed_behavior_once(ablations): + guide = ablated_guide(source("hybrid.md"), "hybrid.md", ablations) + one_step = "batching" in ablations and "wait" in ablations + for name, statement in STATEMENTS.items(): + expected = 1 if name in ablations and not (name == "batching" and one_step) else 0 + assert guide.count(statement) == expected, name + assert guide.count("- Each `act` sends one step.") == (1 if one_step else 0) + # base.md is common policy; no ablation adds to it. + assert ablated_guide(source("base.md"), "base.md", ablations) == source("base.md") + + +@pytest.mark.parametrize("ablations", ALL_SETS, ids=lambda combo: ",".join(combo) or "full") +def test_ablations_only_remove_text_besides_the_statements(ablations): + original = source("hybrid.md") + guide = ablated_guide(original, "hybrid.md", ablations) + for statement in [*STATEMENTS.values(), "- Each `act` sends one step. Send further steps in later calls.\n"]: + guide = guide.replace(statement, "", 1) + # What remains is the original with characters removed, nothing added. + remaining = iter(original) + assert all(character in remaining for character in guide) From 55035a242c2579fb27bc7efafcebb3184326269a Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 05:35:21 +0000 Subject: [PATCH 09/13] ablation: close the gaps the guard, wait, batching and diff switches left guard: the focus check of a focus-bound batch, the select range end and the combobox edit partner now go through the shadow, and a shadowed target keeps the window that holds it. The full arm no longer reads the tree for a shadow it does not use. wait: a rejected wait step points to a fresh observe instead of an input action guessed from its fields, and the no-tree window text no longer offers a wait. batching: input after close is refused as close-not-last without suggesting a wait, and no_focused_window no longer says to start a batch with activate. diff: act errors no longer carry a new wN, the values and windows a wait saw, or a promised observation; the tree reference no longer mentions diffs. A wait-only act without ref or window_ref no longer panics the server when the window list cannot be read. This crash predates the switches and hit every arm that keeps wait. Co-Authored-By: Claude Opus 5.5 --- computer-use/internal/core/ax_ablation.go | 7 ++ .../internal/core/ax_ablation_test.go | 90 +++++++++++++++++++ computer-use/internal/core/ax_batch.go | 3 +- .../internal/core/ax_batch_context.go | 5 ++ .../internal/core/ax_native_window.go | 4 + .../core/ax_partial_inventory_test.go | 21 +++++ .../internal/core/ax_public_request.go | 24 ++++- .../internal/core/ax_snapshot_store.go | 6 +- computer-use/internal/core/ax_wait_batch.go | 29 +++++- .../internal/core/ax_window_inventory.go | 6 +- .../internal/core/ax_window_inventory_test.go | 28 ++++++ .../native/linux/provider/ax_actions.py | 5 ++ .../native/linux/provider/ax_dispatch.py | 19 ++-- .../native/linux/provider/element_resolver.py | 41 +++++++-- .../native/linux/provider/guard_shadow.py | 7 +- .../runtime_tests/test_guard_shadow.py | 63 +++++++++++-- 16 files changed, 326 insertions(+), 32 deletions(-) diff --git a/computer-use/internal/core/ax_ablation.go b/computer-use/internal/core/ax_ablation.go index 496216b..d4ee12f 100644 --- a/computer-use/internal/core/ax_ablation.go +++ b/computer-use/internal/core/ax_ablation.go @@ -106,3 +106,10 @@ func shadowGuard(channel, guard, reason string, detail map[string]any) bool { _, _ = file.Write(append(line, '\n')) return true } + +// axAblatedActionType reports an AX action type that COMPUTER_USE_ABLATE +// removed from the published contract. +func axAblatedActionType(verb string) bool { + return contract.ContainsString(contract.AXActionTypes, verb) && + !contract.ContainsString(contract.PublishedAXActionTypes(), verb) +} diff --git a/computer-use/internal/core/ax_ablation_test.go b/computer-use/internal/core/ax_ablation_test.go index 137b990..033ed0c 100644 --- a/computer-use/internal/core/ax_ablation_test.go +++ b/computer-use/internal/core/ax_ablation_test.go @@ -2,6 +2,7 @@ package core import ( "encoding/json" + "fmt" "os" "path/filepath" "regexp" @@ -167,6 +168,17 @@ func TestBatchingAblationAllowsOneInputStep(t *testing.T) { if !strings.Contains(err.Hint, `{"type":"wait","window":"A"}`) || strings.Contains(err.Hint, "select") { t.Fatalf("hint should keep the steps before the second input: %s", err.Hint) } + // Input after close is refused as close-not-last, without suggesting a + // wait that would put a second input step in the call. + _, err = DecodeAXRequest(decodeTestArguments(t, + `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"click","ref":"r1"}]}`)) + if err == nil || err.rule != contract.ArgumentRuleCloseNotLast || + !strings.HasPrefix(err.Message, "close must be the last step") || strings.Contains(err.Hint, `"gone"`) { + t.Fatalf("input after close: %#v", err) + } + if focus := noFocusMessage(t); strings.Contains(focus, "batch") { + t.Fatalf("no-focus message: %s", focus) + } t.Setenv(contract.AblationEnv, "batching,wait") _, err = DecodeAXRequest(decodeTestArguments(t, @@ -176,6 +188,20 @@ func TestBatchingAblationAllowsOneInputStep(t *testing.T) { } } +// noFocusMessage returns the no_focused_window message for a desktop where +// no window has focus. +func noFocusMessage(t *testing.T) string { + t.Helper() + backend := &recordingPlatformBackend{accessibilityResponse: &ProviderAccessibilityResult{ + OK: true, Apps: editorApps(testWindow{testHandleReport, "Report - Editor", false}), + }} + _, err := NewServiceWithBackend(backend).resolveFocusedWindow() + if err == nil || err.Code != contract.AXCodeNoFocusedWindow { + t.Fatalf("want no_focused_window, got %#v", err) + } + return err.Message +} + func TestBatchingAblationKeepsGUIBatches(t *testing.T) { t.Setenv(contract.AblationEnv, "batching,wait,diff") args := decodeTestArguments(t, `{"actions":[{"type":"click","x":1,"y":2},{"type":"type","text":"a"},{"type":"keypress","keys":["ENTER"]}]}`) @@ -191,6 +217,26 @@ func TestWaitAblationRemovesWaitSteps(t *testing.T) { strings.Contains(err.Hint, "wait") || err.path != "actions[0].type" { t.Fatalf("wait step: %#v", err) } + // A rejected wait points to a fresh read, never to an input action + // guessed from its fields. + for _, step := range []string{ + `{"type":"wait","text":"Saved"}`, + `{"type":"wait","ref":"r1","gone":true}`, + `{"type":"wait","ref":"w1"}`, + `{"type":"wait","keys":["ENTER"]}`, + } { + _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[`+step+`]}`)) + if err == nil || err.Hint != axMinimalCalls[contract.AXModeObserve] { + t.Fatalf("%s: rejected wait got %#v", step, err) + } + } + _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"action":"wait","text":"Saved"}]}`)) + if err == nil || err.Hint != axMinimalCalls[contract.AXModeObserve] { + t.Fatalf("action=wait: %#v", err) + } + if text := (&axNoTreeWindowError{ref: "w2"}).Error(); strings.Contains(text, "wait") { + t.Fatalf("no-tree window text: %s", text) + } _, err = DecodeAXRequest(decodeTestArguments(t, `{"mode":"act","actions":[{"type":"close","ref":"w1"},{"type":"keypress","keys":["ENTER"]}]}`)) if err == nil || err.rule != contract.ArgumentRuleCloseNotLast || strings.Contains(err.Message+err.Hint, "wait") { @@ -242,6 +288,50 @@ func TestDiffAblationReturnsOnlyExecutionEvidence(t *testing.T) { } } +// Under diff, an act's error says how far input went but not what the act +// read: no values, window list, new wN or promised observation. +func TestDiffAblationErrorsCarryNoObservation(t *testing.T) { + s, backend := reportService(t, reportMain()) + window, refs := reportObserve(t, s, "Interface", "Size") + context := &AXBatchContext{AppToken: "Editor"} + if message, _, _ := s.axBatchContinuityRecovery(context); !strings.Contains(message, window) { + t.Fatalf("full HCU names the current window: %s", message) + } + t.Setenv(contract.AblationEnv, contract.AblateDiff) + message, hint, current := s.axBatchContinuityRecovery(context) + if strings.Contains(message+hint, window) || hint != `{"mode":"apps"}` || current != window { + t.Fatalf("recovery under diff: %q %q %q", message, hint, current) + } + if lines := treeUnchangedReferenceLines(3); strings.Contains(lines[0], "diff") { + t.Fatalf("tree reference: %s", lines[0]) + } + + evidence := func(args map[string]any) map[string]any { + t.Helper() + result := FinalizeForModel(contract.PublicToolAX, s.callTool(contract.PublicToolAX, args), false) + var line map[string]any + if len(result.Content) != 1 || json.Unmarshal([]byte(result.Content[0].Text), &line) != nil { + t.Fatalf("want one evidence line: %s", ResultText(result)) + } + return line + } + timedOut := evidence(map[string]any{"mode": "act", "actions": []any{ + map[string]any{"type": "click", "ref": refs["Interface"]}, + map[string]any{"type": "wait", "ref": refs["Size"], "value": "99", "timeout_ms": 200}, + }}) + if timedOut["code"] != contract.AXCodeWaitTimeout || strings.Contains(fmt.Sprint(timedOut), `"12"`) || + strings.Contains(fmt.Sprint(timedOut), "observation") { + t.Fatalf("wait timeout under diff: %v", timedOut) + } + backend.loseReadback = true + lost := evidence(map[string]any{"mode": "act", "actions": []any{ + map[string]any{"type": "click", "ref": refs["Interface"]}, + }}) + if lost["ok"] != false || strings.Contains(fmt.Sprint(lost["message"]), "observation follows") { + t.Fatalf("lost readback under diff: %v", lost) + } +} + func TestDiffAblationKeepsTheErrorOfAFailedAct(t *testing.T) { s, _ := reportService(t, reportMain()) reportObserve(t, s, "Interface") diff --git a/computer-use/internal/core/ax_batch.go b/computer-use/internal/core/ax_batch.go index a73fdfc..042ecae 100644 --- a/computer-use/internal/core/ax_batch.go +++ b/computer-use/internal/core/ax_batch.go @@ -441,7 +441,8 @@ func (s *Service) axBatchRun( observationSummary = "Preparation may have changed state or may complete later. " + "Stop; no recovery observation, wait or later action was attempted." } - if finalObservationReturned { + // Under diff the act result drops the observation it read. + if finalObservationReturned && !contract.Ablated(contract.AblateDiff) { observationSummary = "A fresh current-state observation follows. " + "It does not prove which action caused that state." if result.StructuredContent["action_outcome"] == "not_executed" { diff --git a/computer-use/internal/core/ax_batch_context.go b/computer-use/internal/core/ax_batch_context.go index e40c1bb..f886f8c 100644 --- a/computer-use/internal/core/ax_batch_context.go +++ b/computer-use/internal/core/ax_batch_context.go @@ -271,6 +271,11 @@ func (s *Service) axBatchFocusChanged( if argErr == nil && focus.handle == context.WindowHandle { return ToolCallResult{}, false } + if shadowGuard("ax", "batch_window", "focus_left_bound_window", map[string]any{ + "window_ref": context.WindowRef, "next_action_index": index, + }) { + return ToolCallResult{}, false + } now := "no window has focus" if argErr == nil { now = fmt.Sprintf("focus is now on %s %s (%s)", focus.ref, strconv.Quote(focus.title), focus.appName) diff --git a/computer-use/internal/core/ax_native_window.go b/computer-use/internal/core/ax_native_window.go index fe62feb..4a85f78 100644 --- a/computer-use/internal/core/ax_native_window.go +++ b/computer-use/internal/core/ax_native_window.go @@ -22,6 +22,10 @@ type axNoTreeWindowError struct { } func (err *axNoTreeWindowError) Error() string { + if contract.Ablated(contract.AblateWait) { + return err.ref + " has no accessibility tree. " + + "Read and operate it with GUI; AX can only activate or close this window." + } return err.ref + " has no accessibility tree. " + "Read and operate it with GUI; AX can only activate, " + "close or wait for this window." diff --git a/computer-use/internal/core/ax_partial_inventory_test.go b/computer-use/internal/core/ax_partial_inventory_test.go index 47083cd..7963926 100644 --- a/computer-use/internal/core/ax_partial_inventory_test.go +++ b/computer-use/internal/core/ax_partial_inventory_test.go @@ -80,6 +80,27 @@ func TestPartialInventoryCannotEstablishUniqueTitleFocusOrAbsence(t *testing.T) } } +// A text wait without a window reads the focused window. When the window +// list cannot be read, the call fails with that reason instead of +// crashing the server on a missing window listing. +func TestFocusWaitOnUnreadableInventoryFails(t *testing.T) { + s, backend := partialInventoryService() + for _, broken := range []bool{false, true} { + if broken { + backend.accessibilityResponse.OK = false + backend.accessibilityResponse.Error = "provider timed out" + } + result := PrepareAXEvidence(s.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{map[string]any{"type": "wait", "text": "Saved", "timeout_ms": 300}}, + })) + text, _ := result.StructuredContent["message"].(string) + if !result.IsError || result.StructuredContent["input_emitted"] != false || + !strings.Contains(text, "give window_ref") || strings.Contains(text, "no window has focus") { + t.Fatalf("broken=%v: %#v", broken, result.StructuredContent) + } + } +} + func TestInventoryCompletenessRecoveryPreservesRefs(t *testing.T) { s, backend := partialInventoryService() s.listApps() diff --git a/computer-use/internal/core/ax_public_request.go b/computer-use/internal/core/ax_public_request.go index 9e43392..bf8474c 100644 --- a/computer-use/internal/core/ax_public_request.go +++ b/computer-use/internal/core/ax_public_request.go @@ -110,7 +110,7 @@ var axModeFields = map[string][]string{ } // axModeAllowedFields lists the fields a mode accepts. Without diff, act -// returns no observation, so it takes no observe field. +// always returns the tree, so it takes no observe field. func axModeAllowedFields(mode string) []string { if mode == contract.AXModeAct && contract.Ablated(contract.AblateDiff) { return []string{"mode", "window_ref", "actions"} @@ -366,7 +366,9 @@ func decodeAXActions(raw any) ([]axBatchAction, *axArgError) { return nil, err } if index > 0 && actions[index-1].actionType == "close" && action.actionType != "wait" { - if contract.Ablated(contract.AblateWait) { + // Without wait steps, or with one input step per act, nothing + // can follow close in the same call. + if contract.Ablated(contract.AblateWait) || contract.Ablated(contract.AblateBatching) { return nil, axInvalid(path, "close must be the last step: a closed window cannot take later input", "").withRule(contract.ArgumentRuleCloseNotLast).withFix(axSet("actions", items[:index])) } @@ -419,9 +421,18 @@ func decodeAXAction(object map[string]any, path string) (axBatchAction, *axArgEr } else { message = "action needs type: " + strings.Join(contract.PublishedAXActionTypes(), ", ") } - err := axInvalid(path+".type", message, axActionShapeHint(object)) + hint := axActionShapeHint(object) + verb := actionType + if !hasType { + verb, _ = object["action"].(string) + } + if axAblatedActionType(verb) { + // The tool description sends a removed wait to a fresh read. + hint = axMinimalCalls[contract.AXModeObserve] + } + err := axInvalid(path+".type", message, hint) switch { - case actionType == "settle": + case actionType == "settle" && !axAblatedActionType("wait"): err.Message += `; use type "wait" with settle:true after an input step in the same call` err.Hint = "" err.fix = []axEdit{axSet(path+".type", "wait"), axSet(path+".settle", true)} @@ -616,6 +627,11 @@ func axActionShapeHint(object map[string]any) string { if synonym, known := axActionVerbSynonyms[verb]; known { verb = synonym } + // An action type removed by an ablation has no AX equivalent; guessing + // one from its fields would turn a wait for "Saved" into typing it. + if axAblatedActionType(verb) { + return "" + } if !contract.ContainsString(contract.PublishedAXActionTypes(), verb) { switch { case AXWindowRefPattern.MatchString(ref) && strings.Contains(strings.ToLower(verb), "close"): diff --git a/computer-use/internal/core/ax_snapshot_store.go b/computer-use/internal/core/ax_snapshot_store.go index 29bc85c..967050b 100644 --- a/computer-use/internal/core/ax_snapshot_store.go +++ b/computer-use/internal/core/ax_snapshot_store.go @@ -116,9 +116,13 @@ func deliveredTreeKey(snapshot *AppSnapshot) string { // model's copy is the last full tree plus every diff delivered after it, so // the reference says where the state came from and how to get it again. func treeUnchangedReferenceLines(lineCount int) []string { + source := "last full tree plus later diffs" + if contract.Ablated(contract.AblateDiff) { + source = "the last full tree" + } return []string{fmt.Sprintf( "Tree unchanged: identical to the state of this window already "+ - "delivered by an earlier result (last full tree plus later diffs), "+ + "delivered by an earlier result ("+source+"), "+ "so its %d lines are not repeated; the rN refs shown there still "+ "apply. Call {\"mode\":\"observe\",\"observe\":\"full\"} "+ "to get the whole tree again.", diff --git a/computer-use/internal/core/ax_wait_batch.go b/computer-use/internal/core/ax_wait_batch.go index 5b06c76..5397a83 100644 --- a/computer-use/internal/core/ax_wait_batch.go +++ b/computer-use/internal/core/ax_wait_batch.go @@ -128,6 +128,15 @@ func (s *Service) stopAXBatchAtWait( if unreadable && !strings.Contains(message, "could not be read") { message += " (the state could not be read completely while waiting)" } + // Without diff an act reports no observation; the wait's message would + // otherwise carry the values and windows it saw. + noFeedback := contract.Ablated(contract.AblateDiff) + if noFeedback { + message = fmt.Sprintf("wait_timeout: the wait at actions[%d] did not hold; the steps before it ran and none after it", index) + if unreadable { + message += " (the state could not be read completely while waiting)" + } + } stopped := TextResult(message, true) if len(waitNotes) > 0 { stopped.Content = append([]ContentItem{{Type: "text", Text: strings.Join(waitNotes, "\n")}}, stopped.Content...) @@ -150,6 +159,12 @@ func (s *Service) stopAXBatchAtWait( if corrected, _ := waitResult.StructuredContent["wait_hint"].(string); corrected != "" { hint = corrected } + if noFeedback { + hint = `{"mode":"observe","window_ref":"` + FirstNonEmpty(context.WindowRef, "wN") + `"}` + if wait := actions[index].wait; wait != nil && wait.window != "" { + hint = `{"mode":"apps"}` + } + } stopped.StructuredContent = map[string]any{ contract.AXPublicErrorCodeKey: contract.AXCodeWaitTimeout, contract.AXPublicErrorPathKey: fmt.Sprintf("actions[%d]", index), @@ -267,6 +282,10 @@ func (s *Service) axBatchContinuityRecovery(context *AXBatchContext) (string, st } if ref := snapshot.publicWindowRef; ref != "" { if entry, err := s.WindowRefRegistry.Resolve(ref, ""); err == nil && entry.WindowHandle == snapshot.WindowHandle { + if contract.Ablated(contract.AblateDiff) { + // The new window's wN is a windows report, which diff removes. + return "List current windows, inspect the intended window, then send only the remaining actions.", `{"mode":"apps"}`, ref + } return "The latest read shows " + ref + ". Observe that window before sending only the remaining actions.", axHintJSON(map[string]any{"mode": "observe", "window_ref": ref}), ref } @@ -296,8 +315,14 @@ func (s *Service) axWaitOnly( var argErr *axArgError focus, argErr = s.resolveFocusedWindow() if argErr != nil { - argErr.Message = "a wait without ref or window_ref reads the focused window, and no window has focus; " + - "give window_ref, or wait for a window by title. Windows:" + strings.TrimPrefix(argErr.Message[strings.Index(argErr.Message, "Windows:"):], "Windows:") + // Only a missing focus lists the windows; an unreadable window + // list keeps its own code and message. + if index := strings.Index(argErr.Message, "Windows:"); index >= 0 { + argErr.Message = "a wait without ref or window_ref reads the focused window, and no window has focus; " + + "give window_ref, or wait for a window by title. " + argErr.Message[index:] + } else { + argErr.Message += "; a wait without ref or window_ref reads the focused window, so give window_ref" + } argErr.Hint = `{"mode":"act","window_ref":"w1","actions":[{"type":"wait","text":"Saved"}]}` return axArgErrorResult(argErr, len(actions)) } diff --git a/computer-use/internal/core/ax_window_inventory.go b/computer-use/internal/core/ax_window_inventory.go index 6eff2dd..966f208 100644 --- a/computer-use/internal/core/ax_window_inventory.go +++ b/computer-use/internal/core/ax_window_inventory.go @@ -268,10 +268,14 @@ func (s *Service) resolveFocusedWindow() (*axDesktopWindow, *axArgError) { return &inventory.windows[index], nil } } + start := "Start the batch with activate. " + if contract.Ablated(contract.AblateBatching) { + start = "Activate a window in its own call first. " + } return nil, &axArgError{ Code: contract.AXCodeNoFocusedWindow, Message: "no application window has keyboard focus, so keys or text without ref have no target. " + - "Start the batch with activate. Windows:\n" + inventory.text, + start + "Windows:\n" + inventory.text, Hint: `{"type":"activate","title":"window title"}`, } } diff --git a/computer-use/internal/core/ax_window_inventory_test.go b/computer-use/internal/core/ax_window_inventory_test.go index ba3ff91..fe5b6d4 100644 --- a/computer-use/internal/core/ax_window_inventory_test.go +++ b/computer-use/internal/core/ax_window_inventory_test.go @@ -1,6 +1,8 @@ package core import ( + "os" + "path/filepath" "strings" "testing" @@ -300,6 +302,26 @@ func TestFocusChangeBeforeFirstActionStopsWithoutInput(t *testing.T) { } func TestFocusChangeAfterActionKStopsAtKPlusOne(t *testing.T) { + focusChangeBatch(t, false) +} + +// With guard ablated the focus check only logs, and every key is sent. +func TestGuardAblationLetsFocusBoundBatchContinue(t *testing.T) { + log := filepath.Join(t.TempDir(), "guard.jsonl") + t.Setenv(contract.AblationEnv, contract.AblateGuard) + t.Setenv(contract.GuardLogEnv, log) + focusChangeBatch(t, true) + data, err := os.ReadFile(log) + if err != nil || !strings.Contains(string(data), `"reason":"focus_left_bound_window"`) || + !strings.Contains(string(data), `"next_action_index":2`) { + t.Fatalf("guard log %q err=%v", data, err) + } +} + +// focusChangeBatch sends three keys to the focused window; focus moves to +// another window before the third. +func focusChangeBatch(t *testing.T, guardAblated bool) { + t.Helper() focused := editorApps( testWindow{testHandleReport, "Report - Editor", true}, ) @@ -334,6 +356,12 @@ func TestFocusChangeAfterActionKStopsAtKPlusOne(t *testing.T) { map[string]any{"type": "keypress", "keys": []any{"C"}}, }) result := service.axBatchDecoded(actions, observation{mode: contract.ObserveNone}, "") + if guardAblated { + if result.IsError || len(backend.actionRequests) != 3 { + t.Fatalf("guard ablation stopped the batch after %d keys: %#v", len(backend.actionRequests), result.StructuredContent) + } + return + } if !result.IsError || result.StructuredContent[contract.AXPublicErrorCodeKey] != contract.AXCodeWindowChanged { t.Fatalf("focus change after action 2: %#v", result.StructuredContent) } diff --git a/computer-use/native/linux/provider/ax_actions.py b/computer-use/native/linux/provider/ax_actions.py index 9a72951..084f1e2 100644 --- a/computer-use/native/linux/provider/ax_actions.py +++ b/computer-use/native/linux/provider/ax_actions.py @@ -25,6 +25,7 @@ from .tree_walk import iter_all from .ax_records import combo_edit_target, combo_edit_identity +from .guard_shadow import shadow_guard from .input_method import InputMethodText @@ -46,6 +47,10 @@ def resolve_combo_editor(element, record): return element, False editor = combo_edit_target(element) identity = combo_edit_identity(editor) + if (expected is not None and identity is not None and identity != expected + and shadow_guard("ax", "target_identity", "composite_edit_target_replaced", + expected=expected, live=identity)): + return editor, True if expected is None or identity != expected: raise OperationFailure( "The combobox has no unique unchanged editable partner; no text " diff --git a/computer-use/native/linux/provider/ax_dispatch.py b/computer-use/native/linux/provider/ax_dispatch.py index be947a9..51ff65d 100644 --- a/computer-use/native/linux/provider/ax_dispatch.py +++ b/computer-use/native/linux/provider/ax_dispatch.py @@ -524,10 +524,10 @@ def _resolve_action_target(app, operation, operation_kind, action_input): ) except OperationFailure as refused: shadowed = shadow_semantic_refusal( - app, window_index, bounds, semantic_target, refused) + app, window_index, window, bounds, semantic_target, refused) if shadowed is None: raise - element, element_record = shadowed + element, element_record, window, bounds = shadowed elif semantic_target is None and action_input.get("window_centre"): element = None elif semantic_target is None: @@ -564,10 +564,17 @@ def _resolve_select_end(app, operation, action_input, window, bounds): failure_phase="precondition", input_events_emitted=False, ) - to_element, _record, _note, to_window, _bounds = resolve_semantic_target( - app, window_index, window, bounds, to_target, - operation.get("known_refs"), operation, - ) + try: + to_element, _record, _note, to_window, _bounds = resolve_semantic_target( + app, window_index, window, bounds, to_target, + operation.get("known_refs"), operation, + ) + except OperationFailure as refused: + shadowed = shadow_semantic_refusal( + app, window_index, window, bounds, to_target, refused) + if shadowed is None: + raise + to_element, _record, to_window, _bounds = shadowed if provider_object_path(to_window) != window_path: raise OperationFailure( "select_invalid_range: to_ref is in a different window than ref; " diff --git a/computer-use/native/linux/provider/element_resolver.py b/computer-use/native/linux/provider/element_resolver.py index c616ff0..f86af60 100644 --- a/computer-use/native/linux/provider/element_resolver.py +++ b/computer-use/native/linux/provider/element_resolver.py @@ -14,20 +14,24 @@ ) from .node_geometry import extents from .primitives import name_fingerprint -from .guard_shadow import shadow_guard +from .guard_shadow import guards_shadowed, shadow_guard from .ax_node_properties import ( accessible_id, parse_text_limit, ) from .ax_records import ( + combo_edit_identity, + combo_edit_target, effective_name, effective_role, + gtk_combo_partner, ) from .x11_windows import ( main_window, relative_frame, + window_bounds, ) from .primitives import semantic_text_key from .semantic_window_candidates import ( @@ -550,12 +554,17 @@ def _semantic_resolution_note( } -def shadow_semantic_refusal(app, window_index, bounds, target, refused): - """Return (node, record) at the target's snapshot path, or None. +def shadow_semantic_refusal(app, window_index, window, bounds, target, refused): + """Return (node, record, window, bounds) at the target's snapshot path. - None keeps the refusal: the guard ablation is off, the refusal is not an - identity check, or no live node is left on the path. + The window and bounds are those of the top-level window that holds the + node, as a resolved target has. None keeps the refusal: the guard + ablation is off, the refusal is not an identity check, or no live node is + left on the path. """ + # The full arm must not pay for the AT-SPI reads below. + if not guards_shadowed(): + return None if getattr(refused, "failure_code", None) not in _SHADOWED_TARGET_FAILURES: return None paths = [target.get("runtimeId") or []] @@ -566,17 +575,31 @@ def shadow_semantic_refusal(app, window_index, bounds, target, refused): node = resolve_path(app, path) if node is None or state_value(node, Atspi.StateType.DEFUNCT) is True: continue + role = node_role(node) if not shadow_guard( "ax", "target_identity", refused.failure_code, message=str(refused), selector=target.get("selector"), path=list(path), - live={"role": node_role(node), "name": node_name(node)}, + live={"role": role, "name": node_name(node)}, ): return None + node_window, node_bounds = window, bounds + if path[0] != window_index: + top = safe(lambda: app.get_child_at_index(path[0])) + if top is not None: + node_window = top + node_bounds = window_bounds(app, path[0], top) record = { "runtimeId": list(path), - "controlType": node_role(node), + "controlType": role, "name": node_name(node), - "frame": safe(lambda: relative_frame(node, bounds)), + "frame": safe(lambda: relative_frame(node, node_bounds)), } - return node, record + # A projected combo edits through its partner, as a scanned record + # says (ax_records.py). + if role == "combo box" or gtk_combo_partner(node, role) is not None: + record["controlType"] = "combo box" + identity = combo_edit_identity(combo_edit_target(node)) + if identity is not None: + record["editTarget"] = identity + return node, record, node_window, node_bounds return None diff --git a/computer-use/native/linux/provider/guard_shadow.py b/computer-use/native/linux/provider/guard_shadow.py index ce0a695..66c6cb7 100644 --- a/computer-use/native/linux/provider/guard_shadow.py +++ b/computer-use/native/linux/provider/guard_shadow.py @@ -32,13 +32,18 @@ def guard_ablated(): return "guard" in {name.strip() for name in names} +def guards_shadowed(): + """Report whether a stopped input would be logged and let through.""" + return not _enforced and guard_ablated() + + def shadow_guard(channel, guard, reason, **detail): """Return True when the guard ablation lets the stopped input through. The decision is appended to the guard log first. Without the ablation nothing is written and the caller stops the input as before. """ - if _enforced or not guard_ablated(): + if not guards_shadowed(): return False record = { "at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) diff --git a/computer-use/runtime_tests/test_guard_shadow.py b/computer-use/runtime_tests/test_guard_shadow.py index 3a4912a..b32229d 100644 --- a/computer-use/runtime_tests/test_guard_shadow.py +++ b/computer-use/runtime_tests/test_guard_shadow.py @@ -72,26 +72,36 @@ def test_a_path_without_a_node_stays_stale(self): self.assertIsNone(resolved) self.assertEqual(self.guard_log(), []) - def semantic_refusal(self, code, node): + def semantic_refusal(self, code, node, path=(0, 4), role="push button", app=None, + combo=None, edit_identity=None): refused = runtime.OperationFailure( code + " selector='button:Save'; no input was emitted.", failure_code=code, failure_phase="precondition", input_events_emitted=False) - target = {"selector": "button:Save", "runtimeId": [0, 4]} - with mock.patch.object(element_resolver, "resolve_path", return_value=node), \ + target = {"selector": "button:Save", "runtimeId": list(path)} + with mock.patch.object(element_resolver, "resolve_path", return_value=node) as resolve, \ mock.patch.object(element_resolver, "state_value", return_value=None), \ - mock.patch.object(element_resolver, "node_role", return_value="push button"), \ + mock.patch.object(element_resolver, "node_role", return_value=role), \ mock.patch.object(element_resolver, "node_name", return_value="Cancel"), \ - mock.patch.object(element_resolver, "relative_frame", return_value=None): - return element_resolver.shadow_semantic_refusal(object(), 0, None, target, refused) + mock.patch.object(element_resolver, "relative_frame", return_value=None), \ + mock.patch.object(element_resolver, "window_bounds", return_value=(1, 2, 3, 4)), \ + mock.patch.object(element_resolver, "gtk_combo_partner", return_value=combo), \ + mock.patch.object(element_resolver, "combo_edit_target", return_value=object()), \ + mock.patch.object(element_resolver, "combo_edit_identity", return_value=edit_identity): + self.resolve_path = resolve + return element_resolver.shadow_semantic_refusal( + app or object(), 0, "main", (0, 0, 9, 9), target, refused) def test_semantic_identity_refusal_falls_back_to_the_snapshot_path(self): node = object() self.assertIsNone(self.semantic_refusal("semantic_target_not_found", node)) + # The full arm does not read the tree for a shadow it will not use. + self.resolve_path.assert_not_called() self.ablate_guard() for code in ("semantic_target_not_found", "semantic_target_ambiguous", "stale_element_ref"): - resolved, record = self.semantic_refusal(code, node) + resolved, record, window, bounds = self.semantic_refusal(code, node) self.assertIs(resolved, node) self.assertEqual(record["runtimeId"], [0, 4]) + self.assertEqual((window, bounds), ("main", (0, 0, 9, 9))) self.assertEqual([entry["reason"] for entry in self.guard_log()], ["semantic_target_not_found", "semantic_target_ambiguous", "stale_element_ref"]) # Refusals that are not identity checks, and paths without a node, stay. @@ -99,6 +109,45 @@ def test_semantic_identity_refusal_falls_back_to_the_snapshot_path(self): self.assertIsNone(self.semantic_refusal("stale_element_ref", None)) self.assertEqual(len(self.guard_log()), 3) + def test_shadowed_target_keeps_its_own_window(self): + self.ablate_guard() + dialog = object() + app = mock.Mock() + app.get_child_at_index.return_value = dialog + _node, _record, window, bounds = self.semantic_refusal( + "stale_element_ref", object(), path=(2, 1), app=app) + app.get_child_at_index.assert_called_once_with(2) + self.assertEqual((window, bounds), (dialog, (1, 2, 3, 4))) + + def test_shadowed_combo_keeps_its_edit_partner(self): + self.ablate_guard() + identity = {"providerBusName": ":1.4", "providerObjectPath": "/e", "processGeneration": "g"} + _node, record, _window, _bounds = self.semantic_refusal( + "stale_element_ref", object(), role="toggle button", combo=object(), + edit_identity=identity) + self.assertEqual(record["controlType"], "combo box") + self.assertEqual(record["editTarget"], identity) + + def test_select_end_identity_refusal_is_logged_and_let_through(self): + refused = runtime.OperationFailure( + "stale_element_ref; no input was emitted.", failure_code="stale_element_ref", + failure_phase="precondition", input_events_emitted=False) + window, end = object(), object() + app = mock.Mock() + app.get_child_count.return_value = 1 + app.get_child_at_index.return_value = window + operation = {} + action_input = {"to_target": {"runtimeId": [0, 7]}} + with mock.patch.object(ax_dispatch, "provider_object_path", return_value="/w"), \ + mock.patch.object(ax_dispatch, "resolve_semantic_target", side_effect=refused), \ + mock.patch.object(ax_dispatch, "shadow_semantic_refusal", + wraps=lambda *args: None if not guard_shadow.guards_shadowed() + else (end, {}, window, None)): + with self.assertRaises(runtime.OperationFailure): + ax_dispatch._resolve_select_end(app, operation, action_input, window, None) + self.ablate_guard() + self.assertIs(ax_dispatch._resolve_select_end(app, operation, action_input, window, None), end) + def expected_window_mismatch(self): with mock.patch.object(ax_dispatch, "provider_bus_name", return_value=":1.9"), \ mock.patch.object(ax_dispatch, "process_generation", return_value="g2"), \ From cbb036cf4d3885df2545a47f06cefaeade4cb653 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 05:35:21 +0000 Subject: [PATCH 10/13] ablation: drop the guide passages that the fixed texts contradicted batching removes "start the batch with activate"; diff removes the wait_timeout advice to read the returned state. check_ablation_text.py looks for both. Co-Authored-By: Claude Opus 5.5 --- hcu/osworld/guide_ablation.py | 6 ++++++ scripts/maint/check_ablation_text.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/hcu/osworld/guide_ablation.py b/hcu/osworld/guide_ablation.py index ba0f59c..d5a603f 100644 --- a/hcu/osworld/guide_ablation.py +++ b/hcu/osworld/guide_ablation.py @@ -88,6 +88,10 @@ _WINDOW_CHANGED_READ = """ Read the returned state and send only what remains.""" +_WAIT_TIMEOUT_READ = """ The message says what was seen instead, or that the + state could not be read. Read the returned state and revise the plan; do + not resend the same batch.""" + _APP_STARTING_WAIT = """if the application is still starting, wait once for its window with `{"mode":"act","actions":[{"type":"wait","window":"..."}]}`; otherwise """ @@ -98,6 +102,7 @@ "batching": ( ("hybrid.md", _PLAN_FROM_ONE_READ), ("hybrid.md", _ACT_EXAMPLES), + ("hybrid.md", "- `no_focused_window`: start the batch with `activate`.\n"), ), "wait": ( ("hybrid.md", " or wait again"), @@ -111,6 +116,7 @@ ("hybrid.md", _ACT_REPORT), ("hybrid.md", _SELECT_READBACK), ("hybrid.md", _WINDOW_CHANGED_READ), + ("hybrid.md", _WAIT_TIMEOUT_READ), ), "guard": (), } diff --git a/scripts/maint/check_ablation_text.py b/scripts/maint/check_ablation_text.py index 253b7fc..4460cac 100644 --- a/scripts/maint/check_ablation_text.py +++ b/scripts/maint/check_ablation_text.py @@ -37,8 +37,9 @@ r"\bsettle\b(?!_ms)", r"timeout_ms", r"wait_timeout", r"waits with", r"`wait`"], "diff": [r"\bdiffs?\b", r"(?m)^effect:", r"(?m)^windows:", r"report what changed", - r"reports the selection", r"reports what is now", r"reports each target"], - "batching": [r"ordered steps", r"into one `act`"], + r"reports the selection", r"reports what is now", r"reports each target", + r"(?i)returned state", r"says what was seen"], + "batching": [r"ordered steps", r"into one `act`", r"(?i)start the batch with"], "guard": [], } From b0248e5bdfa528889a6cab73ff138280b4e2a5b2 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 05:35:21 +0000 Subject: [PATCH 11/13] osworld: verify that the guest server applies the ablation The preflight probe sends one call per removed mechanism and requires each to be refused, and requires the server to refuse an unknown COMPUTER_USE_ABLATE value, so a build without the switches fails preparation instead of running full HCU under an ablation label. meta.json records the MCP binary's SHA-256. A task now fails before its agent starts when the gateway writes another token log than the run reads, which happened when ablation groups shared one gateway. Co-Authored-By: Claude Opus 5.5 --- docs/OSWORLD.md | 11 +++++++-- hcu/osworld/guest_lifecycle.py | 3 ++- hcu/osworld/runtime.py | 40 +++++++++++++++++++++++++++++++++ hcu/osworld/staging.py | 6 +++++ hcu/osworld/usage.py | 23 +++++++++++++++++++ tests/test_ablation_switches.py | 2 ++ tests/test_agent_budget.py | 13 +++++++++++ tests/test_public_mcp_probe.py | 35 ++++++++++++++++++++++++++--- 8 files changed, 127 insertions(+), 6 deletions(-) diff --git a/docs/OSWORLD.md b/docs/OSWORLD.md index 1527ec4..c2d342f 100644 --- a/docs/OSWORLD.md +++ b/docs/OSWORLD.md @@ -233,7 +233,12 @@ guides, `--instructions-file ""` and `--code-mode` are refused with `--ablate`, because they can still describe a removed mechanism. With `guard` removed, the server logs the guard decisions in the guest, and the runner copies the log to `guard.jsonl` in the result directory. Without `--ablate` -the harness configurations do not change. +the harness configurations do not change. The preflight probe also checks +that the server applies the ablation: it sends one call per removed +mechanism and requires each to be refused, and it requires the server to +refuse an unknown `COMPUTER_USE_ABLATE` value. A build without the switches +therefore fails preparation instead of running full HCU under an ablation +label. `meta.json` records the SHA-256 of the MCP binary. ## Run a panel @@ -242,7 +247,9 @@ file; omitted paths default to the repository's `artifacts/`, `runs/`, `assets/` and `instructions/`. See [`configs/osworld-v1-codex-queue.example.json`](../configs/osworld-v1-codex-queue.example.json) for the 20-task V1 panel. [`configs/ablation/`](../configs/ablation/) holds -the same panel for each mechanism-ablation group. +the same panel for each mechanism-ablation group. Groups that share one +gateway must share its `--out` file: set the same `token_log` in each config. +A task fails before its agent starts when the gateway writes another file. | Key | Meaning | | --- | --- | diff --git a/hcu/osworld/guest_lifecycle.py b/hcu/osworld/guest_lifecycle.py index 4be4c0c..e31b1e5 100644 --- a/hcu/osworld/guest_lifecycle.py +++ b/hcu/osworld/guest_lifecycle.py @@ -29,7 +29,7 @@ ) from hcu.osworld.staging import GUEST_AGENT_BINARIES, TaskRun, runtime_uploads from hcu.osworld.task import task_value -from hcu.osworld.usage import check_gateway_budget +from hcu.osworld.usage import check_gateway_budget, check_gateway_token_log from desktop_env.cache import run_scoped_cache_dir from desktop_env.desktop_env import DesktopEnv @@ -162,6 +162,7 @@ def prepare_guest(run: TaskRun, env: DesktopEnv) -> str: ) meta["gateway_health"] = health.get("output", "").strip() check_gateway_budget(meta["gateway_health"], args.max_model_calls) + check_gateway_token_log(meta["gateway_health"], run.token_log) agent_version = ( f"{shlex.quote(GUEST_GEMINI_NODE)} {shlex.quote(GUEST_GEMINI_CLI)}" diff --git a/hcu/osworld/runtime.py b/hcu/osworld/runtime.py index a8ce988..5599d01 100644 --- a/hcu/osworld/runtime.py +++ b/hcu/osworld/runtime.py @@ -363,6 +363,46 @@ def call(name, arguments): raise RuntimeError("unexpected public MCP tools: " + repr(names)) report = {"public_tools": names, "requested_protocol": config["protocol"], "negotiated_protocol": negotiated, "child_env_verified": True, "checks": {}} + ablated = [name.strip() for name in config.get("env", {}).get("COMPUTER_USE_ABLATE", "").split(",") if name.strip()] + if ablated: + # The env reached the server; check that the server applies it. A + # build without the switches would run full HCU under an ablation + # label. Each call is refused before input only when its mechanism is + # removed. + uses = { + "wait": ({"mode": "act", "actions": [{"type": "wait", "window": "__hcu_ablation_probe__", "timeout_ms": 100}]}, + {"actions[0].type"}), + "batching": ({"mode": "act", "actions": [{"type": "click", "ref": "r1"}, {"type": "click", "ref": "r2"}]}, + {"actions[1]", "actions"}), + "diff": ({"mode": "observe", "app": "__hcu_ablation_probe__", "observe": "diff"}, {"observe"}), + } + for name in ablated: + if name not in uses or "AX" not in names: + continue + arguments, paths = uses[name] + result = request("tools/call", {"name": "AX", "arguments": arguments}) + evidence = {} + for item in result.get("content") or []: + try: + evidence = json.loads(item.get("text", "")) + break + except (ValueError, TypeError): + continue + if not result.get("isError") or evidence.get("code") != "invalid_argument" or evidence.get("path") not in paths: + raise RuntimeError("MCP server ignores the %s ablation: %s" % (name, json.dumps(evidence)[:300])) + # guard changes no schema; a build that parses the switches refuses + # an unknown one before it answers initialize. + bogus = subprocess.run( + [config["command"], *config.get("args", [])], + input=json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": config["protocol"], "capabilities": {}, + "clientInfo": {"name": "osworld-v2-harness-preflight", "version": "1"}}}) + "\n", + env=dict(child_env, COMPUTER_USE_ABLATE="__hcu_ablation_probe__"), cwd=config.get("cwd"), + capture_output=True, text=True, timeout=30, + ) + if '"result"' in bogus.stdout: + raise RuntimeError("MCP server accepts an unknown COMPUTER_USE_ABLATE value; it cannot apply ablations") + report["checks"]["ablation"] = "passed: " + ",".join(ablated) if "AX" in names: apps, _ = call("AX", {"mode": "apps"}) windows = re.findall(r"^(w[1-9][0-9]*)\s+.*$", apps, re.M) diff --git a/hcu/osworld/staging.py b/hcu/osworld/staging.py index 40e0f33..f19158d 100644 --- a/hcu/osworld/staging.py +++ b/hcu/osworld/staging.py @@ -5,6 +5,7 @@ import argparse from dataclasses import dataclass, field import datetime +import hashlib import json from pathlib import Path import time @@ -396,6 +397,11 @@ def initial_meta(run: TaskRun, docker_runtime: dict) -> dict: else None ), "mcp_binary": str(run.mcp_binary), + # Identifies the build: every build reports the same version. + "mcp_binary_sha256": ( + hashlib.sha256(run.mcp_binary.read_bytes()).hexdigest() + if run.mcp_binary.is_file() else None + ), "mcp_channels": args.channels or None, "ablate": list(run.ablate), "code_mode": bool(getattr(args, "code_mode", False)), diff --git a/hcu/osworld/usage.py b/hcu/osworld/usage.py index bfd51be..f0941af 100644 --- a/hcu/osworld/usage.py +++ b/hcu/osworld/usage.py @@ -233,3 +233,26 @@ def check_gateway_budget(health_output: str, max_model_calls: int) -> None: f"runner={max_model_calls}, gateway={limit!r}. " "Start the gateway with --max-requests-per-task equal to max_model_calls." ) + + +def check_gateway_token_log(health_output: str, token_log: Path) -> None: + """Fail before the agent starts if the gateway logs to another file. + + The runner reads its usage and retry exhaustion from ``token_log``. Runs + that share a gateway with a different ``--out`` would otherwise report + zero usage and never see exhausted retries. + """ + try: + out = json.loads(health_output).get("out") + except (ValueError, AttributeError): + return + # Only the OpenAI gateway reports its log; a relative path is relative + # to the gateway's working directory, which the runner cannot know. + if not isinstance(out, str) or not Path(out).is_absolute(): + return + if Path(out).resolve() != Path(token_log).resolve(): + raise RuntimeError( + f"Gateway writes its token log to {out}, but this run reads {token_log}. " + "Start the gateway with --out set to the run's token_log, or give every " + "run that shares the gateway the same token_log." + ) diff --git a/tests/test_ablation_switches.py b/tests/test_ablation_switches.py index cc7224c..0e32ea0 100644 --- a/tests/test_ablation_switches.py +++ b/tests/test_ablation_switches.py @@ -2,6 +2,7 @@ import argparse import importlib.util +import hashlib import json from pathlib import Path import sys @@ -277,6 +278,7 @@ def test_ablation_run_gives_the_agent_ablated_guides(runner, tmp_path, agent): assert run.base_instructions_copy.read_text(encoding="utf-8").endswith(hybrid) assert "COMPUTER_USE_ABLATE" in run.config_path.read_text(encoding="utf-8") assert run.meta["ablate"] == ["wait", "diff"] + assert run.meta["mcp_binary_sha256"] == hashlib.sha256(run.mcp_binary.read_bytes()).hexdigest() assert run.meta["instructions_file"] == str(guides / "hybrid.md") diff --git a/tests/test_agent_budget.py b/tests/test_agent_budget.py index 8589bec..0259b08 100644 --- a/tests/test_agent_budget.py +++ b/tests/test_agent_budget.py @@ -123,6 +123,19 @@ def test_gateway_cap_is_checked_once_before_the_agent_starts(health, limit, ok): check_gateway_budget(health, limit) +def test_gateway_token_log_must_be_the_runs(tmp_path): + from hcu.osworld.usage import check_gateway_token_log + + log = tmp_path / "run" / "gateway-tokens.jsonl" + check_gateway_token_log(json.dumps({"out": str(log)}), log) + check_gateway_token_log(json.dumps({"out": str(tmp_path / "run" / ".." / "run" / log.name)}), log) + # The Anthropic gateway reports no log; a relative path cannot be checked. + for health in ("{}", "not json", json.dumps({"out": "gateway-tokens.jsonl"})): + check_gateway_token_log(health, log) + with pytest.raises(RuntimeError, match="token log"): + check_gateway_token_log(json.dumps({"out": str(tmp_path / "other.jsonl")}), log) + + def test_rate_limit_waits_extend_the_deadline_and_count_as_progress(polling, monkeypatch): monkeypatch.setattr(agent_loop, "MODEL_PROGRESS_TIMEOUT_SEC", 5) waits = [] diff --git a/tests/test_public_mcp_probe.py b/tests/test_public_mcp_probe.py index ef10d30..58f0198 100644 --- a/tests/test_public_mcp_probe.py +++ b/tests/test_public_mcp_probe.py @@ -14,6 +14,12 @@ SERVER = r''' import json, os, sys scenario = os.environ.get("PROBE_SCENARIO", "ok") +ablated = [n for n in os.environ.get("COMPUTER_USE_ABLATE", "").split(",") if n] +# The "ablated" server applies the switches and refuses unknown ones; the +# "stale" server ignores them, as a build without the switches does. +if scenario == "ablated" and any(n not in ("batching", "wait", "diff", "guard") for n in ablated): + sys.exit("unknown COMPUTER_USE_ABLATE") +REFUSED = {"wait": "actions[0].type", "batching": "actions[1]", "diff": "observe"} for line in sys.stdin: req = json.loads(line) if "id" not in req: @@ -24,7 +30,16 @@ elif method == "tools/list": result = {"tools": [{"name": x} for x in json.loads(sys.argv[1])]} else: - mode = params.get("arguments", {}).get("mode", "screenshot") + arguments = params.get("arguments", {}) + mode = arguments.get("mode", "screenshot") + uses = ("wait" if "wait" in json.dumps(arguments) else + "batching" if len(arguments.get("actions", [])) > 1 else + "diff" if arguments.get("observe") == "diff" else None) + if uses and scenario == "ablated" and uses in ablated: + evidence = {"ok": False, "code": "invalid_argument", "path": REFUSED[uses]} + print(json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": { + "isError": True, "content": [{"type": "text", "text": json.dumps(evidence)}]}}), flush=True) + continue if mode == "apps": text = 'w1 "Fixture" app=fixture active\n' if scenario == "empty": @@ -55,14 +70,17 @@ ''' -def probe(tmp_path, scenario="ok", names=None): +def probe(tmp_path, scenario="ok", names=None, ablate=""): names = ["AX", "GUI"] if names is None else names server = tmp_path / "server.py" server.write_text(SERVER) + env = {"PROBE_SCENARIO": scenario} + if ablate: + env["COMPUTER_USE_ABLATE"] = ablate config = { "command": sys.executable, "args": [str(server), json.dumps(names)], - "env": {"PROBE_SCENARIO": scenario}, + "env": env, "expected_tools": names, "protocol": "2026-07-28", } @@ -96,6 +114,17 @@ def test_probe_rejects_errors_or_missing_evidence(tmp_path, scenario): assert "RuntimeError" in result.stderr +@pytest.mark.parametrize("ablate", ["guard", "wait", "batching,wait,diff,guard"]) +def test_probe_confirms_the_server_applies_the_ablation(tmp_path, ablate): + result = probe(tmp_path, "ablated", ablate=ablate) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["checks"]["ablation"] == "passed: " + ablate + # A build without the switches runs full HCU; the probe must stop it. + stale = probe(tmp_path, "stale", ablate=ablate) + assert stale.returncode != 0 + assert "ignores the" in stale.stderr or "unknown COMPUTER_USE_ABLATE" in stale.stderr + + def test_empty_desktop_does_not_require_creating_a_task_window(tmp_path): result = probe(tmp_path, "empty", ["AX"]) assert result.returncode == 0, result.stderr From 3822a052f8091466be175be5ca184608486428a8 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Sun, 8 Nov 2026 07:14:49 +0000 Subject: [PATCH 12/13] ablation: diff replaces the change report with the whole tree Removing diff now answers what the diff contributes: an act still returns the state after its actions, as the whole current tree of its window, in place of the effect and windows report, the diff and the notes that compare with an earlier read. observe always repeats the tree instead of "Tree unchanged". The tree is read with the same post-action stabilization as a diff, so the ablation changes what is shown, not when it is read. The earlier variant returned only the execution evidence, which also removed every post-act observation; the error-path trimming it needed is reverted. effect_unknown stays in the evidence and a settle wait keeps its message: the first is execution evidence, the second belongs to wait. The guide and tool description say that an act result shows the whole tree instead of a report of what changed, and keep the advice to read the returned state. Without an ablation every model-visible byte is unchanged. Co-Authored-By: Claude Opus 5.5 --- computer-use/internal/contract/ablation.go | 14 +- .../internal/contract/tool_contract.go | 4 +- computer-use/internal/core/ax_ablation.go | 64 ++-- .../internal/core/ax_ablation_test.go | 277 ++++++++++++++---- computer-use/internal/core/ax_act_report.go | 9 +- .../internal/core/ax_action_evidence.go | 9 +- .../internal/core/ax_action_stabilization.go | 4 +- computer-use/internal/core/ax_batch.go | 3 +- .../internal/core/ax_close_observation.go | 10 +- computer-use/internal/core/ax_request.go | 5 +- .../internal/core/ax_snapshot_store.go | 12 +- computer-use/internal/core/ax_wait_batch.go | 19 -- computer-use/internal/core/service.go | 3 - computer-use/internal/core/tool_result.go | 5 - docs/OSWORLD.md | 8 +- hcu/osworld/guide_ablation.py | 16 +- scripts/maint/check_ablation_text.py | 2 +- tests/test_guide_ablation.py | 13 + 18 files changed, 310 insertions(+), 167 deletions(-) diff --git a/computer-use/internal/contract/ablation.go b/computer-use/internal/contract/ablation.go index b5a4650..5118a7f 100644 --- a/computer-use/internal/contract/ablation.go +++ b/computer-use/internal/contract/ablation.go @@ -18,8 +18,14 @@ import ( // still surround it. GUI batches are unchanged. // - wait: AX acts take no wait steps. The tools' own stabilization after // input is unchanged. -// - diff: an AX act returns only its execution evidence, not the effect -// and windows report, the diff or any other observation. +// - diff: an AX act returns its execution evidence and the whole current +// tree of its window, as observe "tree" does, without the effect and +// windows report, the diff, or notes that compare with an earlier read. +// observe always returns the tree, never "unchanged". The tree is read +// with the same post-action stabilization as a diff. Two things stay: +// effect_unknown in the evidence (whether input had a verified effect +// is execution evidence, not a diff) and a settle wait step's message +// (it belongs to wait). // - guard: the target and layout guards still decide, but only log the // input they would have stopped (COMPUTER_USE_GUARD_LOG) and let it // through. Only the Linux provider implements this. @@ -115,8 +121,8 @@ func PublishedAXActionTypes() []string { } // PublishedObserveValues is the observe list the schema publishes and the -// decoder accepts. Without diff, act takes no observe field, so diff and -// none, which only act uses, are gone too. +// decoder accepts. Without diff, act takes no observe field (it always +// returns the tree), so diff and none, which only act uses, are gone too. func PublishedObserveValues() []string { if !Ablated(AblateDiff) { return ObserveValues diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index 596b44d..ebdc88e 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -551,8 +551,8 @@ func publishedAXToolDescription() string { } if diff { // The tools state how the ablated act behaves, as the guide does. - sentence += " Its result reports only whether input was sent and how many steps completed; " + - "see what changed with observe, find, or GUI." + sentence += " Its result shows the whole current tree of its window after its " + + "actions instead of a report of what changed." } text := strings.Replace(axToolDescription, axActFullSentence, sentence, 1) text = strings.Replace(text, axActFullExample, example, 1) diff --git a/computer-use/internal/core/ax_ablation.go b/computer-use/internal/core/ax_ablation.go index d4ee12f..4b1acd0 100644 --- a/computer-use/internal/core/ax_ablation.go +++ b/computer-use/internal/core/ax_ablation.go @@ -14,48 +14,40 @@ import ( // Behavior of the mechanism ablations in the AX service; contract/ablation.go // says what each one removes. -// axActWithoutFeedback runs an act for the diff ablation. The batch runs -// exactly as with diff, including the post-action reads that time its -// return, but the model receives none of what those reads found: -// withoutAXActFeedback keeps only the execution evidence. The reads are not -// recorded as delivered, so a later observe compares with what the model -// actually saw and never answers "unchanged" from a hidden read. -func (s *Service) axActWithoutFeedback(request *axRequest, obs observation) ToolCallResult { - s.observations.hidden = true - defer func() { s.observations.hidden = false }() - result := s.axBatchDecoded(request.actions, obs, request.windowRef). - withEvidenceScope("delivery_or_local_ui") - // Drop images here, before callTool registers a delivered image as the - // GUI frame; the text goes in withoutAXActFeedback, after the evidence - // line has taken the error message from it. - kept := []ContentItem{} - for _, item := range result.Content { - if item.Type == "text" { - kept = append(kept, item) - } +// axActObservation is the observation an act returns. Under the diff +// ablation an act takes no observe field and always returns the whole +// current working tree, as observe "tree" does; withActReport and +// modelFacingAXNotes leave out what changed. +func axActObservation(obs observation) observation { + if contract.Ablated(contract.AblateDiff) { + obs.mode = contract.ObserveTree } - result.Content = kept - return WithAXModeEvidence(result, contract.AXModeAct) + return obs } -// axActEvidenceFields are the public evidence fields an act keeps under the -// diff ablation: whether it succeeded, the error, and how far input went. -var axActEvidenceFields = append(append([]string{}, axPublicFields...), - "failure_phase", "partial_input_possible", "retry_same_request") +// postActionReadStabilizes reports whether the read after an action waits +// for late content and a replaced top-level window. A diff act does; under +// the diff ablation the act's whole tree takes its place and is read the +// same way, so the ablation changes what is shown, not when it is read. +func postActionReadStabilizes(mode string) bool { + return mode == contract.ObserveDiff || + (mode == contract.ObserveTree && contract.Ablated(contract.AblateDiff)) +} -// withoutAXActFeedback reduces a prepared act result to its execution -// evidence: no effect or windows lines, diff, tree, notes or selection -// readback. -func withoutAXActFeedback(result ToolCallResult) ToolCallResult { - kept := map[string]any{} - for _, key := range axActEvidenceFields { - if value, present := result.StructuredContent[key]; present { - kept[key] = value +// axChangeNote reports a model-facing note that compares the state with an +// earlier read: what the diff ablation removes from AX results. +func axChangeNote(note string) bool { + for _, prefix := range []string{ + "[state] semantic_state=unchanged", "[state] visual_change=", "[state] ax_change=", + "[text-attrs]", "[select] ", "[transition] ", + "The accessibility tree structure is unchanged", "The accessibility tree is unchanged", + "[identity] The new top-level window", "The set of top-level windows changed", + } { + if strings.HasPrefix(note, prefix) { + return true } } - result.StructuredContent = kept - result.Content = []ContentItem{} - return result + return false } // axCheckOneInputStep enforces the batching ablation: an act holds at most diff --git a/computer-use/internal/core/ax_ablation_test.go b/computer-use/internal/core/ax_ablation_test.go index 033ed0c..67b6f79 100644 --- a/computer-use/internal/core/ax_ablation_test.go +++ b/computer-use/internal/core/ax_ablation_test.go @@ -109,7 +109,7 @@ func TestAblatedDescriptionStatesTheChangedBehavior(t *testing.T) { statements := map[string]string{ contract.AblateBatching: "act runs one input step with optional wait steps before or after it", contract.AblateWait: "act cannot pause for a later state; to check one, read again with observe or find.", - contract.AblateDiff: "Its result reports only whether input was sent and how many steps completed", + contract.AblateDiff: "Its result shows the whole current tree of its window after its actions instead of a report of what changed.", } for _, set := range ablationSets() { t.Run("ablate="+set, func(t *testing.T) { @@ -260,27 +260,95 @@ func continuityNotesForUnchanged(t *testing.T) []string { return continuityNotes(previous, current) } -func TestDiffAblationReturnsOnlyExecutionEvidence(t *testing.T) { +// modelText is the text a model reads of an AX result. +func modelText(result ToolCallResult) string { + texts := []string{} + for _, item := range FinalizeForModel(contract.PublicToolAX, result, false).Content { + if item.Type == "text" { + texts = append(texts, item.Text) + } + } + return strings.Join(texts, "\n") +} + +// reportActUnder observes the report fixture, sets COMPUTER_USE_ABLATE to +// set, and runs one act whose input applies change. +func reportActUnder(t *testing.T, set string, change func(*reportBackend), act func(map[string]string) map[string]any, names ...string) (*Service, map[string]string, ToolCallResult) { + t.Helper() s, backend := reportService(t, reportMain()) - _, refs := reportObserve(t, s, "Interface") - t.Setenv(contract.AblationEnv, contract.AblateDiff) - backend.onAction = func(b *reportBackend) { b.checked = true } - result := FinalizeForModel(contract.PublicToolAX, s.callTool(contract.PublicToolAX, map[string]any{ - "mode": "act", "actions": []any{map[string]any{"type": "click", "ref": refs["Interface"]}}, - }), false) - if len(result.Content) != 1 { - t.Fatalf("want only the evidence line, got %d items: %s", len(result.Content), ResultText(result)) - } - line := result.Content[0].Text - if line != `{"ok":true,"input_emitted":true,"completed":1,"requested":1}` { - t.Fatalf("evidence line %s", line) - } - // The act's own read was never delivered, so observe returns the tree - // instead of calling it unchanged. - observed := ResultText(s.callTool(contract.PublicToolAX, map[string]any{"mode": "observe", "app": "Editor"})) - if strings.Contains(observed, "Tree unchanged") || !strings.Contains(observed, `"Interface"`) { - t.Fatalf("observe after a hidden act read:\n%s", observed) + _, refs := reportObserve(t, s, names...) + t.Setenv(contract.AblationEnv, set) + backend.onAction = change + return s, refs, s.callTool(contract.PublicToolAX, act(refs)) +} + +func clickOn(name string, extra map[string]any) func(map[string]string) map[string]any { + return func(refs map[string]string) map[string]any { + args := map[string]any{"mode": "act", "actions": []any{map[string]any{"type": "click", "ref": refs[name]}}} + for key, value := range extra { + args[key] = value + } + return args } +} + +// The full HCU must stay as it was: the act reports the effect, the windows +// and a diff, and observe answers an unchanged tree by reference. +func TestFullHCUActOutputIsUnchanged(t *testing.T) { + s, _, result := reportActUnder(t, "", func(b *reportBackend) { b.checked = true }, + clickOn("Interface", nil), "Interface") + want := `{"ok":true,"input_emitted":true,"completed":1,"requested":1,"final_observation_kind":"diff",` + + `"final_observation_returned":true,"next_identical_observation_required":false,"observation_required":false}` + "\n" + + `effect: r1 checkbox "Interface" checked=false→true (only state flags changed in this window)` + "\n" + + "windows: no change\n\n" + + "app: \"Editor\"\nwindow: \"Report\" [ref=w1]\nax_diff:\n" + + ` - checkbox "Interface" [checked=true] [ref=r1]` + "\n" + if got := modelText(result); got != want { + t.Fatalf("full act:\n%s\nwant:\n%s", got, want) + } + observed := modelText(s.callTool(contract.PublicToolAX, map[string]any{"mode": "observe", "app": "Editor"})) + if !strings.Contains(observed, "Note: [state] semantic_state=unchanged") || + !strings.Contains(observed, "Tree unchanged: identical to the state of this window already "+ + "delivered by an earlier result (last full tree plus later diffs)") { + t.Fatalf("full observe:\n%s", observed) + } +} + +// Under diff an act returns its evidence and the whole current tree of its +// window, as observe "tree" does, without the report of what changed. +func TestDiffAblationActReturnsTheWholeTree(t *testing.T) { + check := func(change func(*reportBackend)) (string, string) { + _, _, full := reportActUnder(t, "", change, clickOn("Interface", map[string]any{"observe": "tree"}), "Interface") + s, _, ablated := reportActUnder(t, contract.AblateDiff, change, clickOn("Interface", nil), "Interface") + text := modelText(ablated) + for _, removed := range []string{"effect:", "windows: no change", "ax_diff", "semantic_state", "visual_change", "ax_change"} { + if strings.Contains(text, removed) { + t.Fatalf("act under diff still has %q:\n%s", removed, text) + } + } + // The internally decided evidence is that of the full HCU. + for _, key := range []string{"target_effect_observed", "effect_unknown", "final_observation_kind"} { + if full.StructuredContent[key] != ablated.StructuredContent[key] { + t.Fatalf("%s: full %v, diff %v", key, full.StructuredContent[key], ablated.StructuredContent[key]) + } + } + // The tree was delivered, so a later observe returns it again, + // never "unchanged". + observed := modelText(s.callTool(contract.PublicToolAX, map[string]any{"mode": "observe", "app": "Editor"})) + if strings.Contains(observed, "Tree unchanged") || strings.Contains(observed, "semantic_state") || + !strings.Contains(observed, `checkbox "Interface"`) { + t.Fatalf("observe under diff:\n%s", observed) + } + return text, modelText(full) + } + text, full := check(func(b *reportBackend) { b.checked = true }) + tree := text[strings.Index(text, "app: "):] + if !strings.HasPrefix(text, `{"ok":true,"input_emitted":true,"completed":1,"requested":1,"final_observation_kind":"tree"`) || + !strings.Contains(tree, `checkbox "Interface" [checked=true]`) || !strings.HasSuffix(full, tree) { + t.Fatalf("act under diff:\n%s\nfull with observe tree:\n%s", text, full) + } + check(func(*reportBackend) {}) + _, err := DecodeAXRequest(map[string]any{"mode": "act", "observe": "tree", "actions": []any{map[string]any{"type": "click", "ref": "r1"}}}) if err == nil || err.path != "observe" { @@ -288,66 +356,145 @@ func TestDiffAblationReturnsOnlyExecutionEvidence(t *testing.T) { } } -// Under diff, an act's error says how far input went but not what the act -// read: no values, window list, new wN or promised observation. -func TestDiffAblationErrorsCarryNoObservation(t *testing.T) { - s, backend := reportService(t, reportMain()) - window, refs := reportObserve(t, s, "Interface", "Size") - context := &AXBatchContext{AppToken: "Editor"} - if message, _, _ := s.axBatchContinuityRecovery(context); !strings.Contains(message, window) { - t.Fatalf("full HCU names the current window: %s", message) +// A window the act opened appears with its wN in the tree's window list. +func TestDiffAblationTreeListsAnOpenedWindow(t *testing.T) { + _, _, result := reportActUnder(t, contract.AblateDiff, func(b *reportBackend) { + b.windows[0].active = false + b.windows = append(b.windows, reportWindow{ + handle: reportDialogHandle, role: "dialog", title: "Insert Sheet", path: "/window/2", active: true, + }) + }, clickOn("OK", nil), "OK") + text := modelText(result) + if !regexp.MustCompile(`(?m)^ - dialog "Insert Sheet" \[ref=w[0-9]+\]$`).MatchString(text) || + strings.Contains(text, "opened") { + t.Fatalf("act under diff:\n%s", text) } - t.Setenv(contract.AblationEnv, contract.AblateDiff) +} + +// Error texts belong to other mechanisms (waits, postconditions, recovery) +// and stay as in the full HCU; only the report of what changed goes. +func TestDiffAblationErrorsKeepTheirMessages(t *testing.T) { + s, _ := reportService(t, reportMain()) + reportObserve(t, s, "Interface") + context := &AXBatchContext{AppToken: "Editor"} message, hint, current := s.axBatchContinuityRecovery(context) - if strings.Contains(message+hint, window) || hint != `{"mode":"apps"}` || current != window { - t.Fatalf("recovery under diff: %q %q %q", message, hint, current) - } - if lines := treeUnchangedReferenceLines(3); strings.Contains(lines[0], "diff") { - t.Fatalf("tree reference: %s", lines[0]) + t.Setenv(contract.AblationEnv, contract.AblateDiff) + if m, h, c := s.axBatchContinuityRecovery(context); m != message || h != hint || c != current { + t.Fatalf("recovery under diff: %q %q %q, full: %q %q %q", m, h, c, message, hint, current) } + t.Setenv(contract.AblationEnv, "") - evidence := func(args map[string]any) map[string]any { + evidence := func(result ToolCallResult) map[string]any { t.Helper() - result := FinalizeForModel(contract.PublicToolAX, s.callTool(contract.PublicToolAX, args), false) var line map[string]any - if len(result.Content) != 1 || json.Unmarshal([]byte(result.Content[0].Text), &line) != nil { - t.Fatalf("want one evidence line: %s", ResultText(result)) + if err := json.Unmarshal([]byte(strings.SplitN(modelText(result), "\n", 2)[0]), &line); err != nil { + t.Fatal(err) } return line } - timedOut := evidence(map[string]any{"mode": "act", "actions": []any{ - map[string]any{"type": "click", "ref": refs["Interface"]}, - map[string]any{"type": "wait", "ref": refs["Size"], "value": "99", "timeout_ms": 200}, - }}) - if timedOut["code"] != contract.AXCodeWaitTimeout || strings.Contains(fmt.Sprint(timedOut), `"12"`) || - strings.Contains(fmt.Sprint(timedOut), "observation") { - t.Fatalf("wait timeout under diff: %v", timedOut) - } - backend.loseReadback = true - lost := evidence(map[string]any{"mode": "act", "actions": []any{ - map[string]any{"type": "click", "ref": refs["Interface"]}, - }}) - if lost["ok"] != false || strings.Contains(fmt.Sprint(lost["message"]), "observation follows") { - t.Fatalf("lost readback under diff: %v", lost) + waitAct := func(refs map[string]string) map[string]any { + return map[string]any{"mode": "act", "actions": []any{ + map[string]any{"type": "click", "ref": refs["Interface"]}, + map[string]any{"type": "wait", "ref": refs["Size"], "value": "99", "timeout_ms": 200}, + }} + } + withTree := func(act func(map[string]string) map[string]any) func(map[string]string) map[string]any { + return func(refs map[string]string) map[string]any { + args := act(refs) + args["observe"] = "tree" + return args + } + } + _, _, full := reportActUnder(t, "", nil, withTree(waitAct), "Interface", "Size") + _, _, ablated := reportActUnder(t, contract.AblateDiff, nil, waitAct, "Interface", "Size") + if evidence(ablated)["code"] != contract.AXCodeWaitTimeout || + fmt.Sprint(evidence(ablated)) != fmt.Sprint(evidence(full)) || + !strings.Contains(fmt.Sprint(evidence(ablated)["message"]), `value is "12"`) || + !strings.Contains(modelText(ablated), `textbox "Size"`) || strings.Contains(modelText(ablated), "effect:") { + t.Fatalf("wait timeout under diff:\n%s\nfull:\n%s", modelText(ablated), modelText(full)) + } + + lose := func(b *reportBackend) { b.loseReadback = true } + _, _, full = reportActUnder(t, "", lose, clickOn("Interface", map[string]any{"observe": "tree"}), "Interface") + _, _, ablated = reportActUnder(t, contract.AblateDiff, lose, clickOn("Interface", nil), "Interface") + if line := evidence(ablated); line["code"] != "effect_unknown" || + fmt.Sprint(line) != fmt.Sprint(evidence(full)) || + !strings.Contains(fmt.Sprint(line["message"]), "A fresh current-state observation follows") || + !strings.Contains(modelText(ablated), `checkbox "Interface"`) || strings.Contains(modelText(ablated), "effect:") { + t.Fatalf("lost readback under diff:\n%s\nfull:\n%s", modelText(ablated), modelText(full)) } } func TestDiffAblationKeepsTheErrorOfAFailedAct(t *testing.T) { - s, _ := reportService(t, reportMain()) - reportObserve(t, s, "Interface") + stale := func(map[string]string) map[string]any { + return map[string]any{"mode": "act", "actions": []any{map[string]any{"type": "click", "ref": "r999"}}} + } + _, _, full := reportActUnder(t, "", nil, stale, "Interface") + _, _, ablated := reportActUnder(t, contract.AblateDiff, nil, stale, "Interface") + if !ablated.IsError || modelText(ablated) != modelText(full) { + t.Fatalf("rejected act under diff:\n%s\nfull:\n%s", modelText(ablated), modelText(full)) + } +} + +// Notes that compare with an earlier read go; notes on the current state stay. +func TestDiffAblationDropsChangeNotes(t *testing.T) { + notes := []string{ + "[diagnostic snapshot_unchanged] same tree", "[pixels] persistent_change=false", + "No accessibility change observed: window title, tree, focus and selection match.", + "[text-attrs] Document text formatting changed", `[select] selected 3 chars in 1 elements, from "a" to "c"`, + "[diagnostic snapshot_busy] busy", "[expect] r2 value is \"12\"", + "The accessibility tree structure is unchanged, while accessibility text attributes changed (see the [text-attrs] note above).", + "[transition] A bounded no-input probe observed the transient window finish and returned the current application window.", + "[identity] The new top-level window was refreshed once without input; the current wN/rN refs below are actionable.", + "The set of top-level windows changed but no window took focus within 0.8s. The snapshot below may show a different window.", + } + full := modelFacingAXNotes(notes) t.Setenv(contract.AblationEnv, contract.AblateDiff) - result := FinalizeForModel(contract.PublicToolAX, s.callTool(contract.PublicToolAX, map[string]any{ - "mode": "act", "actions": []any{map[string]any{"type": "click", "ref": "r999"}}, - }), false) - if len(result.Content) != 1 || !result.IsError { - t.Fatalf("want one evidence line for the error: %s", ResultText(result)) - } - var evidence map[string]any - if err := json.Unmarshal([]byte(result.Content[0].Text), &evidence); err != nil { - t.Fatal(err) + got := modelFacingAXNotes(notes) + want := []string{"[state] busy=true recovery=wait_then_observe", "[expect] r2 value is \"12\""} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("notes under diff %q, full %q", got, full) + } + if len(full) <= len(got) { + t.Fatalf("full notes %q", full) } - if evidence["ok"] != false || evidence["code"] == nil || evidence["message"] == nil { - t.Fatalf("error evidence %v", evidence) +} + +// The whole tree an act returns under diff is read with the same post-action +// stabilization as a diff: a value that arrives late is in it. +func TestDiffAblationReadsTheTreeAfterLateContent(t *testing.T) { + for _, set := range []string{"", contract.AblateDiff} { + backend := &recordingPlatformBackend{ + accessibilityResponses: []ProviderAccessibilityResult{ + {OK: true, Snapshot: contentStabilizationSnapshot("v1", "Ready")}, + {OK: true, Snapshot: contentStabilizationSnapshot("v1", "Ready")}, + {OK: true, Snapshot: contentStabilizationSnapshot("v2", "Saved")}, + }, + } + service := NewServiceWithBackend(backend) + off := false + service.getAppState("Editor", nil, nil, nil, nil, &off, nil, contract.TreeViewWorking, "", nil) + t.Setenv(contract.AblationEnv, set) + obs := axActObservation(observation{mode: contract.ObserveDiff}) + result := service.actionResult("Editor", AXServiceRequest{ + App: "Editor", Action: axKeypressCommand{Key: "Tab"}, + Observe: obs.resolvedMode(), IncludeScreenshot: &off, TreeView: contract.TreeViewWorking, + }) + if result.StructuredContent["content_stabilized"] != true || !strings.Contains(ResultText(result), `"Saved"`) { + t.Fatalf("ablate=%q read the transient state: %v\n%s", set, + result.StructuredContent["content_stabilized"], ResultText(result)) + } + } +} + +// A verified close returns the current state, the absent window, not a diff. +func TestDiffAblationClosedWindowIsCurrentState(t *testing.T) { + context := &AXBatchContext{WindowRef: "w3"} + full := ResultText(closedWindowFinalObservation(context, observation{mode: contract.ObserveTree})) + t.Setenv(contract.AblationEnv, contract.AblateDiff) + ablated := ResultText(closedWindowFinalObservation(context, observation{mode: contract.ObserveTree})) + if !strings.HasPrefix(full, "ax_diff:") || ablated != "window: null\nabsent_refs: [w3]" { + t.Fatalf("closed window under diff %q, full %q", ablated, full) } } diff --git a/computer-use/internal/core/ax_act_report.go b/computer-use/internal/core/ax_act_report.go index 59da8b6..34db24a 100644 --- a/computer-use/internal/core/ax_act_report.go +++ b/computer-use/internal/core/ax_act_report.go @@ -103,7 +103,9 @@ func (t *actReportTrace) after(s *Service, index int) *AppSnapshot { } // withActReport puts the report first in an act result. A call rejected -// before any input gets no report: nothing happened. +// before any input gets no report: nothing happened. Under the diff +// ablation the report still decides target_effect_observed but is not +// shown. func (s *Service) withActReport( result ToolCallResult, actions []axBatchAction, @@ -159,6 +161,11 @@ func (s *Service) withActReport( if verified && len(lines) > 1 { result.StructuredContent["target_effect_observed"] = true } + if contract.Ablated(contract.AblateDiff) { + // The diff ablation keeps the evidence the report decides, but + // not the report. + return result + } result.Content = append( []ContentItem{{Type: "text", Text: strings.Join(lines, "\n")}}, result.Content..., diff --git a/computer-use/internal/core/ax_action_evidence.go b/computer-use/internal/core/ax_action_evidence.go index 18d635d..e360af0 100644 --- a/computer-use/internal/core/ax_action_evidence.go +++ b/computer-use/internal/core/ax_action_evidence.go @@ -6,6 +6,8 @@ import ( "sort" "strconv" "strings" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) func unchangedNotes(windowFocusConfirmed bool, pixels pixelVerdict, attrsChanged bool) []string { @@ -350,8 +352,13 @@ func observeNoneNotes(notes []string) []string { func modelFacingAXNotes(notes []string) []string { out := make([]string, 0, len(notes)) seen := map[string]bool{} + withoutChanges := contract.Ablated(contract.AblateDiff) appendOnce := func(value string) { - if value == "" || seen[value] { + if withoutChanges && value == "[state] semantic_state=unchanged busy=true recovery=wait_then_observe" { + // The app's busy state is current state, not a change. + value = "[state] busy=true recovery=wait_then_observe" + } + if value == "" || seen[value] || (withoutChanges && axChangeNote(value)) { return } seen[value] = true diff --git a/computer-use/internal/core/ax_action_stabilization.go b/computer-use/internal/core/ax_action_stabilization.go index b66e204..d0dc1fa 100644 --- a/computer-use/internal/core/ax_action_stabilization.go +++ b/computer-use/internal/core/ax_action_stabilization.go @@ -204,7 +204,7 @@ func postActionWindowIdentityStabilizationTrigger( after *AppSnapshot, obs observation, ) string { - if obs.resolvedMode() != contract.ObserveDiff || + if !postActionReadStabilizes(obs.resolvedMode()) || before == nil || after == nil { return "" @@ -327,7 +327,7 @@ func shouldStabilizePostActionContent( if service == nil || identityStabilized || request.WindowAction != nil || - (observation{mode: request.Observe}).resolvedMode() != contract.ObserveDiff || + !postActionReadStabilizes((observation{mode: request.Observe}).resolvedMode()) || after == nil || after.externalDesktopTransition { return false diff --git a/computer-use/internal/core/ax_batch.go b/computer-use/internal/core/ax_batch.go index 042ecae..a73fdfc 100644 --- a/computer-use/internal/core/ax_batch.go +++ b/computer-use/internal/core/ax_batch.go @@ -441,8 +441,7 @@ func (s *Service) axBatchRun( observationSummary = "Preparation may have changed state or may complete later. " + "Stop; no recovery observation, wait or later action was attempted." } - // Under diff the act result drops the observation it read. - if finalObservationReturned && !contract.Ablated(contract.AblateDiff) { + if finalObservationReturned { observationSummary = "A fresh current-state observation follows. " + "It does not prove which action caused that state." if result.StructuredContent["action_outcome"] == "not_executed" { diff --git a/computer-use/internal/core/ax_close_observation.go b/computer-use/internal/core/ax_close_observation.go index 62a29c8..4088a0f 100644 --- a/computer-use/internal/core/ax_close_observation.go +++ b/computer-use/internal/core/ax_close_observation.go @@ -17,10 +17,12 @@ func closedWindowFinalObservation( if windowRef != "" { refLine = "\n absent_refs: [" + windowRef + "]" } - result := TextResult( - "ax_diff:\n effect: target_window_absent"+refLine, - false, - ) + text := "ax_diff:\n effect: target_window_absent" + refLine + if contract.Ablated(contract.AblateDiff) { + // The current state, without the diff format. + text = "window: null" + strings.Replace(refLine, "\n ", "\n", 1) + } + result := TextResult(text, false) result.StructuredContent = map[string]any{ "full_tree_returned": false, "tree_reference_only": false, diff --git a/computer-use/internal/core/ax_request.go b/computer-use/internal/core/ax_request.go index 1d21ff0..bd4a263 100644 --- a/computer-use/internal/core/ax_request.go +++ b/computer-use/internal/core/ax_request.go @@ -250,12 +250,9 @@ func (s *Service) axAct( request *axRequest, obs observation, ) ToolCallResult { - if contract.Ablated(contract.AblateDiff) { - return s.axActWithoutFeedback(request, obs) - } result := s.axBatchDecoded( request.actions, - obs, + axActObservation(obs), request.windowRef, ).withEvidenceScope("delivery_or_local_ui") return WithAXModeEvidence(result, contract.AXModeAct) diff --git a/computer-use/internal/core/ax_snapshot_store.go b/computer-use/internal/core/ax_snapshot_store.go index 967050b..a9f4534 100644 --- a/computer-use/internal/core/ax_snapshot_store.go +++ b/computer-use/internal/core/ax_snapshot_store.go @@ -116,13 +116,9 @@ func deliveredTreeKey(snapshot *AppSnapshot) string { // model's copy is the last full tree plus every diff delivered after it, so // the reference says where the state came from and how to get it again. func treeUnchangedReferenceLines(lineCount int) []string { - source := "last full tree plus later diffs" - if contract.Ablated(contract.AblateDiff) { - source = "the last full tree" - } return []string{fmt.Sprintf( "Tree unchanged: identical to the state of this window already "+ - "delivered by an earlier result ("+source+"), "+ + "delivered by an earlier result (last full tree plus later diffs), "+ "so its %d lines are not repeated; the rN refs shown there still "+ "apply. Call {\"mode\":\"observe\",\"observe\":\"full\"} "+ "to get the whole tree again.", @@ -144,7 +140,7 @@ func DeliveredSnapshotKey(snapshot *AppSnapshot) string { } func (s *Service) rememberDeliveredTree(key string) { - if key == "" || s.observations.hidden { + if key == "" { return } if len(s.observations.lastFullTree) >= contract.MaxDeliveredTrees && @@ -157,7 +153,7 @@ func (s *Service) rememberDeliveredTree(key string) { } func (s *Service) rememberDeliveredSnapshot(snapshot *AppSnapshot) { - if snapshot == nil || s.observations.hidden { + if snapshot == nil { return } if providerCoverageSummary(snapshot.Annotations).AcquisitionIncomplete > 0 { @@ -265,8 +261,10 @@ func (s *Service) getAppStateResult( providerCoverageSummary(snapshot.Annotations).AcquisitionIncomplete == 0 // A stable provider snapshot can receive new public refs after invalidation. // Only reuse the model's tree when its delivered lines still match. + // The diff ablation always returns the tree. previousDelivery := s.deliveredSnapshotFor(snapshot) if acquisitionComplete && !forceFull && previousDelivery != nil && + !contract.Ablated(contract.AblateDiff) && slices.Equal(previousDelivery.TreeLines, snapshot.TreeLines) && ((snapshot.identicalToPrevious && delivered) || s.deliveredStateMatches(snapshot)) { diff --git a/computer-use/internal/core/ax_wait_batch.go b/computer-use/internal/core/ax_wait_batch.go index 5397a83..d997c62 100644 --- a/computer-use/internal/core/ax_wait_batch.go +++ b/computer-use/internal/core/ax_wait_batch.go @@ -128,15 +128,6 @@ func (s *Service) stopAXBatchAtWait( if unreadable && !strings.Contains(message, "could not be read") { message += " (the state could not be read completely while waiting)" } - // Without diff an act reports no observation; the wait's message would - // otherwise carry the values and windows it saw. - noFeedback := contract.Ablated(contract.AblateDiff) - if noFeedback { - message = fmt.Sprintf("wait_timeout: the wait at actions[%d] did not hold; the steps before it ran and none after it", index) - if unreadable { - message += " (the state could not be read completely while waiting)" - } - } stopped := TextResult(message, true) if len(waitNotes) > 0 { stopped.Content = append([]ContentItem{{Type: "text", Text: strings.Join(waitNotes, "\n")}}, stopped.Content...) @@ -159,12 +150,6 @@ func (s *Service) stopAXBatchAtWait( if corrected, _ := waitResult.StructuredContent["wait_hint"].(string); corrected != "" { hint = corrected } - if noFeedback { - hint = `{"mode":"observe","window_ref":"` + FirstNonEmpty(context.WindowRef, "wN") + `"}` - if wait := actions[index].wait; wait != nil && wait.window != "" { - hint = `{"mode":"apps"}` - } - } stopped.StructuredContent = map[string]any{ contract.AXPublicErrorCodeKey: contract.AXCodeWaitTimeout, contract.AXPublicErrorPathKey: fmt.Sprintf("actions[%d]", index), @@ -282,10 +267,6 @@ func (s *Service) axBatchContinuityRecovery(context *AXBatchContext) (string, st } if ref := snapshot.publicWindowRef; ref != "" { if entry, err := s.WindowRefRegistry.Resolve(ref, ""); err == nil && entry.WindowHandle == snapshot.WindowHandle { - if contract.Ablated(contract.AblateDiff) { - // The new window's wN is a windows report, which diff removes. - return "List current windows, inspect the intended window, then send only the remaining actions.", `{"mode":"apps"}`, ref - } return "The latest read shows " + ref + ". Observe that window before sending only the remaining actions.", axHintJSON(map[string]any{"mode": "observe", "window_ref": ref}), ref } diff --git a/computer-use/internal/core/service.go b/computer-use/internal/core/service.go index 6e42598..c56b560 100644 --- a/computer-use/internal/core/service.go +++ b/computer-use/internal/core/service.go @@ -22,9 +22,6 @@ type failedObservationEntry struct { type observationLedger struct { lastComplete map[string]*AppSnapshot lastFullTree map[string]bool - // hidden is set while an act runs whose observations the model never - // receives (the diff ablation); they are then not recorded as delivered. - hidden bool } // DesktopState is shared by every caller of one desktop: the provider, the diff --git a/computer-use/internal/core/tool_result.go b/computer-use/internal/core/tool_result.go index 841bae2..cb13b81 100644 --- a/computer-use/internal/core/tool_result.go +++ b/computer-use/internal/core/tool_result.go @@ -268,13 +268,8 @@ func FinalizeForModel(name string, result ToolCallResult, structured bool) ToolC func prepareModelEvidence(name string, result ToolCallResult) ToolCallResult { switch name { case contract.PublicToolAX: - actFeedbackHidden := contract.Ablated(contract.AblateDiff) && - result.StructuredContent["ax_mode"] == contract.AXModeAct result = EnforceAXResponseBudget(result) result = PrepareAXEvidence(result) - if actFeedbackHidden { - result = withoutAXActFeedback(result) - } case contract.PublicToolGUI: result = prepareGUIEvidence(result) } diff --git a/docs/OSWORLD.md b/docs/OSWORLD.md index c2d342f..5c2ec1b 100644 --- a/docs/OSWORLD.md +++ b/docs/OSWORLD.md @@ -221,7 +221,13 @@ The gateway defaults to the Docker bridge address at the agent's port; one channel only; `--instructions-file ""` runs without a profile. `--ablate` removes HCU mechanisms from the computer-use server: a -comma-separated subset of `batching`, `wait`, `diff` and `guard`. The runner +comma-separated subset of `batching`, `wait`, `diff` and `guard`. With +`diff` removed, an `act` returns the whole current tree of its window in +place of the change report and diff, and `observe` always repeats the tree. +`effect_unknown` in the act's evidence and the message of a settle wait +step are kept: the first is execution evidence and the second belongs to +`wait`. +The runner sets `COMPUTER_USE_ABLATE` in the MCP server environment of every harness and gives the agent copies of `instructions/base.md` and `instructions/hybrid.md` without the passages that describe the removed mechanisms. For each of diff --git a/hcu/osworld/guide_ablation.py b/hcu/osworld/guide_ablation.py index d5a603f..da6a63b 100644 --- a/hcu/osworld/guide_ablation.py +++ b/hcu/osworld/guide_ablation.py @@ -85,12 +85,9 @@ not resend the same batch. """ -_WINDOW_CHANGED_READ = """ Read the - returned state and send only what remains.""" - -_WAIT_TIMEOUT_READ = """ The message says what was seen instead, or that the - state could not be read. Read the returned state and revise the plan; do - not resend the same batch.""" +# The wait advice names report lines that the diff ablation removes. +_REPORT_NEEDS_NO_WAIT = """ A checkbox that shows `checked=false→true`, or a dialog + listed as `opened`, needs no wait.""" _APP_STARTING_WAIT = """if the application is still starting, wait once for its window with `{"mode":"act","actions":[{"type":"wait","window":"..."}]}`; @@ -115,8 +112,7 @@ "diff": ( ("hybrid.md", _ACT_REPORT), ("hybrid.md", _SELECT_READBACK), - ("hybrid.md", _WINDOW_CHANGED_READ), - ("hybrid.md", _WAIT_TIMEOUT_READ), + ("hybrid.md", _REPORT_NEEDS_NO_WAIT), ), "guard": (), } @@ -137,8 +133,8 @@ `observe` or `find`. """, "diff": """\ -- An `act` result reports only whether input was sent and how many steps - completed. To see what changed, use `observe`, `find`, or a GUI screenshot. +- An `act` result shows the whole current tree of its window after its + actions instead of a report of what changed. """, } # With both batching and wait ablated, an act holds one step and no waits. diff --git a/scripts/maint/check_ablation_text.py b/scripts/maint/check_ablation_text.py index 4460cac..2bc9a27 100644 --- a/scripts/maint/check_ablation_text.py +++ b/scripts/maint/check_ablation_text.py @@ -38,7 +38,7 @@ r"`wait`"], "diff": [r"\bdiffs?\b", r"(?m)^effect:", r"(?m)^windows:", r"report what changed", r"reports the selection", r"reports what is now", r"reports each target", - r"(?i)returned state", r"says what was seen"], + r"checked=false→true", r"listed as `opened`"], "batching": [r"ordered steps", r"into one `act`", r"(?i)start the batch with"], "guard": [], } diff --git a/tests/test_guide_ablation.py b/tests/test_guide_ablation.py index af648be..8c09786 100644 --- a/tests/test_guide_ablation.py +++ b/tests/test_guide_ablation.py @@ -52,3 +52,16 @@ def test_ablations_only_remove_text_besides_the_statements(ablations): # What remains is the original with characters removed, nothing added. remaining = iter(original) assert all(character in remaining for character in guide) + + +def test_diff_guide_keeps_the_returned_state_and_drops_the_report(): + guide = ablated_guide(source("hybrid.md"), "hybrid.md", ("diff",)) + # An act returns the whole tree, so reading the returned state stays. + assert "Read the\n returned state and send only what remains." in guide + assert "Read the returned state and revise the plan" in guide + assert "The message says what was seen instead" in guide + # The report of what changed, and advice that relies on it, goes. + for removed in ("then a diff", "effect: r58", "windows: opened", "checked=false→true", + "listed as `opened`", "reports what is now"): + assert removed not in guide, removed + assert "shows the whole current tree of its window" in STATEMENTS["diff"] From 6bc1d372e90e21218006081f5584b73863c9f8bc Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 18:01:39 +0000 Subject: [PATCH 13/13] ablation: cover batch-window text, read_text and diff-ablation errors Remove the batch-window sentence when batching is ablated, assert that the diff-ablation error keeps its cause, and check that read_text survives every ablation set. --- .../internal/contract/ablation_test.go | 30 +++++++++++++++++++ .../internal/contract/tool_contract.go | 11 ++++--- .../internal/core/ax_ablation_test.go | 3 +- .../internal/core/ax_text_read_test.go | 12 ++++++++ hcu/osworld/guide_ablation.py | 9 ++++-- tests/test_guide_ablation.py | 14 +++++++++ 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/computer-use/internal/contract/ablation_test.go b/computer-use/internal/contract/ablation_test.go index 5aff611..18b6fdf 100644 --- a/computer-use/internal/contract/ablation_test.go +++ b/computer-use/internal/contract/ablation_test.go @@ -77,3 +77,33 @@ func TestPublishedTextWithoutAblationIsTheReleaseText(t *testing.T) { t.Fatalf("instructions: %s", PublishedServerInstructions()) } } + +func TestBatchWindowDescriptionFollowsBatchingAblation(t *testing.T) { + for mask := 0; mask < 1<