From 69ef77b40da1ae928085221fa8680e5ad1bbe0ac Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 07:45:30 +0000 Subject: [PATCH 01/15] fix(ax): verify ref-less typing and report text effects Keep the implicit focused editor through keyboard fallback, stop if activation changes it, and verify the exact text insertion with bounded readback. Report unchanged or unavailable text explicitly without retrying it. Include value evidence for every type step in act reports. Do not treat focus changes or shared batch snapshots as proof of text delivery. Cover rejected, delayed, ambiguous and unreadable input with regression tests. --- CHANGELOG.md | 8 + computer-use/internal/core/ax_act_report.go | 128 +++++++++- .../internal/core/ax_act_report_test.go | 226 +++++++++++++++++- .../native/linux/provider/ax_actions.py | 8 +- .../linux/provider/ax_input_delivery.py | 68 +++++- computer-use/runtime_tests/test_text_input.py | 125 +++++++++- docs/MCP.md | 15 +- 7 files changed, 552 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b2570..5663ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased + +- AX reports text-value evidence for `type` steps without a ref, including + unchanged or unreadable fields. A focus change alone does not confirm typing. +- Linux AX checks focused-field readback after keyboard typing and reports + unconfirmed text explicitly. If window activation changes the resolved text + field, typing stops before keyboard input. + ## 1.0.0 First open-source release of Hybrid Computer Use. diff --git a/computer-use/internal/core/ax_act_report.go b/computer-use/internal/core/ax_act_report.go index f6678b9..4484ca1 100644 --- a/computer-use/internal/core/ax_act_report.go +++ b/computer-use/internal/core/ax_act_report.go @@ -15,8 +15,8 @@ import ( // effect: r58 checkbox "Interface" checked=false→true // windows: opened w5 "Insert Sheet" (dialog, focused); closed w3 "Print"; focus w2 // -// One effect line per action that named an element ref, compared between -// the read before that action and the next read after it. The windows line +// One effect line per type or action that named an element ref, compared +// between the reads around that action. The windows line // compares the application's top-level windows before the first action with // those after the last one. Both come from reads the batch makes anyway; the // report adds no provider call. @@ -98,25 +98,38 @@ func (s *Service) withActReport( } } lines := []string{} - // The effect counts as observed only when every action named an element - // and each of those elements visibly changed; a keypress or a no-change - // action keeps the call's uncertainty. + // The effect counts as observed only when every action has its own + // evidence: a value change for type, a target change for other actions. verified := completed == len(actions) for index, action := range actions { if action.actionType == "wait" { continue } - if !isShortAXRef(action.targetRef) { + if action.actionType != "type" && !isShortAXRef(action.targetRef) { verified = false continue } - if _, ran := trace.before[index]; !ran { - continue - } if index > completed { break } - line, changed := s.actEffectLine(trace, index, action.targetRef) + if _, ran := trace.before[index]; !ran && (action.actionType != "type" || index >= completed) { + continue + } + var line string + var changed bool + if action.actionType == "type" { + switch { + case index > 0 && actions[index-1].actionType != "wait" && + trace.before[index] != nil && trace.before[index] == trace.before[index-1]: + line = axReportEffectPrefix + "type unknown (no separate read before the action)" + case trace.before[index] != nil && index+1 < completed && trace.before[index+1] == nil: + line = axReportEffectPrefix + "type unknown (no separate read after the action)" + default: + line, changed = s.actTypeEffectLine(trace, index, action.targetRef) + } + } else { + line, changed = s.actEffectLine(trace, index, action.targetRef) + } lines = append(lines, line) verified = verified && changed } @@ -131,6 +144,101 @@ func (s *Service) withActReport( return result } +func (s *Service) actTypeEffectLine(trace *actReportTrace, index int, ref string) (string, bool) { + head := axReportEffectPrefix + "type" + beforeSnapshot := trace.before[index] + if beforeSnapshot == nil || beforeSnapshot.AXSnapshot == nil { + return head + " unknown (no read before the action)", false + } + if ref == "" { + var reason string + ref, reason = axReportTypeFocusRef(beforeSnapshot) + if reason != "" { + return head + " unknown (" + reason + ")", false + } + } + head += " " + ref + before := axSnapshotNodeByRef(beforeSnapshot, ref) + if before == nil { + return head + " unknown (the read before the action did not include it)", false + } + head += " " + axReportNodeLabel(before) + afterSnapshot := trace.after(s, index) + if afterSnapshot == nil { + reason := "no read after the action" + if trace.observe == contract.ObserveNone { + reason = `observe "none" skipped the read after the action` + } + return head + " unknown (" + reason + ")", false + } + if afterSnapshot.WindowHandle != beforeSnapshot.WindowHandle { + return head + " unknown (the read after the action is of " + axReportSnapshotWindow(afterSnapshot) + ")", false + } + after := axSnapshotNodeByRef(afterSnapshot, ref) + if after == nil { + return head + " unknown (the read after the action did not include it)", false + } + old, now := axReportTypeValue(before), axReportTypeValue(after) + if old == "" { + line := head + " unknown (value before the action is empty or unavailable)" + if now != "" { + line += "; value after=" + strconv.Quote(truncateRunes(now, 60)) + } + return line, false + } + if now == "" { + return head + " unknown (value after the action is empty or unavailable)", false + } + if old == now { + return head + " no visible change (value unchanged; text input not verified)", false + } + from, to := strconv.Quote(truncateRunes(old, 60)), strconv.Quote(truncateRunes(now, 60)) + if from == to { + return head + " value changed (difference beyond the displayed prefix)", true + } + return head + " value=" + from + "→" + to, true +} + +func axReportTypeFocusRef(snapshot *AppSnapshot) (string, string) { + if !snapshot.AXSnapshot.RefInventoryComplete { + return "", "the read before the action is incomplete; focus uniqueness is unknown" + } + roots := snapshot.AXSnapshot.CanonicalRoots + if len(roots) == 0 { + roots = snapshot.AXSnapshot.Roots + } + var focused *AXNode + count := 0 + var visit func([]*AXNode) + visit = func(nodes []*AXNode) { + for _, node := range nodes { + if node == nil { + continue + } + if node.States.Active == TriTrue { + focused = node + count++ + } + visit(node.Children) + } + } + visit(roots) + if count != 1 || focused.Role != "textbox" || !focused.Capabilities.EditText { + return "", "no unique focused editable textbox in the read before the action" + } + if !isShortAXRef(focused.Ref) { + return "", "the focused editable textbox has no ref" + } + return focused.Ref, "" +} + +func axReportTypeValue(node *AXNode) string { + if node.SourceRecord != nil { + return node.SourceRecord.Value + } + return node.Properties.Value +} + // actEffectLine compares one element between the reads around its action. // changed reports a visible change of the element. func (s *Service) actEffectLine(trace *actReportTrace, index int, ref string) (string, bool) { diff --git a/computer-use/internal/core/ax_act_report_test.go b/computer-use/internal/core/ax_act_report_test.go index 6e323d8..69972ec 100644 --- a/computer-use/internal/core/ax_act_report_test.go +++ b/computer-use/internal/core/ax_act_report_test.go @@ -26,9 +26,10 @@ type reportWindow struct { // reads the window back, as the real provider does. type reportBackend struct { recordingPlatformBackend - windows []reportWindow - checked bool - value string + windows []reportWindow + checked bool + value string + textStates string // onAction mutates the state when input arrives. onAction func(b *reportBackend) // skipReadback makes an action return no snapshot (observe "none"). @@ -47,6 +48,7 @@ func (b *reportBackend) snapshot(selected reportWindow) *ProviderSnapshot { checked = "[checked]" } setValue := true + textStates := FirstNonEmpty(b.textStates, "[editable focusable]") snapshot := &AppSnapshot{ App: AppDescriptor{Name: "Editor", PID: 10, ProviderBusName: ":1.84", ProcessGeneration: "pid:10:start:100", InputCapable: true}, @@ -55,14 +57,14 @@ func (b *reportBackend) snapshot(selected reportWindow) *ProviderSnapshot { TreeLines: []string{ "\t0 " + selected.role + " \"" + selected.title + "\"", "\t\t1 check box \"Interface\" " + checked, - "\t\t2 entry \"Size\" [editable]", + "\t\t2 entry \"Size\" " + textStates, "\t\t3 push button \"OK\" [click]", }, Elements: []ElementRecord{ {Index: 0, RuntimeID: []int{0}, ControlType: selected.role, Name: selected.title}, {Index: 1, RuntimeID: []int{0, 0}, ControlType: "check box", Name: "Interface", States: checked}, {Index: 2, RuntimeID: []int{0, 1}, ControlType: "entry", Name: "Size", Value: b.value, - States: "[editable focusable]", SetValueSupported: &setValue}, + States: textStates, SetValueSupported: &setValue}, {Index: 3, RuntimeID: []int{0, 2}, ControlType: "push button", Name: "OK", States: "[has-click-action]"}, {Index: 4, RuntimeID: []int{0, 3}, ControlType: "label", Name: FirstNonEmpty(b.page, "General")}, {Index: 5, RuntimeID: []int{0, 4}, ControlType: "menu item", Name: "Edit", States: "[has-click-action]"}, @@ -274,6 +276,220 @@ func TestActReportShowsAValueChange(t *testing.T) { } } +func TestActReportClickThenRefLessType(t *testing.T) { + for _, outcome := range []string{"unchanged", "changed"} { + t.Run(outcome, func(t *testing.T) { + changed := outcome == "changed" + service, backend := reportService(t, reportMain()) + backend.value = "1" + _, refs := reportObserve(t, service, "Size") + backend.onAction = func(backend *reportBackend) { + backend.textStates = "[editable focusable focused]" + backend.value = "12" + if len(backend.actionRequests) == 2 && changed { + backend.value = "123" + } + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{ + map[string]any{"type": "click", "ref": refs["Size"]}, + map[string]any{"type": "type", "text": "3"}, + }, + }) + text := ResultText(result) + if result.IsError || len(backend.actionRequests) != 2 { + t.Fatalf("expected both actions to run: %s", text) + } + if !strings.Contains(text, `effect: `+refs["Size"]+` textbox "Size" active=false→true`) { + t.Fatalf("click focus change missing: %s", text) + } + want := "no visible change (value unchanged; text input not verified)" + if changed { + want = `value="12"→"123"` + } + if !strings.Contains(text, `effect: type `+refs["Size"]+` textbox "Size" `+want) { + t.Fatalf("type must compare against the read after click: %s", text) + } + assertTypeReportEvidence(t, result, changed) + }) + } +} + +func reportTypeSnapshot(value string) *AppSnapshot { + record := &ElementRecord{ControlType: "entry", Name: "Input", Value: value, States: "[editable focused]"} + node := &AXNode{ + Ref: "r1", Role: "textbox", Name: "Input", SourceRecord: record, + States: typedAXStates(record), Capabilities: typedAXCapabilities(record, "textbox"), + Properties: AXProperties{Value: normalizedTreeText(value)}, + } + return &AppSnapshot{WindowHandle: reportMainHandle, WindowTitle: "Report", AXSnapshot: &AXSnapshot{ + CanonicalRoots: []*AXNode{node}, Roots: []*AXNode{node}, RefInventoryComplete: true, + }} +} + +func assertTypeReportEvidence(t *testing.T, result ToolCallResult, observed bool) { + t.Helper() + if got := result.StructuredContent["target_effect_observed"] == true; got != observed { + t.Fatalf("target_effect_observed=%v want %v: %s", got, observed, ResultText(result)) + } + public := ResultText(WithAXTextEvidence(PrepareAXEvidence(result))) + if strings.Contains(public, `"effect_unknown":true`) == observed { + t.Fatalf("public uncertainty disagrees with value evidence: %s", public) + } +} + +func TestActReportTypeRequiresValueEvidence(t *testing.T) { + for _, ref := range []string{"", "r1"} { + for _, test := range []struct { + name, before, after, want string + observed bool + }{ + {"changed", "old", "new", `value="old"→"new"`, true}, + {"unchanged", "old", "old", "no visible change (value unchanged; text input not verified)", false}, + {"empty before", "", "new", `unknown (value before the action is empty or unavailable); value after="new"`, false}, + {"empty after", "old", "", "unknown (value after the action is empty or unavailable)", false}, + {"both unavailable", "", "", "unknown (value before the action is empty or unavailable)", false}, + {"whitespace", "old text", "old text", `value="old text"→"old text"`, true}, + {"newline", "old text", "old\ntext", `value="old text"→"old\ntext"`, true}, + {"clipped suffix", strings.Repeat("a", 80) + "b", strings.Repeat("a", 80) + "c", "value changed", true}, + } { + t.Run(ref+"/"+test.name, func(t *testing.T) { + service := NewService() + before, after := reportTypeSnapshot(test.before), reportTypeSnapshot(test.after) + after.AXSnapshot.Roots[0].States.Active = TriFalse + after.AXSnapshot.Roots[0].States.Selected = TriTrue + service.snapshots["editor"] = after + trace := &actReportTrace{appToken: "editor", start: before, before: map[int]*AppSnapshot{0: before}} + result := service.withActReport(ToolCallResult{StructuredContent: map[string]any{ + "input_events_emitted": true, "effect_unknown": true, + }}, []axBatchAction{{actionType: "type", targetRef: ref, text: "text"}}, trace) + assertTypeReportEvidence(t, result, test.observed) + if text := ResultText(result); !strings.Contains(text, `effect: type r1 textbox "Input" `+test.want) { + t.Fatalf("type report lacks value evidence: %s", text) + } + }) + } + } +} + +func TestActReportTypeMissingEvidence(t *testing.T) { + for _, ref := range []string{"", "r1"} { + for _, scenario := range []string{"no before", "no after", "same snapshot", "observe none", "target gone", "other window", "no value source"} { + t.Run(ref+"/"+scenario, func(t *testing.T) { + service := NewService() + before, after := reportTypeSnapshot("old"), reportTypeSnapshot("new") + trace := &actReportTrace{appToken: "editor", start: before, before: map[int]*AppSnapshot{0: before}} + reason := "" + switch scenario { + case "no before": + trace.before[0] = nil + reason = "no read before the action" + case "no after": + after = nil + reason = "no read after the action" + case "same snapshot": + after = before + reason = "no read after the action" + case "observe none": + after = before + trace.observe = contract.ObserveNone + reason = `observe "none" skipped the read after the action` + case "target gone": + after.AXSnapshot.CanonicalRoots, after.AXSnapshot.Roots = nil, nil + reason = "the read after the action did not include it" + case "other window": + after.WindowHandle = reportDialogHandle + reason = "the read after the action is of" + case "no value source": + after.AXSnapshot.Roots[0].SourceRecord = nil + after.AXSnapshot.Roots[0].Properties.Value = "" + reason = "value after the action is empty or unavailable" + } + service.snapshots["editor"] = after + result := service.withActReport(ToolCallResult{StructuredContent: map[string]any{ + "input_events_emitted": true, "effect_unknown": true, + }}, []axBatchAction{{actionType: "type", targetRef: ref, text: "new"}}, trace) + text := ResultText(result) + if !strings.HasPrefix(text, "effect: type ") || !strings.Contains(strings.Split(text, "\n")[0], "unknown ("+reason) { + t.Fatalf("missing reason %q: %s", reason, text) + } + assertTypeReportEvidence(t, result, false) + }) + } + } +} + +func TestActReportRefLessTypeRequiresUniqueFocusedTextbox(t *testing.T) { + for _, scenario := range []string{"unfocused", "not editable", "not textbox", "two focused", "focused button too", "no ref", "incomplete", "projected roots"} { + t.Run(scenario, func(t *testing.T) { + service := NewService() + before, after := reportTypeSnapshot("old"), reportTypeSnapshot("new") + node := before.AXSnapshot.Roots[0] + reason := "no unique focused editable textbox in the read before the action" + observed := false + switch scenario { + case "unfocused": + node.States.Active = TriUnknown + case "not editable": + node.Capabilities.EditText = false + case "not textbox": + node.Role = "button" + case "two focused", "focused button too": + other := *node + other.Ref = "r2" + if scenario == "focused button too" { + other.Role = "button" + } + node.Children = []*AXNode{&other} + case "no ref": + node.Ref = "" + reason = "the focused editable textbox has no ref" + case "incomplete": + before.AXSnapshot.RefInventoryComplete = false + reason = "the read before the action is incomplete; focus uniqueness is unknown" + case "projected roots": + before.AXSnapshot.CanonicalRoots = nil + observed = true + } + service.snapshots["editor"] = after + trace := &actReportTrace{appToken: "editor", start: before, before: map[int]*AppSnapshot{0: before}} + result := service.withActReport(ToolCallResult{StructuredContent: map[string]any{ + "input_events_emitted": true, "effect_unknown": true, + }}, []axBatchAction{{actionType: "type", text: "new"}}, trace) + want := "unknown (" + reason + ")" + if observed { + want = `value="old"→"new"` + } + if text := ResultText(result); !strings.Contains(strings.Split(text, "\n")[0], want) { + t.Fatalf("type report lacks %q: %s", want, text) + } + assertTypeReportEvidence(t, result, observed) + }) + } +} + +func TestActReportTypeDoesNotShareBatchValueEvidence(t *testing.T) { + for _, shared := range []string{"no intermediate snapshot", "reused intermediate snapshot"} { + t.Run(shared, func(t *testing.T) { + service := NewService() + before, after := reportTypeSnapshot("old"), reportTypeSnapshot("oldnew") + service.snapshots["editor"] = after + trace := &actReportTrace{appToken: "editor", start: before, before: map[int]*AppSnapshot{0: before}} + if shared == "reused intermediate snapshot" { + trace.before[1] = before + } + result := service.withActReport(ToolCallResult{StructuredContent: map[string]any{ + "input_events_emitted": true, "effect_unknown": true, + }}, []axBatchAction{{actionType: "type", text: "new"}, {actionType: "type", text: "ignored"}}, trace) + text := ResultText(result) + if strings.Count(text, "effect: type") != 2 || strings.Count(text, " unknown (") != 2 { + t.Fatalf("both types need separate before/after evidence: %s", text) + } + assertTypeReportEvidence(t, result, false) + }) + } +} + // An action whose element did not change says so; the evidence line then // keeps effect_unknown, because nothing confirms the effect. func TestActReportSaysNoVisibleChange(t *testing.T) { diff --git a/computer-use/native/linux/provider/ax_actions.py b/computer-use/native/linux/provider/ax_actions.py index e3fd5c3..e574118 100644 --- a/computer-use/native/linux/provider/ax_actions.py +++ b/computer-use/native/linux/provider/ax_actions.py @@ -398,14 +398,16 @@ def find_editable_text(root): character count stays the same, so the text is silently lost and the tool reports success. - The node must therefore be both EDITABLE and FOCUSED, and exactly one - candidate must exist. Without a clear focus owner nothing is guessed + The node must therefore expose Text and be both EDITABLE and FOCUSED, + and exactly one candidate must exist. EditableText is optional: a + keyboard-only editor still needs text readback. Without a clear focus + owner nothing is guessed from visibility or tree order; the caller uses the keyboard path, which is reported as unverified. """ focused = [] for node in iter_all(root): - if not (has_editable_text_iface(node) and has_text_iface(node)): + if not has_text_iface(node): continue if not state_contains(node, Atspi.StateType.EDITABLE): continue diff --git a/computer-use/native/linux/provider/ax_input_delivery.py b/computer-use/native/linux/provider/ax_input_delivery.py index 558a543..bf560f6 100644 --- a/computer-use/native/linux/provider/ax_input_delivery.py +++ b/computer-use/native/linux/provider/ax_input_delivery.py @@ -19,11 +19,13 @@ focus_target_element, insert_text_outcome_on_node, parse_key, + read_all_text, resolve_combo_editor, scroll_element, send_key, send_text, set_element_value_outcome, + text_insertion_point, ) from .gui_input import send_mouse_click from .input_primitives import ( @@ -484,14 +486,13 @@ def _perform_type( semantic_input_emitted, ) = insert_text_outcome_on_node(element, action_input.get("text", "")) else: + element = find_editable_text(window) ( write_outcome, before_chars, after_chars, semantic_input_emitted, - ) = insert_text_outcome_on_node( - find_editable_text(window), action_input.get("text", "") - ) + ) = insert_text_outcome_on_node(element, action_input.get("text", "")) if semantic_input_emitted: action_evidence["route"] = "editable_text_api" action_evidence["input_emitted"] = True @@ -534,6 +535,18 @@ def _perform_type( if element_record is None: require_window_focus(window, "type") action_evidence["window_focus_verified"] = True + focused = find_editable_text(window) + if element is not None and focused != element: + raise OperationFailure( + "effect_unknown: the focused text field changed during " + "window activation; no keyboard input was emitted. " + "Observe the current focus before retrying.", + failure_code="effect_unknown", + failure_phase="precondition", + input_events_emitted=semantic_input_emitted, + failure_subject="element", + ) + element = focused else: # The semantic write attempt may itself move focus. Re-establish # and verify the exact target immediately before global synthesis. @@ -542,11 +555,21 @@ def _perform_type( "targeted type_text keyboard fallback") action_evidence["window_focus_verified"] = True action_evidence["target_focus_verified"] = True - text_route = send_text(action_input.get("text", "")) + text_iface = safe(element.get_text_iface) if element is not None else None + before_text = read_all_text(text_iface) + expected_text = None + payload = action_input.get("text", "") + if before_text is not None: + offset, selection = text_insertion_point(text_iface) + end = selection[1] if selection else offset + expected_text = before_text[:offset] + payload + before_text[end:] + text_route = send_text(payload) action_evidence["route"] = "keyboard_synthesis" action_evidence["focus_verified"] = True action_evidence["editable_verified"] = None action_evidence["input_emitted"] = True + readback_note = _verify_keyboard_text( + text_iface, before_text, expected_text, action_evidence) notes.append( KEY_CHANNEL + SYNTHESIS @@ -555,12 +578,47 @@ def _perform_type( "verifying the requested element held focus" if element_record is not None else "bringing the window to the foreground", - UNVERIFIED_SYNTHESIS, + readback_note if action_evidence.get("readback_verified") is True + else UNVERIFIED_SYNTHESIS + " " + readback_note, ) + (" " + text_route if text_route else "") ) +def _verify_keyboard_text(text_iface, before_text, expected_text, evidence): + if expected_text is None: + return ( + "Typed text effect is unknown: the focused field's text could not " + "be read before keyboard input.") + deadline = time.monotonic() + 1.0 + while True: + after_text = read_all_text(text_iface) + evidence["readback_sampled"] = True + evidence["readback_verified"] = ( + after_text == expected_text if after_text is not None else None) + if after_text == expected_text: + return ( + "Read back the focused field after keyboard input and " + "confirmed the expected text insertion.") + if time.monotonic() >= deadline: + break + time.sleep(0.02) + if after_text is None: + reason = "the focused field's text readback is unavailable" + elif after_text == before_text: + reason = "the focused field's text is unchanged after keyboard input" + else: + reason = "the focused field does not contain the expected text insertion" + raise OperationFailure( + "effect_unknown: " + reason + "; text was not retried. " + "Observe the field before sending more input.", + failure_code="effect_unknown", + failure_phase="readback", + input_events_emitted=True, + failure_subject="element", + ) + + def _perform_keypress( action_input, window, element, element_record, notes, action_evidence ): diff --git a/computer-use/runtime_tests/test_text_input.py b/computer-use/runtime_tests/test_text_input.py index 5e26b89..515628f 100644 --- a/computer-use/runtime_tests/test_text_input.py +++ b/computer-use/runtime_tests/test_text_input.py @@ -54,7 +54,7 @@ def test_returns_none_when_no_node_has_editable_state(self): self.assertIsNone(runtime.find_editable_text(window)) - def test_ignores_nodes_without_editable_text_iface(self): + def test_finds_focused_editor_without_editable_text_iface_for_readback(self): read_only = FakeNode( states=(STATE.EDITABLE, STATE.FOCUSED), text=FakeTextIface(), @@ -64,7 +64,7 @@ def test_ignores_nodes_without_editable_text_iface(self): states=(STATE.EDITABLE, STATE.SHOWING), text=FakeTextIface()) window = FakeNode(children=(read_only, editor)) - self.assertIsNone(runtime.find_editable_text(window)) + self.assertIs(runtime.find_editable_text(window), read_only) def test_unique_showing_editor_is_not_treated_as_focused(self): editor = FakeNode( @@ -84,6 +84,127 @@ def test_finds_editor_nested_deep_in_the_tree(self): self.assertIs(runtime.find_editable_text(window), editor) +class TypeReadbackTests(AtspiPatchedTestCase): + def setUp(self): + super().setUp() + self.iface = FakeTextIface("prefix", honest=False, caret=6) + self.editor = FakeNode( + states=(STATE.EDITABLE, STATE.FOCUSED), text=self.iface) + self.window = FakeNode(children=(self.editor,)) + self.evidence = {} + self.notes = [] + self.now = 0.0 + self.focus = mock.patch.object(runtime, "require_window_focus").start() + self.send = mock.patch.object(runtime, "send_text", return_value=None).start() + mock.patch.object(runtime.time, "monotonic", side_effect=lambda: self.now).start() + mock.patch.object(runtime.time, "sleep", side_effect=self.advance).start() + self.addCleanup(mock.patch.stopall) + + def advance(self, seconds): + self.now += seconds + + def invoke(self, text="payload"): + runtime._perform_type( + {"text": text}, self.window, None, None, self.notes, self.evidence) + + def test_ignored_keyboard_input_reports_unchanged_after_bounded_readback(self): + with self.assertRaises(runtime.OperationFailure) as raised: + self.invoke() + self.assertEqual(raised.exception.failure_code, "effect_unknown") + self.assertEqual(raised.exception.failure_phase, "readback") + self.assertTrue(raised.exception.input_events_emitted) + self.assertIn("unchanged", str(raised.exception)) + self.assertTrue(self.evidence["readback_sampled"]) + self.assertFalse(self.evidence["readback_verified"]) + self.send.assert_called_once_with("payload") + + def test_delayed_keyboard_input_is_read_back_without_retyping(self): + def delayed(seconds): + self.advance(seconds) + if self.now >= 0.04: + self.iface.text = "prefixpayload" + with mock.patch.object(runtime.time, "sleep", side_effect=delayed): + self.invoke() + self.assertTrue(self.evidence["readback_verified"]) + self.send.assert_called_once_with("payload") + + def test_wrong_same_length_keyboard_result_is_unknown(self): + self.send.side_effect = lambda _text: setattr(self.iface, "text", "prefixinvalid") + with self.assertRaises(runtime.OperationFailure) as raised: + self.invoke() + self.assertIn("expected text", str(raised.exception)) + self.assertFalse(self.evidence["readback_verified"]) + self.send.assert_called_once() + + def test_unreadable_keyboard_result_is_unknown(self): + self.send.side_effect = lambda _text: setattr(self.iface, "text", None) + with self.assertRaises(runtime.OperationFailure) as raised: + self.invoke() + self.assertIn("unavailable", str(raised.exception)) + self.assertIsNone(self.evidence["readback_verified"]) + self.send.assert_called_once() + + def test_focus_change_during_window_activation_stops_keyboard_input(self): + other = FakeNode(states=(STATE.FOCUSED, STATE.EDITABLE), text=FakeTextIface()) + self.focus.side_effect = lambda *_: setattr(self.window, "children", [other]) + with self.assertRaises(runtime.OperationFailure) as raised: + self.invoke() + self.assertIn("focus", str(raised.exception)) + self.assertTrue(raised.exception.input_events_emitted) + self.send.assert_not_called() + self.assertEqual(other._text.text, "") + + def test_editor_without_editable_api_gets_keyboard_readback(self): + self.editor._editable = False + self.send.side_effect = lambda _text: setattr(self.iface, "text", "prefixpayload") + self.invoke() + self.assertTrue(self.evidence["readback_verified"]) + self.send.assert_called_once() + + def test_semantic_typing_preserves_background_focus(self): + self.iface.honest = True + self.invoke() + self.assertEqual(self.iface.text, "prefixpayload") + self.focus.assert_not_called() + self.send.assert_not_called() + self.assertTrue(self.evidence["readback_verified"]) + + def test_keyboard_readback_checks_selection_replacement(self): + self.editor._editable = False + self.iface.selection = (1, 4) + self.send.side_effect = lambda _text: setattr(self.iface, "text", "ppayloadix") + self.invoke() + self.assertTrue(self.evidence["readback_verified"]) + + def test_keyboard_readback_checks_insertion_at_caret(self): + self.editor._editable = False + self.iface.caret = 2 + self.send.side_effect = lambda _text: setattr(self.iface, "text", "prpayloadefix") + self.invoke() + self.assertTrue(self.evidence["readback_verified"]) + + def test_no_readable_focus_reports_unknown_instead_of_claiming_delivery(self): + self.window.children = [] + self.invoke() + self.assertIn("effect is unknown", " ".join(self.notes)) + self.assertIsNot(self.evidence.get("readback_verified"), True) + self.send.assert_called_once() + + def test_activation_focus_change_without_semantic_input_preserves_no_input(self): + self.editor._editable = False + self.focus.side_effect = lambda *_: setattr(self.window, "children", []) + with self.assertRaises(runtime.OperationFailure) as raised: + self.invoke() + self.assertFalse(raised.exception.input_events_emitted) + self.send.assert_not_called() + + def test_multiple_focused_editors_do_not_pick_the_one_with_editable_api(self): + other = FakeNode( + states=(STATE.FOCUSED, STATE.EDITABLE), text=FakeTextIface(), editable=False) + self.window.children.append(other) + self.assertIsNone(runtime.find_editable_text(self.window)) + + def _write_into_window(window, text): """Write the way an untargeted type action does: into the focused editor.""" outcome, _before, _after, _emitted = runtime.insert_text_outcome_on_node( diff --git a/docs/MCP.md b/docs/MCP.md index 96b680a..fe1f239 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -85,6 +85,12 @@ ASCII text without line feeds or tabs is typed as key events on both desktops. When AX falls back to keyboard typing, its note says which way the text was typed. GUI results do not include notes. +Linux AX reads the focused text field after keyboard typing when its text is +accessible. It waits up to one second for the expected insertion. Unchanged, +unexpected or unreadable text produces an `effect_unknown` result without +retyping. If window activation changes the resolved field, AX stops before +sending keys. Without a readable field, the typing effect remains unknown. + To use the input method, applications must inherit the desktop's input-method environment. Existing application processes may need a restart after that environment is corrected. @@ -606,7 +612,14 @@ effect: r58 checkbox "Interface" checked=false→true windows: opened w5 "Insert Sheet" (dialog, focused); closed w3 "Print"; focus w2 ``` -There is one `effect` line per action with an element `ref`. It compares the +Each `type` step reports its text-value evidence. Without a ref, the report +uses the unique focused editable textbox in the pre-action read. A focus +change alone does not confirm typing. Unchanged values are reported as such; +missing, incomplete or unreadable evidence produces `unknown` with a reason. +Empty values remain unknown because the snapshot does not distinguish empty +text from unavailable text. These lines add no provider calls. + +Other actions with an element `ref` also have one `effect` line. It compares the element in the read before that action with the next read after it and names the role, the name, and the state or value fields that changed. When both reads are of the same window and nothing else in it changed, the line ends From c25a5e3c9410ba95242930798e516ecc8ddf5197 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 07:45:30 +0000 Subject: [PATCH 02/15] fix(linux): discover windows without EWMH process IDs Resolve missing window owners through bounded X-Resource queries and retain the existing process-generation and window-handle checks. Report unidentified application windows as incomplete instead of silently omitting them. Preserve pointer-free native activation and cover discovery, failure, and identity refresh paths. --- CHANGELOG.md | 4 + .../internal/provider/linux_runtime_bundle.go | 1 + .../provider/linux_runtime_bundle_test.go | 1 + .../native/linux/provider/native_windows.py | 11 +- .../linux/provider/x11_client_identity.py | 102 +++++++++++++++ .../linux/provider/x11_window_inventory.py | 6 + computer-use/runtime_test.py | 1 + computer-use/runtime_test_support.py | 2 + .../runtime_tests/test_native_windows.py | 55 ++++++++ .../runtime_tests/test_x11_client_identity.py | 120 ++++++++++++++++++ docs/MCP.md | 12 +- 11 files changed, 309 insertions(+), 6 deletions(-) create mode 100644 computer-use/native/linux/provider/x11_client_identity.py create mode 100644 computer-use/runtime_tests/test_x11_client_identity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5663ba3..ab2690a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ - Linux AX checks focused-field readback after keyboard typing and reports unconfirmed text explicitly. If window activation changes the resolved text field, typing stops before keyboard input. +- Linux lists managed application windows that omit `_NET_WM_PID` when + X-Resource can identify their owner. Windows without an accessibility tree + support AX activation without pointer movement. Unverified owners produce + an explicit inventory issue instead of an actionable reference. ## 1.0.0 diff --git a/computer-use/internal/provider/linux_runtime_bundle.go b/computer-use/internal/provider/linux_runtime_bundle.go index b0e55ea..6abf19c 100644 --- a/computer-use/internal/provider/linux_runtime_bundle.go +++ b/computer-use/internal/provider/linux_runtime_bundle.go @@ -13,6 +13,7 @@ var linuxProviderPaths = []string{ "native/linux/provider/x11_windows.py", "native/linux/provider/desktop_layout.py", "native/linux/provider/x11_window_inventory.py", + "native/linux/provider/x11_client_identity.py", "native/linux/provider/window_scale.py", "native/linux/provider/atspi_inventory.py", "native/linux/provider/ax_node_properties.py", diff --git a/computer-use/internal/provider/linux_runtime_bundle_test.go b/computer-use/internal/provider/linux_runtime_bundle_test.go index 87080f6..12e1633 100644 --- a/computer-use/internal/provider/linux_runtime_bundle_test.go +++ b/computer-use/internal/provider/linux_runtime_bundle_test.go @@ -64,6 +64,7 @@ func TestLinuxRuntimeBundleIsComplete(t *testing.T) { "native/linux/provider/window_scale.py", "native/linux/provider/wire.py", "native/linux/provider/wire_request.py", + "native/linux/provider/x11_client_identity.py", "native/linux/provider/x11_window_inventory.py", "native/linux/provider/x11_windows.py", "runtime.py", diff --git a/computer-use/native/linux/provider/native_windows.py b/computer-use/native/linux/provider/native_windows.py index 659df9c..969ad4a 100644 --- a/computer-use/native/linux/provider/native_windows.py +++ b/computer-use/native/linux/provider/native_windows.py @@ -83,7 +83,7 @@ def native_only_windows(managed, accessible_coverage, active_window_id=None, title = str(window.get("title") or "").strip() width = int(window.get("width") or 0) height = int(window.get("height") or 0) - if pid <= 0 or not title: + if not title: continue if width <= 0 or height <= 0: continue @@ -91,13 +91,16 @@ def native_only_windows(managed, accessible_coverage, active_window_id=None, or int(window.get("y") or 0) + height <= 0): continue handle = window_handle_for_pid(pid, window.get("id")) - if not handle: - continue coverage = accessible_coverage.get(pid, {}) - if handle in coverage.get("handles", set()): + if handle and handle in coverage.get("handles", set()): continue if x11_window_type(window.get("id")) not in APPLICATION_WINDOW_TYPES: continue + if not handle: + errors.append( + 'The process identity of window "{}" is unavailable; ' + "it cannot be addressed safely.".format(title)) + continue if not coverage.get("complete", True): errors.append( 'The accessibility coverage of window "{}" is uncertain; ' diff --git a/computer-use/native/linux/provider/x11_client_identity.py b/computer-use/native/linux/provider/x11_client_identity.py new file mode 100644 index 0000000..de0f632 --- /dev/null +++ b/computer-use/native/linux/provider/x11_client_identity.py @@ -0,0 +1,102 @@ +"""Resolve missing window PIDs through the X-Resource extension. + +The query runs in a bounded child process: Xlib's default error handler can +exit on a disappearing client, and an unresponsive server can block a read. +No title, WM_CLASS or process-name matching establishes ownership. +""" + +import ctypes +import json +import subprocess +import sys + + +class _ClientIdSpec(ctypes.Structure): + _fields_ = [("client", ctypes.c_ulong), ("mask", ctypes.c_uint)] + + +class _ClientIdValue(ctypes.Structure): + _fields_ = [("spec", _ClientIdSpec), ("length", ctypes.c_long), + ("value", ctypes.c_void_p)] + + +def _xres_libraries(): + xlib = ctypes.CDLL("libX11.so.6") + xres = ctypes.CDLL("libXRes.so.1") + xlib.XOpenDisplay.argtypes = [ctypes.c_char_p] + xlib.XOpenDisplay.restype = ctypes.c_void_p + xlib.XCloseDisplay.argtypes = [ctypes.c_void_p] + xlib.XCloseDisplay.restype = ctypes.c_int + xres.XResQueryVersion.argtypes = [ + ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int)] + xres.XResQueryVersion.restype = ctypes.c_int + xres.XResQueryClientIds.argtypes = [ + ctypes.c_void_p, ctypes.c_long, ctypes.POINTER(_ClientIdSpec), + ctypes.POINTER(ctypes.c_long), ctypes.POINTER(ctypes.POINTER(_ClientIdValue))] + xres.XResQueryClientIds.restype = ctypes.c_int + xres.XResGetClientPid.argtypes = [ctypes.POINTER(_ClientIdValue)] + xres.XResGetClientPid.restype = ctypes.c_int + xres.XResClientIdsDestroy.argtypes = [ + ctypes.c_long, ctypes.POINTER(_ClientIdValue)] + xres.XResClientIdsDestroy.restype = None + return xlib, xres + + +def _query_client_pids(window_ids): + xlib, xres = _xres_libraries() + display = xlib.XOpenDisplay(None) + if not display: + return {} + try: + major, minor = ctypes.c_int(), ctypes.c_int() + if not xres.XResQueryVersion(display, ctypes.byref(major), ctypes.byref(minor)): + return {} + if (major.value, minor.value) < (1, 2): + return {} + owners = {} + for window_id in window_ids: + xid = int(window_id, 16) + if not 0 < xid <= 0xffffffff: + continue + spec = _ClientIdSpec(xid, 2) + count = ctypes.c_long() + values = ctypes.POINTER(_ClientIdValue)() + status = xres.XResQueryClientIds( + display, 1, ctypes.byref(spec), ctypes.byref(count), ctypes.byref(values)) + if status != 0: + continue + try: + pids = {xres.XResGetClientPid(ctypes.byref(values[index])) + for index in range(count.value)} + pids.discard(-1) + if len(pids) == 1 and next(iter(pids)) > 0: + owners[window_id] = pids.pop() + finally: + xres.XResClientIdsDestroy(count, values) + return owners + finally: + xlib.XCloseDisplay(display) + + +def x11_client_pids(window_ids): + """Return proven local client PIDs; missing entries remain unidentified.""" + if not window_ids: + return {} + try: + result = subprocess.run( + [sys.executable, "-I", __file__, *window_ids], + capture_output=True, text=True, timeout=3, + ) + if result.returncode != 0: + return {} + owners = json.loads(result.stdout) + if not isinstance(owners, dict): + return {} + return {window_id: owners[window_id] for window_id in window_ids + if type(owners.get(window_id)) is int and owners[window_id] > 0} + except (OSError, subprocess.TimeoutExpired, ValueError): + return {} + + +if __name__ == "__main__": + print(json.dumps(_query_client_pids(sys.argv[1:]))) diff --git a/computer-use/native/linux/provider/x11_window_inventory.py b/computer-use/native/linux/provider/x11_window_inventory.py index d28dbea..9e71ae2 100644 --- a/computer-use/native/linux/provider/x11_window_inventory.py +++ b/computer-use/native/linux/provider/x11_window_inventory.py @@ -4,6 +4,7 @@ import subprocess from .foundation import InventoryResult, OperationFailure +from .x11_client_identity import x11_client_pids from .desktop_layout import ( normalize_x11_window_id, parse_xwininfo_layout, @@ -76,6 +77,11 @@ def _x11_managed_windows(): "height": height, "title": parts[8] if len(parts) > 8 else "", }) + missing_pids = [window["id"] for window in windows if window["pid"] <= 0] + owners = x11_client_pids(missing_pids) + for window in windows: + if window["pid"] <= 0: + window["pid"] = owners.get(window["id"], 0) inventory = InventoryResult(windows, complete=True) _x11_window_inventory_cache["all"] = inventory return inventory diff --git a/computer-use/runtime_test.py b/computer-use/runtime_test.py index 7ee2940..f1c8c18 100644 --- a/computer-use/runtime_test.py +++ b/computer-use/runtime_test.py @@ -30,6 +30,7 @@ from runtime_tests.test_a11y_enable import * # noqa: F401,F403 from runtime_tests.test_popup_rendering import * # noqa: F401,F403 from runtime_tests.test_native_windows import * # noqa: F401,F403 +from runtime_tests.test_x11_client_identity import * # noqa: F401,F403 from runtime_tests.test_window_geometry import * # noqa: F401,F403 diff --git a/computer-use/runtime_test_support.py b/computer-use/runtime_test_support.py index 9000672..3a3a2f4 100644 --- a/computer-use/runtime_test_support.py +++ b/computer-use/runtime_test_support.py @@ -70,6 +70,7 @@ x11_windows, desktop_layout, x11_window_inventory, + x11_client_identity, ) PROVIDER_MODULES = ( @@ -84,6 +85,7 @@ x11_windows, desktop_layout, x11_window_inventory, + x11_client_identity, atspi_inventory, ax_node_properties, ax_records, diff --git a/computer-use/runtime_tests/test_native_windows.py b/computer-use/runtime_tests/test_native_windows.py index fd761be..5e67e5c 100644 --- a/computer-use/runtime_tests/test_native_windows.py +++ b/computer-use/runtime_tests/test_native_windows.py @@ -2,6 +2,7 @@ """Windows of processes that export no accessibility tree.""" from runtime_test_support import * # noqa: F401,F403 +from native.linux.provider import x11_window_inventory def _managed(*windows): @@ -80,6 +81,19 @@ def test_listing_gives_them_a_handle_and_no_tree_flag(self): # advice adds no note. self.assertEqual([], inventory["notes"]) + def test_unidentified_application_window_makes_coverage_explicitly_incomplete(self): + windows = runtime.native_only_windows( + [_window(0, "0x1", "Notes"), _window(30, "0x2", "Scene")], {}) + + self.assertEqual(["Scene"], [window["title"] for window in windows]) + self.assertFalse(windows.complete) + self.assertIn('"Notes"', windows.errors[0]) + + def test_unidentified_dock_is_not_an_inventory_gap(self): + windows = runtime.native_only_windows([_window(0, "0x5", "Dock")], {}) + self.assertEqual([], list(windows)) + self.assertTrue(windows.complete) + def test_restart_advice_stays_for_toolkits_that_can_export_a_tree(self): with tempfile.TemporaryDirectory() as root: os.makedirs(os.path.join(root, "20")) @@ -168,6 +182,47 @@ def test_incomplete_empty_root_is_not_evidence_of_absent_accessibility(self): self.assertFalse(windows.complete) +class MissingWindowPidTests(AtspiPatchedTestCase): + def setUp(self): + super().setUp() + x11_window_inventory._x11_window_inventory_cache.clear() + self.addCleanup(x11_window_inventory._x11_window_inventory_cache.clear) + result = subprocess.CompletedProcess([], 0, + "0x00000011 0 0 10 20 800 600 N/A Notes\n" + "0x00000022 0 42 30 40 800 600 host Editor\n") + patcher = mock.patch.object(x11_window_inventory.subprocess, "run", return_value=result) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch.object(x11_window_inventory, "x11_client_pids", + return_value={"0x11": 31}) + self.owners = patcher.start() + self.addCleanup(patcher.stop) + + def test_missing_pid_is_resolved_from_x11_client_not_title(self): + windows = runtime._x11_managed_windows() + + self.assertEqual([31, 42], [window["pid"] for window in windows]) + self.owners.assert_called_once_with(["0x11"]) + self.assertIs(windows, runtime._x11_managed_windows()) + self.owners.assert_called_once() + + def test_unresolved_pid_is_not_invented(self): + self.owners.return_value = {} + windows = runtime._x11_managed_windows() + self.assertEqual([0, 42], [window["pid"] for window in windows]) + + def test_recovered_identity_is_refreshed_with_the_window_inventory(self): + first = runtime._x11_managed_windows()[0] + x11_window_inventory._x11_window_inventory_cache.clear() + self.owners.return_value = {"0x11": 32} + second = runtime._x11_managed_windows()[0] + with mock.patch.object(runtime, "process_generation", return_value="gen"): + old_handle = runtime.window_handle_for_pid(first["pid"], first["id"]) + new_handle = runtime.window_handle_for_pid(second["pid"], second["id"]) + self.assertIsNotNone(old_handle) + self.assertNotEqual(old_handle, new_handle) + + class NativeWindowActionTests(AtspiPatchedTestCase): def setUp(self): super().setUp() diff --git a/computer-use/runtime_tests/test_x11_client_identity.py b/computer-use/runtime_tests/test_x11_client_identity.py new file mode 100644 index 0000000..7dca6bb --- /dev/null +++ b/computer-use/runtime_tests/test_x11_client_identity.py @@ -0,0 +1,120 @@ +"""X-Resource ownership queries fail closed and do not outlive the request.""" + +import ctypes + +from runtime_test_support import * # noqa: F401,F403 +from native.linux.provider import x11_client_identity + + +class ClientPidWorkerTests(unittest.TestCase): + def test_returns_only_requested_positive_integer_owners(self): + result = subprocess.CompletedProcess([], 0, json.dumps({ + "0x1": 71, "0x2": -1, "0x3": 0, "0x4": True, + "0x5": "72", "0x6": 1.2, "0x7": 99, + })) + with mock.patch.object(x11_client_identity.subprocess, "run", return_value=result) as run: + owners = x11_client_identity.x11_client_pids( + ["0x1", "0x2", "0x3", "0x4", "0x5", "0x6"]) + self.assertEqual({"0x1": 71}, owners) + self.assertEqual(3, run.call_args.kwargs["timeout"]) + self.assertEqual([sys.executable, "-I", x11_client_identity.__file__], + run.call_args.args[0][:3]) + + def test_no_missing_pids_needs_no_worker(self): + with mock.patch.object(x11_client_identity.subprocess, "run") as run: + self.assertEqual({}, x11_client_identity.x11_client_pids([])) + run.assert_not_called() + + def test_unavailable_extension_dead_client_timeout_and_bad_output_are_unidentified(self): + for result in ( + subprocess.CompletedProcess([], 1, '{"0x1": 71}'), + subprocess.CompletedProcess([], -11, ""), + subprocess.CompletedProcess([], 0, "not json"), + subprocess.CompletedProcess([], 0, "[]"), + subprocess.CompletedProcess([], 0, "null"), + subprocess.TimeoutExpired("worker", 3), + OSError("cannot start worker"), + ): + with self.subTest(result=result): + with mock.patch.object(x11_client_identity.subprocess, "run") as run: + if isinstance(result, Exception): + run.side_effect = result + else: + run.return_value = result + self.assertEqual({}, x11_client_identity.x11_client_pids(["0x1"])) + + +class XResourceQueryTests(unittest.TestCase): + def setUp(self): + self.xlib, self.xres = mock.Mock(), mock.Mock() + self.xlib.XOpenDisplay.return_value = 100 + self.version = (1, 2) + self.pids = [71] + self.xres.XResQueryVersion.side_effect = self.query_version + self.xres.XResQueryClientIds.side_effect = self.query_ids + self.xres.XResGetClientPid.side_effect = lambda value: value._obj.spec.client + patcher = mock.patch.object(x11_client_identity, "_xres_libraries", + return_value=(self.xlib, self.xres)) + patcher.start() + self.addCleanup(patcher.stop) + + def query_version(self, display, major, minor): + major._obj.value, minor._obj.value = self.version + return 1 + + def query_ids(self, display, count, spec, result_count, result_values): + self.assertEqual(100, display) + self.assertEqual(1, count) + self.assertEqual(0x12, spec._obj.client) + self.assertEqual(2, spec._obj.mask) + self.values = (x11_client_identity._ClientIdValue * len(self.pids))() + for index, pid in enumerate(self.pids): + self.values[index].spec.client = pid + result_count._obj.value = len(self.values) + ctypes.cast(result_values, ctypes.POINTER( + ctypes.POINTER(x11_client_identity._ClientIdValue)))[0] = self.values + return 0 + + def test_query_uses_xid_and_releases_reply_and_display(self): + owners = x11_client_identity._query_client_pids(["0x12"]) + self.assertEqual({"0x12": 71}, owners) + self.xres.XResClientIdsDestroy.assert_called_once() + self.xlib.XCloseDisplay.assert_called_once_with(100) + + def test_no_local_pid_or_multiple_owners_cannot_bind(self): + for pids in ([], [0], [71, 72]): + with self.subTest(pids=pids): + self.pids = pids + self.assertEqual({}, x11_client_identity._query_client_pids(["0x12"])) + self.assertEqual(3, self.xres.XResClientIdsDestroy.call_count) + self.xres.XResGetClientPid.side_effect = lambda value: -1 + self.assertEqual({}, x11_client_identity._query_client_pids(["0x12"])) + + def test_old_extension_does_not_issue_unsupported_request(self): + self.version = (1, 0) + self.assertEqual({}, x11_client_identity._query_client_pids(["0x12"])) + self.xres.XResQueryClientIds.assert_not_called() + self.xlib.XCloseDisplay.assert_called_once_with(100) + + def test_failed_reply_is_not_reused_or_freed_twice(self): + self.xres.XResQueryClientIds.side_effect = None + self.xres.XResQueryClientIds.return_value = 1 + self.assertEqual({}, x11_client_identity._query_client_pids(["0x12"])) + self.xres.XResGetClientPid.assert_not_called() + self.xres.XResClientIdsDestroy.assert_not_called() + self.xlib.XCloseDisplay.assert_called_once_with(100) + + def test_missing_display_or_extension_has_no_owner(self): + self.xlib.XOpenDisplay.return_value = None + self.assertEqual({}, x11_client_identity._query_client_pids(["0x12"])) + self.xlib.XCloseDisplay.assert_not_called() + self.xlib.XOpenDisplay.return_value = 100 + self.xres.XResQueryVersion.side_effect = None + self.xres.XResQueryVersion.return_value = 0 + self.assertEqual({}, x11_client_identity._query_client_pids(["0x12"])) + self.xres.XResQueryClientIds.assert_not_called() + self.xlib.XCloseDisplay.assert_called_once_with(100) + + def test_root_and_invalid_resource_ids_never_query_all_clients(self): + self.assertEqual({}, x11_client_identity._query_client_pids(["0x0", "0x100000000"])) + self.xres.XResQueryClientIds.assert_not_called() diff --git a/docs/MCP.md b/docs/MCP.md index fe1f239..29d8d68 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -59,14 +59,14 @@ XMODIFIERS ``` The provider needs Python 3 with PyGObject and the AT-SPI, GTK and GdkPixbuf -typelibs, plus `xdotool`, `wmctrl` and `x11-utils`. The IBus typelib and +typelibs, plus `xdotool`, `wmctrl`, `x11-utils` and `libxres1`. The IBus typelib and packages are optional; they enable input-method keyboard typing. On Debian or Ubuntu: ```bash sudo apt-get install python3 python3-gi gir1.2-atspi-2.0 gir1.2-gtk-3.0 \ gir1.2-gdkpixbuf-2.0 gir1.2-ibus-1.0 ibus ibus-gtk ibus-gtk3 \ - xdotool wmctrl x11-utils + xdotool wmctrl x11-utils libxres1 ``` When `DISPLAY`, `XDG_RUNTIME_DIR` or `DBUS_SESSION_BUS_ADDRESS` is missing, @@ -795,6 +795,14 @@ close, or wait for them; GUI reads and operates their content. If an AX root cannot be matched safely to a native window, the inventory reports incomplete coverage instead of issuing another reference for that window. +Some X11 applications, including Tk applications, omit `_NET_WM_PID`. +HCU uses the X-Resource extension to identify their local client process. +This requires `libxres1` and X-Resource 1.2 or later on the X server. +If the process identity cannot be verified, the inventory names the window +as an incomplete observation and does not issue an actionable reference. +Activation sends a window-manager request without moving the pointer. +The result confirms the active window, not the focus of an inaccessible control. + AX sees no applications, or `AX apps` reports windows that are not on the accessibility bus: restart those applications after the server has run once (see [Accessibility on Linux](#accessibility-on-linux)). Then check the From 1a6c72dde9aa16258018371cb8cb04371a78fb8e Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 07:45:31 +0000 Subject: [PATCH 03/15] fix(linux): select GTK combo values without opening popups Route native GTK combo set_value through the same Selection path as select. Verify the active item's identity and name after one semantic mutation, and never retry GTK selection through popup or keyboard input. Cover no-op selection, entry-backed controls, refusal, unreadable results and stale ownership. Keep projected LibreOffice editor behavior unchanged. --- CHANGELOG.md | 5 + .../native/linux/provider/ax_dropdown.py | 65 +++- .../linux/provider/ax_input_delivery.py | 9 + .../native/linux/provider/ax_select.py | 3 +- computer-use/runtime_test.py | 1 + computer-use/runtime_tests/test_gtk_combo.py | 343 ++++++++++++++++++ .../runtime_tests/test_select_boundaries.py | 13 +- docs/MCP.md | 12 +- 8 files changed, 439 insertions(+), 12 deletions(-) create mode 100644 computer-use/runtime_tests/test_gtk_combo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ab2690a..395cfd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ X-Resource can identify their owner. Windows without an accessibility tree support AX activation without pointer movement. Unverified owners produce an explicit inventory issue instead of an actionable reference. +- Linux AX `set_value` and `select` choose native GTK combo box options through + accessibility without opening the popup or taking keyboard focus. Both actions + verify the active item and fail explicitly if selection cannot be confirmed. + Combo boxes with an entry use the same option selection; their entry refs + remain available for free-form text editing. ## 1.0.0 diff --git a/computer-use/native/linux/provider/ax_dropdown.py b/computer-use/native/linux/provider/ax_dropdown.py index 3b2cecc..1f46d86 100644 --- a/computer-use/native/linux/provider/ax_dropdown.py +++ b/computer-use/native/linux/provider/ax_dropdown.py @@ -30,6 +30,7 @@ _failure, _quote, _same, + _same_selection_item, _selection_container, _wait_until, text_between, @@ -406,14 +407,68 @@ def _list_selection_first(node): and safe(node.get_selection_iface) is None) +def _select_gtk_option(node, option, wanted): + parent = safe(option.get_parent) + index = safe(option.get_index_in_parent) + + def owned(): + return (parent is not None and index is not None and index >= 0 + and _same_selection_item(safe(parent.get_parent), node) + and _same_selection_item(safe(option.get_parent), parent) + and safe(option.get_index_in_parent) == index + and _same_selection_item( + safe(lambda: parent.get_child_at_index(index)), option)) + + if safe(node.get_selection_iface) is None: + raise _failure("select_unavailable: the element's container does not support " + "selection; no input was emitted.") + if not owned(): + raise _failure("select_unavailable: item ownership changed; " + "no input was emitted.") + + def committed(): + if not owned(): + return False + count = safe(lambda: Atspi.Selection.get_n_selected_children(node)) + selected = (safe(lambda: Atspi.Selection.get_selected_child(node, 0)) + if count == 1 else None) + return (_same_selection_item(selected, option) + and str(safe(selected.get_name, "")).strip() == wanted) + + if committed(): + return wanted + if not all(state_contains(target, state) + for target in (node, option) + for state in (Atspi.StateType.ENABLED, Atspi.StateType.SENSITIVE)): + raise _failure("select_unavailable: the requested option is disabled; " + "no input was emitted.") + action_evidence.update(input_emitted=True, route="selection_api") + try: + Atspi.Selection.select_child(node, int(index)) + except Exception as exc: + raise _failure( + "select_unavailable: the container refused to select the item; " + "the selection may have changed.", code="effect_unknown", + emitted=True, phase="emission") from exc + action_evidence.update(readback_sampled=True) + if not _wait_until(committed, 0.6): + raise _failure( + "select_unverified: the item did not report itself selected.", + code="effect_unknown", emitted=True, phase="readback") + return wanted + + def select_dropdown_option(node, text, window=None): """Choose the option named ``text`` of a closed or open drop-down. Browsers expose an HTML as a combo box whose options are menu - items with a single "select" action; while the list is closed they are - not showing and carry no refs. Native GTK combos use only their own - Selection interface and verify the active item. They never open a popup - or synthesize keys, including when selection fails. Other toolkits try - the option's own action first (Chrome's Selection.select_child returns - False), then the combo box's - own Selection.select_child, then - the option container's Selection.select_child. Gecko content selects - instead activate their separate native popup before any value mutation. A - closed list that holds its own selection is tried through its Selection - first; see _list_selection_first(). + Chromium HTML combos use the option action and fresh selected-state + readback. HTML list box mutations are refused because Chromium exposes + only change, not input events. GTK combos use their Selection interface. + Neither route opens a popup or sends keys. Gecko uses its native popup. A combo box with committed-value readback (GTK and Qt) succeeds only when that value matches the option afterwards; otherwise the @@ -489,7 +545,14 @@ def select_dropdown_option(node, text, window=None): and not _gecko_content_select(node) and state_contains(node, Atspi.StateType.EXPANDED)): node = _gecko_popup_source(node) - options = _dropdown_options(node) + chromium = is_chromium_select(node) + try: + options = _dropdown_options(node) + except OperationFailure: + if not chromium: + raise + raise _failure("select_options_incomplete: the control options could " + "not be read completely; no input was emitted.") from None wanted = str(text).strip() matches = [option for option in options if option_name(option) == wanted] if len(matches) != 1: @@ -502,6 +565,8 @@ def select_dropdown_option(node, text, window=None): ", ".join(_quote(name) for name in names[:30]) or "none readable"), code="invalid_argument") option = matches[0] + if chromium: + return _select_chromium_option(node, option, wanted) if node_role(node) == "combo box" and application_toolkit(node) == "gtk": return _select_gtk_option(node, option, wanted) shows_choice = committed_dropdown_value(node) is not None @@ -575,7 +640,7 @@ def owning_dropdown(node): current = safe(current.get_parent) if current is None: break - if node_role(current) == "combo box": + if node_role(current) == "combo box" or is_chromium_select(current): return current return projected_list_owner(node) diff --git a/computer-use/native/linux/provider/ax_input_delivery.py b/computer-use/native/linux/provider/ax_input_delivery.py index 3732cac..c9bfe98 100644 --- a/computer-use/native/linux/provider/ax_input_delivery.py +++ b/computer-use/native/linux/provider/ax_input_delivery.py @@ -53,6 +53,7 @@ from .ax_snapshot_diagnostics import is_dropdown_item from .ax_records import gtk_combo_partner from .ax_select import perform_select +from .ax_dropdown import is_chromium_select A11Y_CHANNEL = "[a11y]" @@ -658,7 +659,7 @@ def _perform_set_value( ): if element is None: raise RuntimeError("unknown target ref") - if (node_role(element) == "combo box" + if is_chromium_select(element) or (node_role(element) == "combo box" and application_toolkit(element) == "gtk" and gtk_combo_partner(element, "combo box") is None): perform_select({"text": action_input.get("value", "")}, window, diff --git a/computer-use/runtime_test.py b/computer-use/runtime_test.py index 5c392e7..529f712 100644 --- a/computer-use/runtime_test.py +++ b/computer-use/runtime_test.py @@ -26,6 +26,7 @@ from runtime_tests.test_editable_click import * # noqa: F401,F403 from runtime_tests.test_select_boundaries import * # noqa: F401,F403 from runtime_tests.test_gtk_combo import * # noqa: F401,F403 +from runtime_tests.test_chromium_dropdown import * # noqa: F401,F403 from runtime_tests.test_dropdown_navigation import * # noqa: F401,F403 from runtime_tests.test_chromium_windows import * # noqa: F401,F403 from runtime_tests.test_a11y_enable import * # noqa: F401,F403 diff --git a/computer-use/runtime_tests/test_chromium_dropdown.py b/computer-use/runtime_tests/test_chromium_dropdown.py new file mode 100644 index 0000000..5da2ace --- /dev/null +++ b/computer-use/runtime_tests/test_chromium_dropdown.py @@ -0,0 +1,495 @@ +"""Chromium HTML combos verify option actions; list boxes refuse mutations.""" + +from runtime_test_support import ( + AtspiPatchedTestCase, FakeStateSet, FakeTextIface, STATE, _FakeClock, + action_evidence, mock, runtime, +) +from runtime_tests.test_capabilities import _Node + + +class ChromiumDropdownTests(AtspiPatchedTestCase): + def setUp(self): + super().setUp() + action_evidence.reset() + self.addCleanup(action_evidence.reset) + self.clock = _FakeClock() + self.patch(runtime, "time", self.clock) + self.selection = mock.Mock() + self.selection.is_child_selected.return_value = False + self.patch(runtime.Atspi, "Selection", self.selection, create=True) + for name in ("select_child", "deselect_child", "deselect_selected_child", + "clear_selection", "select_all"): + method = getattr(self.selection, name) + method.side_effect = AssertionError("unexpected Selection mutation: " + name) + self.addCleanup(method.assert_not_called) + for name in ( + "send_key", "send_text", "_global_click", "perform_gui", + "emit_keyboard_event", "emit_mouse_event", + "focus_target_element", "require_window_focus", + "_open_popup", "_commit_from_popup"): + fallback = self.patch( + runtime, name, + side_effect=AssertionError("unexpected fallback: " + name)) + self.addCleanup(fallback.assert_not_called) + focus = self.patch( + runtime.Atspi.Component, "grab_focus", + side_effect=AssertionError("unexpected native focus")) + self.addCleanup(focus.assert_not_called) + + def patch(self, target, name, *args, **kwargs): + patcher = mock.patch.object(target, name, *args, **kwargs) + result = patcher.start() + self.addCleanup(patcher.stop) + return result + + def dropdown(self, role="combo box", multiple=False): + control = _Node(role=role, name="City", states=(STATE.ENABLED, STATE.SENSITIVE)) + control.app.get_toolkit_name.return_value = "chromium" + control.get_application = lambda: control.app + control.get_attributes = lambda: {"tag": "select"} + if multiple: + control.states.add(STATE.MULTISELECTABLE) + elif role == "combo box": + control.states.add(STATE.EXPANDABLE) + options = [ + _Node(role="menu item" if role == "combo box" else "list item", + name=name, states=(STATE.ENABLED, STATE.SENSITIVE, STATE.SELECTABLE)) + for name in ("Beijing", "Seattle", "Tokyo") + ] + options[0].states.add(STATE.SELECTED) + group = _Node(role="menu" if role == "combo box" else "section").add(*options) + control.add(group) + control.selection_iface = control + group.selection_iface = group + control.get_n_actions = lambda: 1 + control.get_action_name = lambda _index: "open" + control.do_action = mock.Mock(side_effect=AssertionError("unexpected popup action")) + self.addCleanup(control.do_action.assert_not_called) + for option in options: + option.app = control.app + option.get_application = lambda: control.app + option.get_n_actions = lambda: 2 + option.get_action_name = lambda index: ("press", "select")[index] + option.get_action_iface = lambda option=option: option + + def commit(index, option=option): + if index != 1: + raise AssertionError("only the named select action may be invoked") + if not multiple: + for other in options: + other.states.discard(STATE.SELECTED) + option.states.add(STATE.SELECTED) + return True + + option.do_action = mock.Mock(side_effect=commit) + return control, options + + def invoke(self, intent, control, wanted="Seattle"): + action_evidence.reset() + self.evidence = {} + notes = [] + record = {"controlType": control.role} + if intent == "set_value": + runtime._perform_set_value( + {"value": wanted}, None, control, record, notes, self.evidence) + else: + runtime.perform_select( + {"text": wanted}, None, control, record, None, notes, self.evidence) + return notes, self.evidence + + def assert_precondition(self, intent, control, options, wanted="Seattle", + code="select_unavailable"): + with self.assertRaises(runtime.OperationFailure) as caught: + self.invoke(intent, control, wanted) + self.assertEqual(caught.exception.failure_code, code) + self.assertEqual(caught.exception.failure_phase, "precondition") + self.assertFalse(caught.exception.input_events_emitted) + self.assertFalse(self.evidence.get("input_emitted", False)) + for option in options: + option.do_action.assert_not_called() + return caught.exception + + def assert_unknown(self, intent, control, options, phase="readback"): + with self.assertRaises(runtime.OperationFailure) as caught: + self.invoke(intent, control) + self.assertEqual(caught.exception.failure_code, "effect_unknown") + self.assertEqual(caught.exception.failure_phase, phase) + self.assertTrue(caught.exception.input_events_emitted) + self.assertTrue(self.evidence["input_emitted"]) + self.assertFalse(self.evidence.get("readback_verified", False)) + self.assertEqual(self.evidence["route"], "semantic_action") + options[1].do_action.assert_called_once_with(1) + options[0].do_action.assert_not_called() + options[2].do_action.assert_not_called() + return caught.exception + + def test_named_option_action_commits_even_if_it_returns_false(self): + for intent in ("set_value", "select"): + for accepted in (True, False): + with self.subTest(intent=intent, accepted=accepted): + control, options = self.dropdown() + commit = options[1].do_action.side_effect + + def action(index): + self.assertTrue(self.evidence["input_emitted"]) + commit(index) + return accepted + + options[1].do_action.side_effect = action + notes, evidence = self.invoke(intent, control) + options[1].do_action.assert_called_once_with(1) + options[0].do_action.assert_not_called() + options[2].do_action.assert_not_called() + self.assertEqual( + [item.title for item in options if STATE.SELECTED in item.states], + ["Seattle"]) + self.assertEqual(evidence["route"], "semantic_action") + self.assertTrue(evidence["input_emitted"]) + self.assertTrue(evidence["readback_sampled"]) + self.assertTrue(evidence["readback_verified"]) + self.assertTrue(any("Seattle" in note for note in notes)) + + def test_same_value_needs_no_action_even_without_select_action(self): + for intent in ("set_value", "select"): + for role, multiple in (("combo box", False), ("list box", False), + ("list box", True)): + with self.subTest(intent=intent, role=role, multiple=multiple): + control, options = self.dropdown(role, multiple) + if multiple: + options[2].states.add(STATE.SELECTED) + options[0].get_n_actions = lambda: 0 + before = [set(option.states) for option in options] + notes, evidence = self.invoke(intent, control, "Beijing") + for option in options: + option.do_action.assert_not_called() + self.assertEqual([option.states for option in options], before) + self.assertFalse(evidence["input_emitted"]) + self.assertTrue(evidence["readback_verified"]) + self.assertTrue(any("Beijing" in note for note in notes)) + + def test_list_box_mutation_is_refused_without_action_or_editor_fallback(self): + for intent in ("set_value", "select"): + for multiple in (False, True): + with self.subTest(intent=intent, multiple=multiple): + control, options = self.dropdown("list box", multiple) + if multiple: + options[2].states.add(STATE.SELECTED) + control.states.add(STATE.EDITABLE) + control.text_iface = FakeTextIface("Beijing") + control.get_editable_text_iface = lambda: control.text_iface + before = [set(option.states) for option in options] + with mock.patch.object(runtime, "resolve_combo_editor") as resolve, \ + mock.patch.object(runtime, "set_element_value_outcome") as write: + failure = self.assert_precondition(intent, control, options) + resolve.assert_not_called() + write.assert_not_called() + self.assertIn("input", str(failure).lower()) + self.assertIn("change", str(failure).lower()) + self.assertEqual([option.states for option in options], before) + self.assertEqual(control.text_iface.text, "Beijing") + self.assertEqual(control.text_iface.set_calls, []) + + def test_list_item_ref_cannot_bypass_list_box_mutation_guard(self): + for multiple in (False, True): + for grouped in (False, True): + with self.subTest(multiple=multiple, grouped=grouped): + control, options = self.dropdown("list box", multiple) + if multiple: + options[2].states.add(STATE.SELECTED) + if not grouped: + control.children.clear() + control.add(*options) + before = [set(option.states) for option in options] + action_evidence.reset() + evidence = {} + with self.assertRaises(runtime.OperationFailure) as caught: + runtime.perform_select( + {}, None, options[1], {"controlType": "list item"}, + None, [], evidence) + self.assertEqual(caught.exception.failure_code, "select_unavailable") + self.assertEqual(caught.exception.failure_phase, "precondition") + self.assertFalse(caught.exception.input_events_emitted) + self.assertFalse(evidence.get("input_emitted", False)) + self.assertFalse(evidence.get("readback_verified", False)) + self.assertIn("input", str(caught.exception).lower()) + self.assertIn("change", str(caught.exception).lower()) + self.assertEqual([option.states for option in options], before) + for option in options: + option.do_action.assert_not_called() + + def test_false_or_accepted_action_without_selection_is_unknown(self): + for intent in ("set_value", "select"): + for accepted in (False, True): + with self.subTest(intent=intent, accepted=accepted): + control, options = self.dropdown() + options[1].do_action.side_effect = None + options[1].do_action.return_value = accepted + before = [set(option.states) for option in options] + self.assert_unknown(intent, control, options) + self.assertEqual([option.states for option in options], before) + + def test_lost_action_reply_is_unknown_even_after_commit(self): + for intent in ("set_value", "select"): + for commits in (False, True): + for structured in (False, True): + with self.subTest(intent=intent, commits=commits, structured=structured): + control, options = self.dropdown() + commit = options[1].do_action.side_effect + error = (runtime.OperationFailure( + "lost action reply", failure_code="effect_unknown", + failure_phase="emission", input_events_emitted=True) + if structured else RuntimeError("lost action reply")) + + def disconnect(index): + self.assertTrue(self.evidence["input_emitted"]) + if commits: + commit(index) + raise error + + options[1].do_action.side_effect = disconnect + self.assert_unknown(intent, control, options, phase="emission") + self.assertEqual(STATE.SELECTED in options[1].states, commits) + + def test_readback_waits_for_selected_state_with_fake_clock(self): + for intent in ("set_value", "select"): + with self.subTest(intent=intent): + control, options = self.dropdown() + ready_at = self.clock.monotonic() + 0.2 + + def accept(_index): + options[0].states.discard(STATE.SELECTED) + options[1].get_state_set = lambda: FakeStateSet( + options[1].states | ({STATE.SELECTED} + if self.clock.monotonic() >= ready_at else set())) + return True + + options[1].do_action.side_effect = accept + _notes, evidence = self.invoke(intent, control) + self.assertGreaterEqual(self.clock.monotonic(), ready_at) + self.assertTrue(evidence["readback_verified"]) + options[1].do_action.assert_called_once_with(1) + + def test_fresh_proxy_with_same_identity_and_name_can_confirm_selection(self): + for intent in ("set_value", "select"): + with self.subTest(intent=intent): + control, options = self.dropdown() + replacement = _Node(role="menu item", name="Seattle", states=(STATE.SELECTED,)) + replacement.path = options[1].path + replacement.app = options[1].app + + def commit(_index): + options[0].states.discard(STATE.SELECTED) + control.children[0].children[1] = replacement + replacement.parent = control.children[0] + return True + + options[1].do_action.side_effect = commit + _notes, evidence = self.invoke(intent, control) + self.assertNotIn(STATE.SELECTED, options[1].states) + self.assertTrue(evidence["readback_verified"]) + options[1].do_action.assert_called_once_with(1) + + def test_failed_fresh_readback_never_confirms_cached_option(self): + faults = ( + "removed", "replacement", "foreign_bus", "renamed", "duplicate", + "none_selected", "multiple_selected", "child_count", "missing_child", + "name_disconnected", "role_disconnected", "target_state", "other_state", + ) + for intent in ("set_value", "select"): + for fault in faults: + with self.subTest(intent=intent, fault=fault): + control, options = self.dropdown() + group = control.children[0] + commit = options[1].do_action.side_effect + + def action(index): + commit(index) + if fault == "removed": + group.children.remove(options[1]) + elif fault in ("replacement", "foreign_bus"): + other = _Node(role="menu item", name="Seattle", states=(STATE.SELECTED,)) + if fault == "foreign_bus": + other.path = options[1].path + other.app = mock.Mock(bus_name=":1.99") + group.children[1] = other + other.parent = group + elif fault == "renamed": + options[1].title = "Portland" + elif fault == "duplicate": + group.add(_Node(role="menu item", name="Seattle")) + elif fault == "none_selected": + options[1].states.discard(STATE.SELECTED) + elif fault == "multiple_selected": + options[0].states.add(STATE.SELECTED) + elif fault == "child_count": + group.get_child_count = mock.Mock(side_effect=RuntimeError("disconnected")) + elif fault == "missing_child": + group.get_child_at_index = lambda _index: None + elif fault == "name_disconnected": + options[1].get_name = mock.Mock(side_effect=RuntimeError("disconnected")) + elif fault == "role_disconnected": + options[0].get_role_name = mock.Mock(side_effect=RuntimeError("disconnected")) + else: + target = options[1] if fault == "target_state" else options[0] + target.get_state_set = mock.Mock(side_effect=RuntimeError("disconnected")) + return True + + options[1].do_action.side_effect = action + self.assert_unknown(intent, control, options) + + def test_source_changes_during_fresh_scan_are_rejected_before_action(self): + for intent in ("set_value", "select"): + for fault in ("removed", "replacement", "renamed"): + with self.subTest(intent=intent, fault=fault): + control, options = self.dropdown() + group = control.children[0] + scans = 0 + + def child_count(): + nonlocal scans + scans += 1 + if scans == 2: + if fault == "removed": + group.children.remove(options[1]) + options[1].parent = None + elif fault == "replacement": + other = _Node(role="menu item", name="Seattle", + states=(STATE.ENABLED, STATE.SENSITIVE)) + group.children[1] = other + other.parent = group + options[1].parent = None + else: + options[1].title = "Portland" + return len(group.children) + + group.get_child_count = child_count + self.assert_precondition(intent, control, options) + self.assertGreaterEqual(scans, 2) + self.assertNotIn(STATE.SELECTED, options[1].states) + + def test_disabled_node_or_option_rejected_before_mutation(self): + for intent in ("set_value", "select"): + for target in ("control", "option"): + for state in (STATE.ENABLED, STATE.SENSITIVE): + with self.subTest(intent=intent, target=target, state=state): + control, options = self.dropdown() + node = control if target == "control" else options[1] + node.states.discard(state) + self.assert_precondition(intent, control, options) + + def test_missing_or_duplicate_name_rejected_before_mutation(self): + for intent in ("set_value", "select"): + for wanted, duplicate in (("Missing", False), ("Seattle", True), ("Beijing", True)): + with self.subTest(intent=intent, wanted=wanted, duplicate=duplicate): + control, options = self.dropdown() + if duplicate: + control.children[0].add(_Node(role="menu item", name=wanted)) + self.assert_precondition( + intent, control, options, wanted=wanted, code="invalid_argument") + + def test_incomplete_option_scan_rejected_before_mutation(self): + faults = ("child_count", "negative_count", "missing_child", "child_disconnected", + "name_disconnected", "role_disconnected", "scan_limit") + for intent in ("set_value", "select"): + for fault in faults: + with self.subTest(intent=intent, fault=fault): + control, options = self.dropdown() + group = control.children[0] + if fault == "child_count": + group.get_child_count = mock.Mock(side_effect=RuntimeError("disconnected")) + elif fault == "negative_count": + group.get_child_count = lambda: -1 + elif fault == "missing_child": + group.get_child_at_index = lambda _index: None + elif fault == "child_disconnected": + group.get_child_at_index = mock.Mock(side_effect=RuntimeError("disconnected")) + elif fault == "name_disconnected": + options[0].get_name = mock.Mock(side_effect=RuntimeError("disconnected")) + elif fault == "role_disconnected": + options[0].get_role_name = mock.Mock(side_effect=RuntimeError("disconnected")) + else: + group.get_child_count = lambda: runtime.MAX_DROPDOWN_OPTIONS + 1 + failure = self.assert_precondition(intent, control, options) + self.assertIn("select_options_incomplete", str(failure)) + + def test_missing_select_action_has_no_popup_advice_or_fallback(self): + for intent in ("set_value", "select"): + for fault in ("none", "other_actions", "count_disconnected", "name_disconnected"): + with self.subTest(intent=intent, fault=fault): + control, options = self.dropdown() + if fault == "none": + options[1].get_n_actions = lambda: 0 + elif fault == "other_actions": + options[1].get_action_name = lambda index: ("press", "click")[index] + elif fault == "count_disconnected": + options[1].get_n_actions = mock.Mock(side_effect=RuntimeError("disconnected")) + else: + options[1].get_action_name = mock.Mock(side_effect=RuntimeError("disconnected")) + failure = self.assert_precondition(intent, control, options) + self.assertNotRegex(str(failure).lower(), r"popup|open.*list|keyboard|focus") + + def test_html_select_never_resolves_or_writes_an_editor(self): + for intent in ("set_value", "select"): + with self.subTest(intent=intent): + control, _options = self.dropdown() + control.states.add(STATE.EDITABLE) + control.text_iface = FakeTextIface("Beijing") + control.get_editable_text_iface = lambda: control.text_iface + with mock.patch.object(runtime, "resolve_combo_editor") as resolve, \ + mock.patch.object(runtime, "set_element_value_outcome") as write: + _notes, evidence = self.invoke(intent, control) + resolve.assert_not_called() + write.assert_not_called() + self.assertEqual(control.text_iface.text, "Beijing") + self.assertTrue(evidence["readback_verified"]) + + def test_non_html_selects_keep_editable_value_behavior(self): + cases = ( + ("chromium", "combo box", "input"), + ("chromium", "combo box", "div"), + ("chromium", "combo box", None), + ("chromium", "entry", "select"), + ("chromium", "list box", "div"), + ("qt", "combo box", "select"), + ("gecko", "combo box", "select"), + ("gtk", "list box", "select"), + ) + for toolkit, role, tag in cases: + with self.subTest(toolkit=toolkit, role=role, tag=tag): + control, options = self.dropdown(role) + control.app.get_toolkit_name.return_value = toolkit + control.get_attributes = lambda: {} if tag is None else {"tag": tag} + control.states.add(STATE.EDITABLE) + control.text_iface = FakeTextIface("Beijing") + control.get_editable_text_iface = lambda: control.text_iface + _notes, evidence = self.invoke("set_value", control, "Portland") + self.assertEqual(control.text_iface.text, "Portland") + self.assertEqual(control.text_iface.set_calls, ["Portland"]) + self.assertEqual(evidence["route"], "value_api") + self.assertTrue(evidence["readback_verified"]) + for option in options: + option.do_action.assert_not_called() + + def test_aria_combobox_and_other_toolkits_keep_selection_fallback(self): + for toolkit, tag in (("chromium", "input"), ("chromium", "div"), + ("qt", "select"), ("other", "select")): + with self.subTest(toolkit=toolkit, tag=tag): + control, options = self.dropdown() + control.app.get_toolkit_name.return_value = toolkit + control.get_attributes = lambda: {"tag": tag} + control.children[0].selection_iface = None + options[1].get_n_actions = lambda: 0 + + def commit(owner, index): + self.assertIs(owner, control) + options[0].states.discard(STATE.SELECTED) + options[index].states.add(STATE.SELECTED) + control.title = options[index].title + return True + + with mock.patch.object(self.selection, "select_child", side_effect=commit) as select: + _notes, evidence = self.invoke("select", control) + select.assert_called_once_with(control, 1) + self.assertTrue(evidence["readback_verified"]) + for option in options: + option.do_action.assert_not_called() From 924aa77e8ec7744e35d8e05ceb85808c576af955 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 07:45:31 +0000 Subject: [PATCH 06/15] fix(linux): resolve keypad keys and preserve input failure evidence Resolve case-insensitive named keys from GDK instead of guessing capitalization. Classify unknown keys as invalid_argument and retain spelling guidance through AX batch reporting. Deliver named symbols with their required keyboard layer, preserve hardware modifier and lock-key behavior, and restore NumLock for keypad navigation. Keep partial-input evidence on synthesis and cleanup failures. Add keyboard and public-error regressions, document Linux key names, and refresh the AX/GUI contract golden descriptions. --- CHANGELOG.md | 4 + .../internal/contract/tool_contract.go | 9 +- computer-use/internal/core/ax_evidence.go | 2 +- .../internal/core/ax_runtime_failure.go | 7 + .../internal/core/ax_runtime_failure_test.go | 29 +++ .../native/linux/provider/ax_actions.py | 80 ++++++--- .../native/linux/provider/input_primitives.py | 27 ++- .../runtime_tests/test_window_focus.py | 168 ++++++++++++++++++ .../testdata/contract/public_contract.json | 4 +- docs/MCP.md | 32 +++- 10 files changed, 335 insertions(+), 27 deletions(-) create mode 100644 computer-use/internal/core/ax_runtime_failure_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a77f07..d0c3e59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ window. Both actions read back the selection and fail explicitly when it cannot be confirmed. HTML list box mutations are refused because their accessibility actions do not emit both `input` and `change` events. +- Linux AX and GUI accept case-insensitive keypad and X11 key names. + Named keys use their requested symbol, including keypad digits with NumLock + off. Unknown keys report `invalid_argument` with accepted spelling examples + before the key step takes focus or emits input. - AX reports text-value evidence for `type` steps without a ref, including unchanged or unreadable fields. A focus change alone does not confirm typing. - Linux AX checks focused-field readback after keyboard typing and reports diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index 7f7142f..1604bc6 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -12,6 +12,11 @@ const ( PublicToolGUI = "GUI" ) +const keyChordDescription = "One chord, e.g. [\"CTRL\",\"S\"]. " + + "PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, " + + "including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, " + + "KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'." + const ( GUIMaxActions = 64 GUIMaxScrollDelta = 2000 @@ -227,7 +232,7 @@ func axInputSchema() map[string]any { "keys": map[string]any{ "type": "array", "minItems": 1, "maxItems": GUIMaxKeys, "items": map[string]any{"type": "string", "pattern": "^\\S(.*\\S)?$", "maxLength": GUIMaxKeyLength}, - "description": "One chord, e.g. [\"CTRL\",\"S\"].", + "description": keyChordDescription, }, "value": map[string]any{"type": "string", "maxLength": MaxActionText}, "direction": map[string]any{"type": "string", "enum": []string{"up", "down", "left", "right"}}, @@ -269,7 +274,7 @@ func guiActionProperty(includeScreenshot bool) map[string]any { } keyArray := map[string]any{ "type": "array", - "description": "PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN.", + "description": keyChordDescription, "minItems": 1, "maxItems": GUIMaxKeys, "items": map[string]any{ diff --git a/computer-use/internal/core/ax_evidence.go b/computer-use/internal/core/ax_evidence.go index 8f2299f..5b8a3b5 100644 --- a/computer-use/internal/core/ax_evidence.go +++ b/computer-use/internal/core/ax_evidence.go @@ -571,7 +571,7 @@ func axPublicErrorCode(full map[string]any) string { return contract.AXCodeEffectUnknown case failure == "ax_app_unavailable" || status == "app_unavailable": return contract.AXCodeAppUnavailable - case failure == "ax_app_ambiguous": + case failure == "ax_app_ambiguous", failure == contract.AXCodeInvalidArgument: return contract.AXCodeInvalidArgument case failure == "invalid_argument", failure == "unsupported_action", failure == "input_failed": if axEvidenceReportsInputEmission(full) || full["partial_input_possible"] == true || full["mutation_unknown"] == true { diff --git a/computer-use/internal/core/ax_runtime_failure.go b/computer-use/internal/core/ax_runtime_failure.go index e64471b..db88a4e 100644 --- a/computer-use/internal/core/ax_runtime_failure.go +++ b/computer-use/internal/core/ax_runtime_failure.go @@ -5,6 +5,8 @@ import ( "fmt" "regexp" "strings" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) func (s *Service) providerIdentityConflictResult( @@ -196,6 +198,11 @@ func (s *Service) renderRuntimeReportedFailure( "mutation_unknown": response.MutationUnknown, "failure_facts_complete": response.FailureFactsComplete, } + if response.FailureCode == contract.AXCodeInvalidArgument { + result.StructuredContent[contract.AXPublicErrorDetailKey] = publicRuntimeFailureText(response) + result.StructuredContent[contract.AXPublicErrorHintKey] = + "Correct the invalid argument before retrying; check completed before resending earlier actions." + } if response.UnderlyingFailureCode != "" { result.StructuredContent["underlying_failure_code"] = response.UnderlyingFailureCode diff --git a/computer-use/internal/core/ax_runtime_failure_test.go b/computer-use/internal/core/ax_runtime_failure_test.go new file mode 100644 index 0000000..80b3426 --- /dev/null +++ b/computer-use/internal/core/ax_runtime_failure_test.go @@ -0,0 +1,29 @@ +package core + +import ( + "strings" + "testing" +) + +func TestInvalidKeyKeepsPublicArgumentErrorAndAcceptedSpelling(t *testing.T) { + response := ProviderAccessibilityResult{ + Error: "Unsupported key: KP_AD. Use KP_ADD.", + Failure: &ProviderFailure{ + Code: "invalid_argument", Phase: "precondition", FactsComplete: true, + }, + } + result := NewService().runtimeReportedFailure("", AXServiceRequest{}, response) + result = prependAXBatchSummary(result, "Stopped at action 1/1 (keypress).", map[string]any{ + "actions_requested": 1, "actions_completed_without_call_error": 0, + }) + public := axPublicEvidence(result.StructuredContent, result.IsError, result.Content) + if public["code"] != "invalid_argument" || public["input_emitted"] != false || public["completed"] != 0 { + t.Fatalf("invalid key classification lost: %#v", public) + } + if message, _ := public["message"].(string); !strings.Contains(message, "Use KP_ADD") { + t.Fatalf("accepted spelling lost behind batch summary: %#v", public) + } + if hint, _ := public["hint"].(string); hint == "" { + t.Fatalf("invalid key has no correction hint: %#v", public) + } +} diff --git a/computer-use/native/linux/provider/ax_actions.py b/computer-use/native/linux/provider/ax_actions.py index e574118..9a72951 100644 --- a/computer-use/native/linux/provider/ax_actions.py +++ b/computer-use/native/linux/provider/ax_actions.py @@ -19,6 +19,7 @@ from .input_primitives import ( emit_keyboard_event, emit_mouse_event, + keypad_navigation_numlock_mask, require_window_focus, ) @@ -120,6 +121,21 @@ def normalize_key_expression(key): # Gdk.keyval_from_name returns VoidSymbol, not 0, for an unknown name. GDK_VOID_SYMBOL = 0xFFFFFF +GDK_KEY_NAMES = { + name[4:].lower(): getattr(Gdk, name) + for name in dir(Gdk) if name.startswith("KEY_") +} + + +def invalid_key(message): + return OperationFailure( + message + ". Use CTRL, SHIFT, ALT or SUPER before the main key; " + "use ENTER, TAB, ESC, PAGEUP/PAGE_UP, PAGEDOWN/PAGE_DOWN, " + "KP_ADD, KP_SUBTRACT, KP_ENTER or KP_0 through KP_9. " + "Use PLUS for the '+' key. Linux also accepts mapped X11 key names.", + failure_code="invalid_argument", failure_phase="precondition", + input_events_emitted=False, + ) def keysym(name): @@ -130,18 +146,12 @@ def keysym(name): case ("INSERT", "caps_lock"), because callers write keys the way they appear on the keyboard. """ - candidates = [name] - if len(name) > 1: - candidates.append("_".join(part.capitalize() for part in name.split("_"))) - # Punctuation keysyms are lowercase names ("comma", "plus"). - candidates.append(name.lower()) - for candidate in candidates: - sym = Gdk.keyval_from_name(candidate) - if sym and sym != GDK_VOID_SYMBOL: - return sym + sym = Gdk.keyval_from_name(name) + if sym and sym != GDK_VOID_SYMBOL: + return sym if len(name) == 1: return Gdk.unicode_to_keyval(ord(name)) - return 0 + return GDK_KEY_NAMES.get(name.lower(), 0) def keycode(name): @@ -166,11 +176,16 @@ def keycode(name): raise RuntimeError("Gdk unavailable; cannot resolve keycode for: " + name) sym = keysym(name) if not sym: - raise RuntimeError("Unsupported key: " + name) + raise invalid_key("Unsupported key: " + name) keymap = Gdk.Keymap.get_default() ok, entries = keymap.get_entries_for_keyval(sym) if not ok or not entries: - raise RuntimeError("No hardware keycode mapped for key: " + name) + raise OperationFailure( + "No hardware keycode mapped for key: " + name + + ". Choose a key available in the desktop keyboard layout.", + failure_code="unsupported_action", failure_phase="precondition", + input_events_emitted=False, + ) return int(entries[0].keycode) @@ -182,9 +197,9 @@ def parse_key(key): must fail before the window is raised, not after. """ normalized_expression = normalize_key_expression(key) - parts = [part for part in normalized_expression.split("+") if part] - if not parts: - raise RuntimeError("Unsupported key: " + str(key)) + parts = normalized_expression.split("+") + if not all(parts): + raise invalid_key("Unsupported key expression: " + str(key)) main = parts[-1] codes = [] for modifier in parts[:-1]: @@ -193,7 +208,7 @@ def parse_key(key): # Reject unknown modifiers: skipping them would silently turn a # typo such as "ctrl+shft+s" into a bare "s", which is confusing # and very hard to diagnose. - raise RuntimeError( + raise invalid_key( "Unsupported modifier: {}. Use CTRL, SHIFT, ALT, or SUPER; " "Control_L/Ctrl_L and GTK forms such as s are also accepted." .format(modifier)) @@ -218,6 +233,8 @@ def send_key(key): pressed = [] emitted = False primary_error = None + restore_numlock = 0 + symbol = keysym(normalized) try: for code in codes: emitted = emit_keyboard_event( @@ -236,6 +253,21 @@ def send_key(key): "text key {!r}".format(normalized), emitted, ) + elif len(normalized) > 1 and not ( + Gdk.KEY_Shift_L <= symbol <= Gdk.KEY_Hyper_R + or symbol in (Gdk.KEY_Num_Lock, Gdk.KEY_Scroll_Lock)): + restore_numlock = keypad_navigation_numlock_mask(symbol) + if restore_numlock: + emitted = emit_keyboard_event( + restore_numlock, None, Atspi.KeySynthType.UNLOCKMODIFIERS, + "NumLock preparation", emitted) + emitted = emit_keyboard_event( + symbol, + None, + Atspi.KeySynthType.SYM, + "key {!r}".format(normalized), + emitted, + ) else: emitted = emit_keyboard_event( keycode(normalized), @@ -247,6 +279,14 @@ def send_key(key): except Exception as exc: primary_error = exc + if restore_numlock: + try: + emitted = emit_keyboard_event( + restore_numlock, None, Atspi.KeySynthType.LOCKMODIFIERS, + "NumLock restoration", emitted) + except Exception as exc: + if primary_error is None: + primary_error = exc # Release every pressed modifier even if one release fails. Stopping after # the first failed release can leave the remaining modifiers stuck. release_error = None @@ -262,10 +302,10 @@ def send_key(key): except Exception as exc: if release_error is None: release_error = exc - if primary_error is not None: - raise primary_error - if release_error is not None: - raise release_error + for error in (primary_error, release_error): + if error is not None: + error.events_emitted = bool(emitted or getattr(error, "events_emitted", False)) + raise error # Keep the public keyboard-input bound; input method delivery also has a diff --git a/computer-use/native/linux/provider/input_primitives.py b/computer-use/native/linux/provider/input_primitives.py index 1feea3d..75a9387 100644 --- a/computer-use/native/linux/provider/input_primitives.py +++ b/computer-use/native/linux/provider/input_primitives.py @@ -4,6 +4,7 @@ import time from .foundation import ( Atspi, + Gdk, FOCUS_GRAB_CANDIDATES, node_name, node_pid, @@ -226,6 +227,25 @@ def emit_mouse_event(x, y, event, events_emitted=False): ) +def keypad_navigation_numlock_mask(symbol): + if not Gdk.KEY_KP_Home <= symbol <= Gdk.KEY_KP_Delete: + return 0 + keymap = Gdk.Keymap.get_default() + state = int(Gdk.get_default_root_window().get_pointer().mask) + available, entries = keymap.get_entries_for_keyval(symbol) + for entry in entries if available else []: + for bit in range(3, 8): + mask = 1 << bit + if not state & mask: + continue + translated = keymap.translate_keyboard_state( + entry.keycode, Gdk.ModifierType(mask), entry.group) + if translated[0] and (Gdk.KEY_KP_0 <= translated[1] <= Gdk.KEY_KP_9 + or translated[1] == Gdk.KEY_KP_Decimal): + return mask + return 0 + + def emit_keyboard_event( keyval, keystring, @@ -233,7 +253,12 @@ def emit_keyboard_event( description, events_emitted=False, ): - result = Atspi.generate_keyboard_event(keyval, keystring, synth_type) + try: + result = Atspi.generate_keyboard_event(keyval, keystring, synth_type) + except Exception as exc: + exc.events_emitted = bool(events_emitted or getattr(exc, "events_emitted", False)) + exc.partial_input_possible = True + raise return require_input_emission(result, description, events_emitted) diff --git a/computer-use/runtime_tests/test_window_focus.py b/computer-use/runtime_tests/test_window_focus.py index f361df6..82e2d43 100644 --- a/computer-use/runtime_tests/test_window_focus.py +++ b/computer-use/runtime_tests/test_window_focus.py @@ -549,6 +549,174 @@ def test_unknown_key_still_fails(self): with self.assertRaises(RuntimeError): runtime.keycode("NoSuchKeyName") + def test_keypad_and_mixed_case_names_resolve_in_any_case(self): + names = [name[4:] for name in dir(runtime.Gdk) + if name.startswith("KEY_KP_")] + names.extend(["ISO_Left_Tab", "AudioMute", "Print", "Scroll_Lock"]) + for name in names: + expected = runtime.Gdk.keyval_from_name(name) + self.assertNotIn(expected, (0, runtime.GDK_VOID_SYMBOL)) + for spelling in (name, name.upper(), name.lower()): + with self.subTest(spelling=spelling): + self.assertEqual(runtime.keysym(spelling), expected) + + def test_invalid_key_is_classified_before_focus_or_input(self): + for key in ("KP_AD", "ctrl+shft+s", "+", "ctrl++s"): + with self.subTest(key=key), \ + mock.patch.object(runtime, "require_window_focus") as focus, \ + mock.patch.object(runtime, "focus_target_element") as target, \ + mock.patch.object(runtime, "emit_keyboard_event") as emit, \ + mock.patch.object(runtime.Gdk.Keymap, "get_default") as keymap: + keymap.return_value.get_entries_for_keyval.return_value = ( + True, [mock.Mock(keycode=37)]) + runtime.action_evidence.reset() + try: + runtime._perform_keypress({"key": key}, None, None, None, [], {}) + except runtime.OperationFailure as error: + response = runtime.operation_failure_response(error) + else: + self.fail("invalid key did not raise OperationFailure") + self.assertEqual(response["failureCode"], "invalid_argument") + self.assertEqual(response["failurePhase"], "precondition") + self.assertFalse(response["input_events_emitted"]) + self.assertIn("Use", response["error"]) + focus.assert_not_called() + target.assert_not_called() + emit.assert_not_called() + + def test_named_keys_use_symbols_and_modifiers_use_hardware_codes(self): + for key, expected in (("KP_ADD", "KP_Add"), ("KP_1", "KP_1"), + ("KP_DECIMAL", "KP_Decimal"), ("EXCLAM", "exclam"), + ("TAB", "Tab")): + with self.subTest(key=key), \ + mock.patch.object(runtime, "keycode", return_value=50), \ + mock.patch.object(runtime, "emit_keyboard_event", return_value=True) as emit: + runtime.send_key("SHIFT+" + key) + self.assertEqual(emit.call_args_list[0].args[:3], + (50, None, runtime.Atspi.KeySynthType.PRESS)) + self.assertEqual(emit.call_args_list[1].args[:3], + (runtime.Gdk.keyval_from_name(expected), None, + runtime.Atspi.KeySynthType.SYM)) + self.assertEqual(emit.call_args_list[2].args[:3], + (50, None, runtime.Atspi.KeySynthType.RELEASE)) + + def test_letter_chord_does_not_add_implicit_shift(self): + with mock.patch.object(runtime, "keycode", side_effect=[37, 39, 39]), \ + mock.patch.object(runtime, "emit_keyboard_event", return_value=True) as emit: + runtime.send_key("CTRL+S") + self.assertEqual(emit.call_args_list[1].args[:3], + (39, None, runtime.Atspi.KeySynthType.PRESSRELEASE)) + + def test_known_unmapped_key_is_an_explicit_precondition_failure(self): + with mock.patch.object(runtime.Gdk.Keymap, "get_default") as keymap: + keymap.return_value.get_entries_for_keyval.return_value = (False, []) + with self.assertRaises(runtime.OperationFailure) as caught: + runtime.parse_key("KP_ADD") + self.assertEqual(caught.exception.failure_code, "unsupported_action") + self.assertFalse(caught.exception.input_events_emitted) + self.assertIn("keyboard layout", str(caught.exception)) + + def test_gui_rejects_unknown_key_before_any_action(self): + with mock.patch.object(runtime, "execute_gui_action") as execute, \ + mock.patch.object(runtime, "desktop_bounds") as bounds: + response = runtime.perform_gui([ + {"type": "click", "x": 10, "y": 10}, + {"type": "keypress", "keys": ["KP_AD"]}, + ]) + self.assertFalse(response["ok"]) + self.assertTrue(response["preflight_only"]) + self.assertFalse(response["input_events_emitted"]) + self.assertEqual(response["dispatched"], 0) + self.assertEqual(response["validation_errors"][0]["index"], 1) + self.assertIn("KP_ADD", response["validation_errors"][0]["error"]) + execute.assert_not_called() + bounds.assert_not_called() + + def test_symbol_emission_failure_releases_explicit_modifiers(self): + events = [] + + def generate(keyval, keystring, synth_type): + events.append((keyval, synth_type)) + return synth_type != runtime.Atspi.KeySynthType.SYM + + with mock.patch.object(runtime, "keycode", return_value=50), \ + mock.patch.object(runtime.Atspi, "generate_keyboard_event", side_effect=generate): + with self.assertRaises(runtime.InputEmissionError) as caught: + runtime.send_key("SHIFT+KP_ADD") + self.assertTrue(caught.exception.events_emitted) + self.assertEqual(events, [ + (50, runtime.Atspi.KeySynthType.PRESS), + (runtime.Gdk.KEY_KP_Add, runtime.Atspi.KeySynthType.SYM), + (50, runtime.Atspi.KeySynthType.RELEASE), + ]) + + def test_modifier_and_lock_keys_use_hardware_events(self): + for name in ("CAPS_LOCK", "NUM_LOCK", "SCROLL_LOCK", "CONTROL_R", "SHIFT"): + with self.subTest(name=name), \ + mock.patch.object(runtime, "keycode", return_value=66), \ + mock.patch.object(runtime, "emit_keyboard_event", return_value=True) as emit: + runtime.send_key(name) + self.assertEqual(emit.call_args_list[0].args[:3], + (66, None, runtime.Atspi.KeySynthType.PRESSRELEASE)) + self.assertEqual(emit.call_count, 1) + + def test_keypad_navigation_finds_numlock_mask_from_layout(self): + with mock.patch.object(runtime.Gdk.Keymap, "get_default") as keymap, \ + mock.patch.object(runtime.Gdk, "get_default_root_window") as root: + keymap.return_value.get_entries_for_keyval.return_value = ( + True, [mock.Mock(keycode=79, group=0)]) + keymap.return_value.translate_keyboard_state.return_value = ( + True, runtime.Gdk.KEY_KP_7, 0, 1, 32) + root.return_value.get_pointer.return_value.mask = 32 + self.assertEqual(runtime.keypad_navigation_numlock_mask(runtime.Gdk.KEY_KP_Home), 32) + root.return_value.get_pointer.return_value.mask = 0 + self.assertEqual(runtime.keypad_navigation_numlock_mask(runtime.Gdk.KEY_KP_Home), 0) + keymap.return_value.translate_keyboard_state.assert_called_once_with( + 79, runtime.Gdk.ModifierType(32), 0) + + def test_numlock_cleanup_preserves_failures_and_input_history(self): + for failed_type in (runtime.Atspi.KeySynthType.UNLOCKMODIFIERS, + runtime.Atspi.KeySynthType.SYM, + runtime.Atspi.KeySynthType.LOCKMODIFIERS): + events = [] + + def generate(keyval, keystring, synth_type): + events.append((keyval, synth_type)) + return synth_type != failed_type + + with self.subTest(failed_type=failed_type), \ + mock.patch.object(runtime, "keycode", return_value=79), \ + mock.patch.object(runtime, "keypad_navigation_numlock_mask", return_value=16), \ + mock.patch.object(runtime.Atspi, "generate_keyboard_event", side_effect=generate): + with self.assertRaises(runtime.InputEmissionError) as caught: + runtime.send_key("KP_HOME") + self.assertTrue(caught.exception.events_emitted) + self.assertEqual(events[-1], (16, runtime.Atspi.KeySynthType.LOCKMODIFIERS)) + self.assertEqual(events[0], (16, runtime.Atspi.KeySynthType.UNLOCKMODIFIERS)) + + def test_symbol_transport_exception_keeps_uncertainty_and_cleanup(self): + for key in ("KP_ADD", "SHIFT+KP_ADD"): + events = [] + + def generate(keyval, keystring, synth_type): + events.append((keyval, synth_type)) + if synth_type == runtime.Atspi.KeySynthType.SYM: + raise RuntimeError("keyboard connection lost") + return True + + with self.subTest(key=key), \ + mock.patch.object(runtime, "keycode", return_value=50), \ + mock.patch.object(runtime.Atspi, "generate_keyboard_event", side_effect=generate): + runtime.action_evidence.reset() + with self.assertRaisesRegex(RuntimeError, "keyboard connection lost") as caught: + runtime.send_key(key) + response = runtime.unexpected_operation_failure_response(caught.exception) + self.assertEqual(response["failureCode"], "effect_unknown") + self.assertTrue(response["partial_input_possible"]) + self.assertEqual(response["input_events_emitted"], key.startswith("SHIFT")) + if key.startswith("SHIFT"): + self.assertEqual(events[-1], (50, runtime.Atspi.KeySynthType.RELEASE)) + class FocusDiagnosticTests(AtspiPatchedTestCase): def test_names_the_window_that_actually_holds_focus(self): diff --git a/computer-use/testdata/contract/public_contract.json b/computer-use/testdata/contract/public_contract.json index 1e2e9a4..5a7d2ef 100644 --- a/computer-use/testdata/contract/public_contract.json +++ b/computer-use/testdata/contract/public_contract.json @@ -108,7 +108,7 @@ "type": "boolean" }, "keys": { - "description": "One chord, e.g. [\"CTRL\",\"S\"].", + "description": "One chord, e.g. [\"CTRL\",\"S\"]. PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'.", "items": { "maxLength": 64, "pattern": "^\\S(.*\\S)?$", @@ -616,7 +616,7 @@ "additionalProperties": false, "properties": { "keys": { - "description": "PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN.", + "description": "One chord, e.g. [\"CTRL\",\"S\"]. PAGEUP/PAGE_UP/PAGEDOWN/PAGE_DOWN. Linux named keys are case-insensitive, including KP_ADD, KP_SUBTRACT, KP_MULTIPLY, KP_DIVIDE, KP_DECIMAL, KP_ENTER, KP_0 through KP_9 and mapped X11 names. Use PLUS for '+'.", "items": { "maxLength": 64, "pattern": "\\S", diff --git a/docs/MCP.md b/docs/MCP.md index d03b970..4dae488 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -499,7 +499,7 @@ change canonical text, selectors, refs, or the response budget. - `act` accepts up to 64 ordered steps: actions `{type, ref|title, text|keys|value|direction|pages|to_ref}` and `wait` - steps. Argument errors reject the whole batch before input. The first step + steps. Structural argument errors reject the whole batch before input. The first step that fails stops the batch, and the result says how many ran. - The `rN` or `wN` refs name the window, so an `act` call with refs needs no `window_ref`. `window_ref` only sends ref-less `keypress` and `type` to a @@ -578,6 +578,36 @@ statement inside that follow-up describes the read, not the earlier action. Stale-window follow-ups report the unavailable source reference without repeating an input-delivery statement. +### Linux key names + +AX and GUI `keypress` accept one chord, such as `["CTRL","S"]`. +Named keys are case-insensitive. `KP_ADD`, `kp_add` and `KP_Add` name the +same keypad key. The desktop keyboard layout must map the requested key. + +| Keys | Accepted names | +| --- | --- | +| Modifiers | `CTRL` / `CONTROL`, `SHIFT`, `ALT`, `SUPER` / `WIN` / `CMD` | +| Editing | `ENTER` / `RETURN`, `TAB`, `ESC` / `ESCAPE`, `BACKSPACE`, `DELETE`, `INSERT`, `SPACE` | +| Navigation | `LEFT`, `RIGHT`, `UP`, `DOWN` (also `ARROWLEFT`, `ARROWRIGHT`, `ARROWUP`, `ARROWDOWN`), `HOME`, `END` | +| Pages | `PAGEUP` / `PAGE_UP` / `PRIOR`, `PAGEDOWN` / `PAGE_DOWN` / `NEXT` | +| Function and lock keys | `F1` through `F12`, `CAPS_LOCK`, `NUM_LOCK`, `SCROLL_LOCK`, `PRINT`, `PAUSE`, `MENU` | +| Keypad numbers | `KP_0` through `KP_9` | +| Keypad operations | `KP_ADD`, `KP_SUBTRACT`, `KP_MULTIPLY`, `KP_DIVIDE`, `KP_DECIMAL`, `KP_EQUAL`, `KP_ENTER` | +| Keypad navigation | `KP_HOME`, `KP_END`, `KP_LEFT`, `KP_RIGHT`, `KP_UP`, `KP_DOWN`, `KP_BEGIN`, `KP_INSERT`, `KP_DELETE`, `KP_PRIOR` / `KP_PAGE_UP`, `KP_NEXT` / `KP_PAGE_DOWN` | +| Characters | Letters, digits and punctuation; use `PLUS` for `+`, which separates chord tokens | + +Other X11 key names are accepted when the desktop layout maps them. +Named punctuation includes `COMMA`, `PERIOD`, `MINUS`, `SLASH` and `EQUAL`. +Keypad symbols select their numeric or navigation layer regardless of +NumLock. Explicit modifiers still apply. Letter chords use hardware keycodes, +so `["CTRL","S"]` does not add Shift. + +Unknown names return `invalid_argument` with accepted spelling examples before +that step takes focus or sends input. A known name without a hardware mapping +fails explicitly. GUI validates all key steps before the batch starts; AX +resolves Linux key names before each key step. Check `completed` before retrying +an AX batch that already ran earlier steps. + ### `select` steps A `select` step selects without typing or clicking: From 7903a1e37438e6ce2823eaff459dbaf40179a66b Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 07:45:31 +0000 Subject: [PATCH 07/15] fix(linux): preserve AX refs across nested modal dialogs --- CHANGELOG.md | 3 + .../core/ax_ref_snapshot_assignment_test.go | 121 ++++++++++++++++++ .../internal/core/ax_response_budget.go | 2 +- .../native/linux/provider/ax_snapshot.py | 13 ++ .../runtime_tests/test_popup_rendering.py | 41 ++++++ .../testdata/provider-wire/manifest.json | 1 + .../provider-wire/modal-scope.response.json | 27 ++++ docs/ARCHITECTURE.md | 7 + 8 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 computer-use/internal/core/ax_ref_snapshot_assignment_test.go create mode 100644 computer-use/testdata/provider-wire/modal-scope.response.json diff --git a/CHANGELOG.md b/CHANGELOG.md index d0c3e59..0a04700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ Named keys use their requested symbol, including keypad digits with NumLock off. Unknown keys report `invalid_argument` with accepted spelling examples before the key step takes focus or emits input. +- Linux AX preserves references to controls hidden by an in-window modal dialog. + After the dialog closes, each action still checks the live target; a provider + rebuild requires a unique semantic match in the original runtime path and window. - AX reports text-value evidence for `type` steps without a ref, including unchanged or unreadable fields. A focus change alone does not confirm typing. - Linux AX checks focused-field readback after keyboard typing and reports diff --git a/computer-use/internal/core/ax_ref_snapshot_assignment_test.go b/computer-use/internal/core/ax_ref_snapshot_assignment_test.go new file mode 100644 index 0000000..944f63a --- /dev/null +++ b/computer-use/internal/core/ax_ref_snapshot_assignment_test.go @@ -0,0 +1,121 @@ +package core + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +func modalRefSnapshot(revision string, records ...ElementRecord) *AppSnapshot { + for index := range records { + records[index].Index = index + 1 + records[index].ProviderBusName = ":1.42" + records[index].ProcessGeneration = "epoch1" + records[index].ProviderWindowBusName = ":1.42" + records[index].ProviderWindowObjectPath = "/window" + records[index].ProviderWindowProcessGeneration = "epoch1" + } + return &AppSnapshot{ + App: AppDescriptor{Name: "Messages"}, + WindowHandle: "w2:" + strings.Repeat("a", 32), + SnapshotID: revision, Scope: contract.AXScopeWindow, + publicAppRef: "a1", publicWindowRef: "w1", Elements: records, + } +} + +func assignModalRefSnapshot(t *testing.T, service *Service, source *AppSnapshot) *AXSnapshot { + t.Helper() + tree, err := buildAXSnapshot(source) + if err != nil { + t.Fatal(err) + } + service.RefRegistry.AssignSnapshotRefs(tree) + source.AXSnapshot = tree + service.rememberSnapshot("messages", source) + return tree +} + +func TestModalScopePreservesRefUntilCompleteRead(t *testing.T) { + payload, err := os.ReadFile("../../testdata/provider-wire/modal-scope.response.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Accessibility struct { + Snapshot AppSnapshot `json:"snapshot"` + } `json:"accessibility"` + } + if err := json.Unmarshal(payload, &fixture); err != nil { + t.Fatal(err) + } + for _, outcome := range []string{"same_path", "moved", "renamed", "ambiguous", "absent", "window_replaced"} { + t.Run(outcome, func(t *testing.T) { + service := NewService() + button := ElementRecord{ + RuntimeID: []int{0, 1}, ControlType: "push button", Name: "Invoice", + ProviderObjectPath: "/old/invoice", Actions: []string{"click"}, + } + before := assignModalRefSnapshot(t, service, modalRefSnapshot("before", button)) + ref := before.ByIndex[1].Ref + entry, err := service.RefRegistry.Resolve(ref, "messages") + if err != nil || entry.LocatorKind != AXLocatorSemantic { + t.Fatalf("initial semantic ref: %v, %#v", err, entry) + } + modal := modalRefSnapshot("modal", ElementRecord{ + RuntimeID: []int{0, 0}, ControlType: "push button", Name: "Close", + ProviderObjectPath: "/dialog/close", Actions: []string{"click"}, + }) + modal.Annotations = fixture.Accessibility.Snapshot.Annotations + current := assignModalRefSnapshot(t, service, modal) + if current.RefInventoryComplete || current.AcquisitionIncomplete || !current.IdentityCoverageIncomplete { + t.Fatalf("modal is a coverage boundary, not a failed read: %#v", current) + } + entry, err = service.RefRegistry.Resolve(ref, "messages") + if err != nil || entry.LastSeenRevision != "before" { + t.Fatalf("modal retired or refreshed the hidden ref: %v, %#v", err, entry) + } + target, err := semanticTargetFromRefEntry(entry) + if err != nil { + t.Fatal(err) + } + service.authorizeProviderRebind("messages", entry, target) + if target.AllowProviderRebind { + t.Fatal("hidden target allowed semantic rebind inside modal scope") + } + button.ProviderObjectPath = "/rebuilt/invoice" + after := modalRefSnapshot("after", button) + switch outcome { + case "moved": + after.Elements[0].RuntimeID = []int{0, 2} + case "renamed": + after.Elements[0].Name = "Delete" + case "ambiguous": + peer := after.Elements[0] + peer.Index, peer.RuntimeID, peer.ProviderObjectPath = 2, []int{0, 2}, "/peer/invoice" + after.Elements = append(after.Elements, peer) + case "absent": + after.Elements = nil + case "window_replaced": + after.Elements[0].ProviderWindowObjectPath = "/new/window" + } + current = assignModalRefSnapshot(t, service, after) + entry, err = service.RefRegistry.Resolve(ref, "messages") + if outcome != "same_path" { + if err == nil { + t.Fatalf("%s revived the old ref", outcome) + } + return + } + if err != nil || current.ByIndex[1].Ref != ref || entry.ProviderObjectPath != "/old/invoice" { + t.Fatalf("same-path rebuild lost ref or skipped live validation: %v, %#v", err, entry) + } + service.authorizeProviderRebind("messages", entry, target) + if !target.AllowProviderRebind { + t.Fatal("complete fresh snapshot did not allow guarded live rebind") + } + }) + } +} diff --git a/computer-use/internal/core/ax_response_budget.go b/computer-use/internal/core/ax_response_budget.go index 225c716..a8ac4a9 100644 --- a/computer-use/internal/core/ax_response_budget.go +++ b/computer-use/internal/core/ax_response_budget.go @@ -197,7 +197,7 @@ func providerCoverageSummary( switch reason { case "filtered", "pressure": hasProjection = true - case "collapsed_menu", "provider_children_not_rendered": + case "collapsed_menu", "provider_children_not_rendered", "modal_dialog": hasExpandable = true case "": hasAcquisitionGap = true diff --git a/computer-use/native/linux/provider/ax_snapshot.py b/computer-use/native/linux/provider/ax_snapshot.py index a8a2056..b4cc8c9 100644 --- a/computer-use/native/linux/provider/ax_snapshot.py +++ b/computer-use/native/linux/provider/ax_snapshot.py @@ -231,6 +231,19 @@ def visit_impl(node, depth, path, render_depth=0, parent_text=""): if focus_probe is not None: focus_probe.visit(facts) + if (depth > 0 and facts.has_state(Atspi.StateType.MODAL) + and facts.has_state(Atspi.StateType.SHOWING)): + annotations.append({ + "parentRuntimeId": path[:], + "kind": "hidden_children", + "count": 1, + "reason": "modal_dialog", + "detail": ( + "A modal dialog can hide other controls in this window; " + "their absence does not prove that they were removed." + ), + }) + # Pruning keeps only nodes with an interactive role that are visible # on screen. Only the node's own line is dropped and its children are # still visited: intermediate containers are often the parents of diff --git a/computer-use/runtime_tests/test_popup_rendering.py b/computer-use/runtime_tests/test_popup_rendering.py index 58a7963..890dfd8 100644 --- a/computer-use/runtime_tests/test_popup_rendering.py +++ b/computer-use/runtime_tests/test_popup_rendering.py @@ -33,6 +33,47 @@ def _render(*children): root, None, [0], prune=True, indexer=runtime.StableIndexer({})) +class NestedModalCoverageTests(AtspiPatchedTestCase): + def test_visible_nested_modal_reports_suppressed_window_content(self): + close = _TreeNode(role="push button", name="Close") + dialog = _TreeNode(role="dialog", name="Attachment", kids=(close,)) + dialog.states.add(STATE.MODAL) + for prune in (False, True): + with self.subTest(prune=prune): + root = _TreeNode(role="frame", name="Messages", kids=(dialog,)) + records, _lines, annotations = runtime.render_tree( + root, None, [0], prune=prune) + self.assertIn("Close", [record["name"] for record in records]) + modal = [item for item in annotations + if item.get("reason") == "modal_dialog"] + self.assertEqual(len(modal), 1) + self.assertEqual(modal[0]["parentRuntimeId"], [0, 0]) + fixture = json.loads((Path(__file__).parents[1] / "testdata" / + "provider-wire" / "modal-scope.response.json").read_text()) + encoded = wire._accessibility_result({ + "ok": True, "snapshot": {"annotations": modal}, + }) + self.assertEqual(encoded["snapshot"]["annotations"], + fixture["accessibility"]["snapshot"]["annotations"]) + + def test_root_hidden_and_nonmodal_dialogs_do_not_suppress_inventory(self): + for nested, showing, modal in ((False, True, True), + (True, False, True), + (True, True, False)): + with self.subTest(nested=nested, showing=showing, modal=modal): + dialog = _TreeNode(role="dialog", name="Preferences") + if modal: + dialog.states.add(STATE.MODAL) + if not showing: + dialog.states.discard(STATE.SHOWING) + root = (_TreeNode(role="frame", kids=(dialog,)) + if nested else dialog) + _records, _lines, annotations = runtime.render_tree( + root, None, [0], prune=True) + self.assertFalse(any(item.get("reason") == "modal_dialog" + for item in annotations)) + + class ClosedComboPopupTests(AtspiPatchedTestCase): """Options of a closed combo box are neither rendered nor addressable. diff --git a/computer-use/testdata/provider-wire/manifest.json b/computer-use/testdata/provider-wire/manifest.json index 413cf81..3660c1b 100644 --- a/computer-use/testdata/provider-wire/manifest.json +++ b/computer-use/testdata/provider-wire/manifest.json @@ -24,6 +24,7 @@ {"name": "window-list", "direction": "request", "file": "window-list.request.json"}, {"name": "window-action", "direction": "request", "file": "window-action.request.json"}, {"name": "accessibility-success", "direction": "response", "file": "accessibility-success.response.json"}, + {"name": "modal-scope", "direction": "response", "file": "modal-scope.response.json"}, {"name": "observe-apps-partial", "direction": "response", "file": "observe-apps-partial.response.json"}, {"name": "desktop-success", "direction": "response", "file": "desktop-success.response.json"}, {"name": "window-success", "direction": "response", "file": "window-success.response.json"}, diff --git a/computer-use/testdata/provider-wire/modal-scope.response.json b/computer-use/testdata/provider-wire/modal-scope.response.json new file mode 100644 index 0000000..0a3f826 --- /dev/null +++ b/computer-use/testdata/provider-wire/modal-scope.response.json @@ -0,0 +1,27 @@ +{ + "protocol": "computer-use-provider", + "schema_version": 2, + "request_id": "req-modal", + "operation": "observe", + "accessibility": { + "ok": true, + "snapshot": { + "app": {"name": "Messages", "pid": 42, "inputCapable": false}, + "windowTitle": "Messages", + "windowHandle": "w2:fixture", + "snapshotId": "modal", + "elements": [ + {"index": 0, "runtimeId": [0], "name": "Messages", "controlType": "frame"}, + {"index": 1, "runtimeId": [0, 0], "name": "Attachment", "controlType": "dialog"}, + {"index": 2, "runtimeId": [0, 0, 0], "name": "Close", "controlType": "push button"} + ], + "annotations": [{ + "parentRuntimeId": [0, 0], + "kind": "hidden_children", + "count": 1, + "reason": "modal_dialog", + "detail": "A modal dialog can hide other controls in this window; their absence does not prove that they were removed." + }] + } + } +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a402c61..dc74349 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -296,6 +296,13 @@ resolves its reference against live canonical state. A reference becomes stale when its exact target disappears or is replaced, and batches that would act across windows fail before input. +An in-window modal dialog can hide the rest of the accessibility tree. Linux +marks that observation as scoped, so missing controls do not lose their refs. +After the dialog closes, a complete observation can retain a semantic ref across +a provider-object rebuild in the same runtime path and window. The live resolver +still requires a unique semantic match before input. A moved, renamed, ambiguous +or removed target does not qualify. + The canonical AX tree stays complete. The model sees a compact working projection that drops empty leaves and collapses neutral containers, while `find`, selectors, references and actions keep querying canonical records. From c77c40e1a2cc5eaf053264ef5fc5d4f68e636815 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 07:45:31 +0000 Subject: [PATCH 08/15] fix(ax): keep one public mapping for invalid-argument failures Route provider invalid_argument failures through the shared mapping, so uncertain input still reports effect_unknown and hints stay callable. The rejected key spelling remains in the message. --- computer-use/internal/core/ax_evidence.go | 2 +- computer-use/internal/core/ax_runtime_failure.go | 7 ------- computer-use/internal/core/ax_runtime_failure_test.go | 3 --- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/computer-use/internal/core/ax_evidence.go b/computer-use/internal/core/ax_evidence.go index 5b8a3b5..8f2299f 100644 --- a/computer-use/internal/core/ax_evidence.go +++ b/computer-use/internal/core/ax_evidence.go @@ -571,7 +571,7 @@ func axPublicErrorCode(full map[string]any) string { return contract.AXCodeEffectUnknown case failure == "ax_app_unavailable" || status == "app_unavailable": return contract.AXCodeAppUnavailable - case failure == "ax_app_ambiguous", failure == contract.AXCodeInvalidArgument: + case failure == "ax_app_ambiguous": return contract.AXCodeInvalidArgument case failure == "invalid_argument", failure == "unsupported_action", failure == "input_failed": if axEvidenceReportsInputEmission(full) || full["partial_input_possible"] == true || full["mutation_unknown"] == true { diff --git a/computer-use/internal/core/ax_runtime_failure.go b/computer-use/internal/core/ax_runtime_failure.go index db88a4e..e64471b 100644 --- a/computer-use/internal/core/ax_runtime_failure.go +++ b/computer-use/internal/core/ax_runtime_failure.go @@ -5,8 +5,6 @@ import ( "fmt" "regexp" "strings" - - "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) func (s *Service) providerIdentityConflictResult( @@ -198,11 +196,6 @@ func (s *Service) renderRuntimeReportedFailure( "mutation_unknown": response.MutationUnknown, "failure_facts_complete": response.FailureFactsComplete, } - if response.FailureCode == contract.AXCodeInvalidArgument { - result.StructuredContent[contract.AXPublicErrorDetailKey] = publicRuntimeFailureText(response) - result.StructuredContent[contract.AXPublicErrorHintKey] = - "Correct the invalid argument before retrying; check completed before resending earlier actions." - } if response.UnderlyingFailureCode != "" { result.StructuredContent["underlying_failure_code"] = response.UnderlyingFailureCode diff --git a/computer-use/internal/core/ax_runtime_failure_test.go b/computer-use/internal/core/ax_runtime_failure_test.go index 80b3426..d5a9166 100644 --- a/computer-use/internal/core/ax_runtime_failure_test.go +++ b/computer-use/internal/core/ax_runtime_failure_test.go @@ -23,7 +23,4 @@ func TestInvalidKeyKeepsPublicArgumentErrorAndAcceptedSpelling(t *testing.T) { if message, _ := public["message"].(string); !strings.Contains(message, "Use KP_ADD") { t.Fatalf("accepted spelling lost behind batch summary: %#v", public) } - if hint, _ := public["hint"].(string); hint == "" { - t.Fatalf("invalid key has no correction hint: %#v", public) - } } From 8d724e250db49582b1b518e65ac8c70e1058ab96 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 09:02:52 +0000 Subject: [PATCH 09/15] fix(ax): switch batch targets on explicit activation Preflight refs within activation-delimited window segments and carry each selected target through input and final observation. Keep accidental cross-window refs and unexpected window transitions fail-closed. Route native windows without accessibility trees through the same batch engine. Preserve per-step window evidence, actual final observation identity, wait handoffs, and batching-ablation text. --- CHANGELOG.md | 4 + .../internal/contract/tool_contract.go | 12 +- computer-use/internal/core/ax_act_report.go | 47 +- computer-use/internal/core/ax_batch.go | 63 +- .../internal/core/ax_batch_activation_test.go | 542 ++++++++++++++++++ .../internal/core/ax_batch_context.go | 108 +++- .../internal/core/ax_batch_observation.go | 3 + computer-use/internal/core/ax_batch_test.go | 2 +- .../core/ax_batch_window_error_test.go | 4 +- computer-use/internal/core/ax_evidence.go | 9 + .../internal/core/ax_native_window.go | 142 ++--- .../core/ax_native_window_batch_test.go | 321 +++++++++++ computer-use/internal/core/ax_wait_batch.go | 9 +- .../internal/core/tool_contract_test.go | 10 +- .../testdata/contract/public_contract.json | 6 +- docs/MCP.md | 15 +- instructions/ax-only.md | 7 +- instructions/hybrid-code.md | 3 +- instructions/hybrid.md | 9 +- 19 files changed, 1161 insertions(+), 155 deletions(-) create mode 100644 computer-use/internal/core/ax_batch_activation_test.go create mode 100644 computer-use/internal/core/ax_native_window_batch_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a04700..c729915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- AX `activate` with a window ref or title can switch the target within an + action batch. Later refs and ref-less steps use the selected window; + refs from other windows require another explicit activation. Each step + reports its window, and the final observation uses the last executed window. - AX batch errors expose the failed action's reason and a failure-specific recovery hint while retaining batch progress and input evidence. No-input argument and unsupported-operation failures use `invalid_argument`; diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index 1604bc6..47232e0 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -192,7 +192,7 @@ func axInputSchema() map[string]any { "app": nonBlankString(AppArgumentHelp), "window_ref": map[string]any{ "type": "string", "pattern": "^w[1-9][0-9]*$", - "description": "observe/find: window to read. act: omit when actions have refs, which name the window; it only sends ref-less keys/text to wN instead of the focused window.", + "description": "observe/find: window to read. act: omit when actions have refs, which name the window; otherwise sets the initial target for ref-less steps instead of the focused window.", }, "scope": map[string]any{"type": "string", "enum": AXScopeValues, "description": "observe/find; default window."}, "observe": map[string]any{ @@ -218,7 +218,7 @@ func axInputSchema() map[string]any { "type": map[string]any{"type": "string", "enum": AXActionTypes}, "ref": map[string]any{ "type": "string", "pattern": "^[rw][1-9][0-9]*$", - "description": "rN element; wN for activate/close. Omit for keys/text to the focused window.", + "description": "rN element; wN for activate/close. Omit for keys/text to the batch target (focused window by default).", }, "to_ref": map[string]any{ "type": "string", "pattern": "^r[1-9][0-9]*$", @@ -428,8 +428,12 @@ const axToolDescription = "Accessibility for application windows. Modes: " + "apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with " + "rN refs. find returns matching rN. act runs ordered steps, checks them before input, " + "stops at the first failed step, and reports each target's change, windows " + - "opened/closed/focused, and a diff. Copy rN/wN exactly; stale refs fail. keypress/type " + - "without ref go to the focused window. select marks ref, its text substring, or ref..to_ref " + + "opened/closed/focused, and a diff. " + + "activate with wN or title switches the batch target for later refs and ref-less steps. " + + "Other-window refs fail until another activate selects their window. " + + "Each step reports its window; final observation uses the last executed window. " + + "Copy rN/wN exactly; stale refs fail. keypress/type " + + "without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref " + "(paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. " + "A wait step sends no input and fails the batch on timeout: window (title, any app), " + "query (as find), ref with value/state/text, or text; gone inverts. settle needs no " + diff --git a/computer-use/internal/core/ax_act_report.go b/computer-use/internal/core/ax_act_report.go index 4484ca1..59da8b6 100644 --- a/computer-use/internal/core/ax_act_report.go +++ b/computer-use/internal/core/ax_act_report.go @@ -34,8 +34,10 @@ type actReportTrace struct { observe string // start is the snapshot before any input; before[i] the snapshot just // before action i ran. - start *AppSnapshot - before map[int]*AppSnapshot + start *AppSnapshot + before map[int]*AppSnapshot + afterAction map[int]*AppSnapshot + stepWindows map[int]string } func newActReportTrace(obs observation) *actReportTrace { @@ -61,9 +63,29 @@ func (t *actReportTrace) recordBefore(s *Service, index int) { t.before[index] = s.currentSnapshot(t.appToken) } +func (t *actReportTrace) recordWindow(index int, ref string) { + if t.stepWindows == nil { + t.stepWindows = map[int]string{} + } + t.stepWindows[index] = ref +} + +func (t *actReportTrace) recordAfter(s *Service, index int) { + if t.afterAction == nil { + t.afterAction = map[int]*AppSnapshot{} + } + t.afterAction[index] = s.currentSnapshot(t.appToken) +} + // after returns the first snapshot read after action index, or nil when no // read happened after it. func (t *actReportTrace) after(s *Service, index int) *AppSnapshot { + if after, present := t.afterAction[index]; present { + if after == t.before[index] { + return nil + } + return after + } next := -1 for candidate := range t.before { if candidate > index && (next < 0 || candidate < next) { @@ -606,6 +628,27 @@ func axReportWindows(snapshot *AppSnapshot) []axReportWindow { // actWindowsLine compares the application's windows before the first action // with those after the last one. func (s *Service) actWindowsLine(trace *actReportTrace, result ToolCallResult) string { + windows := map[string]bool{} + indices := make([]int, 0, len(trace.stepWindows)) + for index, ref := range trace.stepWindows { + windows[ref] = true + indices = append(indices, index) + } + if len(windows) > 1 { + sort.Ints(indices) + steps := make([]string, 0, len(indices)) + for _, index := range indices { + steps = append(steps, strconv.Itoa(index+1)+"="+trace.stepWindows[index]) + } + final := trace.windowRef + if ref, ok := result.StructuredContent["final_observation_window_ref"].(string); ok && ref != "" { + final = ref + } else if snapshot := s.currentSnapshot(trace.appToken); snapshot != nil && snapshot.publicWindowRef != "" && + result.StructuredContent["final_observation_returned"] == true { + final = snapshot.publicWindowRef + } + return axReportWindowsPrefix + "step targets " + strings.Join(steps, ", ") + "; final " + final + } closedBound := result.StructuredContent["target_window_absent"] == true || result.StructuredContent["close_effect_verified"] == true final := s.currentSnapshot(trace.appToken) diff --git a/computer-use/internal/core/ax_batch.go b/computer-use/internal/core/ax_batch.go index 9d1ef41..d037eb6 100644 --- a/computer-use/internal/core/ax_batch.go +++ b/computer-use/internal/core/ax_batch.go @@ -31,6 +31,9 @@ func (s *Service) executeAXBatchAction( const noDeclaredTarget = "" switch action.actionType { case "activate", "close": + if window := s.noTreeWindowEntry(action.targetRef); window != nil { + return s.axNoTreeBatchAction(action.actionType, window) + } return s.windowAction( context.AppToken, action.targetRef, @@ -106,9 +109,6 @@ func (s *Service) executeAXBatchAction( } } -// axBatchDecoded runs decoded actions in one window. Window titles and -// the focused window are resolved here, once, before any input; after that -// the batch uses only the resolved wN. func (s *Service) axBatchDecoded( actions []axBatchAction, obs observation, @@ -133,11 +133,8 @@ func (s *Service) axBatchDecoded( "title %s -> %s %s (%s)", strconv.Quote(actions[index].title), window.ref, strconv.Quote(window.title), window.appName)) } - if window := s.axNoTreeBatchWindow(actions, windowRef); window != nil { - return s.axNoTreeWindowBatch(actions, window, bindingNotes, requested) - } var focus *axDesktopWindow - if windowRef == "" && !axBatchHasRef(actions) { + if windowRef == "" && !axBatchHasRef(actions[:axBatchWindowEnd(actions, 0)]) { var argErr *axArgError focus, argErr = s.resolveFocusedWindow() if argErr != nil { @@ -198,10 +195,11 @@ func (s *Service) axBatchRun( return axBatchPreflightError(err, requested) } } - context, err := s.deriveAXBatchContext(windowRef, actions, focus) + contexts, err := s.deriveAXBatchContexts(windowRef, actions, focus) if err != nil { return axBatchPreflightError(err, requested) } + context := contexts[0] context.report = trace trace.bind(s, context) lastInput := -1 @@ -210,12 +208,6 @@ func (s *Service) axBatchRun( lastInput = index continue } - if action.wait.ref == "" { - continue - } - if err := s.preflightAXWaitRef(context, action.wait.ref); err != nil { - return axBatchPreflightError(fmt.Errorf("actions[%d].ref: %w", index, err), requested) - } } executionPlan, err := compileAXBatchExecutionPlan(context, actions, obs) @@ -227,7 +219,7 @@ func (s *Service) axBatchRun( // provider path runs keys back to back. guarded := context.boundByFocus || lastInput < len(actions)-1 for _, action := range actions { - guarded = guarded || action.actionType == "wait" + guarded = guarded || action.actionType == "wait" || action.actionType == "activate" } if _, available := orderedInputBackend(s.platformBackend()); available && !guarded { requests, err := providerOrderedInputRequestsFromPlan(context, executionPlan, actions) @@ -245,16 +237,29 @@ func (s *Service) axBatchRun( waitNotes := []string{} for index := 0; index < len(actions); index++ { action := actions[index] + if action.actionType == "activate" { + pending := context.windowsQueuedInputPending + context = contexts[index] + context.windowsQueuedInputPending = pending + context.report = trace + trace.appToken = context.AppToken + trace.windowRef = context.WindowRef + } if action.actionType == "wait" { // A wait step between actions: it sends no input. trace.recordBefore(s, index) + trace.recordWindow(index, context.WindowRef) waitResult := s.runAXWait(context, action.wait) - actionEvidence = append(actionEvidence, axWaitEvidenceEntry(index, waitResult)) + waitEntry := axWaitEvidenceEntry(index, waitResult) + waitEntry["window_ref"] = context.WindowRef + actionEvidence = append(actionEvidence, waitEntry) if !axPostconditionPassed(waitResult) { return s.stopAXBatchAtWait(context, actions, index, waitResult, actionEvidence, waitNotes, obs) } waitNotes = append(waitNotes, firstLine(ResultText(waitResult))) s.rebindAfterWindowWait(context, action.wait, waitResult) + waitEntry["window_ref"] = context.WindowRef + trace.recordWindow(index, context.WindowRef) if index < len(actions)-1 && actions[index+1].actionType != "wait" { if stopped, changed := s.axBatchContinuityStop(context, actions, index, actionEvidence, obs); changed { return stopped @@ -283,6 +288,7 @@ func (s *Service) axBatchRun( context.settleBaseline = s.readAXSettleState(context, nil) } trace.recordBefore(s, index) + trace.recordWindow(index, context.WindowRef) result := ToolCallResult{} actionEntry := map[string]any{} request, orderedRun := orderedRuns[index] @@ -305,7 +311,9 @@ func (s *Service) axBatchRun( s.renderAXBatchFinalObservation(execution.snapshot, execution.notes, obs)) } } else { - if context.windowsQueuedInputPending && action.actionType == "type" && action.text != "" { + if err := s.validateAXActionWindow(context, action, index); err != nil { + result = axBatchPreflightError(err, requested) + } else if context.windowsQueuedInputPending && action.actionType == "type" && action.text != "" { result = TextResult("Earlier queued input has not been confirmed processed; no text was sent. Observe and confirm focus and selection before sending text separately; do not repeat the earlier input.", true) result.StructuredContent = map[string]any{ "failure_code": "desktop_input_failed", "failure_phase": "preparation", @@ -325,9 +333,9 @@ func (s *Service) axBatchRun( context.windowsQueuedInputPending = true } } - if !result.IsError || + if s.noTreeWindowEntry(context.WindowRef) == nil && (!result.IsError || (action.actionType == "close" && - axEvidenceReportsInputEmission(result.StructuredContent)) { + axEvidenceReportsInputEmission(result.StructuredContent))) { result = s.recoverClosedWindowActionObservation( context, action, @@ -336,6 +344,8 @@ func (s *Service) axBatchRun( ) } actionEntry = batchActionEvidenceEntry(index, action.actionType, result) + actionEntry["window_ref"] = context.WindowRef + trace.recordAfter(s, index) context.settleBaseline.notePixelChange(result) } if result.IsError && index+1 < len(actions) && actions[index+1].actionType == "wait" && @@ -439,13 +449,18 @@ func (s *Service) axBatchRun( // which then shows the state they waited for. for next := index + 1; next < len(actions); next++ { trace.recordBefore(s, next) + trace.recordWindow(next, context.WindowRef) waitResult := s.runAXWait(context, actions[next].wait) - actionEvidence = append(actionEvidence, axWaitEvidenceEntry(next, waitResult)) + waitEntry := axWaitEvidenceEntry(next, waitResult) + waitEntry["window_ref"] = context.WindowRef + actionEvidence = append(actionEvidence, waitEntry) if !axPostconditionPassed(waitResult) { return s.stopAXBatchAtWait(context, actions, next, waitResult, actionEvidence, waitNotes, obs) } waitNotes = append(waitNotes, firstLine(ResultText(waitResult))) s.rebindAfterWindowWait(context, actions[next].wait, waitResult) + waitEntry["window_ref"] = context.WindowRef + trace.recordWindow(next, context.WindowRef) } if index < len(actions)-1 && obs.resolvedMode() != contract.ObserveNone && action.actionType != "close" { @@ -479,6 +494,14 @@ func (s *Service) axBatchRun( if value, ok := result.StructuredContent["final_observation_returned"].(bool); ok { finalObservationReturned = value } + if finalObservationReturned && contexts[0] != contexts[len(contexts)-1] { + if _, present := result.StructuredContent["final_observation_window_ref"]; !present { + if snapshot := s.currentSnapshot(context.AppToken); snapshot != nil && + snapshot.publicWindowRef != "" { + result.StructuredContent["final_observation_window_ref"] = snapshot.publicWindowRef + } + } + } observationRequired := !finalObservationReturned && obs.resolvedMode() != contract.ObserveNone if value, present := diff --git a/computer-use/internal/core/ax_batch_activation_test.go b/computer-use/internal/core/ax_batch_activation_test.go new file mode 100644 index 0000000..3a4f5fb --- /dev/null +++ b/computer-use/internal/core/ax_batch_activation_test.go @@ -0,0 +1,542 @@ +package core + +import ( + "fmt" + "regexp" + "slices" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +type axActivationBackend struct { + recordingPlatformBackend + windows []reportWindow + owners map[string]AppDescriptor + windowRequests []ProviderWindowActionRequest + deliveries []string + failActivate string + transitionAfterInput string + activationModal *reportWindow + inputModal *reportWindow +} + +func (backend *axActivationBackend) snapshot(handle string) *ProviderSnapshot { + owner := backend.owners[handle] + fixture := &reportBackend{value: "12"} + var selected reportWindow + for _, window := range backend.windows { + if backend.owners[window.handle].Name == owner.Name { + fixture.windows = append(fixture.windows, window) + } + if window.handle == handle { + selected = window + } + } + snapshot := fixture.snapshot(selected) + snapshot.App = owner + for index := range snapshot.Elements { + element := &snapshot.Elements[index] + element.ProviderBusName = owner.ProviderBusName + element.ProcessGeneration = owner.ProcessGeneration + element.ProviderWindowBusName = owner.ProviderBusName + element.ProviderWindowProcessGeneration = owner.ProcessGeneration + } + for index := range snapshot.Windows { + window := &snapshot.Windows[index] + window.ProviderBusName = owner.ProviderBusName + window.ProviderProcessGeneration = owner.ProcessGeneration + } + return snapshot +} + +func (backend *axActivationBackend) ObserveApps(ProviderObserveAppsRequest) (ProviderAccessibilityResult, error) { + apps := []AppDescriptor{} + for _, window := range backend.windows { + owner := backend.owners[window.handle] + index := slices.IndexFunc(apps, func(app AppDescriptor) bool { return app.Name == owner.Name }) + if index < 0 { + index = len(apps) + apps = append(apps, owner) + } + apps[index].Windows = append(apps[index].Windows, ProviderAppWindow{ + Title: window.title, Role: window.role, Handle: window.handle, + ProviderBusName: owner.ProviderBusName, ProviderObjectPath: window.path, + ProcessGeneration: owner.ProcessGeneration, Active: window.active, InputTarget: true, + }) + } + return ProviderAccessibilityResult{OK: true, Apps: apps}, nil +} + +func (backend *axActivationBackend) Observe(request ProviderObserveRequest) (ProviderAccessibilityResult, error) { + backend.observeRequests = append(backend.observeRequests, request) + if _, exists := backend.owners[request.WindowHandle]; !exists { + return ProviderAccessibilityResult{}, fmt.Errorf("unexpected observation handle %q", request.WindowHandle) + } + return ProviderAccessibilityResult{OK: true, Snapshot: backend.snapshot(request.WindowHandle)}, nil +} + +func (backend *axActivationBackend) ResolveAndAct(request ProviderActionRequest) (ProviderAccessibilityResult, error) { + backend.actionRequests = append(backend.actionRequests, request) + handle := request.WindowHandle + if request.Target != nil && request.Target.WindowHandle != "" { + if handle != "" && handle != request.Target.WindowHandle { + return ProviderAccessibilityResult{}, fmt.Errorf("input and target disagree: %q, %q", handle, request.Target.WindowHandle) + } + handle = request.Target.WindowHandle + } + backend.deliveries = append(backend.deliveries, string(request.Intent)+"@"+handle) + if _, exists := backend.owners[handle]; !exists { + return ProviderAccessibilityResult{}, fmt.Errorf("unexpected input handle %q", handle) + } + if backend.transitionAfterInput != "" { + handle = backend.transitionAfterInput + backend.transitionAfterInput = "" + } + if backend.inputModal != nil { + modal := *backend.inputModal + backend.windows = append(backend.windows, modal) + backend.owners[modal.handle] = backend.owners[handle] + handle = modal.handle + backend.inputModal = nil + } + return ProviderAccessibilityResult{ + OK: true, Snapshot: backend.snapshot(handle), + Mutation: &ProviderMutationFacts{InputEventsEmitted: true, DesktopStateMutated: true}, + ActionEvidence: &ProviderActionEvidence{InputEmitted: true, Route: "semantic_action"}, + }, nil +} + +func (backend *axActivationBackend) ActOnWindow(request ProviderWindowActionRequest) (ProviderWindowResult, error) { + backend.windowRequests = append(backend.windowRequests, request) + handle := request.WindowHandle + if request.Target != nil && request.Target.WindowHandle != "" { + if handle != "" && handle != request.Target.WindowHandle { + return ProviderWindowResult{}, fmt.Errorf("activation and target disagree: %q, %q", handle, request.Target.WindowHandle) + } + handle = request.Target.WindowHandle + } + backend.deliveries = append(backend.deliveries, string(request.Action)+"@"+handle) + if handle == backend.failActivate { + return ProviderWindowResult{ + OK: false, Error: "activation failed before input", + Failure: &ProviderFailure{Code: "input_failed", Phase: "emit", Subject: "window", FactsComplete: true}, + Mutation: &ProviderMutationFacts{}, + }, nil + } + if request.Action != "activate" { + return ProviderWindowResult{}, fmt.Errorf("unexpected window action %q", request.Action) + } + if _, exists := backend.owners[handle]; !exists { + return ProviderWindowResult{}, fmt.Errorf("unexpected activation handle %q", handle) + } + if backend.activationModal != nil { + modal := *backend.activationModal + backend.windows = append(backend.windows, modal) + backend.owners[modal.handle] = backend.owners[handle] + handle = modal.handle + backend.activationModal = nil + } + for index := range backend.windows { + backend.windows[index].active = backend.windows[index].handle == handle + } + return ProviderWindowResult{ + OK: true, Snapshot: backend.snapshot(handle), + Mutation: &ProviderMutationFacts{InputEventsEmitted: true, DesktopStateMutated: true}, + ActionEvidence: &ProviderActionEvidence{InputEmitted: true, Route: "native_window_command"}, + }, nil +} + +type axActivationRefs struct { + window string + button string + entry string +} + +func axActivationService(t *testing.T, crossApp bool) (*Service, *axActivationBackend, axActivationRefs, axActivationRefs) { + t.Helper() + first := AppDescriptor{Name: "Editor", PID: 10, ProviderBusName: ":1.84", + ProcessGeneration: "pid:10:start:100", InputCapable: true} + second := first + if crossApp { + second = AppDescriptor{Name: "Viewer", PID: 20, ProviderBusName: ":1.85", + ProcessGeneration: "pid:20:start:200", InputCapable: true} + } + backend := &axActivationBackend{ + windows: []reportWindow{ + reportMain(), + {handle: reportOtherHandle, role: "frame", title: "Notes", path: "/window/2"}, + }, + owners: map[string]AppDescriptor{reportMainHandle: first, reportOtherHandle: second}, + } + service := NewServiceWithBackend(backend) + listed := service.callTool(contract.PublicToolAX, map[string]any{"mode": "apps"}) + if listed.IsError { + t.Fatalf("list fixture windows: %s", ResultText(listed)) + } + refs := make([]axActivationRefs, 2) + for index, window := range backend.windows { + match := regexp.MustCompile(`(?m)^\s*(w[0-9]+)[^\n]*"` + regexp.QuoteMeta(window.title) + `"`).FindStringSubmatch(ResultText(listed)) + if match == nil { + t.Fatalf("missing window %q in %s", window.title, ResultText(listed)) + } + refs[index].window = match[1] + observed := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "observe", "window_ref": match[1], "observe": "tree", + }) + if observed.IsError { + t.Fatalf("observe fixture window: %s", ResultText(observed)) + } + for name, destination := range map[string]*string{"OK": &refs[index].button, "Size": &refs[index].entry} { + match := regexp.MustCompile(`"` + name + `"[^\n]*\[ref=(r[0-9]+)\]`).FindStringSubmatch(ResultText(observed)) + if match == nil { + t.Fatalf("missing element %q in %s", name, ResultText(observed)) + } + *destination = match[1] + } + } + return service, backend, refs[0], refs[1] +} + +func axActivationEvidence(t *testing.T, result ToolCallResult, windows ...string) { + t.Helper() + batch := result.StructuredContent + if nested, exists := batch["a11y_batch_result"]; exists { + var valid bool + batch, valid = nested.(map[string]any) + if !valid { + t.Fatalf("invalid batch evidence: %#v", nested) + } + } + actions, _ := batch["actions"].([]map[string]any) + if len(actions) != len(windows) { + t.Fatalf("action evidence count = %d, want %d: %#v", len(actions), len(windows), batch) + } + for index, window := range windows { + if actions[index]["window_ref"] != window { + t.Errorf("action %d window evidence = %#v, want %s", index, actions[index], window) + } + } +} + +func TestAXBatchActivationSwitchesTarget(t *testing.T) { + for _, scenario := range []struct { + name string + crossApp, byTitle, explicit bool + }{ + {name: "same_app_ref"}, + {name: "same_app_title_overrides_window", byTitle: true, explicit: true}, + {name: "cross_app_ref_overrides_window", crossApp: true, explicit: true}, + {name: "cross_app_title", crossApp: true, byTitle: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + service, backend, first, second := axActivationService(t, scenario.crossApp) + activation := map[string]any{"type": "activate", "ref": second.window} + if scenario.byTitle { + activation = map[string]any{"type": "activate", "title": "Notes"} + } + args := map[string]any{"mode": "act", "observe": "tree", "actions": []any{ + map[string]any{"type": "click", "ref": first.button}, + activation, + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + map[string]any{"type": "type", "text": "untargeted"}, + map[string]any{"type": "click", "ref": second.button}, + map[string]any{"type": "keypress", "ref": second.entry, "keys": []any{"TAB"}}, + map[string]any{"type": "type", "ref": second.entry, "text": "targeted"}, + }} + if scenario.explicit { + args["window_ref"] = first.window + } + result := service.callTool(contract.PublicToolAX, args) + if result.IsError { + t.Fatalf("explicit switch rejected: %s", ResultText(result)) + } + want := []string{"click@" + reportMainHandle, "activate@" + reportOtherHandle, + "keypress@" + reportOtherHandle, "type@" + reportOtherHandle, "click@" + reportOtherHandle, + "keypress@" + reportOtherHandle, "type@" + reportOtherHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("delivery order/handles = %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, first.window, second.window, second.window, + second.window, second.window, second.window, second.window) + owner := backend.owners[reportOtherHandle] + snapshot := service.currentSnapshot(owner.Name) + if snapshot == nil || snapshot.WindowHandle != reportOtherHandle { + t.Fatalf("final snapshot does not belong to B: %#v", snapshot) + } + if !strings.Contains(ResultText(result), `window: "Notes" [ref=`+second.window+`]`) { + t.Errorf("final observation does not identify B: %s", ResultText(result)) + } + if result.StructuredContent["final_observation_returned"] != true { + t.Errorf("missing final observation evidence: %#v", result.StructuredContent) + } + }) + } +} + +func TestAXBatchActivationReturnsToOriginalWindow(t *testing.T) { + for _, crossApp := range []bool{false, true} { + t.Run(fmt.Sprintf("cross_app_%t", crossApp), func(t *testing.T) { + service, backend, first, second := axActivationService(t, crossApp) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": first.window, "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": first.window}, + map[string]any{"type": "click", "ref": first.button}, + map[string]any{"type": "activate", "ref": second.window}, + map[string]any{"type": "click", "ref": second.button}, + map[string]any{"type": "activate", "ref": first.window}, + map[string]any{"type": "click", "ref": first.button}, + }, + }) + if result.IsError { + t.Fatalf("A -> B -> A rejected: %s", ResultText(result)) + } + want := []string{"activate@" + reportMainHandle, "click@" + reportMainHandle, + "activate@" + reportOtherHandle, "click@" + reportOtherHandle, + "activate@" + reportMainHandle, "click@" + reportMainHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("delivery order/handles = %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, first.window, first.window, second.window, + second.window, first.window, first.window) + if !strings.Contains(ResultText(result), `window: "Report" [ref=`+first.window+`]`) { + t.Errorf("final observation did not return to A: %s", ResultText(result)) + } + }) + } +} + +func TestAXBatchActivationFirstActionOverridesWindow(t *testing.T) { + service, backend, first, second := axActivationService(t, false) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": first.window, "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": second.window}, + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + map[string]any{"type": "type", "text": "only B receives input"}, + }, + }) + if result.IsError { + t.Fatalf("first activation did not override window_ref: %s", ResultText(result)) + } + want := []string{"activate@" + reportOtherHandle, "keypress@" + reportOtherHandle, "type@" + reportOtherHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("first activation delivery targets = %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, second.window, second.window, second.window) + if !strings.Contains(ResultText(result), `window: "Notes" [ref=`+second.window+`]`) { + t.Errorf("final observation does not identify B: %s", ResultText(result)) + } +} + +func TestAXBatchActivationBindsInitialRefLessInputToFocus(t *testing.T) { + for _, crossApp := range []bool{false, true} { + t.Run(fmt.Sprintf("cross_app_%t", crossApp), func(t *testing.T) { + service, backend, first, second := axActivationService(t, crossApp) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "observe": "tree", "actions": []any{ + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + map[string]any{"type": "type", "text": "focused A"}, + map[string]any{"type": "activate", "ref": second.window}, + map[string]any{"type": "keypress", "keys": []any{"TAB"}}, + map[string]any{"type": "type", "text": "activated B"}, + }, + }) + if result.IsError { + t.Fatalf("focus-bound initial segment rejected: %s", ResultText(result)) + } + want := []string{"keypress@" + reportMainHandle, "type@" + reportMainHandle, + "activate@" + reportOtherHandle, "keypress@" + reportOtherHandle, "type@" + reportOtherHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("focus/activation delivery targets = %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, first.window, first.window, second.window, second.window, second.window) + if !strings.Contains(ResultText(result), `window: "Notes" [ref=`+second.window+`]`) { + t.Errorf("final observation does not identify B: %s", ResultText(result)) + } + }) + } +} + +func TestAXBatchActivationStopsWhenActivationOpensModal(t *testing.T) { + service, backend, first, second := axActivationService(t, true) + backend.activationModal = &reportWindow{ + handle: reportDialogHandle, role: "dialog", title: "Confirm changes", path: "/window/3", + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "observe": "tree", "actions": []any{ + map[string]any{"type": "click", "ref": first.button}, + map[string]any{"type": "activate", "ref": second.window}, + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + map[string]any{"type": "type", "text": "must not reach modal"}, + }, + }) + public := PrepareAXEvidence(result).StructuredContent + if !result.IsError || public["code"] != contract.AXCodeWindowChanged { + t.Errorf("unexpected modal lost window_changed guard: %#v\n%s", public, ResultText(result)) + } + want := []string{"click@" + reportMainHandle, "activate@" + reportOtherHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("input continued after activation opened a modal: %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, first.window, second.window) + if public["completed"] != 2 || result.StructuredContent["stopped_before_action_index"] != 2 { + t.Errorf("modal stop lost completed/next step: %#v", result.StructuredContent) + } +} + +func TestAXBatchActivationRejectsWrongWindowRefsBeforeInput(t *testing.T) { + for _, scenario := range []struct { + name, field string + afterSwitch bool + }{ + {"old_ref_after_switch", "ref", true}, + {"early_ref_before_switch", "ref", false}, + {"old_wait_ref_after_switch", "wait", true}, + {"early_wait_ref_before_switch", "wait", false}, + {"old_to_ref_after_switch", "to_ref", true}, + {"early_to_ref_before_switch", "to_ref", false}, + } { + t.Run(scenario.name, func(t *testing.T) { + service, backend, first, second := axActivationService(t, false) + current, wrong := first, second + actions := []any{map[string]any{"type": "click", "ref": first.button}} + if scenario.afterSwitch { + current, wrong = second, first + actions = append(actions, map[string]any{"type": "activate", "ref": second.window}) + } + badIndex := len(actions) + field := "ref" + badAction := map[string]any{"type": "click", "ref": wrong.button} + switch scenario.field { + case "wait": + badAction = map[string]any{"type": "wait", "ref": wrong.entry, "state": "visible", "timeout_ms": 1} + case "to_ref": + field = "to_ref" + badAction = map[string]any{"type": "select", "ref": current.entry, "to_ref": wrong.entry} + } + actions = append(actions, badAction) + if !scenario.afterSwitch { + actions = append(actions, map[string]any{"type": "activate", "ref": second.window}) + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": actions, + }) + path := fmt.Sprintf("actions[%d].%s", badIndex, field) + public := PrepareAXEvidence(result).StructuredContent + if !result.IsError || public["path"] != path { + t.Errorf("wrong-window ref must fail at %s: %#v", path, public) + } + if len(backend.actionRequests) != 0 || len(backend.windowRequests) != 0 || len(backend.desktopRequests) != 0 { + t.Errorf("ownership preflight reached input backend: actions=%#v, windows=%#v, desktop=%#v", + backend.actionRequests, backend.windowRequests, backend.desktopRequests) + } + if public["input_emitted"] != false || public["completed"] != 0 || public["code"] != contract.AXCodeInvalidArgument { + t.Errorf("wrong-window rejection evidence: %#v", public) + } + }) + } +} + +func TestAXBatchActivationFailureStopsFollowingInput(t *testing.T) { + service, backend, first, second := axActivationService(t, true) + backend.failActivate = reportOtherHandle + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "observe": "tree", "actions": []any{ + map[string]any{"type": "click", "ref": first.button}, + map[string]any{"type": "activate", "ref": second.window}, + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + map[string]any{"type": "type", "ref": second.entry, "text": "must not run"}, + }, + }) + if !result.IsError || !strings.Contains(ResultText(result), "activation failed before input") { + t.Errorf("activation failure missing: %s", ResultText(result)) + } + want := []string{"click@" + reportMainHandle, "activate@" + reportOtherHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("failure did not stop immediately: %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, first.window, second.window) + batch, _ := result.StructuredContent["a11y_batch_result"].(map[string]any) + if batch["failed_action_index"] != 1 || batch["actions_completed_without_call_error"] != 1 { + t.Errorf("failure location/completed count: %#v", batch) + } +} + +func TestAXBatchActivationKeepsImplicitWindowChangeGuard(t *testing.T) { + service, backend, first, _ := axActivationService(t, false) + backend.transitionAfterInput = reportOtherHandle + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": first.window, "observe": "tree", "actions": []any{ + map[string]any{"type": "click", "ref": first.button}, + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + }, + }) + public := PrepareAXEvidence(result).StructuredContent + if !result.IsError || public["code"] != contract.AXCodeWindowChanged { + t.Errorf("implicit switch lost window_changed guard: %#v\n%s", public, ResultText(result)) + } + if want := []string{"click@" + reportMainHandle}; !slices.Equal(backend.deliveries, want) { + t.Errorf("input continued after implicit switch: %v", backend.deliveries) + } +} + +func TestAXBatchActivationRejectsOldWindowCloseAfterWait(t *testing.T) { + service, backend, first, _ := axActivationService(t, false) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{ + map[string]any{"type": "activate", "ref": first.window}, + map[string]any{"type": "wait", "window": "Notes"}, + map[string]any{"type": "close", "ref": first.window}, + }, + }) + public := PrepareAXEvidence(result).StructuredContent + if !result.IsError || public["code"] != contract.AXCodeInvalidArgument || + public["path"] != "actions[2].ref" || public["completed"] != 2 { + t.Fatalf("old window close was not rejected after wait: %#v", public) + } + if len(backend.deliveries) != 1 { + t.Fatalf("old window received input after wait: %v", backend.deliveries) + } +} + +func TestAXBatchActivationFinalObservationIdentifiesModal(t *testing.T) { + service, backend, first, second := axActivationService(t, true) + backend.inputModal = &reportWindow{ + handle: reportDialogHandle, role: "dialog", title: "Opened document", path: "/window/3", active: true, + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": first.window}, + map[string]any{"type": "activate", "ref": second.window}, + map[string]any{"type": "keypress", "keys": []any{"CTRL", "O"}}, + }, + }) + if result.IsError { + t.Fatal(ResultText(result)) + } + snapshot := service.currentSnapshot(backend.owners[reportDialogHandle].Name) + if result.StructuredContent["final_observation_window_ref"] != snapshot.publicWindowRef || + !strings.Contains(ResultText(result), "final "+snapshot.publicWindowRef) { + t.Fatalf("modal observation has the wrong identity: %#v\n%s", result.StructuredContent, ResultText(result)) + } + axActivationEvidence(t, result, first.window, second.window, second.window) +} + +func TestAXBatchActivationFinalWaitDoesNotInventWindowLifecycle(t *testing.T) { + service, _, first, second := axActivationService(t, true) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": first.window}, + map[string]any{"type": "wait", "window": "Notes"}, + }, + }) + text := ResultText(result) + if result.IsError || strings.Contains(text, "opened "+second.window) || strings.Contains(text, "closed "+first.window) { + t.Fatalf("wait fabricated window lifecycle: %s", text) + } + if !strings.Contains(text, "final "+second.window) { + t.Fatalf("wait report lost final window: %s", text) + } + axActivationEvidence(t, result, first.window, second.window) +} diff --git a/computer-use/internal/core/ax_batch_context.go b/computer-use/internal/core/ax_batch_context.go index df5f5e7..e40c1bb 100644 --- a/computer-use/internal/core/ax_batch_context.go +++ b/computer-use/internal/core/ax_batch_context.go @@ -34,16 +34,65 @@ const ( axWindowUnknown axWindowContinuity = "unknown" ) -// deriveAXBatchContext binds a batch to one logical window. The window -// comes from the actions' rN/wN refs, from window_ref, or, for a batch whose -// actions carry no ref, from the window that holds keyboard focus. A -// window_ref that names the same window as the refs is accepted; a -// different one is rejected. focus is the focused window read before any -// input; it is used only when nothing else names a window. +func axBatchWindowEnd(actions []axBatchAction, start int) int { + for index := start + 1; index < len(actions); index++ { + if actions[index].actionType == "activate" { + return index + } + } + return len(actions) +} + +func (s *Service) deriveAXBatchContexts( + explicitWindowRef string, + actions []axBatchAction, + focus *axDesktopWindow, +) ([]*AXBatchContext, error) { + contexts := make([]*AXBatchContext, len(actions)) + for start := 0; start < len(actions); { + end := axBatchWindowEnd(actions, start) + windowRef := explicitWindowRef + if actions[start].actionType == "activate" { + windowRef = actions[start].targetRef + } + context, err := s.deriveAXWindowContext(windowRef, actions[start:end], focus, start) + if err != nil { + return nil, err + } + if err := s.validateAXNoTreeSegment(context, actions[start:end], start); err != nil { + return nil, err + } + for index := start; index < end; index++ { + contexts[index] = context + if actions[index].wait != nil && actions[index].wait.ref != "" { + if err := s.preflightAXWaitRef(context, actions[index].wait.ref); err != nil { + return nil, axBatchRefError(index, "ref", err) + } + } + if actions[index].toRef != "" { + if err := s.preflightAXWaitRef(context, actions[index].toRef); err != nil { + return nil, axBatchRefError(index, "to_ref", err) + } + } + } + start = end + } + return contexts, nil +} + func (s *Service) deriveAXBatchContext( explicitWindowRef string, actions []axBatchAction, focus *axDesktopWindow, +) (*AXBatchContext, error) { + return s.deriveAXWindowContext(explicitWindowRef, actions, focus, 0) +} + +func (s *Service) deriveAXWindowContext( + explicitWindowRef string, + actions []axBatchAction, + focus *axDesktopWindow, + actionOffset int, ) (*AXBatchContext, error) { context := &AXBatchContext{} bindWindow := func(ref, handle, appRef, appIdentity string) error { @@ -51,9 +100,9 @@ func (s *Service) deriveAXBatchContext( handle = strings.TrimSpace(handle) if context.WindowHandle != "" && context.WindowHandle != handle { return axInvalid("", fmt.Sprintf( - "batch spans two windows (%s and %s); one act call works in one window, so split it", - context.WindowRef, ref), - "Split the actions into separate act calls, one window per call.", + "ref belongs to window %s, not the current batch window %s; activate that window before using its refs", + ref, context.WindowRef), + "Use refs from the current window, or activate the other window before acting in it.", ) } if context.WindowRef == "" { @@ -93,7 +142,7 @@ func (s *Service) deriveAXBatchContext( if err == nil { err = bindWindow(entry.LogicalWindowRef, entry.WindowHandle, entry.AppRef, entry.AppIdentity) - if err != nil && explicitWindowRef != "" && index == 0 { + if err != nil && explicitWindowRef != "" && index == 0 && actionOffset == 0 { err = axInvalid("window_ref", fmt.Sprintf("%s belongs to window %s, not window_ref %s; omit window_ref", address, entry.LogicalWindowRef, explicitWindowRef), "Omit window_ref so the action refs determine the window, or use refs from window_ref.") @@ -107,11 +156,7 @@ func (s *Service) deriveAXBatchContext( } } if err != nil { - var argument *axArgError - if errors.As(err, &argument) && argument.path == "" { - argument.path = fmt.Sprintf("actions[%d].ref", index) - } - return nil, fmt.Errorf("actions[%d].ref: %w", index, err) + return nil, axBatchRefError(actionOffset+index, "ref", err) } } if context.WindowHandle == "" && !hasTarget && focus != nil { @@ -143,12 +188,24 @@ func (s *Service) deriveAXBatchContext( return context, nil } +func axBatchRefError(index int, field string, err error) error { + path := fmt.Sprintf("actions[%d].%s", index, field) + var argument *axArgError + if errors.As(err, &argument) && argument.path == "" { + argument.path = path + } + return fmt.Errorf("%s: %w", path, err) +} + func (s *Service) axBatchWindowContinuity( context *AXBatchContext, ) axWindowContinuity { if s == nil || context == nil || strings.TrimSpace(context.WindowHandle) == "" { return axWindowUnknown } + if continuity, native := s.axNoTreeBatchWindowContinuity(context); native { + return continuity + } snapshot := s.currentSnapshot(context.AppToken) if snapshot == nil { return axWindowUnknown @@ -182,6 +239,27 @@ func (s *Service) axBatchWindowContinuity( return axWindowSame } +func (s *Service) validateAXActionWindow(context *AXBatchContext, action axBatchAction, index int) error { + for _, target := range []struct{ field, ref string }{{"ref", action.targetRef}, {"to_ref", action.toRef}} { + if isShortAXRef(target.ref) { + if err := s.preflightAXWaitRef(context, target.ref); err != nil { + return axBatchRefError(index, target.field, err) + } + } else if isShortAXWindowRef(target.ref) && action.actionType != "activate" { + entry, err := s.resolveShortAXWindowRefOwned(target.ref) + if err != nil { + return axBatchRefError(index, target.field, err) + } + if entry.WindowHandle != context.WindowHandle { + return axBatchRefError(index, target.field, axInvalid("", fmt.Sprintf( + "%s belongs to another window, not the current batch window %s", target.ref, context.WindowRef), + "Activate that window before acting in it.")) + } + } + } + return nil +} + // axBatchFocusChanged reports whether keyboard focus left the window a // focus-bound batch was bound to, and if so returns the stop result. func (s *Service) axBatchFocusChanged( diff --git a/computer-use/internal/core/ax_batch_observation.go b/computer-use/internal/core/ax_batch_observation.go index 5e0cd6d..aefba6c 100644 --- a/computer-use/internal/core/ax_batch_observation.go +++ b/computer-use/internal/core/ax_batch_observation.go @@ -91,6 +91,9 @@ func (s *Service) axBatchFinalObservation( if context == nil || obs.resolvedMode() == contract.ObserveNone { return ToolCallResult{} } + if window := s.noTreeWindowEntry(context.WindowRef); window != nil { + return axNoTreeBatchObservation(window) + } request := AXServiceRequest{ Operation: axServiceObserve, App: context.AppToken, diff --git a/computer-use/internal/core/ax_batch_test.go b/computer-use/internal/core/ax_batch_test.go index e685950..808b2a8 100644 --- a/computer-use/internal/core/ax_batch_test.go +++ b/computer-use/internal/core/ax_batch_test.go @@ -275,7 +275,7 @@ func TestCrossWindowBatchFailsBeforeInput(t *testing.T) { {actionType: "click", targetRef: secondRef}, }, nil, ) - if err == nil || !strings.Contains(err.Error(), "batch spans two windows") { + if err == nil || !strings.Contains(err.Error(), "not the current batch window") { t.Fatalf("expected cross-window rejection, got %v", err) } } diff --git a/computer-use/internal/core/ax_batch_window_error_test.go b/computer-use/internal/core/ax_batch_window_error_test.go index 0fcc2de..00b5038 100644 --- a/computer-use/internal/core/ax_batch_window_error_test.go +++ b/computer-use/internal/core/ax_batch_window_error_test.go @@ -24,9 +24,9 @@ func TestAXCrossWindowPreflightReportsArgumentError(t *testing.T) { map[string]any{"type": "click", "ref": second}, } args := map[string]any{"mode": "act", "actions": actions} - wantPath, wantHint := "actions[1].ref", "separate" + wantPath, wantHint := "actions[1].ref", "activate" if kind == "window ref action" { - actions[1] = map[string]any{"type": "activate", "ref": secondWindow} + actions[1] = map[string]any{"type": "close", "ref": secondWindow} } if kind == "explicit window" { args["window_ref"] = secondWindow diff --git a/computer-use/internal/core/ax_evidence.go b/computer-use/internal/core/ax_evidence.go index 8f2299f..329803b 100644 --- a/computer-use/internal/core/ax_evidence.go +++ b/computer-use/internal/core/ax_evidence.go @@ -84,11 +84,20 @@ func compactAXBatchResult(value any) map[string]any { } } } + windows := map[string]bool{} + for _, action := range rawActions { + if ref, ok := action["window_ref"].(string); ok && ref != "" { + windows[ref] = true + } + } actions := make([]map[string]any, 0, len(rawActions)) for _, action := range rawActions { entry := map[string]any{ "index": action["action_index"], } + if len(windows) > 1 { + entry["window_ref"] = action["window_ref"] + } actionability, _ := action["actionability"].(map[string]any) if actionability != nil { if resolution := strings.TrimSpace(fmt.Sprint( diff --git a/computer-use/internal/core/ax_native_window.go b/computer-use/internal/core/ax_native_window.go index 90e5436..fe62feb 100644 --- a/computer-use/internal/core/ax_native_window.go +++ b/computer-use/internal/core/ax_native_window.go @@ -49,93 +49,75 @@ func (s *Service) noTreeWindowEntry(ref string) *AXWindowRefEntry { return entry } -// axNoTreeBatchWindow returns the window without a tree that a batch names -// through window_ref or an activate or close target, or nil. -func (s *Service) axNoTreeBatchWindow(actions []axBatchAction, windowRef string) *AXWindowRefEntry { - if entry := s.noTreeWindowEntry(windowRef); entry != nil { - return entry - } - for _, action := range actions { - if entry := s.noTreeWindowEntry(action.targetRef); entry != nil { - return entry - } +func (s *Service) validateAXNoTreeSegment(context *AXBatchContext, actions []axBatchAction, start int) error { + if context == nil || s.noTreeWindowEntry(context.WindowRef) == nil { + return nil } - return nil -} - -// axNoTreeWindowBatch runs a batch on a window without an accessibility -// tree. Only activate, close and window waits can work there; anything -// else is rejected before the first action, so no input is sent. -func (s *Service) axNoTreeWindowBatch( - actions []axBatchAction, - window *AXWindowRefEntry, - notes []string, - requested int, -) ToolCallResult { for index, action := range actions { switch action.actionType { case "activate", "close": - if action.targetRef == "" || s.noTreeWindowEntry(action.targetRef) != nil { - continue - } - return axArgErrorResult(axInvalid(fmt.Sprintf("actions[%d]", index), - "one call cannot act on a window without an accessibility tree "+ - "and on other windows; split it into separate calls", - `{"mode":"act","actions":[{"type":"activate","ref":"`+window.Ref+`"}]}`), requested) + continue case "wait": if action.wait != nil && action.wait.window != "" { continue } } - return axNoTreeWindowResult(&axNoTreeWindowError{ref: window.Ref}, - fmt.Sprintf("actions[%d]", index), requested) - } - lines := append([]string{}, notes...) - emitted := false - closed := map[string]bool{} - for index, action := range actions { - if action.actionType == "wait" { - result := s.runAXWait(&AXBatchContext{WindowRef: window.Ref}, action.wait) - if !axPostconditionPassed(result) { - return axNoTreeBatchStop(result, index, requested, emitted, lines) - } - lines = append(lines, firstLine(ResultText(result))) - continue - } - target := window - if action.targetRef != "" { - target = s.noTreeWindowEntry(action.targetRef) + return &axArgError{ + Code: contract.AXCodeAppUnavailable, + path: fmt.Sprintf("actions[%d]", start+index), + Message: (&axNoTreeWindowError{ref: context.WindowRef}).Error(), + Hint: GUIScreenshotCall, } - result := s.axNoTreeWindowCommand(action.actionType, target) - if result.IsError { - return axNoTreeBatchStop(result, index, requested, emitted, lines) - } - emitted = true - verb := "activated" - if action.actionType == "close" { - verb = "closed" - closed[target.Ref] = true - } - lines = append(lines, verb+" "+target.Ref) } - if !closed[window.Ref] { - lines = append(lines, window.Ref+" "+axNoTreeWindowMark) - } - result := TextResult(strings.Join(lines, "\n"), false) + return nil +} + +func (s *Service) axNoTreeBatchAction(action string, window *AXWindowRefEntry) ToolCallResult { + result := s.axNoTreeWindowCommand(action, window) + if result.IsError { + return result + } + result = axNoTreeBatchObservation(window) + verb := "activated" + if action == "close" { + verb = "closed" + } + result.Content = append([]ContentItem{{Type: "text", Text: verb + " " + window.Ref}}, result.Content...) + result.StructuredContent["input_events_emitted"] = true + return result +} + +func axNoTreeBatchObservation(window *AXWindowRefEntry) ToolCallResult { + result := TextResult(window.Ref+" "+axNoTreeWindowMark, false) result.StructuredContent = map[string]any{ - "input_events_emitted": emitted, + "input_events_emitted": false, "final_observation_returned": false, "final_observation_kind": "none", "observation_required": false, "evidence_scope": "delivery_or_local_ui", - "a11y_batch_result": map[string]any{ - "actions_requested": requested, - "actions_completed_without_call_error": len(actions), - }, } return result } +func (s *Service) axNoTreeBatchWindowContinuity(context *AXBatchContext) (axWindowContinuity, bool) { + if s == nil || context == nil || s.noTreeWindowEntry(context.WindowRef) == nil { + return axWindowUnknown, false + } + if !validInternalWindowHandle(context.WindowHandle) { + return axWindowUnknown, true + } + inventory, _ := s.listDesktopWindows(false) + if inventory == nil { + return axWindowUnknown, true + } + for _, window := range inventory.windows { + if window.handle == context.WindowHandle { + return axWindowSame, true + } + } + return axWindowChanged, true +} + // axNoTreeWindowCommand asks the provider to activate or close the native // window and to confirm the effect with the window manager. func (s *Service) axNoTreeWindowCommand(action string, window *AXWindowRefEntry) ToolCallResult { @@ -182,32 +164,6 @@ func (s *Service) axNoTreeWindowCommand(action string, window *AXWindowRefEntry) return result } -func axNoTreeBatchStop( - result ToolCallResult, - index, requested int, - emitted bool, - lines []string, -) ToolCallResult { - if result.StructuredContent == nil { - result.StructuredContent = map[string]any{} - } - if emitted { - result.StructuredContent["input_events_emitted"] = true - } - result.StructuredContent["a11y_batch_result"] = map[string]any{ - "actions_requested": requested, - "actions_completed_without_call_error": index, - "failed_action_index": index, - } - if len(lines) > 0 { - result.Content = append( - []ContentItem{{Type: "text", Text: strings.Join(lines, "\n")}}, - result.Content..., - ) - } - return result -} - // asNoTreeWindowError reports whether err is a read of a window without a tree. func asNoTreeWindowError(err error) (*axNoTreeWindowError, bool) { var noTree *axNoTreeWindowError diff --git a/computer-use/internal/core/ax_native_window_batch_test.go b/computer-use/internal/core/ax_native_window_batch_test.go new file mode 100644 index 0000000..dd34fac --- /dev/null +++ b/computer-use/internal/core/ax_native_window_batch_test.go @@ -0,0 +1,321 @@ +package core + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +type axNoTreeActivationBackend struct { + *axActivationBackend + listings int + issues []ProviderInventoryIssue + inventoryError error + nativeReads int +} + +func (backend *axNoTreeActivationBackend) ObserveApps(request ProviderObserveAppsRequest) (ProviderAccessibilityResult, error) { + backend.listings++ + if backend.inventoryError != nil { + return ProviderAccessibilityResult{}, backend.inventoryError + } + result, err := backend.axActivationBackend.ObserveApps(request) + result.InventoryIssues = backend.issues + for appIndex := range result.Apps { + for windowIndex := range result.Apps[appIndex].Windows { + window := &result.Apps[appIndex].Windows[windowIndex] + window.NoAccessibilityTree = window.Handle == reportMainHandle + } + } + return result, err +} + +func (backend *axNoTreeActivationBackend) Observe(request ProviderObserveRequest) (ProviderAccessibilityResult, error) { + if request.WindowHandle == reportMainHandle { + backend.nativeReads++ + return ProviderAccessibilityResult{}, fmt.Errorf("native window has no accessibility tree") + } + return backend.axActivationBackend.Observe(request) +} + +func axNoTreeActivationService(t *testing.T) (*Service, *axNoTreeActivationBackend, axActivationRefs, axActivationRefs) { + t.Helper() + service, activation, native, tree := axActivationService(t, true) + backend := &axNoTreeActivationBackend{axActivationBackend: activation} + service.Backend = backend + if listed := service.listApps(); listed.IsError { + t.Fatalf("list native window: %s", ResultText(listed)) + } + return service, backend, native, tree +} + +func TestAXNoTreeMixedActivationToTree(t *testing.T) { + for _, byTitle := range []bool{false, true} { + t.Run(fmt.Sprintf("title_%t", byTitle), func(t *testing.T) { + service, backend, native, tree := axNoTreeActivationService(t) + activation := map[string]any{"type": "activate", "ref": tree.window} + if byTitle { + activation = map[string]any{"type": "activate", "title": "Notes"} + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": native.window, "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": native.window}, + activation, + map[string]any{"type": "type", "text": "untargeted"}, + map[string]any{"type": "type", "ref": tree.entry, "text": "targeted"}, + }, + }) + if result.IsError { + t.Fatalf("native -> tree rejected: %s", ResultText(result)) + } + want := []string{"activate@" + reportMainHandle, "activate@" + reportOtherHandle, "type@" + reportOtherHandle, "type@" + reportOtherHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("deliveries = %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, native.window, tree.window, tree.window, tree.window) + if backend.nativeReads != 0 || result.StructuredContent["final_observation_returned"] != true || !strings.Contains(ResultText(result), `window: "Notes" [ref=`+tree.window+`]`) { + t.Errorf("wrong final observation: native reads %d, result %#v\n%s", backend.nativeReads, result.StructuredContent, ResultText(result)) + } + }) + } +} + +func TestAXNoTreeMixedActivationFinalNative(t *testing.T) { + service, backend, native, tree := axNoTreeActivationService(t) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": tree.window, "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": tree.window}, + map[string]any{"type": "type", "ref": tree.entry, "text": "before switch"}, + map[string]any{"type": "activate", "title": "Report"}, + }, + }) + if result.IsError { + t.Fatalf("tree -> native rejected: %s", ResultText(result)) + } + want := []string{"activate@" + reportOtherHandle, "type@" + reportOtherHandle, "activate@" + reportMainHandle} + if !slices.Equal(backend.deliveries, want) { + t.Errorf("deliveries = %v, want %v", backend.deliveries, want) + } + axActivationEvidence(t, result, tree.window, tree.window, native.window) + if backend.nativeReads != 0 || !strings.Contains(ResultText(result), native.window+" "+axNoTreeWindowMark) { + t.Errorf("native final observation: reads %d, text %s", backend.nativeReads, ResultText(result)) + } + for key, want := range map[string]any{"input_events_emitted": true, "final_observation_returned": false, "final_observation_kind": "none", "observation_required": false} { + if result.StructuredContent[key] != want { + t.Errorf("%s = %v, want %v", key, result.StructuredContent[key], want) + } + } +} + +func TestAXNoTreeMixedInputRejectedBeforeAnyDelivery(t *testing.T) { + for _, action := range []map[string]any{ + {"type": "keypress", "keys": []any{"ENTER"}}, + {"type": "type", "text": "must not be sent"}, + {"type": "wait", "text": "not a window wait", "timeout_ms": 200}, + } { + t.Run(fmt.Sprint(action["type"]), func(t *testing.T) { + service, backend, native, tree := axNoTreeActivationService(t) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{ + map[string]any{"type": "activate", "ref": tree.window}, + map[string]any{"type": "activate", "ref": native.window}, + action, + map[string]any{"type": "activate", "ref": tree.window}, + }, + }) + if !result.IsError || result.StructuredContent[contract.AXPublicErrorCodeKey] != contract.AXCodeAppUnavailable || result.StructuredContent[contract.AXPublicErrorPathKey] != "actions[2]" || !strings.Contains(ResultText(result), "no accessibility tree") { + t.Errorf("want native input preflight error, got %#v\n%s", result.StructuredContent, ResultText(result)) + } + if len(backend.deliveries) != 0 || axEvidenceReportsInputEmission(result.StructuredContent) { + t.Errorf("invalid batch emitted input: %v, %#v", backend.deliveries, result.StructuredContent) + } + }) + } +} + +func TestAXNoTreeSegmentValidation(t *testing.T) { + service, backend, native, tree := axNoTreeActivationService(t) + allowed := []axBatchAction{ + {actionType: "activate", targetRef: native.window}, + {actionType: "wait", wait: &axWait{window: "Report"}}, + {actionType: "close", targetRef: native.window}, + } + if err := service.validateAXNoTreeSegment(&AXBatchContext{WindowRef: native.window}, allowed, 4); err != nil { + t.Fatalf("native window operations rejected: %v", err) + } + for _, action := range []axBatchAction{ + {actionType: "keypress", key: "ENTER"}, + {actionType: "type", text: "unsent"}, + {actionType: "click", targetRef: tree.button}, + {actionType: "scroll", direction: "down"}, + {actionType: "set_value", targetRef: tree.entry, value: "unsent"}, + {actionType: "select", targetRef: tree.entry, toRef: tree.entry}, + {actionType: "wait"}, + {actionType: "wait", wait: &axWait{ref: tree.entry}}, + {actionType: "wait", wait: &axWait{query: map[string]any{"role": "button"}}}, + {actionType: "wait", wait: &axWait{settle: true}}, + } { + actions := []axBatchAction{allowed[0], action} + err := service.validateAXNoTreeSegment(&AXBatchContext{WindowRef: native.window}, actions, 4) + var argument *axArgError + if !errors.As(err, &argument) || argument.Code != contract.AXCodeAppUnavailable || argument.path != "actions[5]" || argument.Hint != GUIScreenshotCall { + t.Errorf("%s rejection = %v", action.actionType, err) + } + if err := service.validateAXNoTreeSegment(&AXBatchContext{WindowRef: tree.window}, actions, 4); err != nil { + t.Errorf("tree segment rejected by native validation: %v", err) + } + } + if len(backend.deliveries) != 0 || backend.nativeReads != 0 { + t.Errorf("segment validation reached input/tree: %v, %d", backend.deliveries, backend.nativeReads) + } +} + +func TestAXNoTreeBatchWindowContinuityUsesFreshStrongHandle(t *testing.T) { + for _, scenario := range []struct { + name string + mutate func(*axNoTreeActivationBackend) + want axWindowContinuity + }{ + {name: "same", want: axWindowSame}, + {name: "renamed", want: axWindowSame, mutate: func(backend *axNoTreeActivationBackend) { backend.windows[0].title = "Renamed" }}, + {name: "other_window_focused", want: axWindowSame, mutate: func(backend *axNoTreeActivationBackend) { backend.windows[1].active = true }}, + {name: "gone", want: axWindowChanged, mutate: func(backend *axNoTreeActivationBackend) { backend.windows = backend.windows[1:] }}, + {name: "same_title_new_handle", want: axWindowChanged, mutate: func(backend *axNoTreeActivationBackend) { + backend.owners[testHandleScene] = backend.owners[reportMainHandle] + backend.windows[0].handle = testHandleScene + }}, + {name: "incomplete_present", want: axWindowUnknown, mutate: func(backend *axNoTreeActivationBackend) { + backend.issues = []ProviderInventoryIssue{{Reason: "enumeration incomplete"}} + }}, + {name: "incomplete_absent", want: axWindowUnknown, mutate: func(backend *axNoTreeActivationBackend) { + backend.windows = backend.windows[1:] + backend.issues = []ProviderInventoryIssue{{Reason: "enumeration incomplete"}} + }}, + {name: "failed", want: axWindowUnknown, mutate: func(backend *axNoTreeActivationBackend) { backend.inventoryError = errors.New("inventory unavailable") }}, + } { + t.Run(scenario.name, func(t *testing.T) { + service, backend, native, _ := axNoTreeActivationService(t) + if scenario.mutate != nil { + scenario.mutate(backend) + } + before := backend.listings + state, handled := service.axNoTreeBatchWindowContinuity(&AXBatchContext{WindowRef: native.window, WindowHandle: reportMainHandle}) + if !handled || state != scenario.want || backend.listings != before+1 { + t.Errorf("continuity = %s/%t, listings %d -> %d; want %s and one fresh inventory", state, handled, before, backend.listings, scenario.want) + } + if backend.nativeReads != 0 || len(backend.deliveries) != 0 { + t.Errorf("continuity read native tree or sent input: %d, %v", backend.nativeReads, backend.deliveries) + } + }) + } + service, backend, native, tree := axNoTreeActivationService(t) + before := backend.listings + if _, handled := service.axNoTreeBatchWindowContinuity(&AXBatchContext{WindowRef: tree.window, WindowHandle: reportOtherHandle}); handled || backend.listings != before { + t.Errorf("ordinary tree continuity intercepted: %t, listings %d -> %d", handled, before, backend.listings) + } + state, handled := service.axNoTreeBatchWindowContinuity(&AXBatchContext{WindowRef: native.window}) + if !handled || state != axWindowUnknown || backend.listings != before { + t.Errorf("missing strong handle should stay unknown without enumeration: %s/%t", state, handled) + } +} + +func TestAXNoTreeBatchActionAndObservationEvidence(t *testing.T) { + for _, action := range []string{"activate", "close"} { + t.Run(action, func(t *testing.T) { + service, backend := windowScriptService(sceneApps()) + window := service.noTreeWindowEntry(noTreeWindowRef(t, service)) + backend.windowResponse = &ProviderWindowResult{OK: true} + result := service.axNoTreeBatchAction(action, window) + if result.IsError || !slices.Equal(backend.calls, []string{"window_" + action}) || !strings.Contains(ResultText(result), axNoTreeWindowMark) { + t.Fatalf("native action failed: %v, %s", backend.calls, ResultText(result)) + } + for key, want := range map[string]any{"input_events_emitted": true, "final_observation_returned": false, "final_observation_kind": "none", "observation_required": false} { + if result.StructuredContent[key] != want { + t.Errorf("%s = %v, want %v", key, result.StructuredContent[key], want) + } + } + observation := axNoTreeBatchObservation(window) + if observation.IsError || observation.StructuredContent["input_events_emitted"] != false || len(backend.calls) != 1 { + t.Errorf("native final observation emitted input: %#v, %v", observation, backend.calls) + } + }) + } +} + +func TestAXNoTreeBatchActionPreservesProviderFailure(t *testing.T) { + for _, emitted := range []bool{false, true} { + t.Run(fmt.Sprintf("emitted_%t", emitted), func(t *testing.T) { + service, backend := windowScriptService(sceneApps()) + window := service.noTreeWindowEntry(noTreeWindowRef(t, service)) + backend.windowResponse = &ProviderWindowResult{ + Error: "provider could not confirm activation", + Failure: &ProviderFailure{Code: "input_failed", Phase: "emit", FactsComplete: true}, + Mutation: &ProviderMutationFacts{InputEventsEmitted: emitted}, + } + want := service.axNoTreeWindowCommand("activate", window) + got := service.axNoTreeBatchAction("activate", window) + if !got.IsError || !reflect.DeepEqual(got, want) || got.StructuredContent["input_events_emitted"] != emitted { + t.Errorf("failure facts changed: got %#v, want %#v", got, want) + } + }) + } +} + +func TestAXNoTreeBatchKeepsNativeWindowBinding(t *testing.T) { + for _, wrongClose := range []bool{false, true} { + t.Run(fmt.Sprintf("wrong_close_%t", wrongClose), func(t *testing.T) { + apps := sceneApps() + nativeApp := &apps[len(apps)-1] + second := nativeApp.Windows[0] + second.Handle = testHandleNotes + second.Title = "Render - Paint3D" + nativeApp.Windows = append(nativeApp.Windows, second) + service, backend := windowScriptService(apps) + backend.windowResponse = &ProviderWindowResult{OK: true} + firstRef := noTreeWindowRef(t, service) + inventory, failure := service.listDesktopWindows(false) + if inventory == nil { + t.Fatalf("list native windows: %s", ResultText(failure)) + } + secondRef := "" + for _, window := range inventory.windows { + if window.handle == second.Handle { + secondRef = window.ref + } + } + if secondRef == "" { + t.Fatal("missing second native window") + } + closeRef := secondRef + if wrongClose { + closeRef = firstRef + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": firstRef, "observe": "tree", "actions": []any{ + map[string]any{"type": "activate", "ref": firstRef}, + map[string]any{"type": "activate", "title": "Render"}, + map[string]any{"type": "wait", "window": "Render", "timeout_ms": 200}, + map[string]any{"type": "close", "ref": closeRef}, + }, + }) + if wrongClose { + if !result.IsError || len(backend.calls) != 0 || axEvidenceReportsInputEmission(result.StructuredContent) || result.StructuredContent[contract.AXPublicErrorCodeKey] != contract.AXCodeInvalidArgument { + t.Fatalf("close in previous native window was not rejected before input: %v, %#v", backend.calls, result) + } + return + } + if result.IsError || !slices.Equal(backend.calls, []string{"window_activate", "window_activate", "window_close"}) { + t.Fatalf("all native batch failed: %v, %s", backend.calls, ResultText(result)) + } + axActivationEvidence(t, result, firstRef, secondRef, secondRef, secondRef) + if result.StructuredContent["input_events_emitted"] != true || result.StructuredContent["final_observation_returned"] != false || result.StructuredContent["final_observation_kind"] != "none" || result.StructuredContent["observation_required"] != false { + t.Errorf("native final evidence changed: %#v", result.StructuredContent) + } + }) + } +} diff --git a/computer-use/internal/core/ax_wait_batch.go b/computer-use/internal/core/ax_wait_batch.go index c3cc59b..2398ef4 100644 --- a/computer-use/internal/core/ax_wait_batch.go +++ b/computer-use/internal/core/ax_wait_batch.go @@ -32,8 +32,9 @@ func (s *Service) preflightAXWaitRef(context *AXBatchContext, ref string) error return err } if context != nil && entry.WindowHandle != context.WindowHandle { - return fmt.Errorf("%s belongs to window %s, not the batch window %s; wait for it in its own act call", - ref, entry.LogicalWindowRef, context.WindowRef) + return axInvalid("", fmt.Sprintf("%s belongs to window %s, not the current batch window %s; activate that window before using its refs", + ref, entry.LogicalWindowRef, context.WindowRef), + "Use refs from the current window, or activate the other window before acting in it.") } return nil } @@ -88,6 +89,10 @@ func (s *Service) rebindAfterWindowWait( Role: entry.ProviderWindowRole, Name: entry.ProviderWindowName, } + if context.report != nil { + context.report.appToken = context.AppToken + context.report.windowRef = context.WindowRef + } // One read of the new window makes it the current state of its app, so // the next step's window check and its diff start from that window. noScreenshot := false diff --git a/computer-use/internal/core/tool_contract_test.go b/computer-use/internal/core/tool_contract_test.go index 435d10f..832b3bf 100644 --- a/computer-use/internal/core/tool_contract_test.go +++ b/computer-use/internal/core/tool_contract_test.go @@ -64,8 +64,14 @@ func TestMCPInstructionsRouteDesktopInputThroughTheTools(t *testing.T) { } } ax := findToolDefinition(t, "AX") - if !strings.Contains(ax.Description, "keypress/type without ref go to the focused window") { - t.Fatal("AX description must say where ref-less keys go") + for _, fragment := range []string{ + "keypress/type without ref use the batch target", + "initially the focused window unless refs or window_ref select it", + "activate with wN or title switches the batch target for later refs and ref-less steps", + } { + if !strings.Contains(ax.Description, fragment) { + t.Fatalf("AX description must explain ref-less input routing: missing %q", fragment) + } } } diff --git a/computer-use/testdata/contract/public_contract.json b/computer-use/testdata/contract/public_contract.json index 5a7d2ef..4aac468 100644 --- a/computer-use/testdata/contract/public_contract.json +++ b/computer-use/testdata/contract/public_contract.json @@ -46,7 +46,7 @@ "openWorldHint": false, "title": "Accessibility" }, - "description": "Accessibility for application windows. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. Copy rN/wN exactly; stale refs fail. keypress/type without ref go to the focused window. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", + "description": "Accessibility for application windows. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. activate with wN or title switches the batch target for later refs and ref-less steps. Other-window refs fail until another activate selects their window. Each step reports its window; final observation uses the last executed window. Copy rN/wN exactly; stale refs fail. keypress/type without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", "inputSchema": { "$defs": { "query": { @@ -127,7 +127,7 @@ "$ref": "#/$defs/query" }, "ref": { - "description": "rN element; wN for activate/close. Omit for keys/text to the focused window.", + "description": "rN element; wN for activate/close. Omit for keys/text to the batch target (focused window by default).", "pattern": "^[rw][1-9][0-9]*$", "type": "string" }, @@ -249,7 +249,7 @@ "type": "string" }, "window_ref": { - "description": "observe/find: window to read. act: omit when actions have refs, which name the window; it only sends ref-less keys/text to wN instead of the focused window.", + "description": "observe/find: window to read. act: omit when actions have refs, which name the window; otherwise sets the initial target for ref-less steps instead of the focused window.", "pattern": "^w[1-9][0-9]*$", "type": "string" } diff --git a/docs/MCP.md b/docs/MCP.md index 4dae488..1998a8b 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -502,17 +502,20 @@ change canonical text, selectors, refs, or the response budget. steps. Structural argument errors reject the whole batch before input. The first step that fails stops the batch, and the result says how many ran. - The `rN` or `wN` refs name the window, so an `act` call with refs needs no - `window_ref`. `window_ref` only sends ref-less `keypress` and `type` to a - window other than the focused one. `keypress` and `type` without `ref` go - to the window that had focus when the call started. + `window_ref`. For ref-less steps, `window_ref` selects the initial batch + target; otherwise it defaults to the window focused when the call started. - `activate` and `close` accept a window title substring, resolved to one - `wN` before any input. + `wN` before any input. `activate` with a `wN` or title switches the batch + target. Later refs and ref-less steps use that window. AX rejects refs + from another window without an explicit `activate` first. To reuse refs + from the previous window, activate that window again; refs must still be + valid. Each step reports its window. - The decoder accepts `type.value` as `type.text` and non-empty `set_value.text` as `set_value.value`. Supplying both fields is rejected before input. The published schema advertises only the canonical names. - `observe` selects the evidence returned after the batch. The default is a - compact `diff`. An explicit `tree` returns the complete final window tree, - and `full` also returns a screenshot. These requests have the same + compact `diff` of the last executed window. An explicit `tree` returns + that window's complete tree, and `full` also returns a screenshot. These requests have the same meaning after a mutation, an ordered input sequence, or a final wait; evidence is captured after the wait finishes. - The removed `until` field, and its older spellings `postcondition` and diff --git a/instructions/ax-only.md b/instructions/ax-only.md index 3f63422..9857182 100644 --- a/instructions/ax-only.md +++ b/instructions/ax-only.md @@ -37,8 +37,11 @@ option. Follow the current tool schema for argument shapes and actions. - Actions take current `rN` element refs or `wN` window refs, copied exactly. `activate` and `close` also take a window title substring. `keypress` and `type` without `ref` go to the window that has focus. -- Batch known actions within one window while later refs and focus should - remain valid. End the batch at an action that can open, close, replace, or +- Batch known actions while later refs and focus should remain valid. + `activate` with a `wN` or title switches the batch target for later refs + and ref-less steps. Before using refs from another window, activate that + window; activate the previous window again to reuse its refs. + End the batch at an action that can open, close, replace, or navigate a window; `close` is last. - Semantic click may activate a selectable item, not merely select it. When a later action depends on a state that may arrive late, put a `wait` step diff --git a/instructions/hybrid-code.md b/instructions/hybrid-code.md index 7cf5d90..13db176 100644 --- a/instructions/hybrid-code.md +++ b/instructions/hybrid-code.md @@ -105,7 +105,8 @@ shapes are: {mode: "find", window_ref: "w2", query: {role: "button", name_contains: "Save"}} {mode: "act", actions: [{type: "set_value", ref: "r7", value: "42"}, {type: "activate", ref: "w3"}]} // act step types: click, type, keypress, set_value, select, scroll, activate, close, wait -// activate and close take ref wN or title; one act works in one window +// activate and close take ref wN or title; activate switches the batch target +// later refs and ref-less steps use that window; activate back before reusing old-window refs // query fields: role, name_equals, name_contains, text_equals, text_contains, state {actions: [{type: "scroll", x: 640, y: 412, scroll_y: -3}]} // GUI; scroll_x or scroll_y, non-zero ``` diff --git a/instructions/hybrid.md b/instructions/hybrid.md index bdb5ceb..4dd71ec 100644 --- a/instructions/hybrid.md +++ b/instructions/hybrid.md @@ -35,8 +35,13 @@ fits which state, and how to combine them. - Once the source, target, and operation are known, act. Further checks must answer a concrete blocker. - Plan from one read. After `observe` or `find`, put every action you can - already determine for that window into one `act`, while later refs and - focus remain valid. Split when the next action needs a new observation. + already determine into one `act`, while later refs and focus remain valid. + Split when the next action needs a new observation. +- Use `activate` with a `wN` or title to switch the batch target. + Later refs and ref-less steps use that window. AX rejects refs from another + window without explicit `activate`; activate back before reusing the + previous window's refs. Each step reports its window; the final observation + uses the last executed window. - Every `act` result starts with what happened, then a diff: ```text From 86fe18b8d239d8061ebad8840c6b410c308590cf Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 09:02:52 +0000 Subject: [PATCH 10/15] fix(linux): guard global AX input and prefer semantic click and scroll Select actionless tabs/items through AT-SPI Selection and scroll ancestor ranges with verified readback. Add per-action allow_global_input for Linux click/scroll, reject unsafe fallback before input, and retain synthesis warnings through result compaction and batches. Update public contract, provider-wire fixtures and recovery guidance. Cover semantic readback, clean refusal, fractional/boundary scrolling, policy serialization, platform validation and partial-batch evidence. --- CHANGELOG.md | 17 ++ .../internal/contract/contract_describe.go | 27 +- .../internal/contract/published_values.go | 35 +-- .../internal/contract/tool_contract.go | 14 +- .../internal/core/ax_action_evidence.go | 7 +- .../internal/core/ax_action_evidence_test.go | 16 ++ .../internal/core/ax_action_service.go | 22 +- .../internal/core/ax_argument_hint_test.go | 1 + computer-use/internal/core/ax_batch.go | 41 ++- computer-use/internal/core/ax_batch_result.go | 3 +- .../internal/core/ax_batch_result_test.go | 67 +++++ computer-use/internal/core/ax_batch_test.go | 141 +++++++++ computer-use/internal/core/ax_evidence.go | 20 +- .../internal/core/ax_public_request.go | 12 +- computer-use/internal/core/ax_request_test.go | 32 +++ computer-use/internal/core/provider_types.go | 29 +- .../internal/core/provider_wire_test.go | 69 +++++ computer-use/internal/core/runtime_backend.go | 11 + .../internal/core/runtime_backend_test.go | 22 ++ .../internal/core/runtime_protocol.go | 42 +-- .../internal/core/runtime_protocol_test.go | 69 +++++ .../internal/core/schema_consistency_test.go | 4 + .../internal/provider/linux_runtime_bundle.go | 1 + .../provider/linux_runtime_bundle_test.go | 1 + .../linux/provider/ax_input_delivery.py | 65 ++++- .../linux/provider/ax_semantic_scroll.py | 90 ++++++ .../native/linux/provider/wire_request.py | 7 +- computer-use/runtime_test.py | 1 + computer-use/runtime_test_support.py | 2 + .../runtime_tests/test_background_input.py | 269 ++++++++++++++++++ .../testdata/contract/public_contract.json | 5 + .../background-click.request.json | 25 ++ .../background-scroll.request.json | 24 ++ .../provider-wire/linux-input-manifest.json | 9 + docs/MCP.md | 57 +++- instructions/hybrid.md | 7 + 36 files changed, 1179 insertions(+), 85 deletions(-) create mode 100644 computer-use/internal/core/runtime_protocol_test.go create mode 100644 computer-use/native/linux/provider/ax_semantic_scroll.py create mode 100644 computer-use/runtime_tests/test_background_input.py create mode 100644 computer-use/testdata/provider-wire/background-click.request.json create mode 100644 computer-use/testdata/provider-wire/background-scroll.request.json create mode 100644 computer-use/testdata/provider-wire/linux-input-manifest.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c729915..ee23be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ action batch. Later refs and ref-less steps use the selected window; refs from other windows require another explicit activation. Each step reports its window, and the final observation uses the last executed window. +- Linux AX `click` and `scroll` accept `allow_global_input`, defaulting to + `true`. Setting it to `false` forbids driver focus/raise and synthesized + pointer/keyboard input, including editor focus preparation and Qt state + synchronization. Semantic actions can still change application focus. + Unsupported uses reject the whole batch with `invalid_argument` before + input. A required global route returns `global_input_forbidden` without + input from that action. Under this policy, an AT-SPI action returning + false after invocation reports `effect_unknown` without a global retry. +- Linux AX tab clicks prefer the parent container's accessibility selection + operation. Supported list and table items without their own action also + use parent selection. Failed selection verification returns + `effect_unknown` without global input fallback. +- Linux AX scroll can use accessible scrollbar ranges with readback. + A page means five minimum range increments, not pixels; pages remain + approximate, and a clamped boundary no-op is verified. AX results retain + synthesis route notes and warnings without claiming successful effects. + - AX batch errors expose the failed action's reason and a failure-specific recovery hint while retaining batch progress and input evidence. No-input argument and unsupported-operation failures use `invalid_argument`; diff --git a/computer-use/internal/contract/contract_describe.go b/computer-use/internal/contract/contract_describe.go index a8f40d4..3eec298 100644 --- a/computer-use/internal/contract/contract_describe.go +++ b/computer-use/internal/contract/contract_describe.go @@ -9,7 +9,7 @@ import ( // tool schemas, the records envelope and the error table below. Adding an // optional field or operation raises the minor number; removing or changing // one raises the major number. -const ContractVersion = "1.0" +const ContractVersion = "1.1" // AXCodeGUIFrameStale is the GUI code for coordinates that refer to a // screenshot that no longer matches the desktop layout. Nothing was sent. @@ -30,18 +30,19 @@ type PublicError struct { } var PublicErrorCodes = map[string]PublicError{ - AXCodeInvalidArgument: {2, "no", "fix the argument named by path; hint is a corrected call"}, - AXCodeWaitTimeout: {3, "maybe", "observe again; completed counts the steps that ran and the message says what the wait saw"}, - AXCodeStaleRef: {4, "no", "observe or find again and use the new ref; refs never rebind"}, - AXCodeGUIFrameStale: {4, "no", "take a new GUI screenshot and plan coordinates from it"}, - AXCodeWindowChanged: {5, "maybe", "observe before retrying; do not repeat the batch blindly"}, - AXCodeEffectUnknown: {5, "maybe", "observe before retrying; input may have taken effect"}, - AXCodeAppUnavailable: {6, "no", "list apps; the application is not running or not accessible"}, - AXCodeAmbiguousWindow: {6, "no", "use a longer title or the wN from apps"}, - AXCodeWindowNotFound: {6, "no", "list apps and use a wN or an exact title"}, - AXCodeNoFocusedWindow: {6, "no", "activate a window first or pass window_ref"}, - AXCodeRuntimeError: {10, "maybe", "retry reads later; observe before retrying input"}, - GUICodePermissionDenied: {6, "no", GUIPermissionHint}, + AXCodeInvalidArgument: {2, "no", "fix the argument named by path; hint is a corrected call"}, + AXCodeWaitTimeout: {3, "maybe", "observe again; completed counts the steps that ran and the message says what the wait saw"}, + AXCodeStaleRef: {4, "no", "observe or find again and use the new ref; refs never rebind"}, + AXCodeGUIFrameStale: {4, "no", "take a new GUI screenshot and plan coordinates from it"}, + AXCodeWindowChanged: {5, "maybe", "observe before retrying; do not repeat the batch blindly"}, + AXCodeEffectUnknown: {5, "maybe", "observe before retrying; input may have taken effect"}, + AXCodeGlobalInputForbidden: {5, "maybe", "the refused action sent no input; check completed and observe before replanning; keep allow_global_input false unless global input is authorized"}, + AXCodeAppUnavailable: {6, "no", "list apps; the application is not running or not accessible"}, + AXCodeAmbiguousWindow: {6, "no", "use a longer title or the wN from apps"}, + AXCodeWindowNotFound: {6, "no", "list apps and use a wN or an exact title"}, + AXCodeNoFocusedWindow: {6, "no", "activate a window first or pass window_ref"}, + AXCodeRuntimeError: {10, "maybe", "retry reads later; observe before retrying input"}, + GUICodePermissionDenied: {6, "no", GUIPermissionHint}, } // DescribeContract is the machine-readable contract: what `schema` prints diff --git a/computer-use/internal/contract/published_values.go b/computer-use/internal/contract/published_values.go index 303603c..e789152 100644 --- a/computer-use/internal/contract/published_values.go +++ b/computer-use/internal/contract/published_values.go @@ -24,23 +24,24 @@ const ( // Public AX error codes. Internal failure codes stay in telemetry; the model // sees only these. const ( - AXCodeInvalidArgument = "invalid_argument" - AXCodeStaleRef = "stale_ref" - AXCodeWindowChanged = "window_changed" - AXCodeWaitTimeout = "wait_timeout" - AXCodeEffectUnknown = "effect_unknown" - AXCodeAppUnavailable = "app_unavailable" - AXCodeRuntimeError = "runtime_error" - AXCodeAmbiguousWindow = "ambiguous_window" - AXCodeWindowNotFound = "window_not_found" - AXCodeNoFocusedWindow = "no_focused_window" - AXModeApps = "apps" - AXDefaultFindPageSize = 20 - AXDefaultWaitTimeout = 5000 - AXPublicErrorCodeKey = "public_error_code" - AXPublicErrorPathKey = "public_error_path" - AXPublicErrorHintKey = "public_error_hint" - AXPublicErrorDetailKey = "public_error_message" + AXCodeInvalidArgument = "invalid_argument" + AXCodeStaleRef = "stale_ref" + AXCodeWindowChanged = "window_changed" + AXCodeWaitTimeout = "wait_timeout" + AXCodeEffectUnknown = "effect_unknown" + AXCodeGlobalInputForbidden = "global_input_forbidden" + AXCodeAppUnavailable = "app_unavailable" + AXCodeRuntimeError = "runtime_error" + AXCodeAmbiguousWindow = "ambiguous_window" + AXCodeWindowNotFound = "window_not_found" + AXCodeNoFocusedWindow = "no_focused_window" + AXModeApps = "apps" + AXDefaultFindPageSize = 20 + AXDefaultWaitTimeout = 5000 + AXPublicErrorCodeKey = "public_error_code" + AXPublicErrorPathKey = "public_error_path" + AXPublicErrorHintKey = "public_error_hint" + AXPublicErrorDetailKey = "public_error_message" ) var AXActionTypes = []string{ diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index 47232e0..8e22643 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -237,11 +237,15 @@ func axInputSchema() map[string]any { "value": map[string]any{"type": "string", "maxLength": MaxActionText}, "direction": map[string]any{"type": "string", "enum": []string{"up", "down", "left", "right"}}, "pages": map[string]any{"type": "number", "exclusiveMinimum": 0, "maximum": MaxScrollPages}, - "window": nonBlankString("wait: a top-level window whose title contains this, on the whole desktop."), - "query": map[string]any{"$ref": "#/$defs/query"}, - "state": nonBlankString("wait: ref state, e.g. checked."), - "gone": map[string]any{"type": "boolean", "description": "wait until it is absent."}, - "settle": map[string]any{"type": "boolean", "description": "wait: true waits until the UI changes after the step before it, then stops changing."}, + "allow_global_input": map[string]any{ + "type": "boolean", "default": true, + "description": "Linux click/scroll only. false forbids driver focus/raise and pointer/keyboard synthesis. Semantic actions may themselves change app focus. Unsupported uses reject the batch before input.", + }, + "window": nonBlankString("wait: a top-level window whose title contains this, on the whole desktop."), + "query": map[string]any{"$ref": "#/$defs/query"}, + "state": nonBlankString("wait: ref state, e.g. checked."), + "gone": map[string]any{"type": "boolean", "description": "wait until it is absent."}, + "settle": map[string]any{"type": "boolean", "description": "wait: true waits until the UI changes after the step before it, then stops changing."}, "timeout_ms": map[string]any{ "type": "integer", "minimum": 1, "maximum": MaxVerifyTimeout, "description": "wait: default 5000.", diff --git a/computer-use/internal/core/ax_action_evidence.go b/computer-use/internal/core/ax_action_evidence.go index 62167cd..18d635d 100644 --- a/computer-use/internal/core/ax_action_evidence.go +++ b/computer-use/internal/core/ax_action_evidence.go @@ -328,7 +328,8 @@ func observeNoneNotes(notes []string) []string { if phase == "immediate" && category != "" { continue } - if strings.HasPrefix(note, "[expect]") || + if strings.HasPrefix(note, "[a11y][synthesis]") || + strings.HasPrefix(note, "[expect]") || strings.HasPrefix(note, "[semantic-resolution]") || strings.HasPrefix(note, "[canonical-address]") || strings.Contains(note, `"predicate_verified":`) || @@ -404,6 +405,10 @@ func modelFacingAXNotes(notes []string) []string { } continue } + if strings.HasPrefix(note, "[a11y][synthesis]") { + appendOnce(note) + continue + } if strings.HasPrefix(note, "[a11y]") { if offset := strings.Index(note, "[state]"); offset >= 0 { fact := strings.TrimSpace(note[offset:]) diff --git a/computer-use/internal/core/ax_action_evidence_test.go b/computer-use/internal/core/ax_action_evidence_test.go index 49b2825..cbda391 100644 --- a/computer-use/internal/core/ax_action_evidence_test.go +++ b/computer-use/internal/core/ax_action_evidence_test.go @@ -65,6 +65,22 @@ func TestUnchangedNotesReportOnlyObservedChange(t *testing.T) { } } +func TestAXSynthesisNotesSurviveNormalization(t *testing.T) { + note := "[a11y][synthesis] Synthesized wheel input. Global input may change focus; verify the effect." + for _, notes := range [][]string{ + modelFacingAXNotes([]string{note, note}), + modelFacingAXNotes(observeNoneNotes([]string{note})), + } { + if len(notes) != 1 || notes[0] != note { + t.Fatalf("synthesis warning lost or rewritten: %#v", notes) + } + } + result := compactAXActionContent(TextResult("Note: "+note, false)) + if ResultText(result) != "Note: "+note { + t.Fatalf("batch compaction lost the warning: %s", ResultText(result)) + } +} + func TestFindOmitsInternalContinuityDiagnostics(t *testing.T) { notes := compactFindNotes([]string{ "SNAPSHOT UNCHANGED: a long explanation that should not be repeated.", diff --git a/computer-use/internal/core/ax_action_service.go b/computer-use/internal/core/ax_action_service.go index b87ab1f..3bff9c2 100644 --- a/computer-use/internal/core/ax_action_service.go +++ b/computer-use/internal/core/ax_action_service.go @@ -368,6 +368,10 @@ func applyActionElementResolution( } func (s *Service) click(app, elementIndex, declared string, x, y *float64, clickCount int, mouseButton, clickMethod string, obs observation) ToolCallResult { + return s.clickWithGlobalInput(app, elementIndex, declared, x, y, clickCount, mouseButton, clickMethod, obs, nil) +} + +func (s *Service) clickWithGlobalInput(app, elementIndex, declared string, x, y *float64, clickCount int, mouseButton, clickMethod string, obs observation, allowGlobalInput *bool) ToolCallResult { if app == "" { return TextResult("Missing required argument: app", true) } @@ -402,7 +406,8 @@ func (s *Service) click(app, elementIndex, declared string, x, y *float64, click request := AXServiceRequest{ App: app, Action: AXClickCommand{ - X: x, Y: y, ClickCount: clickCount, + AllowGlobalInput: allowGlobalInput, + X: x, Y: y, ClickCount: clickCount, MouseButton: mouseButton, ClickMethod: clickMethod, }, @@ -448,6 +453,16 @@ func (s *Service) scroll( expectedWindow *ProviderWindowExpectation, pages float64, obs observation, +) ToolCallResult { + return s.scrollWithGlobalInput(app, direction, elementIndex, declared, targetWindowHandle, expectedWindow, pages, obs, nil) +} + +func (s *Service) scrollWithGlobalInput( + app, direction, elementIndex, declared, targetWindowHandle string, + expectedWindow *ProviderWindowExpectation, + pages float64, + obs observation, + allowGlobalInput *bool, ) ToolCallResult { if app == "" { return TextResult("Missing required argument: app", true) @@ -470,7 +485,8 @@ func (s *Service) scroll( return s.actionResult(app, AXServiceRequest{ App: app, Action: axScrollCommand{ - Direction: normalized, Pages: pages, WindowCentre: true, + AllowGlobalInput: allowGlobalInput, + Direction: normalized, Pages: pages, WindowCentre: true, }, TargetWindowHandle: targetWindowHandle, ExpectedWindow: expectedWindow, @@ -485,7 +501,7 @@ func (s *Service) scroll( } request := AXServiceRequest{ App: app, - Action: axScrollCommand{Direction: normalized, Pages: pages}, + Action: axScrollCommand{Direction: normalized, Pages: pages, AllowGlobalInput: allowGlobalInput}, TargetWindowHandle: targetWindowHandle, IncludeScreenshot: obs.resolvedScreenshot(), Observe: obs.resolvedMode(), CaptureSnapshot: obs.captureSnapshot, LightReadback: obs.lightReadback, diff --git a/computer-use/internal/core/ax_argument_hint_test.go b/computer-use/internal/core/ax_argument_hint_test.go index e2a73f6..d069326 100644 --- a/computer-use/internal/core/ax_argument_hint_test.go +++ b/computer-use/internal/core/ax_argument_hint_test.go @@ -77,6 +77,7 @@ var axBadCalls = []string{ `{"mode":"act","actions":[{"type":"activate","ref":"w3","window_ref":"w3"}]}`, `{"mode":"act","actions":[{"type":"click","ref":"r??"}]}`, `{"mode":"act","actions":[{"type":"click","ref":"w1"}]}`, + `{"mode":"act","actions":[{"type":"click","ref":"r1","allow_global_input":"false"}]}`, `{"mode":"act","actions":[{"type":"close","ref":"r1"}]}`, `{"mode":"act","actions":[{"type":"click"}]}`, `{"mode":"act","actions":[{"type":"set_value","ref":"r1"}]}`, diff --git a/computer-use/internal/core/ax_batch.go b/computer-use/internal/core/ax_batch.go index d037eb6..a73fdfc 100644 --- a/computer-use/internal/core/ax_batch.go +++ b/computer-use/internal/core/ax_batch.go @@ -9,8 +9,9 @@ import ( ) type axBatchAction struct { - actionType string - targetRef string + actionType string + targetRef string + allowGlobalInput *bool // toRef is the range end of a select action. toRef string direction string @@ -42,7 +43,7 @@ func (s *Service) executeAXBatchAction( obs, ).withEvidenceScope("delivery_or_local_ui") case "click": - return s.click( + return s.clickWithGlobalInput( context.AppToken, action.targetRef, noDeclaredTarget, @@ -52,6 +53,7 @@ func (s *Service) executeAXBatchAction( "left", "auto", obs, + action.allowGlobalInput, ).withEvidenceScope("delivery_or_local_ui") case "keypress": return s.pressKey( @@ -64,7 +66,7 @@ func (s *Service) executeAXBatchAction( obs, ).withEvidenceScope("delivery_or_local_ui") case "scroll": - return s.scroll( + return s.scrollWithGlobalInput( context.AppToken, action.direction, action.targetRef, @@ -73,6 +75,7 @@ func (s *Service) executeAXBatchAction( context.expectedWindow, action.pages, obs, + action.allowGlobalInput, ).withEvidenceScope("delivery_or_local_ui") case "set_value": return s.setValue( @@ -115,6 +118,16 @@ func (s *Service) axBatchDecoded( windowRef string, ) ToolCallResult { requested := len(actions) + if s.platformBackend().HostInfo().Platform != "linux" { + for index, action := range actions { + if action.allowGlobalInput != nil { + return axArgErrorResult(axInvalid( + fmt.Sprintf("actions[%d].allow_global_input", index), + "allow_global_input is supported only for Linux AX click and scroll", "", + ), requested) + } + } + } if axBatchOnlyWaits(actions) { return s.axWaitOnly(actions, obs, windowRef) } @@ -186,7 +199,17 @@ func (s *Service) axBatchRun( focus *axDesktopWindow, requested int, trace *actReportTrace, -) ToolCallResult { +) (batchResult ToolCallResult) { + var synthesisNotes []string + defer func() { + text := ResultText(batchResult) + for _, note := range synthesisNotes { + if !strings.Contains(text, note) { + batchResult.Content = append(batchResult.Content, ContentItem{Type: "text", Text: note}) + text += "\n" + note + } + } + }() if err := s.preflightPlatformAXKeys(actions); err != nil { return axBatchPreflightError(err, requested) } @@ -357,7 +380,15 @@ func (s *Service) axBatchRun( result.IsError = false actionEntry["call_error"] = false } + for _, line := range strings.Split(ResultText(result), "\n") { + if note := strings.TrimSpace(line); strings.HasPrefix(note, "Note: [a11y][synthesis]") { + synthesisNotes = append(synthesisNotes, note) + } + } if result.IsError { + if axPublicErrorCode(result.StructuredContent) == contract.AXCodeGlobalInputForbidden { + result.StructuredContent[contract.AXPublicErrorCodeKey] = contract.AXCodeGlobalInputForbidden + } actionEvidence = append(actionEvidence, actionEntry) anyInputEmitted, anyPartialPossible, anyMutationUnknown := batchMutationSummary( diff --git a/computer-use/internal/core/ax_batch_result.go b/computer-use/internal/core/ax_batch_result.go index cca45ef..664f51b 100644 --- a/computer-use/internal/core/ax_batch_result.go +++ b/computer-use/internal/core/ax_batch_result.go @@ -83,7 +83,8 @@ func compactAXActionContent(result ToolCallResult) ToolCallResult { kept := make([]string, 0, len(lines)) for _, line := range lines { trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "Note:") { + if strings.HasPrefix(trimmed, "Note:") && + !strings.HasPrefix(strings.TrimSpace(strings.TrimPrefix(trimmed, "Note:")), "[a11y][synthesis]") { drop := false for _, marker := range routine { if strings.Contains(trimmed, marker) { diff --git a/computer-use/internal/core/ax_batch_result_test.go b/computer-use/internal/core/ax_batch_result_test.go index 4258275..1092195 100644 --- a/computer-use/internal/core/ax_batch_result_test.go +++ b/computer-use/internal/core/ax_batch_result_test.go @@ -22,6 +22,73 @@ func (backend *envelopeFailureBackend) ResolveAndAct(request ProviderActionReque return backend.reportBackend.ResolveAndAct(request) } +func TestGlobalInputForbiddenKeepsCauseAndBatchFacts(t *testing.T) { + for _, unknown := range []bool{false, true} { + for _, failAt := range []int{1, 2} { + service, desktop := reportService(t, reportMain()) + _, refs := reportObserve(t, service, "OK") + code, phase := contract.AXCodeGlobalInputForbidden, "precondition" + if unknown { + code, phase = contract.AXCodeEffectUnknown, "emission" + } + backend := &envelopeFailureBackend{ + reportBackend: desktop, failAt: failAt, + failure: ProviderAccessibilityResult{ + Error: "semantic route unavailable or unverified; no global fallback", + Failure: &ProviderFailure{Code: code, Phase: phase, FactsComplete: true}, + Mutation: &ProviderMutationFacts{InputEventsEmitted: unknown, MutationUnknown: unknown}, + }, + } + service.Backend = &globalInputPolicyBackend{backend, "linux"} + result := PrepareAXEvidence(service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{ + map[string]any{"type": "click", "ref": refs["OK"], "allow_global_input": false}, + map[string]any{"type": "click", "ref": refs["OK"], "allow_global_input": false}, + }, + })) + public := result.StructuredContent + if !result.IsError || public["code"] != code || public["completed"] != failAt-1 || + public["input_emitted"] != (unknown || failAt > 1) { + t.Fatalf("failed action/prefix evidence lost: %#v, %s", public, ResultText(result)) + } + if backend.attempts != failAt { + t.Fatalf("failure retried or later action ran: %d attempts", backend.attempts) + } + } + } +} + +type synthesisNoteBackend struct { + *reportBackend + note string +} + +func (backend *synthesisNoteBackend) ResolveAndAct(request ProviderActionRequest) (ProviderAccessibilityResult, error) { + result, err := backend.reportBackend.ResolveAndAct(request) + if len(backend.actionRequests) == 1 { + result.Notes = []string{backend.note} + } + return result, err +} + +func TestAXBatchRetainsEarlierSynthesisWarning(t *testing.T) { + for _, observe := range []string{"none", "diff", "tree"} { + service, desktop := reportService(t, reportMain()) + _, refs := reportObserve(t, service, "OK", "Size") + note := "[a11y][synthesis] Synthesized click. Global input may change focus; verify the effect." + service.Backend = &synthesisNoteBackend{desktop, note} + result := PrepareAXEvidence(service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "observe": observe, "actions": []any{ + map[string]any{"type": "click", "ref": refs["OK"]}, + map[string]any{"type": "set_value", "ref": refs["Size"], "value": "14"}, + }, + })) + if result.IsError || strings.Count(ResultText(result), note) != 1 { + t.Fatalf("%s lost or duplicated earlier synthesis warning: %s", observe, ResultText(result)) + } + } +} + func TestRuntimeFailureEnvelopeKeepsCauseAndBatchFacts(t *testing.T) { for _, failure := range []struct { code, phase, reason, publicCode string diff --git a/computer-use/internal/core/ax_batch_test.go b/computer-use/internal/core/ax_batch_test.go index 808b2a8..93ab3e1 100644 --- a/computer-use/internal/core/ax_batch_test.go +++ b/computer-use/internal/core/ax_batch_test.go @@ -8,6 +8,147 @@ import ( "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) +type globalInputPolicyBackend struct { + PlatformBackend + platform string +} + +func (backend *globalInputPolicyBackend) HostInfo() HostBackendInfo { + return HostBackendInfo{Platform: backend.platform} +} + +func TestAXGlobalInputPolicyRejectsWholeInvalidBatch(t *testing.T) { + tests := []struct { + name, platform string + action map[string]any + }{} + for _, actionType := range contract.AXActionTypes { + if actionType == "click" || actionType == "scroll" { + continue + } + for _, policy := range []bool{false, true} { + tests = append(tests, struct { + name, platform string + action map[string]any + }{actionType + "/" + contract.CompactJSON(policy), "linux", map[string]any{"type": actionType, "allow_global_input": policy}}) + } + } + for _, actionType := range []string{"click", "scroll"} { + for _, value := range []any{nil, "false", 0.0, 1.0, []any{}, map[string]any{}} { + tests = append(tests, struct { + name, platform string + action map[string]any + }{actionType + "/" + contract.CompactJSON(value), "linux", map[string]any{ + "type": actionType, "allow_global_input": value, + }}) + } + for _, platform := range []string{"darwin", "windows", "unknown"} { + for _, policy := range []bool{false, true} { + action := map[string]any{"type": actionType, "ref": "r1", "allow_global_input": policy} + if actionType == "scroll" { + action["direction"] = "down" + } + tests = append(tests, struct { + name, platform string + action map[string]any + }{platform + "/" + actionType + "/" + contract.CompactJSON(policy), platform, action}) + } + } + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &recordingPlatformBackend{} + service := NewServiceWithBackend(&globalInputPolicyBackend{backend, test.platform}) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "actions": []any{ + map[string]any{"type": "activate", "title": "Editor"}, test.action, + }, + }) + evidence := axPublicEvidence(result.StructuredContent, result.IsError, result.Content) + if !result.IsError || evidence["code"] != contract.AXCodeInvalidArgument || + evidence["path"] != "actions[1].allow_global_input" || evidence["completed"] != 0 || evidence["input_emitted"] != false { + t.Fatalf("invalid batch was not rejected before input: %#v, %s", evidence, ResultText(result)) + } + if len(backend.calls) != 0 || len(backend.actionRequests) != 0 { + t.Fatalf("invalid batch reached the provider: %v, %v", backend.calls, backend.actionRequests) + } + }) + } +} + +func TestGlobalInputPolicyRejectsWrongToolOrTopLevel(t *testing.T) { + for _, tool := range []string{contract.PublicToolAX, contract.PublicToolGUI} { + for _, topLevel := range []bool{false, true} { + if tool == contract.PublicToolAX && !topLevel { + continue + } + backend := &recordingPlatformBackend{} + service := NewServiceWithBackend(&globalInputPolicyBackend{backend, "linux"}) + action := map[string]any{"type": "click", "x": 1.0, "y": 1.0} + args := map[string]any{"actions": []any{action}} + if tool == contract.PublicToolAX { + args["mode"] = "act" + args["actions"] = []any{map[string]any{"type": "click", "ref": "r1"}} + } + if topLevel { + args["allow_global_input"] = false + } else { + action["allow_global_input"] = false + } + result := service.callTool(tool, args) + if !result.IsError || !strings.Contains(ResultText(result), "allow_global_input") || len(backend.calls) != 0 { + t.Fatalf("unsupported %s policy accepted: %s, calls %v", tool, ResultText(result), backend.calls) + } + } + } +} + +func TestAXGlobalInputPolicyReachesProvider(t *testing.T) { + for _, kind := range []string{"click", "targeted scroll", "window scroll"} { + for _, policy := range []any{nil, false, true} { + t.Run(kind+"/"+contract.CompactJSON(policy), func(t *testing.T) { + service, backend := reportService(t, reportMain()) + service.Backend = &globalInputPolicyBackend{backend, "linux"} + window, refs := reportObserve(t, service, "OK") + action := map[string]any{"type": "scroll", "direction": "down"} + args := map[string]any{"mode": "act", "actions": []any{action}} + if kind == "click" { + action = map[string]any{"type": "click", "ref": refs["OK"]} + args["actions"] = []any{action} + } else if kind == "targeted scroll" { + action["ref"] = refs["OK"] + } else { + args["window_ref"] = window + } + if policy != nil { + action["allow_global_input"] = policy + } + result := service.callTool(contract.PublicToolAX, args) + if result.IsError || len(backend.actionRequests) != 1 { + t.Fatalf("action failed: %s, requests %v", ResultText(result), backend.actionRequests) + } + request := backend.actionRequests[0] + var got *bool + if kind == "click" { + got = request.Click.AllowGlobalInput + } else { + got = request.Scroll.AllowGlobalInput + if request.Scroll.WindowCentre != (kind == "window scroll") { + t.Fatalf("lost scroll target: %#v", request.Scroll) + } + } + if policy == nil { + if got != nil { + t.Fatal("omitted policy was not preserved") + } + } else if got == nil || *got != policy.(bool) { + t.Fatalf("provider lost policy %v: %v", policy, got) + } + }) + } + } +} + func TestCompactAXActionContentDropsRoutineRouteNotes(t *testing.T) { result := compactAXActionContent(ToolCallResult{ Content: []ContentItem{{Type: "text", Text: strings.Join([]string{ diff --git a/computer-use/internal/core/ax_evidence.go b/computer-use/internal/core/ax_evidence.go index 329803b..64aac33 100644 --- a/computer-use/internal/core/ax_evidence.go +++ b/computer-use/internal/core/ax_evidence.go @@ -553,12 +553,13 @@ var axPublicFields = []string{ // axPublicHints are the default corrected calls per error code. var axPublicHints = map[string]string{ - contract.AXCodeStaleRef: `{"mode":"find","window_ref":"wN","query":{"name_contains":"..."}}`, - contract.AXCodeWindowChanged: `{"mode":"observe","window_ref":"wN"}`, - contract.AXCodeWaitTimeout: `{"mode":"observe","window_ref":"wN"}`, - contract.AXCodeEffectUnknown: `{"mode":"observe","window_ref":"wN"}`, - contract.AXCodeAppUnavailable: `{"mode":"apps"}`, - contract.AXCodeRuntimeError: `{"mode":"apps"}`, + contract.AXCodeStaleRef: `{"mode":"find","window_ref":"wN","query":{"name_contains":"..."}}`, + contract.AXCodeWindowChanged: `{"mode":"observe","window_ref":"wN"}`, + contract.AXCodeWaitTimeout: `{"mode":"observe","window_ref":"wN"}`, + contract.AXCodeEffectUnknown: `{"mode":"observe","window_ref":"wN"}`, + contract.AXCodeGlobalInputForbidden: `{"mode":"observe","window_ref":"wN"}`, + contract.AXCodeAppUnavailable: `{"mode":"apps"}`, + contract.AXCodeRuntimeError: `{"mode":"apps"}`, } // axPublicErrorCode maps internal failure facts to one public code. @@ -578,6 +579,13 @@ func axPublicErrorCode(full map[string]any) string { return contract.AXCodeWaitTimeout case failure == "effect_unknown": return contract.AXCodeEffectUnknown + case failure == contract.AXCodeGlobalInputForbidden: + if axEvidenceReportsInputEmission(full) || full["partial_input_possible"] == true || full["mutation_unknown"] == true { + return contract.AXCodeEffectUnknown + } + if phase == "precondition" { + return contract.AXCodeGlobalInputForbidden + } case failure == "ax_app_unavailable" || status == "app_unavailable": return contract.AXCodeAppUnavailable case failure == "ax_app_ambiguous": diff --git a/computer-use/internal/core/ax_public_request.go b/computer-use/internal/core/ax_public_request.go index 9ddbf1b..2712303 100644 --- a/computer-use/internal/core/ax_public_request.go +++ b/computer-use/internal/core/ax_public_request.go @@ -25,6 +25,7 @@ var axPublicErrorCodes = []string{ contract.AXCodeInvalidArgument, contract.AXCodeStaleRef, contract.AXCodeWindowChanged, contract.AXCodeWaitTimeout, contract.AXCodeEffectUnknown, contract.AXCodeAppUnavailable, contract.AXCodeRuntimeError, contract.AXCodeAmbiguousWindow, contract.AXCodeWindowNotFound, contract.AXCodeNoFocusedWindow, + contract.AXCodeGlobalInputForbidden, } // axArgError is a rejected argument. It is returned before any input. @@ -109,12 +110,12 @@ var axModeFields = map[string][]string{ // axActionFields lists the fields each action type accepts besides type. var axActionFields = map[string][]string{ - "click": {"ref"}, + "click": {"ref", "allow_global_input"}, "type": {"ref", "text"}, "keypress": {"ref", "keys"}, "set_value": {"ref", "value"}, "select": {"ref", "to_ref", "text"}, - "scroll": {"ref", "direction", "pages"}, + "scroll": {"ref", "direction", "pages", "allow_global_input"}, "activate": {"ref", "title"}, "close": {"ref", "title"}, "wait": {"window", "query", "ref", "value", "state", "text", "gone", "settle", "timeout_ms"}, @@ -413,6 +414,13 @@ func decodeAXAction(object map[string]any, path string) (axBatchAction, *axArgEr } return action, err.withRule(axRuleFieldForType) } + if raw, present := object["allow_global_input"]; present { + allowed, ok := raw.(bool) + if !ok { + return action, axInvalid(path+".allow_global_input", "allow_global_input must be a boolean", "") + } + action.allowGlobalInput = &allowed + } if actionType == "wait" { wait, err := decodeAXWait(object, path) if err != nil { diff --git a/computer-use/internal/core/ax_request_test.go b/computer-use/internal/core/ax_request_test.go index 32a345e..f51cff5 100644 --- a/computer-use/internal/core/ax_request_test.go +++ b/computer-use/internal/core/ax_request_test.go @@ -7,6 +7,38 @@ import ( "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) +func TestDecodeAXGlobalInputPolicy(t *testing.T) { + for _, actionType := range []string{"click", "scroll"} { + for _, policy := range []any{nil, false, true} { + t.Run(actionType+"/"+contract.CompactJSON(policy), func(t *testing.T) { + action := map[string]any{"type": actionType, "ref": "r1"} + if actionType == "scroll" { + action["direction"] = "down" + } + if policy != nil { + action["allow_global_input"] = policy + } + args := map[string]any{"mode": "act", "actions": []any{action}} + if err := contract.ValidateToolArguments(contract.PublicToolAX, args); err != nil { + t.Fatalf("schema rejected policy: %v", err) + } + request, err := DecodeAXRequest(args) + if err != nil { + t.Fatal(err) + } + got := request.actions[0].allowGlobalInput + if policy == nil { + if got != nil { + t.Fatalf("omitted policy became explicit: %v", *got) + } + } else if got == nil || *got != policy.(bool) { + t.Fatalf("lost explicit policy %v: %v", policy, got) + } + }) + } + } +} + func TestDecodeAXRequestAcceptsEachMode(t *testing.T) { for name, args := range map[string]map[string]any{ "apps": {"mode": "apps"}, diff --git a/computer-use/internal/core/provider_types.go b/computer-use/internal/core/provider_types.go index b7787f1..4fbb2cd 100644 --- a/computer-use/internal/core/provider_types.go +++ b/computer-use/internal/core/provider_types.go @@ -153,17 +153,19 @@ func (limit *ProviderTextLimit) UnmarshalJSON(data []byte) error { } type ProviderClickInput struct { - X *float64 `json:"x,omitempty"` - Y *float64 `json:"y,omitempty"` - ClickCount int `json:"click_count,omitempty"` - MouseButton string `json:"mouse_button,omitempty"` - ClickMethod string `json:"click_method,omitempty"` + AllowGlobalInput *bool `json:"allow_global_input,omitempty"` + X *float64 `json:"x,omitempty"` + Y *float64 `json:"y,omitempty"` + ClickCount int `json:"click_count,omitempty"` + MouseButton string `json:"mouse_button,omitempty"` + ClickMethod string `json:"click_method,omitempty"` } type ProviderScrollInput struct { - Direction string `json:"direction,omitempty"` - Pages float64 `json:"pages,omitempty"` - WindowCentre bool `json:"window_centre,omitempty"` + AllowGlobalInput *bool `json:"allow_global_input,omitempty"` + Direction string `json:"direction,omitempty"` + Pages float64 `json:"pages,omitempty"` + WindowCentre bool `json:"window_centre,omitempty"` } type ProviderTypeInput struct { @@ -295,6 +297,17 @@ func (request *ProviderActionRequest) UnmarshalJSON(data []byte) error { if len(wire.Input) == 0 { return errors.New("provider action input is required") } + if wire.Intent == AccessibilityClick || wire.Intent == AccessibilityScroll { + var policy struct { + AllowGlobalInput json.RawMessage `json:"allow_global_input"` + } + if err := json.Unmarshal(wire.Input, &policy); err != nil { + return err + } + if string(policy.AllowGlobalInput) == "null" { + return errors.New("provider allow_global_input must be a boolean") + } + } return DecodeProviderWireJSON(wire.Input, target) } switch wire.Intent { diff --git a/computer-use/internal/core/provider_wire_test.go b/computer-use/internal/core/provider_wire_test.go index 655d5e0..668426a 100644 --- a/computer-use/internal/core/provider_wire_test.go +++ b/computer-use/internal/core/provider_wire_test.go @@ -45,6 +45,75 @@ func readProviderFixture(t *testing.T, name string) []byte { return data } +func TestProviderLinuxInputFixtureCorpus(t *testing.T) { + var manifest struct { + SchemaVersion int `json:"schema_version"` + Protocol string `json:"protocol"` + Platforms map[string]bool `json:"allow_global_input_platforms"` + Fixtures []providerFixtureEntry `json:"fixtures"` + } + if err := strictFixtureDecode(readProviderFixture(t, "linux-input-manifest.json"), &manifest); err != nil { + t.Fatal(err) + } + if manifest.Protocol != ProviderProtocolName || manifest.SchemaVersion != ProviderProtocolVersion || + !reflect.DeepEqual(manifest.Platforms, map[string]bool{"linux": true, "darwin": false, "windows": false}) { + t.Fatalf("invalid Linux input fixture contract: %#v", manifest) + } + seen := map[AccessibilityOperation]bool{} + for _, fixture := range manifest.Fixtures { + t.Run(fixture.Name, func(t *testing.T) { + data := readProviderFixture(t, fixture.File) + request, err := DecodeProviderWireRequest(data) + if err != nil { + t.Fatal(err) + } + if fixture.Direction != "request" || request.Operation != ProviderWireResolveAndAct || request.Action == nil { + t.Fatal("Linux input fixture must exercise an accessibility action") + } + assertProviderFixtureRoundTrip(t, data, request) + var policy *bool + switch request.Action.Intent { + case AccessibilityClick: + policy = request.Action.Click.AllowGlobalInput + case AccessibilityScroll: + policy = request.Action.Scroll.AllowGlobalInput + if request.Action.Scroll.Pages != 1.5 { + t.Fatal("scroll fixture must preserve fractional pages") + } + default: + t.Fatalf("unexpected fixture intent %s", request.Action.Intent) + } + if policy == nil || *policy { + t.Fatal("fixture must retain explicit false") + } + seen[request.Action.Intent] = true + for platform, supported := range manifest.Platforms { + for _, allowed := range []bool{false, true} { + *policy = allowed + backend := &recordingPlatformBackend{accessibilityResponse: &ProviderAccessibilityResult{OK: true}} + service := NewServiceWithBackend(&globalInputPolicyBackend{backend, platform}) + result, err := service.callProviderAction(*request.Action) + if err != nil { + t.Fatal(err) + } + if supported { + if !result.OK || len(backend.actionRequests) != 1 { + t.Fatalf("Linux fixture was not dispatched: %#v", result) + } + } else if result.OK || len(backend.actionRequests) != 0 || result.Failure == nil || + result.Failure.Code != "invalid_argument" || result.Failure.Phase != "precondition" || + result.Mutation == nil || result.Mutation.InputEventsEmitted { + t.Fatalf("%s silently accepted policy %t: %#v", platform, allowed, result) + } + } + } + }) + } + if !reflect.DeepEqual(seen, map[AccessibilityOperation]bool{AccessibilityClick: true, AccessibilityScroll: true}) { + t.Fatalf("missing Linux input fixtures: %v", seen) + } +} + func TestProviderNativeFixtureCorpus(t *testing.T) { var manifest providerNativeFixtureManifest if err := strictFixtureDecode(readProviderFixture(t, "native-manifest.json"), &manifest); err != nil { diff --git a/computer-use/internal/core/runtime_backend.go b/computer-use/internal/core/runtime_backend.go index 28354f3..f97cd86 100644 --- a/computer-use/internal/core/runtime_backend.go +++ b/computer-use/internal/core/runtime_backend.go @@ -98,6 +98,17 @@ func (s *Service) callProviderAction( request ProviderActionRequest, ) (*ProviderAccessibilityResult, error) { defer s.countProviderTime(time.Now()) + if s.platformBackend().HostInfo().Platform != "linux" && + ((request.Click != nil && request.Click.AllowGlobalInput != nil) || + (request.Scroll != nil && request.Scroll.AllowGlobalInput != nil)) { + result := &ProviderAccessibilityResult{ + Error: "allow_global_input is supported only for Linux AX click and scroll", + Failure: &ProviderFailure{Code: "invalid_argument", Phase: "precondition", FactsComplete: true}, + Mutation: &ProviderMutationFacts{}, + } + result.PrepareForService() + return result, nil + } result, err := s.platformBackend().ResolveAndAct(request) result.PrepareForService() return &result, err diff --git a/computer-use/internal/core/runtime_backend_test.go b/computer-use/internal/core/runtime_backend_test.go index 84e7200..e4d2826 100644 --- a/computer-use/internal/core/runtime_backend_test.go +++ b/computer-use/internal/core/runtime_backend_test.go @@ -8,6 +8,28 @@ import ( "testing" ) +func TestProviderGlobalInputPolicyRejectsNonLinuxBeforeDispatch(t *testing.T) { + for _, platform := range []string{"darwin", "windows", "unknown"} { + for _, policy := range []bool{false, true} { + for _, intent := range []AccessibilityOperation{AccessibilityClick, AccessibilityScroll} { + backend := &recordingPlatformBackend{} + service := NewServiceWithBackend(&globalInputPolicyBackend{backend, platform}) + request := ProviderActionRequest{Intent: intent} + if intent == AccessibilityClick { + request.Click = &ProviderClickInput{AllowGlobalInput: &policy} + } else { + request.Scroll = &ProviderScrollInput{AllowGlobalInput: &policy} + } + result, err := service.callProviderAction(request) + if err != nil || result.OK || result.Failure == nil || result.Failure.Code != "invalid_argument" || + result.Failure.Phase != "precondition" || result.Mutation.InputEventsEmitted || len(backend.actionRequests) != 0 { + t.Fatalf("%s dispatched unsupported %s policy: %#v, %v", platform, intent, result, err) + } + } + } + } +} + func TestRuntimeCallsRouteByPlatformDomain(t *testing.T) { backend := &recordingPlatformBackend{ accessibilityResponse: &ProviderAccessibilityResult{OK: true}, diff --git a/computer-use/internal/core/runtime_protocol.go b/computer-use/internal/core/runtime_protocol.go index 7801102..d4d6330 100644 --- a/computer-use/internal/core/runtime_protocol.go +++ b/computer-use/internal/core/runtime_protocol.go @@ -66,11 +66,12 @@ func (axResolveTargetCommand) applyProviderAction(*ProviderActionRequest) {} func (axResolveTargetCommand) appendTraceFields(map[string]any) {} type AXClickCommand struct { - X *float64 - Y *float64 - ClickCount int - MouseButton string - ClickMethod string + AllowGlobalInput *bool + X *float64 + Y *float64 + ClickCount int + MouseButton string + ClickMethod string } func (AXClickCommand) Intent() AccessibilityOperation { @@ -81,15 +82,19 @@ func (command AXClickCommand) applyProviderAction( request *ProviderActionRequest, ) { request.Click = &ProviderClickInput{ - X: command.X, - Y: command.Y, - ClickCount: command.ClickCount, - MouseButton: command.MouseButton, - ClickMethod: command.ClickMethod, + AllowGlobalInput: command.AllowGlobalInput, + X: command.X, + Y: command.Y, + ClickCount: command.ClickCount, + MouseButton: command.MouseButton, + ClickMethod: command.ClickMethod, } } func (command AXClickCommand) appendTraceFields(fields map[string]any) { + if command.AllowGlobalInput != nil { + fields["allow_global_input"] = *command.AllowGlobalInput + } fields["x"] = command.X fields["y"] = command.Y fields["click_count"] = command.ClickCount @@ -98,9 +103,10 @@ func (command AXClickCommand) appendTraceFields(fields map[string]any) { } type axScrollCommand struct { - Direction string - Pages float64 - WindowCentre bool + AllowGlobalInput *bool + Direction string + Pages float64 + WindowCentre bool } func (axScrollCommand) Intent() AccessibilityOperation { @@ -111,13 +117,17 @@ func (command axScrollCommand) applyProviderAction( request *ProviderActionRequest, ) { request.Scroll = &ProviderScrollInput{ - Direction: command.Direction, - Pages: command.Pages, - WindowCentre: command.WindowCentre, + AllowGlobalInput: command.AllowGlobalInput, + Direction: command.Direction, + Pages: command.Pages, + WindowCentre: command.WindowCentre, } } func (command axScrollCommand) appendTraceFields(fields map[string]any) { + if command.AllowGlobalInput != nil { + fields["allow_global_input"] = *command.AllowGlobalInput + } fields["direction"] = command.Direction fields["pages"] = command.Pages fields["window_centre"] = command.WindowCentre diff --git a/computer-use/internal/core/runtime_protocol_test.go b/computer-use/internal/core/runtime_protocol_test.go new file mode 100644 index 0000000..6239ed2 --- /dev/null +++ b/computer-use/internal/core/runtime_protocol_test.go @@ -0,0 +1,69 @@ +package core + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +func TestAXGlobalInputPolicyWireRoundTrip(t *testing.T) { + for _, intent := range []AccessibilityOperation{AccessibilityClick, AccessibilityScroll} { + for _, value := range []*bool{nil, boolPointer(false), boolPointer(true)} { + var command axActionCommand = AXClickCommand{AllowGlobalInput: value} + if intent == AccessibilityScroll { + command = axScrollCommand{AllowGlobalInput: value, Direction: "down", Pages: 1} + } + request := ProviderActionRequest{ + ProtocolVersion: ProviderProtocolVersion, Intent: intent, App: "Editor", + Target: &ProviderSemanticTarget{Selector: `button "Save"`, HasName: true}, + } + command.applyProviderAction(&request) + wire := NewProviderWireRequest("policy", ProviderWireResolveAndAct) + wire.Action = &request + data, err := json.Marshal(wire) + if err != nil { + t.Fatal(err) + } + var raw struct { + Action struct { + Input map[string]any `json:"input"` + } `json:"action"` + } + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatal(err) + } + serialized, present := raw.Action.Input["allow_global_input"] + if (value != nil) != present || (value != nil && serialized != *value) { + t.Fatalf("wire lost %s policy %v: %s", intent, value, data) + } + decoded, err := DecodeProviderWireRequest(data) + if err != nil { + t.Fatal(err) + } + var actual *bool + if intent == AccessibilityClick { + actual = decoded.Action.Click.AllowGlobalInput + } else { + actual = decoded.Action.Scroll.AllowGlobalInput + } + if !reflect.DeepEqual(actual, value) { + t.Fatalf("wire round trip lost %s policy: got %v, want %v", intent, actual, value) + } + trace := map[string]any{} + command.appendTraceFields(trace) + traced, present := trace["allow_global_input"] + if (value != nil) != present || (value != nil && traced != *value) { + t.Fatalf("trace lost %s policy: %v", intent, trace) + } + if value != nil && !*value { + for _, invalid := range []string{"null", " null ", "0", `"false"`, "[]", "{}"} { + invalidData := strings.Replace(string(data), `"allow_global_input":false`, `"allow_global_input":`+invalid, 1) + if _, err := DecodeProviderWireRequest([]byte(invalidData)); err == nil { + t.Fatalf("wire accepted invalid %s policy %s", intent, invalid) + } + } + } + } + } +} diff --git a/computer-use/internal/core/schema_consistency_test.go b/computer-use/internal/core/schema_consistency_test.go index 33c11ac..9c9f016 100644 --- a/computer-use/internal/core/schema_consistency_test.go +++ b/computer-use/internal/core/schema_consistency_test.go @@ -104,6 +104,10 @@ type probeReached struct{} type probeBackend struct{ *recordingPlatformBackend } +func (probeBackend) HostInfo() HostBackendInfo { + return HostBackendInfo{Platform: "linux"} +} + func (probeBackend) ObserveApps(ProviderObserveAppsRequest) (ProviderAccessibilityResult, error) { panic(probeReached{}) } diff --git a/computer-use/internal/provider/linux_runtime_bundle.go b/computer-use/internal/provider/linux_runtime_bundle.go index 6abf19c..52ebead 100644 --- a/computer-use/internal/provider/linux_runtime_bundle.go +++ b/computer-use/internal/provider/linux_runtime_bundle.go @@ -43,6 +43,7 @@ var linuxProviderPaths = []string{ "native/linux/provider/input_method.py", "native/linux/provider/ax_actions.py", "native/linux/provider/ax_select.py", + "native/linux/provider/ax_semantic_scroll.py", "native/linux/provider/ax_dropdown.py", "native/linux/provider/ax_dispatch.py", "native/linux/provider/ax_input_delivery.py", diff --git a/computer-use/internal/provider/linux_runtime_bundle_test.go b/computer-use/internal/provider/linux_runtime_bundle_test.go index 12e1633..9b434da 100644 --- a/computer-use/internal/provider/linux_runtime_bundle_test.go +++ b/computer-use/internal/provider/linux_runtime_bundle_test.go @@ -28,6 +28,7 @@ func TestLinuxRuntimeBundleIsComplete(t *testing.T) { "native/linux/provider/ax_node_properties.py", "native/linux/provider/ax_records.py", "native/linux/provider/ax_select.py", + "native/linux/provider/ax_semantic_scroll.py", "native/linux/provider/ax_snapshot.py", "native/linux/provider/ax_snapshot_diagnostics.py", "native/linux/provider/ax_target_notes.py", diff --git a/computer-use/native/linux/provider/ax_input_delivery.py b/computer-use/native/linux/provider/ax_input_delivery.py index c9bfe98..427f440 100644 --- a/computer-use/native/linux/provider/ax_input_delivery.py +++ b/computer-use/native/linux/provider/ax_input_delivery.py @@ -52,7 +52,8 @@ ) from .ax_snapshot_diagnostics import is_dropdown_item from .ax_records import gtk_combo_partner -from .ax_select import perform_select +from .ax_select import perform_select, select_item +from .ax_semantic_scroll import semantic_scroll from .ax_dropdown import is_chromium_select @@ -62,6 +63,37 @@ SYNTHESIS = "[synthesis] " +def _require_global_input(action_input): + if action_input.get("allow_global_input") is False: + raise OperationFailure( + "global_input_forbidden: no usable semantic route is available; " + "this action requires driver focus or global pointer/keyboard input. " + "No input was emitted.", + failure_code="global_input_forbidden", failure_phase="precondition", + input_events_emitted=False, failure_subject="element") + + +def _selection_click(element, notes, evidence): + if node_role(element) not in {"page tab", "list item", "table row", "table cell"}: + return False + parent = safe(lambda: element.get_parent()) + if parent is None or safe(lambda: parent.get_selection_iface()) is None: + return False + try: + select_item(element) + except OperationFailure as error: + if error.input_events_emitted: + evidence.update(route="semantic_selection", input_emitted=True, + readback_sampled=True, readback_verified=False) + raise + evidence.update(route="semantic_selection", input_emitted=True, + readback_sampled=True, readback_verified=True) + notes.append(A11Y_CHANNEL + SEMANTIC + "Selected the element through its " + "AT-SPI container and verified selection; no focus or pointer input " + "was sent.") + return True + + def _perform_click( action_input, window, @@ -123,7 +155,11 @@ def _accessibility_click( "click_method 'accessibility' cannot represent click_count > 1; " "use click_method 'auto' or 'global' for a real multi-click" ) - if not do_action_by_index(element, preferred_action_index(element)): + action_index = preferred_action_index(element) + if (action_index is None or node_role(element) == "page tab") and _selection_click( + element, notes, action_evidence): + return + if not do_action_by_index(element, action_index): raise RuntimeError( "click_method 'accessibility' could not click the requested element" ) @@ -147,6 +183,7 @@ def _global_click( notes, action_evidence, ): + _require_global_input(action_input) element_record, _ = require_stable_coordinate_geometry( element, element_record, bounds ) @@ -171,7 +208,7 @@ def _global_click( addressing_channel(element_record) + SYNTHESIS + "Synthesized {} coordinate click(s) {} after bringing the window " - "to the foreground. {}{}" + "to the foreground and moving the real pointer. {}{}" .format( click_count, frame_point_text(x, y), @@ -205,6 +242,9 @@ def _auto_click( and not is_dropdown_item(element, window) ) action_index = preferred_action_index(element) if semantic_click else None + if (semantic_click and (action_index is None or node_role(element) == "page tab") + and _selection_click(element, notes, action_evidence)): + return element_record if ( action_index is not None and node_role(element) in {"entry", "text", "password text"} @@ -217,6 +257,7 @@ def _auto_click( # Entry.activate means submit/Enter, not editor focus. In a freshly # opened dialog it can leave focus on an outer row, consuming the # start of a following untargeted type action outside the editor. + _require_global_input(action_input) try: focus_target_element( window, element, element_record, "editable text click") @@ -243,6 +284,7 @@ def _auto_click( state_only_note = "" if semantic_click: if action_only_sets_state(element, action_index): + _require_global_input(action_input) pointer_record, refused = _pointer_click_for_state_only_action( window, bounds, element, element_record, notes, action_evidence) @@ -256,6 +298,14 @@ def _auto_click( "changes.".format(refused) ) handled = do_action_by_index(element, action_index) + if action_index is not None and not handled and action_input.get("allow_global_input") is False: + action_evidence.update(route="semantic_action", input_emitted=True, + readback_verified=False) + raise OperationFailure( + "The AT-SPI action was attempted but did not confirm delivery. " + "No global input was sent. Observe before retrying.", + failure_code="effect_unknown", failure_phase="emission", + input_events_emitted=True, failure_subject="element") if handled: action_evidence["route"] = "semantic_action" action_evidence["input_emitted"] = True @@ -289,6 +339,7 @@ def _auto_click( failure_phase="precondition", input_events_emitted=False, ) + _require_global_input(action_input) element_record, _ = require_stable_coordinate_geometry( element, element_record, bounds ) @@ -334,6 +385,7 @@ def _auto_click( addressing_channel(element_record) + SYNTHESIS + reason + + "This route brings the window to the foreground and moves the real pointer. " + UNVERIFIED_SYNTHESIS ) return element_record @@ -384,7 +436,7 @@ def _pointer_click_for_state_only_action( A11Y_CHANNEL + SYNTHESIS + "Clicked the element with the pointer {} after checking that " - "the window is in front " + "the window is in front; this moves the real pointer " "and the point is on this element. Its only AT-SPI action is a " "toolkit toggle that changes the checked state without the click the " "application listens for, so a real click was needed. {}".format( @@ -404,6 +456,9 @@ def _perform_scroll( ): direction = action_input.get("direction", "down") pages = action_input.get("pages", 1) + if semantic_scroll(element, direction, pages, notes, action_evidence): + return element_record + _require_global_input(action_input) point = None if action_input.get("window_centre") and bounds is not None: point = ( @@ -437,7 +492,7 @@ def _perform_scroll( "element you named. Second, one page is APPROXIMATED as {} notches; " "the real distance is whatever the application's scroll settings say, " "so do not treat pages=1 as exactly one Page_Down. This also moves the " - "real pointer. {}".format( + "real pointer and brings the window to the foreground. {}".format( int(math.ceil(float(pages or 1))) * WHEEL_CLICKS_PER_PAGE, frame_point_text(*point), # State where the position came from: without a target ref diff --git a/computer-use/native/linux/provider/ax_semantic_scroll.py b/computer-use/native/linux/provider/ax_semantic_scroll.py new file mode 100644 index 0000000..fcd325b --- /dev/null +++ b/computer-use/native/linux/provider/ax_semantic_scroll.py @@ -0,0 +1,90 @@ +"""Relative scrolling through an ancestor scrollbar, with exact value readback.""" + +import math + +from .foundation import Atspi, OperationFailure, node_role, safe, state_contains +from .ax_actions import WHEEL_CLICKS_PER_PAGE +from .ax_select import _wait_until + + +def _scrollbar(node, direction): + orientation = (Atspi.StateType.HORIZONTAL if direction in ("left", "right") + else Atspi.StateType.VERTICAL) + current = node + for _ in range(16): + if current is None: + break + role = node_role(current) + if role == "scroll bar": + candidates = [current] + elif role in {"scroll pane", "viewport"}: + count = safe(lambda: int(current.get_child_count()), 0) + if count > 64: + return None + candidates = [safe(lambda: current.get_child_at_index(index)) + for index in range(count)] + else: + candidates = [] + matches = [candidate for candidate in candidates + if candidate is not None and node_role(candidate) == "scroll bar" + and state_contains(candidate, orientation)] + if len(matches) == 1: + return matches[0] + if len(matches) > 1 or role in {"scroll pane", "viewport", "frame", "dialog", "window", "application"}: + return None + current = safe(lambda: current.get_parent()) + return None + + +def semantic_scroll(element, direction, pages, notes, evidence): + scrollbar = _scrollbar(element, direction) + value = safe(lambda: scrollbar.get_value_iface()) if scrollbar is not None else None + if value is None: + return False + try: + minimum = float(Atspi.Value.get_minimum_value(value)) + maximum = float(Atspi.Value.get_maximum_value(value)) + before = float(Atspi.Value.get_current_value(value)) + increment = float(Atspi.Value.get_minimum_increment(value)) + except Exception: + return False + if (not all(math.isfinite(number) for number in (minimum, maximum, before, increment)) + or maximum < minimum or not minimum <= before <= maximum or increment <= 0): + return False + sign = -1 if direction in ("up", "left") else 1 + requested = min(maximum, max(minimum, before + sign * float(pages) * + WHEEL_CLICKS_PER_PAGE * increment)) + evidence.update(route="semantic_value", readback_sampled=True, + readback_verified=False, input_emitted=False) + if requested == before: + if before != (minimum if sign < 0 else maximum): + raise OperationFailure( + "The requested scroll amount is too small to change the range value. " + "No input was emitted.", failure_code="invalid_argument", + failure_phase="precondition", input_events_emitted=False) + evidence["readback_verified"] = True + notes.append("[a11y][state] scroll_boundary=true. " + "Verified the scrollbar value. No input was emitted.") + return True + evidence["input_emitted"] = True + def readback_matches(): + after = float(Atspi.Value.get_current_value(value)) + return after != before and math.isclose(after, requested, rel_tol=1e-9, abs_tol=1e-9) + + try: + accepted = bool(Atspi.Value.set_current_value(value, requested)) + verified = accepted and _wait_until(readback_matches, 0.6) + except Exception: + verified = False + if not verified: + raise OperationFailure( + "The scrollbar value update was attempted but its requested value could " + "not be verified. No global input was sent. Observe before retrying.", + failure_code="effect_unknown", failure_phase="readback", + input_events_emitted=True, failure_subject="element") + evidence["readback_verified"] = True + notes.append("[a11y][semantic] Scrolled through the ancestor scrollbar Value " + "interface and verified {} -> {}. Each page approximates {} range " + "increments. No focus or pointer input was sent.".format( + before, requested, WHEEL_CLICKS_PER_PAGE)) + return True diff --git a/computer-use/native/linux/provider/wire_request.py b/computer-use/native/linux/provider/wire_request.py index 1bfdc82..a4e1266 100644 --- a/computer-use/native/linux/provider/wire_request.py +++ b/computer-use/native/linux/provider/wire_request.py @@ -83,8 +83,8 @@ def _strict_object( PREDICATE_FIELDS = {"kind", "value"} ACTION_INPUT_CONTRACTS = { "resolve_target": (set(), set()), - "click": ({"x", "y", "click_count", "mouse_button", "click_method"}, set()), - "scroll": ({"direction", "pages", "window_centre"}, {"direction", "pages"}), + "click": ({"x", "y", "click_count", "mouse_button", "click_method", "allow_global_input"}, set()), + "scroll": ({"direction", "pages", "window_centre", "allow_global_input"}, {"direction", "pages"}), "type": ({"text"}, {"text"}), "keypress": ({"key"}, {"key"}), "set_value": ({"value", "require_semantic_write"}, {"value"}), @@ -474,6 +474,9 @@ def _validate_action_input(payload): "invalid_provider_request", "untargeted action.{} requires window_handle".format(intent), ) + if "allow_global_input" in input_object and not isinstance(input_object["allow_global_input"], bool): + raise ProviderRequestError( + "invalid_provider_request", "action.input.allow_global_input must be a boolean") if intent == "scroll": if input_object["direction"] not in {"up", "down", "left", "right"}: raise ProviderRequestError( diff --git a/computer-use/runtime_test.py b/computer-use/runtime_test.py index 529f712..cb83a43 100644 --- a/computer-use/runtime_test.py +++ b/computer-use/runtime_test.py @@ -24,6 +24,7 @@ from runtime_tests.test_capabilities import * # noqa: F401,F403 from runtime_tests.test_combo_edit import * # noqa: F401,F403 from runtime_tests.test_editable_click import * # noqa: F401,F403 +from runtime_tests.test_background_input import * from runtime_tests.test_select_boundaries import * # noqa: F401,F403 from runtime_tests.test_gtk_combo import * # noqa: F401,F403 from runtime_tests.test_chromium_dropdown import * # noqa: F401,F403 diff --git a/computer-use/runtime_test_support.py b/computer-use/runtime_test_support.py index 3a3a2f4..ae058f3 100644 --- a/computer-use/runtime_test_support.py +++ b/computer-use/runtime_test_support.py @@ -30,6 +30,7 @@ ax_dispatch, ax_dropdown, ax_input_delivery, + ax_semantic_scroll, ax_actions, ax_node_properties, ax_records, @@ -116,6 +117,7 @@ ax_dropdown, ax_dispatch, ax_input_delivery, + ax_semantic_scroll, ax_snapshot_diagnostics, ax_target_notes, ax_transition_evidence, diff --git a/computer-use/runtime_tests/test_background_input.py b/computer-use/runtime_tests/test_background_input.py new file mode 100644 index 0000000..0bd3cf3 --- /dev/null +++ b/computer-use/runtime_tests/test_background_input.py @@ -0,0 +1,269 @@ +"""Semantic click/scroll delivery and pre-emission global-input refusal.""" + +from runtime_test_support import * +from runtime_tests.test_capabilities import _Node + + +class BackgroundInputTests(AtspiPatchedTestCase): + def setUp(self): + super().setUp() + self.notes, self.evidence = [], {} + self.focus = mock.Mock() + self.pointer = mock.Mock() + self.wheel = mock.Mock(return_value="wheel") + self.geometry = mock.Mock(side_effect=lambda node, record, bounds: (record, True)) + for name, replacement in ( + ("require_coordinate_delivery", self.focus), + ("require_window_focus", self.focus), + ("focus_target_element", self.focus), + ("send_mouse_click", self.pointer), + ("scroll_element", self.wheel), + ("screen_point", mock.Mock(return_value=(10, 20))), + ("require_stable_coordinate_geometry", self.geometry), + ("preferred_action_index", mock.Mock(return_value=None)), + ("do_action_by_index", mock.Mock(return_value=False)), + ("is_dropdown_item", mock.Mock(return_value=False)), + ): + patch = mock.patch.object(runtime, name, replacement) + patch.start() + self.addCleanup(patch.stop) + + def click(self, node, **kwargs): + return runtime._perform_click(kwargs, object(), {}, node, + {"frame": {"x": 0}}, self.notes, self.evidence) + + def scroll(self, node, **kwargs): + return runtime._perform_scroll(kwargs, object(), {}, node, + {"frame": {"x": 0}}, self.notes, self.evidence) + + def assert_quiet(self): + self.geometry.assert_not_called() + self.focus.assert_not_called() + self.pointer.assert_not_called() + self.wheel.assert_not_called() + + def tab(self): + parent = _Node(role="page tab list") + parent.get_selection_iface = lambda: parent + tab = _Node(role="page tab", states=(STATE.SELECTABLE,), parent=parent) + parent.add(tab) + selection = mock.Mock() + selection.select_child.side_effect = lambda *_: tab.states.add(STATE.SELECTED) or True + selection.is_child_selected.side_effect = lambda *_: STATE.SELECTED in tab.states + patch = mock.patch.object(runtime.Atspi, "Selection", selection, create=True) + patch.start() + self.addCleanup(patch.stop) + return tab, selection + + def range_node(self, orientation=STATE.VERTICAL, current=20, increment=3): + pane = _Node(role="scroll pane") + target = _Node(role="table cell", parent=pane) + bar = _Node(role="scroll bar", states=(orientation,), parent=pane) + pane.add(target) + pane.add(bar) + value = FakeValueIface(current) + bar.get_value_iface = lambda: value + for name, result in (("get_minimum_value", 0), ("get_maximum_value", 100), + ("get_minimum_increment", increment)): + patch = mock.patch.object(runtime.Atspi.Value, name, return_value=result, create=True) + patch.start() + self.addCleanup(patch.stop) + return target, bar, value + + def test_tab_without_action_uses_selection_and_reads_back(self): + tab, selection = self.tab() + self.click(tab, allow_global_input=False) + selection.select_child.assert_called_once() + self.assertIn(STATE.SELECTED, tab.states) + self.assertTrue(self.evidence["readback_verified"]) + self.assert_quiet() + + def test_lying_selection_never_retries_with_pointer(self): + tab, selection = self.tab() + selection.select_child.side_effect = None + selection.select_child.return_value = True + with mock.patch.object(runtime, "_wait_until", side_effect=lambda predicate, timeout: predicate()): + with self.assertRaises(runtime.OperationFailure) as caught: + self.click(tab) + self.assertEqual(caught.exception.failure_code, "effect_unknown") + self.assertTrue(caught.exception.input_events_emitted) + self.assert_quiet() + + def test_pointer_fallback_is_refused_before_focus_or_geometry(self): + for method in ("auto", "global"): + with self.subTest(method=method): + with self.assertRaises(runtime.OperationFailure) as caught: + self.click(FakeNode(), click_method=method, allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "global_input_forbidden") + self.assertFalse(caught.exception.input_events_emitted) + self.assert_quiet() + + def test_editor_focus_is_forbidden_before_grab_focus(self): + editor = FakeNode(states=(STATE.EDITABLE,), text=FakeTextIface("old")) + editor.get_role_name = lambda: "entry" + editor.get_action_name = lambda index: "activate" + with mock.patch.object(runtime, "preferred_action_index", return_value=0): + with self.assertRaises(runtime.OperationFailure) as caught: + self.click(editor, allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "global_input_forbidden") + self.assert_quiet() + + def test_state_only_pointer_route_is_forbidden(self): + with mock.patch.object(runtime, "preferred_action_index", return_value=0), \ + mock.patch.object(runtime, "action_only_sets_state", return_value=True): + with self.assertRaises(runtime.OperationFailure) as caught: + self.click(FakeNode(), allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "global_input_forbidden") + self.assert_quiet() + + def test_scroll_ancestor_range_reads_back_without_global_input(self): + target, bar, value = self.range_node() + self.scroll(target, direction="down", pages=1.5, allow_global_input=False) + self.assertEqual(value.current, 42.5) + self.assertEqual(value.set_calls, [42.5]) + self.assertTrue(self.evidence["readback_verified"]) + self.assert_quiet() + + def test_horizontal_scroll_uses_horizontal_range(self): + target, bar, value = self.range_node(STATE.HORIZONTAL) + self.scroll(target, direction="left", pages=1, allow_global_input=False) + self.assertEqual(value.current, 5) + self.assert_quiet() + + def test_scroll_clamps_to_boundary_without_repeating_input(self): + target, bar, value = self.range_node(current=96) + self.scroll(target, direction="down", pages=1) + self.scroll(target, direction="down", pages=1) + self.assertEqual(value.set_calls, [100]) + self.assertFalse(self.evidence["input_emitted"]) + self.assertTrue(self.evidence["readback_verified"]) + self.assert_quiet() + + def test_lying_range_returns_unknown_without_global_retry(self): + target, bar, value = self.range_node() + value.honest = False + with mock.patch.object(runtime, "_wait_until", side_effect=lambda predicate, timeout: predicate()): + with self.assertRaises(runtime.OperationFailure) as caught: + self.scroll(target, direction="down", pages=1) + self.assertEqual(caught.exception.failure_code, "effect_unknown") + self.assertTrue(caught.exception.input_events_emitted) + self.assertFalse(self.evidence["readback_verified"]) + self.assert_quiet() + + def test_missing_or_unusable_range_refuses_without_input(self): + target, bar, value = self.range_node(increment=float("nan")) + for node in (FakeNode(), target): + with self.subTest(node=node): + with self.assertRaises(runtime.OperationFailure) as caught: + self.scroll(node, allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "global_input_forbidden") + self.assertFalse(caught.exception.input_events_emitted) + self.assertEqual(value.set_calls, []) + self.assert_quiet() + + def test_default_keeps_explicit_foreground_fallback(self): + self.scroll(FakeNode()) + self.focus.assert_called_once() + self.wheel.assert_called_once() + self.assertTrue(any("[synthesis]" in note for note in self.notes)) + + def test_attempted_action_failure_is_not_a_no_input_refusal(self): + with mock.patch.object(runtime, "preferred_action_index", return_value=0): + with self.assertRaises(runtime.OperationFailure) as caught: + self.click(FakeNode(), allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "effect_unknown") + self.assertTrue(caught.exception.input_events_emitted) + self.assert_quiet() + + def test_ambiguous_scrollbars_refuse_without_choosing_one(self): + target, bar, value = self.range_node() + target.parent.add(_Node(role="scroll bar", states=(STATE.VERTICAL,))) + with self.assertRaises(runtime.OperationFailure): + self.scroll(target, allow_global_input=False) + self.assertEqual(value.set_calls, []) + self.assert_quiet() + + def test_nearest_scroll_pane_does_not_bubble_to_outer_scrollbar(self): + target, bar, value = self.range_node() + inner = _Node(role="scroll pane", parent=target.parent) + nested = _Node(role="table cell", parent=inner) + with self.assertRaises(runtime.OperationFailure): + self.scroll(nested, allow_global_input=False) + self.assertEqual(value.set_calls, []) + self.assert_quiet() + + def test_range_mutates_then_throws_retains_emission_evidence(self): + target, bar, value = self.range_node() + def lost_reply(iface, requested): + iface.current = requested + raise RuntimeError("reply lost") + with mock.patch.object(runtime.Atspi.Value, "set_current_value", side_effect=lost_reply): + with self.assertRaises(runtime.OperationFailure) as caught: + self.scroll(target, allow_global_input=False) + self.assertTrue(caught.exception.input_events_emitted) + self.assertEqual(caught.exception.failure_code, "effect_unknown") + self.assertEqual(value.current, 35) + self.assert_quiet() + + def test_sibling_range_in_generic_panel_is_not_scroll_ownership(self): + target, bar, value = self.range_node() + target.parent.role = "panel" + with self.assertRaises(runtime.OperationFailure): + self.scroll(target, allow_global_input=False) + self.assertEqual(value.set_calls, []) + self.assert_quiet() + + def test_unrepresentable_delta_is_not_reported_as_a_boundary(self): + target, bar, value = self.range_node() + with self.assertRaises(runtime.OperationFailure) as caught: + self.scroll(target, direction="down", pages=1e-300, allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "invalid_argument") + self.assertFalse(caught.exception.input_events_emitted) + self.assertEqual(value.set_calls, []) + self.assert_quiet() + + def test_unchanged_value_within_tolerance_is_not_verified(self): + target, bar, value = self.range_node(increment=1e-11) + value.honest = False + with mock.patch.object(runtime, "_wait_until", side_effect=lambda predicate, timeout: predicate()): + with self.assertRaises(runtime.OperationFailure) as caught: + self.scroll(target, direction="down", pages=1, allow_global_input=False) + self.assertEqual(caught.exception.failure_code, "effect_unknown") + self.assertTrue(caught.exception.input_events_emitted) + self.assert_quiet() + + +class BackgroundInputWireTests(unittest.TestCase): + def test_linux_policy_fixtures_reach_dispatch_unchanged(self): + root = Path(__file__).parents[1] / "testdata" / "provider-wire" + manifest = json.loads((root / "linux-input-manifest.json").read_text()) + self.assertTrue(manifest["allow_global_input_platforms"]["linux"]) + for fixture in manifest["fixtures"]: + with self.subTest(fixture=fixture["name"]): + request = json.loads((root / fixture["file"]).read_text()) + with mock.patch.object(wire, "_initialize_provider"), \ + mock.patch.object(wire, "perform_provider_request", return_value={"ok": True}) as dispatch: + response = wire._execute_serialized_request(json.dumps(request)) + self.assertTrue(response["accessibility"]["ok"]) + dispatch.assert_called_once_with("resolve_and_act", request["action"]) + self.assertIs(dispatch.call_args.args[1]["input"]["allow_global_input"], False) + + def test_policy_requires_a_boolean(self): + for intent in ("click", "scroll"): + for policy in (None, "false", 0, 1, [], {}): + with self.subTest(intent=intent, policy=policy): + payload = {"intent": intent, "target": {}, "input": { + "allow_global_input": policy}} + if intent == "scroll": + payload["input"].update(direction="down", pages=1) + with self.assertRaises(wire_request.ProviderRequestError): + wire_request._validate_action_input(payload) + + def test_policy_absent_true_and_false_are_accepted(self): + for intent in ("click", "scroll"): + for policy in ({}, {"allow_global_input": True}, {"allow_global_input": False}): + with self.subTest(intent=intent, policy=policy): + payload = {"intent": intent, "target": {}, "input": dict(policy)} + if intent == "scroll": + payload["input"].update(direction="down", pages=1) + wire_request._validate_action_input(payload) diff --git a/computer-use/testdata/contract/public_contract.json b/computer-use/testdata/contract/public_contract.json index 4aac468..c7c1cc1 100644 --- a/computer-use/testdata/contract/public_contract.json +++ b/computer-use/testdata/contract/public_contract.json @@ -94,6 +94,11 @@ "items": { "additionalProperties": false, "properties": { + "allow_global_input": { + "default": true, + "description": "Linux click/scroll only. false forbids driver focus/raise and pointer/keyboard synthesis. Semantic actions may themselves change app focus. Unsupported uses reject the batch before input.", + "type": "boolean" + }, "direction": { "enum": [ "up", diff --git a/computer-use/testdata/provider-wire/background-click.request.json b/computer-use/testdata/provider-wire/background-click.request.json new file mode 100644 index 0000000..bb7b7f5 --- /dev/null +++ b/computer-use/testdata/provider-wire/background-click.request.json @@ -0,0 +1,25 @@ +{ + "protocol": "computer-use-provider", + "schema_version": 2, + "request_id": "background-click", + "operation": "resolve_and_act", + "action": { + "protocol_version": 2, + "intent": "click", + "app": "Editor", + "target": { + "selector": "page tab \"Controls\"", + "windowHandle": "w2:0123456789abcdef0123456789abcdef", + "role": "page tab", + "name": "Controls", + "hasName": true + }, + "input": { + "allow_global_input": false, + "click_count": 1, + "mouse_button": "left", + "click_method": "auto" + }, + "observation": {} + } +} diff --git a/computer-use/testdata/provider-wire/background-scroll.request.json b/computer-use/testdata/provider-wire/background-scroll.request.json new file mode 100644 index 0000000..9086193 --- /dev/null +++ b/computer-use/testdata/provider-wire/background-scroll.request.json @@ -0,0 +1,24 @@ +{ + "protocol": "computer-use-provider", + "schema_version": 2, + "request_id": "background-scroll", + "operation": "resolve_and_act", + "action": { + "protocol_version": 2, + "intent": "scroll", + "app": "Editor", + "target": { + "selector": "scroll bar \"Items\"", + "windowHandle": "w2:0123456789abcdef0123456789abcdef", + "role": "scroll bar", + "name": "Items", + "hasName": true + }, + "input": { + "allow_global_input": false, + "direction": "down", + "pages": 1.5 + }, + "observation": {} + } +} diff --git a/computer-use/testdata/provider-wire/linux-input-manifest.json b/computer-use/testdata/provider-wire/linux-input-manifest.json new file mode 100644 index 0000000..d45bb88 --- /dev/null +++ b/computer-use/testdata/provider-wire/linux-input-manifest.json @@ -0,0 +1,9 @@ +{ + "schema_version": 2, + "protocol": "computer-use-provider", + "allow_global_input_platforms": {"linux": true, "darwin": false, "windows": false}, + "fixtures": [ + {"name": "background-click", "direction": "request", "file": "background-click.request.json"}, + {"name": "background-scroll", "direction": "request", "file": "background-scroll.request.json"} + ] +} diff --git a/docs/MCP.md b/docs/MCP.md index 1998a8b..1bf477a 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -498,7 +498,7 @@ change canonical text, selectors, refs, or the response budget. ### `act` steps - `act` accepts up to 64 ordered steps: actions - `{type, ref|title, text|keys|value|direction|pages|to_ref}` and `wait` + `{type, ref|title, text|keys|value|direction|pages|to_ref, allow_global_input}` and `wait` steps. Structural argument errors reject the whole batch before input. The first step that fails stops the batch, and the result says how many ran. - The `rN` or `wN` refs name the window, so an `act` call with refs needs no @@ -581,6 +581,60 @@ statement inside that follow-up describes the read, not the earlier action. Stale-window follow-ups report the unavailable source reference without repeating an input-delivery statement. +### Linux click and scroll input policy + +Linux AX `click` and `scroll` accept an optional per-action boolean +`allow_global_input`. It defaults to `true`. Set it to `false` to forbid +driver-requested focus changes, window activation or raising, and synthesized +pointer or keyboard input for that action. This includes focus preparation +for editors and pointer input used only to synchronize Qt control state. +The policy does not promise that application focus stays unchanged. An +accessibility action can itself open or raise a dialog or change app focus. + +```json +{"mode":"act","actions":[{"type":"click","ref":"r12","allow_global_input":false},{"type":"scroll","ref":"r15","direction":"down","pages":1,"allow_global_input":false}]} +``` + +The field is supported only on Linux and only inside `click` or `scroll` +actions. A non-boolean value, another action type, a top-level field, or use +on another platform returns `invalid_argument`. These errors reject the +whole batch before any action runs, even when the supplied value is `true`. +Omitting the field preserves the existing behavior on every platform. +The policy applies to each action separately; it does not restrict other +actions in the same batch. + +Tab clicks prefer the parent container's accessibility selection operation, +even when the tab exposes its own action. List items, table rows and table +cells use parent selection only when they have no own action. + +Semantic scroll uses a direct scrollbar or the unique directional scrollbar +of the nearest scroll pane or viewport. It does not select an unrelated +sibling panel's scrollbar. Each requested page represents five minimum +range increments, clamped to the range bounds. +Fractional pages retain their fractional range increment. +`pages` is approximate for both range changes and wheel input. Range units +are not assumed to be pixels. The provider reads back the exact clamped +target; an unchanged value at a boundary is a verified no-op. + +If a semantic route is unavailable, global input is allowed only when +`allow_global_input` is `true`. Otherwise the action returns +`global_input_forbidden`, with provider phase `precondition` and +`input_events_emitted=false`. + +An attempted selection or range mutation whose verification fails returns +`effect_unknown` with `input_emitted=true`, without a global retry. +With global input forbidden, an AT-SPI action that was called but returned +false also reports `effect_unknown` with `input_emitted=true`, without a +global retry. An AT-SPI action returning true keeps the existing post-action +evidence behavior; the return value does not prove the requested effect. +When global input is allowed, unsuccessful generic actions keep their +existing fallback behavior. +This input flag includes attempted semantic mutations, not just global events. +Check `completed` and `input_emitted`: earlier batch steps may already have run. + +Results retain `[a11y][synthesis]` route notes and their warnings. A route +note describes input delivery, not proof that the application accepted it. + ### Linux key names AX and GUI `keypress` accept one chord, such as `["CTRL","S"]`. @@ -709,6 +763,7 @@ Every AX and GUI result starts with one JSON line: `ok`, and on failure | `wait_timeout` | AX | maybe | Observe again; `completed` counts the steps that ran. | | `window_changed` | AX, GUI | maybe | Observe before retrying; do not repeat the batch blindly. | | `effect_unknown` | AX, GUI | maybe | Observe before retrying; input may have taken effect. | +| `global_input_forbidden` | AX | only earlier batch steps | The refused action sent no input. Check `completed` and observe before replanning; permit global input only when authorized. | | `app_unavailable` | AX | no | List apps; the application is not running or not accessible. | | `ambiguous_window` | AX | no | Use a longer title or the `wN` from `apps`. | | `window_not_found` | AX | no | List apps and use a `wN` or an exact title. | diff --git a/instructions/hybrid.md b/instructions/hybrid.md index 4dd71ec..a97c50d 100644 --- a/instructions/hybrid.md +++ b/instructions/hybrid.md @@ -42,6 +42,10 @@ fits which state, and how to combine them. window without explicit `activate`; activate back before reusing the previous window's refs. Each step reports its window; the final observation uses the last executed window. +- On Linux, set `allow_global_input:false` on each AX `click` or `scroll` + when driver focus changes or global pointer/keyboard input are forbidden. + Semantic actions can still cause the application to raise a dialog or take + focus; this field does not guarantee complete background isolation. - Every `act` result starts with what happened, then a diff: ```text @@ -116,6 +120,9 @@ Never repeat the request when `retry_same_request=false`. Next step by code: not resend the same batch. - `effect_unknown`: input may have been sent. Inspect state before any retry, so that nothing is applied twice. +- `global_input_forbidden`: this step sent no input; earlier steps may have + run. Keep `allow_global_input:false` unless global input is authorized. + Observe and choose an action compatible with the user's constraints. - `ambiguous_window`, `window_not_found`: choose a `wN` from the listed windows. - `no_focused_window`: start the batch with `activate`. - `app_unavailable`: if the application is still starting, wait once for From 284df1593275c53e67bca59f67ecf0e1aee7e136 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 09:02:52 +0000 Subject: [PATCH 11/15] fix(linux): verify set_value keyboard fallback on the original control Read back the requested control after keyboard synthesis. Report effect_unknown with input evidence when its value is unconfirmed, stop later batch steps, and warn against retyping text left in an editor. Do not send an implicit commit key. Cover unchanged, unreadable, normalized, delayed and exact readback in provider tests; cover batch failure propagation despite focus and pixel changes in Go. Document the control-value and application-commit boundary. --- CHANGELOG.md | 3 + .../internal/core/ax_runtime_failure_test.go | 158 ++++++++++++++++++ .../linux/provider/ax_input_delivery.py | 48 ++++-- computer-use/runtime_test.py | 1 + .../runtime_tests/test_set_value_readback.py | 108 ++++++++++++ docs/MCP.md | 10 ++ 6 files changed, 312 insertions(+), 16 deletions(-) create mode 100644 computer-use/runtime_tests/test_set_value_readback.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ee23be1..ea8c5af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ A page means five minimum range increments, not pixels; pages remain approximate, and a clamped boundary no-op is verified. AX results retain synthesis route notes and warnings without claiming successful effects. +- Linux AX `set_value` reads back the requested control after keyboard fallback. + Unconfirmed values return `effect_unknown` and stop the batch, preserving input + evidence and warning against retyping text that may remain in an editor. - AX batch errors expose the failed action's reason and a failure-specific recovery hint while retaining batch progress and input evidence. No-input diff --git a/computer-use/internal/core/ax_runtime_failure_test.go b/computer-use/internal/core/ax_runtime_failure_test.go index d5a9166..89119ac 100644 --- a/computer-use/internal/core/ax_runtime_failure_test.go +++ b/computer-use/internal/core/ax_runtime_failure_test.go @@ -1,10 +1,168 @@ package core import ( + "bytes" + "encoding/base64" + "fmt" + "image" + "image/color" + "image/png" "strings" "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) +type unconfirmedSetValueBackend struct { + *reportBackend + failure ProviderAccessibilityResult + failAt int + snapshotInFailure bool +} + +func (*unconfirmedSetValueBackend) HostInfo() HostBackendInfo { + return HostBackendInfo{Platform: "linux"} +} + +func (backend *unconfirmedSetValueBackend) ResolveAndAct(request ProviderActionRequest) (ProviderAccessibilityResult, error) { + response, err := backend.reportBackend.ResolveAndAct(request) + if len(backend.actionRequests) != backend.failAt { + return response, err + } + failure := backend.failure + if backend.snapshotInFailure { + failure.Snapshot = response.Snapshot + } + return failure, nil +} + +func TestSetValueUnconfirmedEditorStopsBatchDespiteObservedChanges(t *testing.T) { + for _, scenario := range []struct { + name string + focusChanged bool + pixelsChanged bool + snapshotInFailure bool + }{ + {name: "unchanged"}, + {name: "focus_changed", focusChanged: true}, + {name: "editor_and_pixels_changed", focusChanged: true, pixelsChanged: true}, + {name: "changed_failure_snapshot", focusChanged: true, pixelsChanged: true, snapshotInFailure: true}, + } { + for _, failAt := range []int{1, 2} { + for _, observe := range []string{contract.ObserveNone, contract.ObserveFull} { + t.Run(fmt.Sprintf("%s/action_%d/%s", scenario.name, failAt, observe), func(t *testing.T) { + service, desktop := reportService(t, reportMain()) + window, refs := reportObserve(t, service, "Size", "Interface") + before := desktop.snapshot(reportMain()) + desktop.desktopResponse = testDesktopFrame(t, "before", 40, 30) + beforePixels := desktop.desktopResponse.Snapshot.ScreenshotPNGBase64 + response, err := DecodeProviderWireResponse([]byte(fmt.Sprintf(`{ + "protocol":%q,"schema_version":%d,"request_id":"set-value","operation":"resolve_and_act", + "accessibility":{"ok":false, + "error":"effect_unknown: the original editor did not confirm the requested value; text was not retried. Observe before sending more input.", + "failure":{"code":"effect_unknown","phase":"readback","subject":"element"}, + "action_evidence":{"route":"keyboard_synthesis","input_emitted":true,"geometry_sampled":false,"readback_sampled":true,"readback_verified":false}} + }`, ProviderProtocolName, ProviderProtocolVersion))) + if err != nil { + t.Fatalf("decode existing failure wire: %v", err) + } + backend := &unconfirmedSetValueBackend{reportBackend: desktop, + failure: *response.Accessibility, failAt: failAt, snapshotInFailure: scenario.snapshotInFailure} + service.Backend = backend + desktop.onAction = func(current *reportBackend) { + if len(current.actionRequests) != failAt { + current.checked = true + return + } + if scenario.focusChanged { + current.textStates = "[editable focusable focused]" + } + if scenario.pixelsChanged { + current.page = "Editing" + pixels := image.NewRGBA(image.Rect(0, 0, 40, 30)) + pixels.SetRGBA(10, 10, color.RGBA{R: 255, A: 255}) + var encoded bytes.Buffer + if err := png.Encode(&encoded, pixels); err != nil { + t.Fatal(err) + } + current.desktopResponse = testDesktopFrame(t, "after", 40, 30) + current.desktopResponse.Snapshot.ScreenshotPNGBase64 = base64.StdEncoding.EncodeToString(encoded.Bytes()) + backend.failure.Notes = []string{"[pixels] 1% of the window changed on screen."} + } + } + actions := []any{} + if failAt == 2 { + actions = append(actions, map[string]any{"type": "click", "ref": refs["Interface"]}) + } + actions = append(actions, + map[string]any{"type": "set_value", "ref": refs["Size"], "value": "16"}, + map[string]any{"type": "wait", "ref": refs["Size"], "value": "12", "timeout_ms": 1.0}, + map[string]any{"type": "keypress", "keys": []any{"ENTER"}}, + ) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": window, "observe": observe, "actions": actions, + }) + if len(backend.actionRequests) != failAt { + t.Fatalf("failed input was retried or later input was sent: %#v", backend.actionRequests) + } + if request := backend.actionRequests[failAt-1]; request.Intent != AccessibilitySetValue || request.SetValue == nil || request.SetValue.Value == nil || *request.SetValue.Value != "16" { + t.Fatalf("failure did not exercise set_value: %#v", request) + } + full := result.StructuredContent + batch, _ := full["a11y_batch_result"].(map[string]any) + entries, _ := batch["actions"].([]map[string]any) + if len(entries) != failAt || entries[failAt-1]["call_error"] != true || + batch["failed_action_index"] != failAt-1 || batch["failed_action_type"] != "set_value" { + t.Fatalf("failure or batch stop was lost: %#v", batch) + } + for _, evidence := range []map[string]any{full, entries[failAt-1]} { + runtime, ok := evidence["runtime_action_evidence"].(RuntimeActionEvidence) + if !ok || runtime.Route != "keyboard_synthesis" || !runtime.InputEmitted || + !runtime.ReadbackSampled || runtime.ReadbackVerified == nil || *runtime.ReadbackVerified || + evidence["input_events_emitted"] != true || evidence["mutation_unknown"] != true { + t.Fatalf("unconfirmed runtime input evidence was lost: %#v", evidence) + } + } + prepared := PrepareAXEvidence(result) + public := prepared.StructuredContent + if !prepared.IsError || public["ok"] != false || public["code"] != contract.AXCodeEffectUnknown || + public["completed"] != failAt-1 || public["requested"] != len(actions) || public["input_emitted"] != true || + public["failure_phase"] != "readback" { + t.Fatalf("changed state overrode provider failure: %#v\n%s", public, ResultText(prepared)) + } + continuation, _ := public["continuation"].(map[string]any) + if continuation["state"] != "verify_only" || continuation["retry_same_request"] != false { + t.Fatalf("unsafe continuation after unconfirmed input: %#v", continuation) + } + for _, required := range []string{"state=verify_only", "retry_same_request=false", "text was not retried", "Observe before sending more input"} { + if !strings.Contains(ResultText(prepared), required) { + t.Errorf("model text lost %q: %s", required, ResultText(prepared)) + } + } + after := desktop.snapshot(reportMain()) + if desktop.value != "12" || (after.Elements[2].States != before.Elements[2].States) != scenario.focusChanged || + (desktop.desktopResponse.Snapshot.ScreenshotPNGBase64 != beforePixels) != scenario.pixelsChanged { + t.Fatal("fixture did not keep the original value while changing only the requested focus/pixels") + } + if observe == contract.ObserveFull { + if full["recovery_observation_returned"] != true || full["task_effect_verified"] != false || full["cause_unattributed"] != true { + t.Fatalf("fresh observation falsely confirmed the failed action: %#v", full) + } + if onlyImage(t, prepared).Data != desktop.desktopResponse.Snapshot.ScreenshotPNGBase64 { + t.Fatal("failure did not return the current pixels") + } + if scenario.pixelsChanged && !strings.Contains(ResultText(prepared), "Editing") { + t.Fatalf("failure did not return the changed editor: %s", ResultText(prepared)) + } + } else if len(desktop.observeRequests) != 1 || len(desktop.desktopRequests) != 0 { + t.Fatal("observe none unexpectedly read the desktop or ran the wait") + } + }) + } + } + } +} + func TestInvalidKeyKeepsPublicArgumentErrorAndAcceptedSpelling(t *testing.T) { response := ProviderAccessibilityResult{ Error: "Unsupported key: KP_AD. Use KP_ADD.", diff --git a/computer-use/native/linux/provider/ax_input_delivery.py b/computer-use/native/linux/provider/ax_input_delivery.py index 427f440..882175a 100644 --- a/computer-use/native/linux/provider/ax_input_delivery.py +++ b/computer-use/native/linux/provider/ax_input_delivery.py @@ -709,6 +709,30 @@ def _perform_keypress( ) +def _verify_keyboard_value(element, value, evidence): + deadline = time.monotonic() + 1.0 + while True: + after_text = read_all_text(safe(element.get_text_iface)) + evidence["readback_sampled"] = True + evidence["readback_verified"] = ( + after_text == value if after_text is not None else None) + if after_text == value: + return + if time.monotonic() >= deadline: + break + time.sleep(0.02) + raise OperationFailure( + "effect_unknown: text was typed, but the requested control value " + "could not be confirmed. The text may still be in an uncommitted " + "editor. Do not retype the value. Inspect the control, complete any " + "pending edit through its normal commit action, then read it back.", + failure_code="effect_unknown", + failure_phase="readback", + input_events_emitted=True, + failure_subject="element", + ) + + def _perform_set_value( action_input, window, element, element_record, notes, action_evidence ): @@ -743,34 +767,26 @@ def _perform_set_value( "fallback was not attempted. Read the control before retrying.", failure_code="effect_unknown", failure_phase="readback", input_events_emitted=True, failure_subject="element") - # A refused semantic write does not mean the control is read-only: - # Blink text inputs, for example, implement neither the AT-SPI - # Value nor the EditableText interface. As click and type_text do, - # fall back to synthesis: focus, select all, then type. The note - # must state that this path has no read-back confirmation and that - # keyboard synthesis is global, so delivery is not guaranteed. focus_target_element( window, element, element_record, "targeted set_value keyboard fallback", ) - send_key("ctrl+a") - time.sleep(0.05) - text_route = send_text(value) action_evidence["route"] = "keyboard_synthesis" action_evidence["window_focus_verified"] = True action_evidence["target_focus_verified"] = True action_evidence["focus_verified"] = True - action_evidence["readback_verified"] = False + send_key("ctrl+a") action_evidence["input_emitted"] = True + time.sleep(0.05) + text_route = send_text(value) + _verify_keyboard_value(element, value, action_evidence) notes.append( addressing_channel(element_record) + SYNTHESIS - + "The AT-SPI value API refused this element (Blink and some toolkits " - "do not implement it on text inputs), so this fell back to focusing " - "the control, selecting all, and typing. There is no read-back " - "confirmation on this path, and keyboard synthesis is global — verify " - "from the tree that the control now holds what you wanted. {}".format( - UNVERIFIED_SYNTHESIS) + + "The AT-SPI value API did not confirm a change, so this fell back " + "to focusing the control, selecting all, and typing through global " + "keyboard synthesis. Exact readback confirmed the requested control " + "value. The application may still require OK/Apply." + (" " + text_route if text_route else "") ) else: diff --git a/computer-use/runtime_test.py b/computer-use/runtime_test.py index cb83a43..2167b7a 100644 --- a/computer-use/runtime_test.py +++ b/computer-use/runtime_test.py @@ -5,6 +5,7 @@ from runtime_tests.test_core import * # noqa: F401,F403 from runtime_tests.test_text_input import * # noqa: F401,F403 +from runtime_tests.test_set_value_readback import * # noqa: F401,F403 from runtime_tests.test_input_method import * # noqa: F401,F403 from runtime_tests.test_window_focus import * # noqa: F401,F403 from runtime_tests.test_action_notes import * # noqa: F401,F403 diff --git a/computer-use/runtime_tests/test_set_value_readback.py b/computer-use/runtime_tests/test_set_value_readback.py new file mode 100644 index 0000000..74c4c19 --- /dev/null +++ b/computer-use/runtime_tests/test_set_value_readback.py @@ -0,0 +1,108 @@ +"""Keyboard value writes require readback from the requested control.""" + +from runtime_test_support import * # noqa: F401,F403 + + +class SetValueReadbackTests(AtspiPatchedTestCase): + def setUp(self): + super().setUp() + self.clock = 0.0 + self.control = FakeTextIface(text="pending", honest=False) + self.editor = FakeTextIface(text="") + self.target = FakeNode(text=self.control) + self.evidence = runtime._new_action_evidence() + self.notes = [] + self.key = mock.Mock() + self.send = mock.Mock(side_effect=self.type_into_editor) + for name, replacement in ( + ("focus_target_element", mock.Mock()), + ("send_key", self.key), + ("send_text", self.send), + ("time", mock.Mock(monotonic=lambda: self.clock, + sleep=self.advance_clock)), + ): + patcher = mock.patch.object(runtime, name, replacement) + patcher.start() + self.addCleanup(patcher.stop) + + def advance_clock(self, duration): + self.clock += duration + + def type_into_editor(self, value): + self.editor.text = value + + def write(self, value="matched"): + runtime._perform_set_value( + {"value": value}, None, self.target, {}, self.notes, self.evidence) + + def assert_unknown(self, value="matched"): + with self.assertRaises(runtime.OperationFailure) as caught: + self.write(value) + failure = caught.exception + self.assertEqual(failure.failure_code, "effect_unknown") + self.assertEqual(failure.failure_phase, "readback") + self.assertTrue(failure.input_events_emitted) + self.assertIn("typed", str(failure)) + self.assertIn("Do not retype", str(failure)) + self.assertEqual(self.evidence["route"], "keyboard_synthesis") + self.assertTrue(self.evidence["input_emitted"]) + self.assertTrue(self.evidence["readback_sampled"]) + self.assertIsNot(self.evidence["readback_verified"], True) + self.key.assert_called_once_with("ctrl+a") + self.send.assert_called_once_with(value) + + def test_editor_text_does_not_confirm_the_control_value(self): + self.assert_unknown() + self.assertEqual(self.editor.text, "matched") + self.assertEqual(self.control.text, "pending") + self.assertFalse(self.evidence["readback_verified"]) + + def test_unreadable_control_does_not_confirm_even_an_empty_value(self): + self.target = FakeNode() + for value in ("matched", ""): + with self.subTest(value=value): + self.key.reset_mock() + self.send.reset_mock() + self.assert_unknown(value) + self.assertIsNone(self.evidence["readback_verified"]) + + def test_different_control_value_is_not_verified_or_retried(self): + def normalize(value): + self.control.text = value.upper() + self.send.side_effect = normalize + self.assert_unknown() + self.assertEqual(self.control.text, "MATCHED") + + def test_keyboard_write_confirms_exact_target_text_without_enter(self): + for value in ("matched", "", "中文🙂", " matched "): + with self.subTest(value=value): + self.key.reset_mock() + self.send.reset_mock() + self.control.text = "pending" + self.send.side_effect = lambda payload: setattr(self.control, "text", payload) + self.write(value) + self.assertEqual(self.control.text, value) + self.assertTrue(self.evidence["readback_verified"]) + self.assertTrue(self.evidence["readback_sampled"]) + self.key.assert_called_once_with("ctrl+a") + self.send.assert_called_once_with(value) + + def test_waits_for_delayed_control_update_without_retyping(self): + def commit_after_delay(duration): + self.advance_clock(duration) + if self.clock >= 0.1: + self.control.text = self.editor.text + with mock.patch.object(runtime.time, "sleep", side_effect=commit_after_delay): + self.write() + self.assertTrue(self.evidence["readback_verified"]) + self.assertGreaterEqual(self.clock, 0.1) + self.send.assert_called_once_with("matched") + + def test_verified_semantic_write_needs_no_keyboard_or_commit(self): + self.control.honest = True + self.write() + self.assertEqual(self.control.text, "matched") + self.assertEqual(self.evidence["route"], "value_api") + self.assertTrue(self.evidence["readback_verified"]) + self.key.assert_not_called() + self.send.assert_not_called() diff --git a/docs/MCP.md b/docs/MCP.md index 1bf477a..8ee7bbc 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -91,6 +91,16 @@ unexpected or unreadable text produces an `effect_unknown` result without retyping. If window activation changes the resolved field, AX stops before sending keys. Without a readable field, the typing effect remains unknown. +After keyboard fallback, Linux AX `set_value` reads the requested control for +up to one second. Only an exact value match confirms the write. Text in a +separate editor and changes to focus or selection do not confirm the control +value. If readback fails, AX returns `effect_unknown`, retains input evidence, +and stops later batch steps. Do not retype that value. Inspect the control, +complete any pending edit through its normal commit action, then read it back. +AX does not automatically press Enter because Enter can submit a form or insert +a newline in other controls. Control readback does not prove that the document +was saved or that a dialog applied its changes. + To use the input method, applications must inherit the desktop's input-method environment. Existing application processes may need a restart after that environment is corrected. From 885347160a85ccd98bb35a1ffa2014fb6e4ca5cd Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 09:02:52 +0000 Subject: [PATCH 12/15] fix(linux): add bounded AX text reads and recover missing body previews Read Text, name, and description pages through fresh ref resolution without desktop input. Preserve compact trees while reporting full lengths, truncation, and continuation calls, including content-only nodes. Keep MCP, records, SDK, ablations, docs, and wire fixtures aligned. Cover Unicode paging, provider failures, stale refs, response budgets, and display deduplication; verify GTK, Chromium, and Electron reads against an unchanged background sentinel. --- CHANGELOG.md | 14 + .../internal/contract/tool_contract.go | 12 +- computer-use/internal/core/app_snapshot.go | 2 + .../internal/core/ax_argument_hint_test.go | 11 + computer-use/internal/core/ax_evidence.go | 4 + .../internal/core/ax_find_delivery.go | 3 + .../internal/core/ax_public_request.go | 18 +- .../core/ax_ref_snapshot_assignment.go | 4 + computer-use/internal/core/ax_renderer.go | 40 ++- .../internal/core/ax_renderer_test.go | 47 +++ computer-use/internal/core/ax_request.go | 3 + .../internal/core/ax_response_budget.go | 5 +- computer-use/internal/core/ax_snapshot.go | 10 +- .../internal/core/ax_snapshot_build.go | 8 +- computer-use/internal/core/ax_text_read.go | 171 ++++++++++ .../internal/core/ax_text_read_test.go | 171 ++++++++++ .../internal/core/provider_request_builder.go | 1 + computer-use/internal/core/provider_types.go | 16 + computer-use/internal/core/provider_wire.go | 15 + .../internal/core/provider_wire_test.go | 25 ++ computer-use/internal/core/records.go | 4 +- .../internal/core/runtime_protocol.go | 1 + computer-use/internal/core/tool_result.go | 6 + .../native/linux/provider/ax_dispatch.py | 9 +- .../linux/provider/ax_node_properties.py | 134 +++++++- .../native/linux/provider/ax_records.py | 61 ++-- computer-use/native/linux/provider/wire.py | 1 + .../native/linux/provider/wire_request.py | 48 ++- .../runtime_tests/test_node_properties.py | 299 ++++++++++++++++++ computer-use/runtime_tests/test_protocol.py | 135 ++++++++ .../testdata/contract/public_contract.json | 34 +- .../provider-wire/text-read.request.json | 25 ++ .../provider-wire/text-read.response.json | 10 + docs/MCP.md | 82 ++++- instructions/hybrid.md | 33 ++ sdk/python/hcu_sdk/__init__.py | 5 +- sdk/python/tests/test_sdk.py | 1 + 37 files changed, 1413 insertions(+), 55 deletions(-) create mode 100644 computer-use/internal/core/ax_text_read.go create mode 100644 computer-use/internal/core/ax_text_read_test.go create mode 100644 computer-use/testdata/provider-wire/text-read.request.json create mode 100644 computer-use/testdata/provider-wire/text-read.response.json diff --git a/CHANGELOG.md b/CHANGELOG.md index ea8c5af..da9ec61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,20 @@ Unconfirmed values return `effect_unknown` and stop the batch, preserving input evidence and warning against retyping text that may remain in an editor. +- Linux AX `observe` accepts `read_text` for an existing element `root`. + It pages AT-SPI Text values and complete accessible names or descriptions + in at most 4000 Unicode code points per call. Results include raw text, + offsets, total length, and a next-page call; structured text pages retain + the raw text too. Invalid ranges, owner mismatches, and unavailable Text + fail explicitly. Lower inline budgets reduce page size without raising + tree caps. Each page reads live content; detected length changes fail that + call, but cross-page and same-length edits have no snapshot guarantee. +- Linux AX trees mark clipped fields with their full lengths and read calls. + Non-interactive Text content gets an 80-code-point preview and a bindable + ref. Existing Go cell, find excerpt, name, and description rules remain. +- Text paging and the revised hybrid guide change the model-visible contract + and require a new evaluation attempt. Keep instruction revisions separate; + the interface changes do not establish evaluation performance. - AX batch errors expose the failed action's reason and a failure-specific recovery hint while retaining batch progress and input evidence. No-input argument and unsupported-operation failures use `invalid_argument`; diff --git a/computer-use/internal/contract/tool_contract.go b/computer-use/internal/contract/tool_contract.go index 8e22643..fcacda9 100644 --- a/computer-use/internal/contract/tool_contract.go +++ b/computer-use/internal/contract/tool_contract.go @@ -32,6 +32,7 @@ const ( GUIMaxSettleMS = 3000 AXBatchMaxActions = 64 AXBatchMaxText = 20000 + AXMaxReadText = 4000 MaxClickCount = 10 MaxScrollPages = 100 MaxVerifyTimeout = 60000 @@ -203,6 +204,15 @@ func axInputSchema() map[string]any { "type": "string", "pattern": "^(focused|r[1-9][0-9]*)$", "description": "observe: subtree at rN or the focused element.", }, + "read_text": map[string]any{ + "type": "object", "additionalProperties": false, "required": []string{"field"}, + "description": "Linux observe only: read one live root rN without input or a tree. The ref identifies its window; omit app, scope and observe. Offsets count Unicode code points. Follow next_read to continue; pages are live reads, not a frozen document.", + "properties": map[string]any{ + "field": map[string]any{"type": "string", "enum": []string{"name", "value", "description"}, "description": "value reads the node's AT-SPI Text interface; unavailable text returns an error. name and description read those attributes."}, + "start": map[string]any{"type": "integer", "minimum": 0, "maximum": 2147483647, "description": "Start offset, inclusive; default 0."}, + "limit": map[string]any{"type": "integer", "minimum": 1, "maximum": AXMaxReadText, "description": "Maximum code points; default 4000."}, + }, + }, "query": map[string]any{"$ref": "#/$defs/query"}, "limit": map[string]any{"type": "integer", "minimum": 1, "maximum": MaxAXFindPageSize, "description": "find page size; default 20."}, "cursor": map[string]any{ @@ -428,7 +438,7 @@ func GUIInputSchema() map[string]any { // axToolDescription is model-visible on every client, including ones that // compact the input schema. Clients may truncate descriptions at 1,000 bytes. -const axToolDescription = "Accessibility for application windows. Modes: " + +const axToolDescription = "Accessibility for application windows. Linux: observe with root rN and read_text reads bounded pages of full name, value (AT-SPI Text), or description; follow next_read. Modes: " + "apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with " + "rN refs. find returns matching rN. act runs ordered steps, checks them before input, " + "stops at the first failed step, and reports each target's change, windows " + diff --git a/computer-use/internal/core/app_snapshot.go b/computer-use/internal/core/app_snapshot.go index 13d2b6a..376c3f7 100644 --- a/computer-use/internal/core/app_snapshot.go +++ b/computer-use/internal/core/app_snapshot.go @@ -64,6 +64,8 @@ type windowSelection struct { } type ElementRecord struct { + TextTruncated []string `json:"textTruncated,omitempty"` + TextLengths map[string]int `json:"textLengths,omitempty"` Index int `json:"index"` RuntimeID []int `json:"runtimeId,omitempty"` RuntimeIDAliases [][]int `json:"runtimeAliases,omitempty"` diff --git a/computer-use/internal/core/ax_argument_hint_test.go b/computer-use/internal/core/ax_argument_hint_test.go index d069326..2dac3a5 100644 --- a/computer-use/internal/core/ax_argument_hint_test.go +++ b/computer-use/internal/core/ax_argument_hint_test.go @@ -14,6 +14,17 @@ import ( // TestEveryAXArgumentErrorHintDecodes checks that the table reaches every // axInvalid call in ax_public_request.go. var axBadCalls = []string{ + `{"mode":"observe","read_text":{"field":"value"}}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value"},"scope":"window"}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value"},"app":"Editor"}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value"},"observe":"tree"}`, + `{"mode":"observe","root":"r1","read_text":null}`, + `{"mode":"observe","root":"r1","read_text":{}}`, + `{"mode":"observe","root":"r1","read_text":{"field":"bad"}}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value","start":-1}}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value","limit":4001}}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value","limit":true}}`, + `{"mode":"observe","root":"r1","read_text":{"field":"value","extra":true}}`, // mode `{}`, `{"mode":"assert_ax"}`, diff --git a/computer-use/internal/core/ax_evidence.go b/computer-use/internal/core/ax_evidence.go index 64aac33..937deea 100644 --- a/computer-use/internal/core/ax_evidence.go +++ b/computer-use/internal/core/ax_evidence.go @@ -170,6 +170,10 @@ func compactAXModelEvidence( ) map[string]any { compact := map[string]any{} mode := strings.TrimSpace(fmt.Sprint(full["ax_mode"])) + if page := full["text_page"]; page != nil { + compact["text_page"] = page + compact["text_read"] = full["text_read"] + } if mode == contract.AXModeFind { for _, key := range []string{ "find_match_count", "find_page_start", "find_page_count", "next_cursor", diff --git a/computer-use/internal/core/ax_find_delivery.go b/computer-use/internal/core/ax_find_delivery.go index fd5008f..db3001b 100644 --- a/computer-use/internal/core/ax_find_delivery.go +++ b/computer-use/internal/core/ax_find_delivery.go @@ -63,6 +63,9 @@ func FindMatchUnit(snapshot *AXSnapshot, record ElementRecord, ordinal int) stri display := findDisplayNode(node) fmt.Fprintf(&out, " - match=%d %s\n", ordinal, renderAXNodeLabel(display)) for _, property := range renderAXProperties(display, AXRenderOptions{}) { + if property[0] == "/text" { + continue + } out.WriteString(renderAXTreePropertyLine(property, 2) + "\n") } } else { diff --git a/computer-use/internal/core/ax_public_request.go b/computer-use/internal/core/ax_public_request.go index 2712303..3b82fda 100644 --- a/computer-use/internal/core/ax_public_request.go +++ b/computer-use/internal/core/ax_public_request.go @@ -92,6 +92,7 @@ type axRequest struct { limit int cursor string actions []axBatchAction + readText *ProviderTextRead } var ( @@ -103,7 +104,7 @@ var ( var axModeFields = map[string][]string{ contract.AXModeApps: {"mode"}, - contract.AXModeObserve: {"mode", "app", "window_ref", "scope", "observe", "root"}, + contract.AXModeObserve: {"mode", "app", "window_ref", "scope", "observe", "root", "read_text"}, contract.AXModeFind: {"mode", "app", "window_ref", "scope", "query", "limit", "cursor"}, contract.AXModeAct: {"mode", "window_ref", "actions", "observe"}, } @@ -208,11 +209,24 @@ func decodeAXCall(args map[string]any) (*axRequest, *axArgError) { } switch mode { case contract.AXModeObserve: + if raw, present := args["read_text"]; present { + if request.readText, err = decodeAXTextRead(raw); err != nil { + return nil, err + } + if !AXElementRefPattern.MatchString(request.root) { + return nil, axInvalid("root", "read_text requires an element root rN", "").withRule(axRuleRequiredForMode).withFix(axSet("root", "r1")) + } + for _, field := range []string{"app", "scope", "observe"} { + if _, present := args[field]; present { + return nil, axInvalid(field, "read_text returns only the requested text page; omit "+field, "").withRule(axRuleFieldForMode).withFix(axRemove(field)) + } + } + } if request.observe == contract.ObserveNone { return nil, axInvalid("observe", `observe "none" returns nothing; to wait without input, send act with only wait steps`, "").withRule(axRuleObserveNone).withFix(axRemove("observe")) } - if request.app == "" && request.windowRef == "" { + if request.app == "" && request.windowRef == "" && request.readText == nil { return nil, axInvalid("app", "observe needs app or window_ref", "").withRule(axRuleRequiredForMode).withFix(axSet("window_ref", "w1")) } diff --git a/computer-use/internal/core/ax_ref_snapshot_assignment.go b/computer-use/internal/core/ax_ref_snapshot_assignment.go index 87c1a72..4acdc7e 100644 --- a/computer-use/internal/core/ax_ref_snapshot_assignment.go +++ b/computer-use/internal/core/ax_ref_snapshot_assignment.go @@ -121,6 +121,10 @@ func axNodeRefEligibility( } if !axNodeHasInteractionSemantics(node) { + if axNodeProviderIdentityComplete(node) && + (node.Properties.Value != "" || len(node.Properties.TextTruncated) > 0) { + return true, "", AXLocatorProviderBound + } return false, "interaction_semantics_unknown", "" } if axNodeIsUnnamedEditor(node) { diff --git a/computer-use/internal/core/ax_renderer.go b/computer-use/internal/core/ax_renderer.go index 81db4e1..fc202bf 100644 --- a/computer-use/internal/core/ax_renderer.go +++ b/computer-use/internal/core/ax_renderer.go @@ -166,7 +166,7 @@ func axInlineCellValue(node *AXNode) (string, bool) { return "", false } value := strings.TrimSpace(strings.ToValidUTF8(node.Properties.Value, "\uFFFD")) - if comparableTreeText(value) == "" || semanticTextEqual(value, node.Name) || + if comparableTreeText(value) == "" || axValueRepeatsName(node) || pureNonSemanticUnicode(value, node) { return "", false } @@ -175,9 +175,21 @@ func axInlineCellValue(node *AXNode) (string, bool) { func renderAXProperties(node *AXNode, options AXRenderOptions) [][2]string { properties := make([][2]string, 0, 8) + valuePreview := node.Properties.Value + if !roleExposesWorkingValue(node.Role) { + if node.Role == "cell" || node.Role == "gridcell" { + valuePreview, _ = axInlineCellValue(node) + } else { + valuePreview, _ = shortenAtWord(valuePreview, maxFindTextRunes) + if valuePreview != "" && !axValueRepeatsName(node) && + !pureNonSemanticUnicode(node.Properties.Value, node) { + properties = append(properties, [2]string{"/text", valuePreview}) + } + } + } if node.Properties.Value != "" && (roleExposesWorkingValue(node.Role) && - !semanticTextEqual(node.Properties.Value, node.Name) && + !axValueRepeatsName(node) && !pureNonSemanticUnicode(node.Properties.Value, node)) { properties = append(properties, [2]string{"/value", node.Properties.Value}) } @@ -190,6 +202,22 @@ func renderAXProperties(node *AXNode, options AXRenderOptions) [][2]string { !semanticTextEqual(node.Properties.Description, node.Properties.Value)) { properties = append(properties, [2]string{"/description", node.Properties.Description}) } + for _, field := range []struct{ name, preview string }{ + {"name", node.Name}, {"value", valuePreview}, {"description", node.Properties.Description}, + } { + if field.name == "value" && axValueRepeatsName(node) { + continue + } + total := node.Properties.TextLengths[field.name] + if field.preview == "" || !axTextPreviewTruncated(node, field.name, field.preview) { + continue + } + hint := "use find to obtain a ref, then observe with root and read_text.field=" + field.name + if node.Ref != "" { + hint = contract.CompactJSON(axTextReadCall("", node.Ref, field.name, 0, contract.AXMaxReadText)) + } + properties = append(properties, [2]string{"/text-truncated", fmt.Sprintf("… %s length=%d codepoints; Linux: %s", field.name, total, hint)}) + } if node.Capabilities.SetValue { properties = append(properties, [2]string{"/set-value", "true"}) } @@ -221,6 +249,14 @@ func semanticTextEqual(left, right string) bool { comparableTreeText(left) == comparableTreeText(right) } +func axValueRepeatsName(node *AXNode) bool { + if semanticTextEqual(node.Properties.Value, node.Name) { + return true + } + record := node.SourceRecord + return record != nil && semanticTextEqual(record.Value, record.Name) +} + func pureNonSemanticUnicode(value string, node *AXNode) bool { if node == nil || node.Name != "" || node.Properties.Description != "" || diff --git a/computer-use/internal/core/ax_renderer_test.go b/computer-use/internal/core/ax_renderer_test.go index 03ae68b..772253b 100644 --- a/computer-use/internal/core/ax_renderer_test.go +++ b/computer-use/internal/core/ax_renderer_test.go @@ -7,6 +7,53 @@ import ( "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" ) +func TestObserveShowsTextInterfaceOnlyParagraph(t *testing.T) { + snapshot := cellValueSnapshot() + snapshot.Elements[2].ControlType = "paragraph" + snapshot.Elements[2].Name = "" + snapshot.Elements[2].Value = "The paragraph body comes from Text, not the accessible name." + text := typedTreeTextForTest(t, snapshot) + if !strings.Contains(text, snapshot.Elements[2].Value) { + t.Fatalf("Text-only body missing: %s", text) + } +} + +func TestClippedCellTextReportsLengthAndReadRoute(t *testing.T) { + snapshot := cellValueSnapshot() + text := typedTreeTextForTest(t, snapshot) + for _, want := range []string{"/text-truncated:", "value length=", "read_text"} { + if !strings.Contains(text, want) { + t.Fatalf("missing %q: %s", want, text) + } + } + if strings.Contains(text, snapshot.Elements[3].Value) { + t.Fatal("the ordinary tree must not include the full long value") + } +} + +func TestFindDoesNotRepeatTruncationForIdenticalNameAndValue(t *testing.T) { + snapshot := cellValueSnapshot() + record := &snapshot.Elements[2] + record.ControlType = "label" + record.Name = strings.Repeat("x", 500) + "..." + record.Value = record.Name + record.TextLengths = map[string]int{"name": 501, "value": 501} + record.TextTruncated = []string{"name", "value"} + typed, err := buildAXSnapshot(snapshot) + if err != nil { + t.Fatal(err) + } + text := FindMatchUnit(typed, *record, 1) + if strings.Count(text, "/text-truncated:") != 1 || !strings.Contains(text, "name length=501") { + t.Fatalf("find duplicated the truncation hint after shortening the name: %s", text) + } + for _, property := range renderAXProperties(findDisplayNode(typed.ByIndex[record.Index]), AXRenderOptions{}) { + if property[0] == "/text" || property[0] == "/value" { + t.Fatalf("find context duplicated a value that equals the canonical name: %v", property) + } + } +} + func TestStructuredAXYAMLSupportsValueAndChildren(t *testing.T) { root := &AXNode{ ID: "0", diff --git a/computer-use/internal/core/ax_request.go b/computer-use/internal/core/ax_request.go index cb207c1..d627c07 100644 --- a/computer-use/internal/core/ax_request.go +++ b/computer-use/internal/core/ax_request.go @@ -126,6 +126,9 @@ func (s *Service) axDecodedRequest(request *axRequest) ToolCallResult { contract.AXModeApps, ) case contract.AXModeObserve: + if request.readText != nil { + return WithAXModeEvidence(s.axReadText(request), contract.AXModeObserve) + } return s.axObserve(request, obs) case contract.AXModeFind: return s.axFind(request) diff --git a/computer-use/internal/core/ax_response_budget.go b/computer-use/internal/core/ax_response_budget.go index a8ac4a9..0a7af1b 100644 --- a/computer-use/internal/core/ax_response_budget.go +++ b/computer-use/internal/core/ax_response_budget.go @@ -96,6 +96,9 @@ func EnforceAXResponseBudget(result ToolCallResult) ToolCallResult { limit := MaxInlineChars() const evidenceReserve = 2048 alreadyTruncated := resultContainsInlineTruncation(result) + if result.StructuredContent["text_page"] != nil { + alreadyTruncated = false + } if result.StructuredContent["find_delivery_complete"] == true { // Application text is not truncation evidence for atomic find units. alreadyTruncated = result.StructuredContent["truncation_reason"] == "response_budget" @@ -157,7 +160,7 @@ func EnforceAXResponseBudget(result ToolCallResult) ToolCallResult { reason = "structural_budget" } treeComplete := !truncated && !structuralBudgetApplied && - coverageIncomplete == 0 + coverageIncomplete == 0 && result.StructuredContent["text_page"] == nil responseChars := textContentChars(result) if result.StructuredContent != nil { result.StructuredContent["response_chars"] = responseChars diff --git a/computer-use/internal/core/ax_snapshot.go b/computer-use/internal/core/ax_snapshot.go index 6557605..7f680b3 100644 --- a/computer-use/internal/core/ax_snapshot.go +++ b/computer-use/internal/core/ax_snapshot.go @@ -61,10 +61,12 @@ type AXCapabilities struct { } type AXProperties struct { - Value string `json:"value,omitempty"` - Description string `json:"description,omitempty"` - Placeholder string `json:"placeholder,omitempty"` - Omitted string `json:"omitted,omitempty"` + TextTruncated []string `json:"textTruncated,omitempty"` + TextLengths map[string]int `json:"textLengths,omitempty"` + Value string `json:"value,omitempty"` + Description string `json:"description,omitempty"` + Placeholder string `json:"placeholder,omitempty"` + Omitted string `json:"omitted,omitempty"` } type AXProviderIdentity struct { diff --git a/computer-use/internal/core/ax_snapshot_build.go b/computer-use/internal/core/ax_snapshot_build.go index 5c3670e..3528788 100644 --- a/computer-use/internal/core/ax_snapshot_build.go +++ b/computer-use/internal/core/ax_snapshot_build.go @@ -231,9 +231,11 @@ func newAXNodeFromRecord(snapshot *AppSnapshot, record *ElementRecord, binding a Name: normalizedTreeText(record.Name), States: typedAXStates(record), Properties: AXProperties{ - Value: normalizedTreeText(record.Value), - Description: normalizedTreeText(record.Description), - Placeholder: normalizedTreeText(record.Placeholder), + TextTruncated: append([]string(nil), record.TextTruncated...), + TextLengths: axTextLengths(record), + Value: normalizedTreeText(record.Value), + Description: normalizedTreeText(record.Description), + Placeholder: normalizedTreeText(record.Placeholder), }, Relations: record.Relations, NameProvenance: AXNameProvenance{ diff --git a/computer-use/internal/core/ax_text_read.go b/computer-use/internal/core/ax_text_read.go new file mode 100644 index 0000000..540d90c --- /dev/null +++ b/computer-use/internal/core/ax_text_read.go @@ -0,0 +1,171 @@ +package core + +import ( + "fmt" + "runtime" + "unicode/utf8" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +func decodeAXTextRead(raw any) (*ProviderTextRead, *axArgError) { + object, ok := raw.(map[string]any) + if !ok { + return nil, axInvalid("read_text", "read_text must be an object", "").withFix(axSet("read_text", map[string]any{"field": "value"})) + } + if err := axCheckFields(object, []string{"field", "start", "limit"}, "read_text", "read_text"); err != nil { + return nil, err + } + field, err := axOptionalEnum(object, "field", "read_text", []string{"name", "value", "description"}) + if err != nil { + return nil, err + } + if field == "" { + return nil, axInvalid("read_text.field", "read_text.field is required", "").withFix(axSet("read_text.field", "value")) + } + read := &ProviderTextRead{Field: field, Limit: contract.AXMaxReadText} + for _, item := range []struct { + name string + minimum int + maximum int + target *int + }{{"start", 0, int(^uint32(0) >> 1), &read.Start}, {"limit", 1, contract.AXMaxReadText, &read.Limit}} { + if value, present := object[item.name]; present { + number, valid := axInteger(value) + if !valid || number < item.minimum || number > item.maximum { + path := "read_text." + item.name + return nil, axInvalid(path, fmt.Sprintf("%s must be an integer from %d to %d", path, item.minimum, item.maximum), "").withFix(axNumberEdit(path, value, float64(item.minimum), float64(item.maximum), true)) + } + *item.target = number + } + } + return read, nil +} + +func axTextReadError(message string) ToolCallResult { + result := TextResult(message, true) + result.StructuredContent = map[string]any{ + "code": contract.AXCodeRuntimeError, "message": message, + "input_events_emitted": false, "desktop_state_mutated": false, + "mutation_unknown": false, "evidence_scope": "error_only", + } + return result +} + +func (s *Service) axReadText(request *axRequest) ToolCallResult { + if runtime.GOOS != "linux" { + return axTextReadError("read_text is supported only by the Linux provider") + } + identity, err := s.resolveAXReadIdentity(request.app, request.windowRef, request.root) + if err != nil { + return axIdentityErrorResult(err) + } + entry, err := s.resolveShortAXRefOwned(request.root) + if err != nil { + return axIdentityErrorResult(err) + } + target, err := semanticTargetFromRefEntry(entry) + if err != nil { + return axStaleRefResult(request.root, err) + } + read := *request.readText + available := MaxInlineChars() - 3072 + if available < 1 { + return axTextReadError("read_text needs COMPUTER_USE_MAX_INLINE_CHARS above 3072 to deliver a text page and its continuation") + } + if read.Limit > available { + read.Limit = available + } + no := false + operation := AXServiceRequest{ + Operation: axServiceObserve, App: identity.appToken, + TargetWindowHandle: identity.windowHandle, SemanticTarget: target, + SourceRef: request.root, Action: axResolveTargetCommand{}, + TextRead: &read, CaptureSnapshot: &no, IncludeScreenshot: &no, + } + if err := s.bindRuntimeAppIdentity(&operation); err != nil { + return axIdentityErrorResult(err) + } + response, err := s.callProviderAction(providerActionRequestFromOperation(operation)) + if err != nil { + return axTextReadError(err.Error()) + } + if !response.OK { + return s.runtimeReportedFailure(identity.appToken, operation, *response) + } + page := response.TextRead + if page == nil || page.Field != read.Field || page.Start != read.Start || + page.End < page.Start || page.End > page.Total || + page.End-page.Start > read.Limit || !utf8.ValidString(page.Text) || + utf8.RuneCountInString(page.Text) != page.End-page.Start || + (page.End < page.Total && page.End-page.Start != read.Limit) { + return axTextReadError("the provider did not return a complete, valid text page") + } + metadata := map[string]any{ + "ref": request.root, "window_ref": identity.windowRef, + "field": page.Field, "start": page.Start, "end": page.End, "total": page.Total, + "truncated": page.End < page.Total, + } + if page.End < page.Total { + metadata["next_read"] = axTextReadCall(identity.windowRef, request.root, page.Field, page.End, request.readText.Limit) + } + result := TextResult(contract.CompactJSON(metadata)+"\n"+page.Text, false) + result.StructuredContent = map[string]any{ + "text_page": page, "text_read": metadata, + "input_events_emitted": false, "desktop_state_mutated": false, + "mutation_unknown": false, "evidence_scope": "independent_ui_read", + } + return result +} + +func axTextReadCall(window, ref, field string, start, limit int) map[string]any { + call := map[string]any{ + "mode": "observe", "root": ref, + "read_text": map[string]any{"field": field, "start": start, "limit": limit}, + } + if window != "" { + call["window_ref"] = window + } + return call +} + +func axTextLengths(record *ElementRecord) map[string]int { + lengths := make(map[string]int, 3) + for field, value := range map[string]string{"name": record.Name, "value": record.Value, "description": record.Description} { + length := utf8.RuneCountInString(value) + if full, known := record.TextLengths[field]; known { + length = full + } + if length > 0 { + lengths[field] = length + } + } + return lengths +} + +func axTextPreviewTruncated(node *AXNode, field, preview string) bool { + if contract.ContainsString(node.Properties.TextTruncated, field) { + return true + } + original := "" + if record := node.SourceRecord; record != nil { + switch field { + case "name": + original = record.Name + case "value": + original = record.Value + case "description": + original = record.Description + } + } else { + switch field { + case "name": + original = node.Name + case "value": + original = node.Properties.Value + case "description": + original = node.Properties.Description + } + } + return utf8.RuneCountInString(normalizedTreeText(original)) > utf8.RuneCountInString(preview) +} diff --git a/computer-use/internal/core/ax_text_read_test.go b/computer-use/internal/core/ax_text_read_test.go new file mode 100644 index 0000000..9ed62c0 --- /dev/null +++ b/computer-use/internal/core/ax_text_read_test.go @@ -0,0 +1,171 @@ +package core + +import ( + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +func textReadService(t *testing.T) (*Service, *recordingPlatformBackend, string) { + t.Helper() + snapshot := windowTransitionSnapshot("w2:0123456789abcdef0123456789abcdef", "Document", true) + backend := &recordingPlatformBackend{accessibilityResponse: &ProviderAccessibilityResult{OK: true, Snapshot: snapshot}} + service := NewServiceWithBackend(backend) + result := service.callTool("AX", map[string]any{"mode": "observe", "app": "Editor"}) + if result.IsError { + t.Fatal(ResultText(result)) + } + ref := service.currentSnapshot("Editor").AXSnapshot.ByIndex[3].Ref + if ref == "" { + t.Fatal("setup did not issue a ref") + } + backend.calls = nil + return service, backend, ref +} + +func TestTextReadPagesKeepUnicodeAndLiveIdentity(t *testing.T) { + service, backend, ref := textReadService(t) + body := strings.Repeat("中🙂e\u0301", 1100) + runes := []rune(body) + backend.accessibilityResponses = []ProviderAccessibilityResult{ + {OK: true, TextRead: &ProviderTextPage{Field: "value", Start: 0, End: 4000, Total: len(runes), Text: string(runes[:4000])}}, + {OK: true, TextRead: &ProviderTextPage{Field: "value", Start: 4000, End: len(runes), Total: len(runes), Text: string(runes[4000:])}}, + } + result := service.callTool("AX", axTextReadCall("", ref, "value", 0, 4000)) + if result.IsError { + t.Fatal(ResultText(result)) + } + page := result.StructuredContent["text_page"].(*ProviderTextPage) + metadata := result.StructuredContent["text_read"].(map[string]any) + next := metadata["next_read"].(map[string]any) + result = service.callTool("AX", next) + if result.IsError { + t.Fatal(ResultText(result)) + } + last := result.StructuredContent["text_page"].(*ProviderTextPage) + if page.Text+last.Text != body || result.StructuredContent["text_read"].(map[string]any)["next_read"] != nil { + t.Fatal("text paging lost characters or left a false continuation") + } + if len(backend.calls) != 2 || len(backend.actionRequests) != 2 { + t.Fatalf("text reads took unrelated snapshots: %v", backend.calls) + } + for _, request := range backend.actionRequests { + if request.Intent != AccessibilityResolveTarget || request.Target.ProviderObjectPath != "/dialog/3" || + request.Target.WindowHandle == "" || request.AppIdentity == nil || + request.Observation.CaptureSnapshot == nil || *request.Observation.CaptureSnapshot || + request.Observation.IncludeScreenshot == nil || *request.Observation.IncludeScreenshot { + t.Fatalf("read lost identity or requested side effects: %#v", request) + } + } +} + +func TestTextReadRecordsUseTheSameBoundedProviderRead(t *testing.T) { + service, backend, ref := textReadService(t) + backend.accessibilityResponse = &ProviderAccessibilityResult{OK: true, TextRead: &ProviderTextPage{Field: "value", Start: 0, End: 3, Total: 3, Text: "中🙂e"}} + record := service.CallRecord("AX", axTextReadCall("", ref, "value", 0, 3), CallOptions{}) + if !record.OK || record.Evidence["text_page"] == nil || len(backend.calls) != 1 || backend.calls[0] != "resolve_and_act" { + t.Fatalf("records path did not read the same text page: %#v %v", record, backend.calls) + } +} + +func TestTextReadRejectsWrongWindowAndStaleRefBeforeProvider(t *testing.T) { + service, backend, ref := textReadService(t) + for _, call := range []map[string]any{ + axTextReadCall("w999", ref, "value", 0, 1), + axTextReadCall("", "r999", "value", 0, 1), + } { + result := service.callTool("AX", call) + if !result.IsError || len(backend.calls) != 0 { + t.Fatalf("invalid owner reached provider: %#v %v", result, backend.calls) + } + } +} + +func TestTextReadRejectsAbsentShortOrMalformedProviderPages(t *testing.T) { + for _, page := range []*ProviderTextPage{ + nil, + {Field: "value", Start: 0, End: 0, Total: 5}, + {Field: "value", Start: 0, End: 3, Total: 3, Text: "ab"}, + {Field: "name", Start: 0, End: 3, Total: 3, Text: "abc"}, + {Field: "value", Start: 1, End: 3, Total: 3, Text: "ab"}, + } { + service, backend, ref := textReadService(t) + backend.accessibilityResponse = &ProviderAccessibilityResult{OK: true, TextRead: page} + result := service.callTool("AX", axTextReadCall("", ref, "value", 0, 3)) + if !result.IsError || result.StructuredContent["code"] != contract.AXCodeRuntimeError { + t.Fatalf("invalid page reported success: %#v", result) + } + } +} + +func TestTextReadPropagatesProviderFailures(t *testing.T) { + service, backend, ref := textReadService(t) + backend.err = errors.New("Text interface read failed") + result := service.callTool("AX", axTextReadCall("", ref, "value", 0, 2)) + if !result.IsError || !strings.Contains(ResultText(result), "Text interface read failed") { + t.Fatalf("read error hidden: %#v", result) + } + backend.err = nil + backend.accessibilityResponse = &ProviderAccessibilityResult{OK: false, Error: "stale target", Failure: &ProviderFailure{Code: "stale_target", Phase: "precondition"}} + result = service.callTool("AX", axTextReadCall("", ref, "value", 0, 2)) + if !result.IsError || axPublicErrorCode(result.StructuredContent) != contract.AXCodeStaleRef { + t.Fatalf("stale live target hidden: %#v", result) + } +} + +func TestTextReadBudgetDoesNotCutDeliveredPage(t *testing.T) { + t.Setenv("COMPUTER_USE_MAX_INLINE_CHARS", "4096") + service, backend, ref := textReadService(t) + body := "[TRUNCATED: this is application text]\n" + strings.Repeat("🙂", 1100) + runes := []rune(body) + backend.accessibilityResponse = &ProviderAccessibilityResult{OK: true, TextRead: &ProviderTextPage{ + Field: "value", Start: 0, End: 1024, Total: len(runes), Text: string(runes[:1024]), + }} + result := service.callTool("AX", axTextReadCall("", ref, "value", 0, 4000)) + if result.IsError || backend.actionRequests[0].Observation.TextRead.Limit != 1024 || + result.StructuredContent["truncation_reason"] == "response_budget" || + !strings.Contains(ResultText(result), string(runes[:1024])) || utf8.RuneCountInString(ResultText(result)) > 4096 { + t.Fatalf("budget damaged page or continuation: %#v", result) + } + for _, structured := range []bool{false, true} { + final := FinalizeForModel("AX", result, structured) + if !strings.Contains(ResultText(final), string(runes[:1024])) || utf8.RuneCountInString(ResultText(final)) > 4096 { + t.Fatalf("model finalization damaged text: %#v", final) + } + if structured && final.StructuredContent["text_page"] == nil { + t.Fatal("structured consumers lost the text page") + } + } +} + +func TestProviderTruncationIsMarkedEvenWhenEllipsisFillsTheGap(t *testing.T) { + for _, total := range []int{501, 502, 503} { + record := &ElementRecord{Name: strings.Repeat("x", 500) + "...", TextLengths: map[string]int{"name": total}, TextTruncated: []string{"name"}} + node := &AXNode{Name: record.Name, Ref: "r1", SourceRecord: record, Properties: AXProperties{TextLengths: axTextLengths(record), TextTruncated: record.TextTruncated}} + found := false + for _, property := range renderAXProperties(node, AXRenderOptions{}) { + if property[0] == "/text-truncated" { + found = true + if !strings.Contains(property[1], "read_text") { + t.Fatal("missing recovery route") + } + } + } + if !found { + t.Fatalf("total=%d was silently clipped", total) + } + } +} + +func TestWhitespaceNormalizationIsNotReportedAsTruncation(t *testing.T) { + record := &ElementRecord{Value: "hello\n\nworld", TextLengths: map[string]int{"value": 12}} + node := &AXNode{Role: "paragraph", SourceRecord: record, Properties: AXProperties{Value: "hello world", TextLengths: axTextLengths(record)}} + for _, property := range renderAXProperties(node, AXRenderOptions{}) { + if property[0] == "/text-truncated" { + t.Fatal("whitespace normalization was reported as truncation") + } + } +} diff --git a/computer-use/internal/core/provider_request_builder.go b/computer-use/internal/core/provider_request_builder.go index 077d6df..50aa88e 100644 --- a/computer-use/internal/core/provider_request_builder.go +++ b/computer-use/internal/core/provider_request_builder.go @@ -92,6 +92,7 @@ func providerObservationOptionsFromOperation( // the read (attachDesktopFrame), so the provider never captures one. noImage := false return ProviderObservationOptions{ + TextRead: operation.TextRead, TextLimit: providerTextLimit(operation.TextLimit), MaxTreeNodes: maxTreeNodes, MaxTreeDepth: operation.MaxTreeDepth, diff --git a/computer-use/internal/core/provider_types.go b/computer-use/internal/core/provider_types.go index 4fbb2cd..e86ca7e 100644 --- a/computer-use/internal/core/provider_types.go +++ b/computer-use/internal/core/provider_types.go @@ -85,7 +85,22 @@ type ProviderSemanticTarget struct { SourceIndex *int `json:"-"` } +type ProviderTextRead struct { + Field string `json:"field"` + Start int `json:"start"` + Limit int `json:"limit"` +} + +type ProviderTextPage struct { + Field string `json:"field"` + Start int `json:"start"` + End int `json:"end"` + Total int `json:"total"` + Text string `json:"text"` +} + type ProviderObservationOptions struct { + TextRead *ProviderTextRead `json:"text_read,omitempty"` TextLimit *ProviderTextLimit `json:"text_limit,omitempty"` MaxTreeNodes int `json:"max_tree_nodes,omitempty"` MaxTreeDepth int `json:"max_tree_depth,omitempty"` @@ -448,6 +463,7 @@ type ProviderInventoryIssue struct { } type ProviderAccessibilityResult struct { + TextRead *ProviderTextPage `json:"text_read,omitempty"` OK bool `json:"ok"` Text string `json:"text,omitempty"` Error string `json:"error,omitempty"` diff --git a/computer-use/internal/core/provider_wire.go b/computer-use/internal/core/provider_wire.go index 226ebf7..58e3d9a 100644 --- a/computer-use/internal/core/provider_wire.go +++ b/computer-use/internal/core/provider_wire.go @@ -413,6 +413,9 @@ func ValidateProviderObserveAppsRequest(request ProviderObserveAppsRequest) erro } func ValidateProviderObserveRequest(request ProviderObserveRequest) error { + if request.Observation.TextRead != nil { + return errors.New("text_read requires resolve_target") + } if err := validateProviderProtocolVersion(request.ProtocolVersion); err != nil { return err } @@ -425,6 +428,12 @@ func ValidateProviderObserveRequest(request ProviderObserveRequest) error { func validateProviderObservationOptions( observation ProviderObservationOptions, ) error { + if read := observation.TextRead; read != nil { + if !contract.ContainsString([]string{"name", "value", "description"}, read.Field) || + read.Start < 0 || read.Start > 2147483647 || read.Limit < 1 || read.Limit > contract.AXMaxReadText { + return errors.New("invalid provider text_read field, start, or limit") + } + } if observation.TextLimit != nil && !observation.TextLimit.Max && observation.TextLimit.Value <= 0 { @@ -471,6 +480,9 @@ func ValidateProviderActionRequest(request ProviderActionRequest) error { if !validAccessibilityOperation(request.Intent) { return fmt.Errorf("unsupported provider accessibility intent %q", request.Intent) } + if request.Observation.TextRead != nil && request.Intent != AccessibilityResolveTarget { + return errors.New("text_read requires resolve_target") + } if strings.TrimSpace(request.App) == "" { return errors.New("provider accessibility app is required") } @@ -735,6 +747,9 @@ func ValidateProviderWindowListRequest(request ProviderWindowListRequest) error func ValidateProviderWindowActionRequest( request ProviderWindowActionRequest, ) error { + if request.Observation.TextRead != nil { + return errors.New("text_read requires resolve_target") + } if err := validateProviderProtocolVersion(request.ProtocolVersion); err != nil { return err } diff --git a/computer-use/internal/core/provider_wire_test.go b/computer-use/internal/core/provider_wire_test.go index 668426a..e26c3d2 100644 --- a/computer-use/internal/core/provider_wire_test.go +++ b/computer-use/internal/core/provider_wire_test.go @@ -13,6 +13,31 @@ import ( var providerFixtureFS = os.DirFS("../..").(fs.ReadFileFS) +func TestLinuxTextReadWireFixtures(t *testing.T) { + requestData := readProviderFixture(t, "text-read.request.json") + request, err := DecodeProviderWireRequest(requestData) + if err != nil { + t.Fatal(err) + } + assertProviderFixtureRoundTrip(t, requestData, request) + responseData := readProviderFixture(t, "text-read.response.json") + response, err := DecodeProviderWireResponse(responseData) + if err != nil { + t.Fatal(err) + } + assertProviderFixtureRoundTrip(t, responseData, response) + for _, field := range []string{"value", "name", "description"} { + request.Action.Observation.TextRead.Field = field + if err := request.Validate(); err != nil { + t.Fatal(err) + } + } + request.Action.Observation.TextRead.Limit = contract.AXMaxReadText + 1 + if err := request.Validate(); err == nil { + t.Fatal("oversize text read accepted") + } +} + type providerFixtureManifest struct { SchemaVersion int `json:"schema_version"` Protocol string `json:"protocol"` diff --git a/computer-use/internal/core/records.go b/computer-use/internal/core/records.go index 6fe5695..f53f25b 100644 --- a/computer-use/internal/core/records.go +++ b/computer-use/internal/core/records.go @@ -320,8 +320,8 @@ func (s *Service) observeRecords(args map[string]any, since string) (Record, boo if bad { return errorRecord(contract.PublicToolAX, rejected), true } - if request.observe == contract.ObserveScreenshot || request.observe == contract.ObserveFull { - // Screenshots go through the shared path and come back as files. + if request.readText != nil || request.observe == contract.ObserveScreenshot || request.observe == contract.ObserveFull { + // Text pages and screenshots use the shared read path. return Record{}, false } readContext, identityErr := s.resolveAXReadIdentity(request.app, request.windowRef, "") diff --git a/computer-use/internal/core/runtime_protocol.go b/computer-use/internal/core/runtime_protocol.go index d4d6330..4339d8c 100644 --- a/computer-use/internal/core/runtime_protocol.go +++ b/computer-use/internal/core/runtime_protocol.go @@ -17,6 +17,7 @@ type AXServiceRequest struct { WindowAction *WindowOperation `json:"-"` WindowBounds *frame `json:"windowBounds,omitempty"` TextLimit any `json:"text_limit,omitempty"` + TextRead *ProviderTextRead `json:"-"` MaxTreeNodes int `json:"max_tree_nodes,omitempty"` MaxTreeDepth int `json:"max_tree_depth,omitempty"` Prune *bool `json:"prune,omitempty"` diff --git a/computer-use/internal/core/tool_result.go b/computer-use/internal/core/tool_result.go index a28a18f..cb13b81 100644 --- a/computer-use/internal/core/tool_result.go +++ b/computer-use/internal/core/tool_result.go @@ -3,6 +3,7 @@ package core import ( "encoding/base64" "fmt" + "maps" "os" "path/filepath" "strconv" @@ -182,6 +183,11 @@ func init() { // WithAXTextEvidence puts the AX evidence line first: one JSON object that // starts with ok, then code, path, message, hint and counts. func WithAXTextEvidence(result ToolCallResult) ToolCallResult { + if result.StructuredContent["text_page"] != nil { + result.StructuredContent = maps.Clone(result.StructuredContent) + delete(result.StructuredContent, "text_page") + delete(result.StructuredContent, "text_read") + } return withEvidenceLine(result, axPublicFields) } diff --git a/computer-use/native/linux/provider/ax_dispatch.py b/computer-use/native/linux/provider/ax_dispatch.py index cb8fd22..f53fff8 100644 --- a/computer-use/native/linux/provider/ax_dispatch.py +++ b/computer-use/native/linux/provider/ax_dispatch.py @@ -21,6 +21,7 @@ from .ax_node_properties import ( parse_text_limit, positive_int, + read_element_text, ) from .x11_windows import ( @@ -298,7 +299,7 @@ def _resolve_target_operation(operation): input_events_emitted=False, ) ( - _element, + element, element_record, _semantic_note, _window, @@ -312,10 +313,14 @@ def _resolve_target_operation(operation): operation.get("known_refs"), operation, ) - return { + response = { "ok": True, "resolvedElement": element_record, } + text_read = (operation.get("observation") or {}).get("text_read") + if text_read is not None: + response["textRead"] = read_element_text(element, text_read) + return response def _perform_ax_action( diff --git a/computer-use/native/linux/provider/ax_node_properties.py b/computer-use/native/linux/provider/ax_node_properties.py index 54af2e1..be60fa4 100644 --- a/computer-use/native/linux/provider/ax_node_properties.py +++ b/computer-use/native/linux/provider/ax_node_properties.py @@ -7,6 +7,7 @@ DEFAULT_TEXT_LIMIT, NON_SELF_ACTIONS, OBJECT_REPLACEMENT, + OperationFailure, UNIVERSAL_ACTIONS, child_count, limit_text, @@ -80,7 +81,8 @@ def accessible_id(node): return str(safe(getter, "") or "") if callable(getter) else "" -def text_value_with_status(node, text_limit=DEFAULT_TEXT_LIMIT): +def text_value_with_status(node, text_limit=DEFAULT_TEXT_LIMIT, text_lengths=None, + text_truncated=None): """Return text plus whether the provider read completed reliably.""" try: text_iface = node.get_text_iface() @@ -89,8 +91,12 @@ def text_value_with_status(node, text_limit=DEFAULT_TEXT_LIMIT): if text_iface is None: return "", True try: - count = int(Atspi.Text.get_character_count(text_iface) or 0) - if count <= 0: + count = Atspi.Text.get_character_count(text_iface) + if type(count) is not int or count < 0: + return "", False + if count == 0: + if text_lengths is not None: + text_lengths["value"] = 0 return "", True end_offset = count if text_limit is None else min(count, text_limit + 1) value = str( @@ -98,16 +104,103 @@ def text_value_with_status(node, text_limit=DEFAULT_TEXT_LIMIT): ) except Exception: return "", False + if len(value) == end_offset: + if text_lengths is not None: + text_lengths["value"] = count + if (text_truncated is not None and text_limit is not None + and (count > end_offset + or len(value.replace(OBJECT_REPLACEMENT, "")) > text_limit)): + text_truncated.append("value") return limit_text(value, text_limit=text_limit), True -def text_value(node, text_limit=DEFAULT_TEXT_LIMIT): +def text_value(node, text_limit=DEFAULT_TEXT_LIMIT, text_lengths=None, + text_truncated=None): value, _complete = text_value_with_status( - node, text_limit=text_limit + node, text_limit=text_limit, text_lengths=text_lengths, + text_truncated=text_truncated, ) return value +def read_element_text(node, options): + """Read one codepoint page from the resolved node without emitting input.""" + field = options["field"] + start = options["start"] + limit = options["limit"] + + def refuse(reason): + raise OperationFailure( + "runtime_error: {} text read failed: {}; no input was emitted.".format( + field, reason), + failure_code="runtime_error", + failure_phase="readback", + input_events_emitted=False, + ) + + try: + if (field not in ("name", "value", "description") + or type(start) is not int or not 0 <= start <= 2147483647 + or type(limit) is not int or not 1 <= limit <= 4000): + refuse("invalid text range") + role = node.get_role_name() + if not isinstance(role, str) or not role: + refuse("unreadable role") + if role.strip().lower().replace("_", " ") == "password text": + refuse("password text is protected") + if field == "value": + text_iface = node.get_text_iface() + if text_iface is None: + refuse("Text interface is unavailable") + total = Atspi.Text.get_character_count(text_iface) + if type(total) is not int or total < 0: + refuse("invalid character count") + if start > total: + refuse("text start exceeds character count") + end = min(total, start + limit) + text = Atspi.Text.get_text(text_iface, start, end) + after = Atspi.Text.get_character_count(text_iface) + if type(after) is not int or after != total: + refuse("character count changed during read") + if not isinstance(text, str) or len(text) != end - start: + refuse("text read length does not match range") + else: + if field == "name": + text = node.get_name() + if not isinstance(text, str): + refuse("unreadable name") + from .ax_records import ( + combo_label, content_name, gtk_combo_partner, labelled_by_name, + ) + + combo = gtk_combo_partner(node, role) + if combo is not None: + text = combo_label(combo, node)[0] + elif not text.strip(): + text = labelled_by_name(node) or content_name(node, role) + else: + text = node.get_description() + if not isinstance(text, str): + refuse("unreadable description") + text = plain_text_from_rich_text(text) + total = len(text) + if start > total: + refuse("text start exceeds character count") + end = min(total, start + limit) + text = text[start:end] + except OperationFailure: + raise + except Exception as exc: + raise OperationFailure( + "runtime_error: {} text read failed ({}); no input was emitted.".format( + field, type(exc).__name__), + failure_code="runtime_error", + failure_phase="readback", + input_events_emitted=False, + ) from exc + return {"field": field, "start": start, "end": end, "total": total, "text": text} + + def numeric_value(node): value_iface = safe(node.get_value_iface) if value_iface is None: @@ -241,24 +334,45 @@ def embedded_editor_text(node, text_limit=DEFAULT_TEXT_LIMIT): return limit_text("\n".join(lines), text_limit=text_limit) -def element_value(node, text_limit=DEFAULT_TEXT_LIMIT, role=None): +def element_value(node, text_limit=DEFAULT_TEXT_LIMIT, role=None, text_lengths=None, + text_truncated=None): """Return the node's text or numeric value; see selected_option_name(). ``role`` passes a role the caller already read, so it is not read from the provider a second time. """ - direct = text_value(node, text_limit=text_limit) or numeric_value(node) + if role is None: + role = node_role(node) + if str(role).strip().lower().replace("_", " ") == "password text": + return "" + direct_lengths = {} + direct_truncated = [] + direct = text_value( + node, text_limit=text_limit, text_lengths=direct_lengths, + text_truncated=direct_truncated, + ) if direct: + if text_lengths is not None: + text_lengths.update(direct_lengths) + if text_truncated is not None: + text_truncated.extend(direct_truncated) return direct + numeric = numeric_value(node) + if numeric: + return numeric embedded = embedded_editor_text(node, text_limit=text_limit) if embedded: return embedded # Only dropdown roles pay for the descendant scan. - if role is None: - role = node_role(node) role = str(role or "") if role in ("combo box", "list box"): - return limit_text(selected_option_name(node), text_limit=text_limit) + selected = limit_text(selected_option_name(node), text_limit=text_limit) + if selected: + return selected + if text_lengths is not None: + text_lengths.update(direct_lengths) + if text_truncated is not None: + text_truncated.extend(direct_truncated) return direct diff --git a/computer-use/native/linux/provider/ax_records.py b/computer-use/native/linux/provider/ax_records.py index b89e2b2..bbaf6b0 100644 --- a/computer-use/native/linux/provider/ax_records.py +++ b/computer-use/native/linux/provider/ax_records.py @@ -6,6 +6,7 @@ from .foundation import ( Atspi, # pyright: ignore[reportAttributeAccessIssue] DEFAULT_TEXT_LIMIT, + OBJECT_REPLACEMENT, TOP_LEVEL_CONTAINER_ROLES, has_editable_text_iface, has_text_iface, @@ -26,13 +27,16 @@ accessible_id, element_value, node_actions, - node_description, + plain_text_from_rich_text, placeholder_text, state_segment, text_attributes, ) +RELATION_TARGET_LIMIT = 16 + + class StableIndexer: """Assign element indices that survive across snapshots. @@ -104,7 +108,7 @@ def labelled_by_name(node): kind = safe(lambda: relation.get_relation_type()) if kind != Atspi.RelationType.LABELLED_BY: continue - count = safe(lambda: relation.get_n_targets(), 0) or 0 + count = min(int(safe(lambda: relation.get_n_targets(), 0) or 0), RELATION_TARGET_LIMIT) for index in range(count): target = safe(lambda: relation.get_target(index)) if target is None: @@ -420,7 +424,7 @@ def editable_text_role(node, role): return "entry" -def semantic_relations(node, text_limit=DEFAULT_TEXT_LIMIT): +def semantic_relations(node, text_limit=DEFAULT_TEXT_LIMIT, labelled_names=None): """Return bounded relation facts for retained nodes only. Relations are evidence, not a replacement address. Targets stay descriptive @@ -453,7 +457,7 @@ def semantic_relations(node, text_limit=DEFAULT_TEXT_LIMIT): label = wanted.get(kind) if not label: continue - count = min(int(safe(lambda: relation.get_n_targets(), 0) or 0), 16) + count = min(int(safe(lambda: relation.get_n_targets(), 0) or 0), RELATION_TARGET_LIMIT) targets = [] for index in range(count): try: @@ -464,21 +468,25 @@ def semantic_relations(node, text_limit=DEFAULT_TEXT_LIMIT): if target is None: complete = False continue + full_name = str(safe(lambda: node_name(target), "") or "") + if label == "labelled-by" and labelled_names is not None and full_name.strip(): + labelled_names.append(full_name.strip()) + relation_limit = 160 if text_limit is None else min(text_limit, 160) targets.append({ "role": str(safe(lambda: node_role(target), "") or ""), "name": limit_text( str( - safe(lambda: node_name(target), "") + full_name or safe( lambda: element_value( target, - text_limit=min(text_limit, 160), + text_limit=relation_limit, ), "", ) or "" ), - text_limit=min(text_limit, 160), + text_limit=relation_limit, ), "providerBusName": provider_bus_name(target), "providerObjectPath": provider_object_path(target), @@ -505,14 +513,10 @@ def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, # LibreOffice ATK bridge. actions, has_click_action = node_actions(node) own_name = str(facts.value("name", "") or "") + labelled_names = [] relations, relations_complete = semantic_relations( - node, text_limit=text_limit + node, text_limit=text_limit, labelled_names=labelled_names ) - labelled_names = [ - str(target.get("name") or "").strip() - for target in relations.get("labelled-by", []) - if str(target.get("name") or "").strip() - ] combo = gtk_combo_partner(node, role) role = top_level_window_role(node, editable_text_role(node, role)) if combo is not None: @@ -533,6 +537,22 @@ def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, full_name = content_name(node, role) name_source = "content" if full_name else "" name = limit_text(full_name, text_limit=text_limit) + description_read = (None, "unavailable") + if callable(getattr(node, "get_description", None)): + description_read = facts.strict("description") + full_description = plain_text_from_rich_text(str(description_read[0] or "")) + text_lengths = {} + name_read = facts.strict("name") + if name_read[1] is None and isinstance(name_read[0], str): + text_lengths["name"] = len(full_name) + if description_read[1] is None and isinstance(description_read[0], str): + text_lengths["description"] = len(full_description) + text_truncated = [ + field for field, full_text in (("name", full_name), ("description", full_description)) + if text_limit is not None + and len(full_text.replace(OBJECT_REPLACEMENT, "")) > text_limit + ] + protected = role.strip().lower().replace("_", " ") == "password text" record = { "index": index, "runtimeId": path[:], @@ -549,7 +569,9 @@ def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, "controlType": role, "localizedControlType": role, "className": str(safe(node.get_toolkit_name, "") or ""), - "value": element_value(node, text_limit=text_limit, role=role), + "value": "" if protected else element_value( + node, text_limit=text_limit, role=role, text_lengths=text_lengths, + text_truncated=text_truncated), "nativeWindowHandle": 0, "frame": bounds, "actions": actions, @@ -557,9 +579,9 @@ def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, node, has_click_action=has_click_action, borrowed_name=borrowed, facts=facts, ), - "description": node_description( - node, text_limit=text_limit, facts=facts - ), + "description": limit_text(full_description, text_limit=text_limit), + "textLengths": text_lengths, + "textTruncated": text_truncated, "placeholder": placeholder_text( node, text_limit=text_limit, role=role ), @@ -569,7 +591,7 @@ def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, # them from the focused text node does not work because the # LibreOffice root pane also reports FOCUSED, and a separate full-tree # scan is slow and could miss the target paragraph. - "textAttributes": text_attributes(node), + "textAttributes": None if protected else text_attributes(node), "relations": relations, "relationReadComplete": relations_complete, "nameSource": name_source, @@ -581,6 +603,9 @@ def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, if identity is not None: record["editTarget"] = identity record["value"] = element_value(editor, text_limit=text_limit) + text_lengths.pop("value", None) + if "value" in text_truncated: + text_truncated.remove("value") # When the name is truncated, also store a hash of the full name. # # Identity checks compare the live full name, which never equals the diff --git a/computer-use/native/linux/provider/wire.py b/computer-use/native/linux/provider/wire.py index b2a9e79..5d5be58 100644 --- a/computer-use/native/linux/provider/wire.py +++ b/computer-use/native/linux/provider/wire.py @@ -98,6 +98,7 @@ def _accessibility_result(response): ("apps", "apps"), ("inventory_issues", "inventory_issues"), ("resolvedElement", "resolved_element"), + ("textRead", "text_read"), ("candidates", "candidates"), ("candidateCount", "candidate_count"), ("candidatesTruncated", "candidates_truncated"), diff --git a/computer-use/native/linux/provider/wire_request.py b/computer-use/native/linux/provider/wire_request.py index a4e1266..604e265 100644 --- a/computer-use/native/linux/provider/wire_request.py +++ b/computer-use/native/linux/provider/wire_request.py @@ -53,6 +53,7 @@ def _strict_object( OBSERVATION_FIELDS = { "text_limit", + "text_read", "max_tree_nodes", "max_tree_depth", "prune", @@ -99,7 +100,7 @@ def _strict_object( "controlType", "localizedControlType", "className", "value", "nativeWindowHandle", "frame", "actions", "states", "description", "textAttributes", "placeholder", "relations", "relationReadComplete", - "nameSource", + "nameSource", "textLengths", "textTruncated", } POINT_FIELDS = {"x", "y"} DESKTOP_ACTION_CONTRACTS = { @@ -157,10 +158,30 @@ def _validate_app_identity(value, path): ) -def _validate_observation(value, path): +def _validate_observation(value, path, allow_text_read=False): if value is None: return observation = _strict_object(value, path, OBSERVATION_FIELDS) + if "text_read" in observation: + if not allow_text_read: + raise ProviderRequestError( + "invalid_provider_request", + "{}.text_read requires resolve_target".format(path), + ) + text_read = _strict_object( + observation["text_read"], path + ".text_read", + {"field", "start", "limit"}, {"field", "start", "limit"}, + ) + if (text_read["field"] not in ("name", "value", "description") + or type(text_read["start"]) is not int + or not 0 <= text_read["start"] <= 2147483647 + or type(text_read["limit"]) is not int + or not 1 <= text_read["limit"] <= 4000): + raise ProviderRequestError( + "invalid_provider_request", + "{}.text_read requires field name/value/description, " + "start 0..2147483647 and limit 1..4000 integers".format(path), + ) text_limit = observation.get("text_limit") if text_limit is not None: valid_limit = ( @@ -432,6 +453,24 @@ def _validate_element(value, path): if value is None: return element = _strict_object(value, path, ELEMENT_FIELDS, ("index",)) + if "textTruncated" in element: + truncated = element["textTruncated"] + if (not isinstance(truncated, list) + or any(field not in ("name", "value", "description") for field in truncated)): + raise ProviderRequestError( + "invalid_provider_request", + "{}.textTruncated must be an array of name/value/description".format(path), + ) + if "textLengths" in element: + lengths = _strict_object( + element["textLengths"], path + ".textLengths", + {"name", "value", "description"}, + ) + if any(type(length) is not int or length < 0 for length in lengths.values()): + raise ProviderRequestError( + "invalid_provider_request", + "{}.textLengths must contain non-negative integers".format(path), + ) if element.get("frame") is not None: _strict_object( element["frame"], path + ".frame", FRAME_FIELDS, FRAME_FIELDS @@ -592,7 +631,10 @@ def _validate_payload(operation, payload): FRAME_FIELDS, FRAME_FIELDS, ) - _validate_observation(payload.get("observation"), "action.observation") + _validate_observation( + payload.get("observation"), "action.observation", + allow_text_read=payload["intent"] == "resolve_target", + ) return if operation == "desktop_execute_batch": payload = _strict_object( diff --git a/computer-use/runtime_tests/test_node_properties.py b/computer-use/runtime_tests/test_node_properties.py index ecdf4bb..f2e5b50 100644 --- a/computer-use/runtime_tests/test_node_properties.py +++ b/computer-use/runtime_tests/test_node_properties.py @@ -331,6 +331,305 @@ def test_empty_and_none_are_safe(self): self.assertEqual(runtime.plain_text_from_rich_text(""), "") +class TextReadTests(AtspiPatchedTestCase): + def setUp(self): + super().setUp() + self.node = _TreeNode(role="entry", name="Document") + self.node._text = FakeTextIface("A🙂e\u0301\ufffc中文\n" * 1000) + + def read(self, field="value", start=0, limit=4000): + return runtime.read_element_text( + self.node, {"field": field, "start": start, "limit": limit}) + + def test_pages_preserve_codepoints_and_reconstruct_text(self): + chunks = [] + offset = 0 + while offset < len(self.node._text.text): + page = self.read(start=offset, limit=997) + self.assertEqual(page["total"], len(self.node._text.text)) + self.assertEqual(page["start"], offset) + self.assertEqual(page["end"], offset + len(page["text"])) + self.assertLessEqual(len(page["text"]), 997) + chunks.append(page["text"]) + offset = page["end"] + self.assertEqual("".join(chunks), self.node._text.text) + + def test_exact_end_and_empty_text_are_valid_empty_pages(self): + for text in ("", "🙂中"): + self.node._text.text = text + self.assertEqual(self.read(start=len(text)), { + "field": "value", "start": len(text), "end": len(text), + "total": len(text), "text": "", + }) + + def test_reads_only_requested_bounded_span(self): + with mock.patch.object(FakeAtspi.Text, "get_text", wraps=FakeAtspi.Text.get_text) as read: + page = self.read(start=1) + read.assert_called_once_with(self.node._text, 1, 4001) + self.assertEqual(page["text"], self.node._text.text[1:4001]) + + def test_invalid_range_fails_without_reading(self): + for start, limit in ((-1, 1), (False, 1), (0, True), (0, 0), + (0, 4001), (0.5, 1), (99999, 10), (2147483648, 1)): + with self.subTest(start=start, limit=limit), \ + mock.patch.object(FakeAtspi.Text, "get_text") as read: + with self.assertRaises(runtime.OperationFailure) as raised: + self.read(start=start, limit=limit) + self.assertEqual(raised.exception.failure_code, "runtime_error") + self.assertFalse(raised.exception.input_events_emitted) + read.assert_not_called() + + def test_maximum_start_is_accepted_at_exact_end(self): + with mock.patch.object(FakeAtspi.Text, "get_character_count", return_value=2147483647), \ + mock.patch.object(FakeAtspi.Text, "get_text", return_value="") as read: + page = self.read(start=2147483647, limit=1) + self.assertEqual(page["start"], 2147483647) + self.assertEqual(page["end"], 2147483647) + self.assertEqual(page["text"], "") + read.assert_called_once_with(self.node._text, 2147483647, 2147483647) + + def test_missing_text_never_uses_numeric_or_descendant_fallback(self): + self.node._text = None + self.node.value = FakeValueIface(42) + with mock.patch.object(runtime, "element_value") as fallback: + with self.assertRaises(runtime.OperationFailure): + self.read() + fallback.assert_not_called() + + def test_unreadable_interface_and_counts_fail(self): + with mock.patch.object(self.node, "get_text_iface", side_effect=RuntimeError("offline")): + with self.assertRaises(runtime.OperationFailure): + self.read() + for count in (None, -1, True, 3.5): + with self.subTest(count=count), \ + mock.patch.object(FakeAtspi.Text, "get_character_count", return_value=count): + with self.assertRaises(runtime.OperationFailure): + self.read() + with mock.patch.object(FakeAtspi.Text, "get_character_count", side_effect=RuntimeError("offline")): + with self.assertRaises(runtime.OperationFailure): + self.read() + + def test_short_long_and_unreadable_text_fail(self): + for text in (None, "", "x", "abcd", b"abc"): + with self.subTest(text=text), \ + mock.patch.object(FakeAtspi.Text, "get_text", return_value=text): + with self.assertRaises(runtime.OperationFailure): + self.read(limit=3) + with mock.patch.object(FakeAtspi.Text, "get_text", side_effect=RuntimeError("offline")): + with self.assertRaises(runtime.OperationFailure): + self.read() + + def test_count_changes_and_readback_failures_fail(self): + for after in (7999, 8001, None, True, RuntimeError("offline")): + with self.subTest(after=after), mock.patch.object( + FakeAtspi.Text, "get_character_count", side_effect=[8000, after]): + with self.assertRaises(runtime.OperationFailure): + self.read(limit=3) + + def test_full_name_and_normalized_description_do_not_read_value(self): + self.node.title = "🙂Name\ufffc" * 900 + self.node.description = "

" + "中 & text
" * 900 + "

" + expected_description = runtime.plain_text_from_rich_text(self.node.description) + with mock.patch.object(self.node, "get_text_iface", side_effect=AssertionError("value read")): + for field, full in (("name", self.node.title), ("description", expected_description)): + page = self.read(field, start=501, limit=1000) + self.assertEqual(page["text"], full[501:1501]) + self.assertEqual(page["total"], len(full)) + + def test_unreadable_name_and_description_fail(self): + for field, getter in (("name", "get_name"), ("description", "get_description")): + with self.subTest(field=field), mock.patch.object( + self.node, getter, side_effect=RuntimeError("offline")): + with self.assertRaises(runtime.OperationFailure): + self.read(field) + + def test_password_fields_refuse_before_reading_content(self): + self.node.role = "password text" + with mock.patch.object(self.node, "get_text_iface") as text, \ + mock.patch.object(self.node, "get_name") as name, \ + mock.patch.object(self.node, "get_description") as description: + for field in ("name", "value", "description"): + with self.assertRaises(runtime.OperationFailure) as raised: + self.read(field) + self.assertIn("protected", str(raised.exception)) + text.assert_not_called() + name.assert_not_called() + description.assert_not_called() + + +class TextLengthMetadataTests(AtspiPatchedTestCase): + def test_explicit_truncation_survives_ellipsis_length_overlap(self): + for total in (500, 501, 502, 503): + with self.subTest(total=total): + full = "🙂" * total + node = _TreeNode(role="entry", name=full, description=full) + node._text = FakeTextIface(full) + with mock.patch.object(FakeAtspi.Text, "get_character_count", wraps=FakeAtspi.Text.get_character_count) as count, \ + mock.patch.object(FakeAtspi.Text, "get_text", wraps=FakeAtspi.Text.get_text) as read: + record = runtime.record_for(node, 0, [0], None) + expected = ["name", "value", "description"] if total > 500 else [] + self.assertCountEqual(record["textTruncated"], expected) + count.assert_called_once_with(node._text) + read.assert_called_once_with(node._text, 0, min(total, 501)) + for field in ("name", "value", "description"): + self.assertEqual(record["textLengths"][field], total) + self.assertEqual(record[field], full[:500] + ("..." if total > 500 else "")) + if total > 500: + self.assertLessEqual(total, len(record[field])) + + def test_complete_text_with_literal_ellipsis_is_not_marked(self): + full = "x" * 497 + "..." + node = _TreeNode(role="entry", name=full, description=full) + node._text = FakeTextIface(full) + self.assertEqual(runtime.record_for(node, 0, [0], None)["textTruncated"], []) + node.title = node.description = node._text.text = "x" * 600 + self.assertEqual(runtime.record_for(node, 0, [0], None, text_limit=None)["textTruncated"], []) + + def test_value_truncation_uses_raw_count_and_actual_preview(self): + node = _TreeNode(role="entry") + node._text = FakeTextIface("\ufffc" + "x" * 500) + record = runtime.record_for(node, 0, [0], None) + self.assertEqual(record["value"], "x" * 500) + self.assertNotIn("value", record["textTruncated"]) + node._text.text += "z" + record = runtime.record_for(node, 0, [0], None) + self.assertEqual(record["value"], "x" * 500) + self.assertIn("value", record["textTruncated"]) + with mock.patch.object(FakeAtspi.Text, "get_text", return_value="x" * 500): + record = runtime.record_for(node, 0, [0], None) + self.assertNotIn("value", record["textTruncated"]) + self.assertNotIn("value", record["textLengths"]) + + def test_fallback_values_do_not_inherit_raw_text_truncation(self): + node = _TreeNode(role="entry") + node._text = FakeTextIface("\ufffc" * 600) + node.value = FakeValueIface(42) + record = runtime.record_for(node, 0, [0], None) + self.assertEqual(record["value"], "42.0") + self.assertNotIn("value", record["textTruncated"]) + node.value = None + with mock.patch.object(runtime, "embedded_editor_text", return_value="aggregate"): + record = runtime.record_for(node, 0, [0], None) + self.assertNotIn("value", record["textTruncated"]) + + def test_full_lengths_share_reads_and_keep_default_preview(self): + node = _TreeNode(role="entry", name="🙂中" * 600, description="说明 " * 600) + node._text = FakeTextIface("🙂正文" * 100000) + facts = runtime.NodeFacts(node) + with mock.patch.object(node, "get_name", wraps=node.get_name) as name, \ + mock.patch.object(node, "get_description", wraps=node.get_description) as description, \ + mock.patch.object(FakeAtspi.Text, "get_character_count", wraps=FakeAtspi.Text.get_character_count) as count, \ + mock.patch.object(FakeAtspi.Text, "get_text", wraps=FakeAtspi.Text.get_text) as text: + facts.value("name") + facts.value("description") + record = runtime.record_for(node, 0, [0], None, facts=facts) + name.assert_called_once_with() + description.assert_called_once_with() + count.assert_called_once_with(node._text) + text.assert_called_once_with(node._text, 0, 501) + expected = {"name": node.title, "value": node._text.text, + "description": runtime.plain_text_from_rich_text(node.description)} + for field, full in expected.items(): + self.assertEqual(record[field], full[:500] + "...") + self.assertEqual(record["textLengths"][field], len(full)) + + def test_labelled_name_uses_full_labels_without_reading_their_values(self): + label = _TreeNode(role="label", name="Long 🙂 label " * 90 + ":") + node = _TreeNode(role="entry") + node.get_relation_set = lambda: [_FakeRelation(FakeAtspi.RelationType.LABELLED_BY, [label])] + expected = label.title.rstrip(":").strip() + with mock.patch.object(label, "get_name", wraps=label.get_name) as read, \ + mock.patch.object(label, "get_text_iface", side_effect=AssertionError("value read")): + record = runtime.record_for(node, 0, [0], None) + read.assert_called_once_with() + page = runtime.read_element_text(node, {"field": "name", "start": 500, "limit": 900}) + self.assertEqual(record["name"], expected[:500] + "...") + self.assertEqual(record["textLengths"]["name"], len(expected)) + self.assertEqual(record["nameHash"], runtime.name_fingerprint(expected)) + self.assertIn("name", record["textTruncated"]) + self.assertEqual(page["text"], expected[500:1400]) + self.assertEqual(page["total"], len(expected)) + + def test_value_length_is_not_numeric_or_aggregated_length(self): + node = _TreeNode(role="entry") + for iface in (None, FakeTextIface("")): + node._text = iface + node.value = FakeValueIface(42) + record = runtime.record_for(node, 0, [0], None) + self.assertEqual(record["value"], "42.0") + self.assertNotIn("value", record["textLengths"]) + node.value = None + with mock.patch.object(runtime, "embedded_editor_text", return_value="line one\nline two"): + record = runtime.record_for(node, 0, [0], None) + self.assertNotIn("value", record["textLengths"]) + node.role = "combo box" + with mock.patch.object(runtime, "selected_option_name", return_value="Option"): + record = runtime.record_for(node, 0, [0], None) + self.assertEqual(record["value"], "Option") + self.assertNotIn("value", record["textLengths"]) + + def test_borrowed_name_pages_follow_the_same_bounded_relation_read(self): + labels = [_TreeNode(role="label", name="Label " + str(index)) for index in range(20)] + relation = _FakeRelation(FakeAtspi.RelationType.LABELLED_BY, labels) + node = _TreeNode(role="entry") + node.get_relation_set = lambda: [relation] + record = runtime.record_for(node, 0, [0], None) + with mock.patch.object(relation, "get_target", wraps=relation.get_target) as targets: + page = runtime.read_element_text(node, {"field": "name", "start": 0, "limit": 4000}) + self.assertEqual(page["text"], record["name"]) + self.assertEqual(page["total"], record["textLengths"]["name"]) + self.assertEqual(targets.call_count, 16) + + def test_projected_combo_value_does_not_claim_its_editors_length(self): + toggle = _TreeNode(role="toggle button") + toggle._text = FakeTextIface("toggle text" * 100) + combo = _TreeNode(role="combo box", name="Choice") + editor = _TreeNode(role="entry") + editor._text = FakeTextIface("actual editor text") + with mock.patch.object(runtime, "gtk_combo_partner", return_value=combo), \ + mock.patch.object(runtime, "combo_edit_target", return_value=editor), \ + mock.patch.object(runtime, "combo_edit_identity", return_value={"providerBusName": ":1.42"}): + record = runtime.record_for(toggle, 0, [0], None) + page = runtime.read_element_text(toggle, {"field": "name", "start": 0, "limit": 4000}) + self.assertEqual(record["value"], editor._text.text) + self.assertNotIn("value", record["textLengths"]) + self.assertNotIn("value", record["textTruncated"]) + self.assertEqual(page["text"], combo.title) + + def test_content_name_pages_use_full_visible_labels(self): + label = _TreeNode(role="label", name="Visible label " * 100) + node = _TreeNode(role="list item", kids=[label]) + record = runtime.record_for(node, 0, [0], None) + page = runtime.read_element_text(node, {"field": "name", "start": 500, "limit": 900}) + self.assertEqual(record["nameSource"], "content") + self.assertEqual(page["text"], label.title.strip()[500:1400]) + self.assertEqual(page["total"], record["textLengths"]["name"]) + + def test_empty_text_keeps_zero_and_unreadable_text_has_no_length(self): + node = _TreeNode(role="entry") + node._text = FakeTextIface("") + self.assertEqual(runtime.record_for(node, 0, [0], None)["textLengths"]["value"], 0) + node._text.text = "abcd" + with mock.patch.object(FakeAtspi.Text, "get_text", return_value="a"): + record = runtime.record_for(node, 0, [0], None) + self.assertNotIn("value", record["textLengths"]) + + def test_unreadable_description_has_no_claimed_length(self): + node = _TreeNode(role="entry") + with mock.patch.object(node, "get_description", side_effect=RuntimeError("offline")): + record = runtime.record_for(node, 0, [0], None) + self.assertNotIn("description", record["textLengths"]) + + def test_password_record_never_reads_text(self): + node = _TreeNode(role="password text", name="Password") + node._text = FakeTextIface("secret") + with mock.patch.object(node, "get_text_iface", side_effect=AssertionError("password read")) as text: + record = runtime.record_for(node, 0, [0], None) + text.assert_not_called() + self.assertEqual(record["value"], "") + self.assertNotIn("value", record["textLengths"]) + + class SaneRectFrameTests(unittest.TestCase): """Screen frames of content scrolled far away are real. diff --git a/computer-use/runtime_tests/test_protocol.py b/computer-use/runtime_tests/test_protocol.py index 84d3372..d696169 100644 --- a/computer-use/runtime_tests/test_protocol.py +++ b/computer-use/runtime_tests/test_protocol.py @@ -7,6 +7,141 @@ from runtime_test_support import * # noqa: F401,F403 +class TextReadProtocolTests(AtspiPatchedTestCase): + def test_linux_text_read_wire_fixtures(self): + fixtures = Path(__file__).resolve().parents[1] / "testdata" / "provider-wire" + request = json.loads((fixtures / "text-read.request.json").read_text(encoding="utf-8")) + response = json.loads((fixtures / "text-read.response.json").read_text(encoding="utf-8")) + runtime.validate_request(request) + payload = response["accessibility"] + converted = runtime.provider_response(request, { + "ok": payload["ok"], "textRead": payload["text_read"], + }) + self.assertEqual(converted["accessibility"], payload) + self.assertEqual(payload["text_read"], { + "field": "value", "start": 1, "end": 4, "total": 6, "text": "中🙂e", + }) + + def request(self): + return { + "protocol": "computer-use-provider", "schema_version": 2, + "request_id": "req-text", "operation": "resolve_and_act", + "action": { + "protocol_version": 2, "intent": "resolve_target", "app": "Editor", + "target": { + "selector": 'entry "Document"', "hasName": True, + "role": "entry", "name": "Document", + "providerBusName": ":1.42", "providerObjectPath": "/document", + "processGeneration": "12345", + }, + "observation": {"text_read": {"field": "value", "start": 0, "limit": 4000}}, + }, + } + + def test_text_read_requires_resolve_target(self): + request = self.request() + runtime.validate_request(request) + for intent in ("click", "set_value"): + request = self.request() + request["action"]["intent"] = intent + request["action"]["input"] = {} if intent == "click" else {"value": "x"} + with self.subTest(intent=intent), self.assertRaises(runtime.ProviderRequestError): + runtime.validate_request(request) + request = self.request() + observation = request.pop("action")["observation"] + request["operation"] = "observe" + request["observe"] = {"protocol_version": 2, "app": "Editor", "observation": observation} + with self.assertRaises(runtime.ProviderRequestError): + runtime.validate_request(request) + + def test_range_validation_is_strict(self): + invalid = [None, {}, [], {"field": "value", "start": 0}, + {"field": "value", "start": 0, "limit": 1, "extra": True}] + for field, values in (("field", ("text", None, [], 1)), + ("start", (-1, True, 0.5, "0", None, 2147483648)), + ("limit", (0, 4001, True, 1.5, "1", None))): + for value in values: + options = {"field": "value", "start": 0, "limit": 4000} + options[field] = value + invalid.append(options) + for options in invalid: + request = self.request() + request["action"]["observation"]["text_read"] = options + with self.subTest(options=options), self.assertRaises(runtime.ProviderRequestError): + runtime.validate_request(request) + for field in ("name", "value", "description"): + request = self.request() + request["action"]["observation"]["text_read"] = {"field": field, "start": 1, "limit": 1} + runtime.validate_request(request) + request["action"]["observation"]["text_read"]["start"] = 2147483647 + runtime.validate_request(request) + + def test_old_records_accept_only_valid_text_lengths(self): + request = self.request() + request["action"]["element"] = {"index": 1, "textLengths": {"value": 0, "name": 1234}} + runtime.validate_request(request) + for lengths in (None, [], {"value": -1}, {"value": True}, {"name": 1.5}, {"extra": 3}): + request["action"]["element"]["textLengths"] = lengths + with self.subTest(lengths=lengths), self.assertRaises(runtime.ProviderRequestError): + runtime.validate_request(request) + + def test_old_records_accept_only_valid_truncated_field_arrays(self): + request = self.request() + for fields in ([], ["name"], ["name", "value", "description"]): + request["action"]["element"] = {"index": 1, "textTruncated": fields} + runtime.validate_request(request) + for fields in (None, "value", {}, [None], [[]], [False], [1], ["placeholder"]): + request["action"]["element"]["textTruncated"] = fields + with self.subTest(fields=fields), self.assertRaises(runtime.ProviderRequestError): + runtime.validate_request(request) + + def test_wire_read_uses_resolved_node_without_capture_or_input(self): + node = _TreeNode(role="entry", name="Document") + node._text = FakeTextIface("🙂中 actual value") + window = _TreeNode(role="frame", name="Editor") + record = {"index": 1, "value": "stale preview", "textLengths": {"value": 9999}, + "textTruncated": ["value"], "_private": True} + request = self.request() + with mock.patch.object(runtime, "resolve_operation_app", return_value=object()), \ + mock.patch.object(runtime, "main_window", return_value=(0, window)), \ + mock.patch.object(runtime, "window_bounds", return_value=None), \ + mock.patch.object(runtime, "resolve_semantic_target", return_value=(node, record, "", window, None)) as resolve, \ + mock.patch.object(runtime, "build_snapshot") as snapshot, \ + mock.patch.object(runtime, "capture_window_pixels") as capture, \ + mock.patch.object(runtime, "_perform_ax_action") as action, \ + mock.patch.object(runtime, "_execute_desktop_batch") as desktop: + response = runtime.execute(request) + payload = response["accessibility"] + self.assertTrue(payload["ok"]) + self.assertEqual(payload["text_read"], { + "field": "value", "start": 0, "end": len(node._text.text), + "total": len(node._text.text), "text": node._text.text, + }) + self.assertEqual(payload["resolved_element"]["textLengths"], {"value": 9999}) + self.assertEqual(payload["resolved_element"]["textTruncated"], ["value"]) + self.assertNotIn("_private", payload["resolved_element"]) + self.assertEqual(resolve.call_args.args[4], request["action"]["target"]) + snapshot.assert_not_called() + capture.assert_not_called() + action.assert_not_called() + desktop.assert_not_called() + + def test_failed_read_has_explicit_failure_without_text_or_recovery(self): + node = _TreeNode(role="entry", name="Document") + window = _TreeNode(role="frame", name="Editor") + with mock.patch.object(runtime, "resolve_operation_app", return_value=object()), \ + mock.patch.object(runtime, "main_window", return_value=(0, window)), \ + mock.patch.object(runtime, "window_bounds", return_value=None), \ + mock.patch.object(runtime, "resolve_semantic_target", return_value=(node, {"index": 1}, "", window, None)), \ + mock.patch.object(runtime, "bounded_recovery_context") as recovery: + response = runtime.execute(self.request()) + self.assertFalse(response["accessibility"]["ok"]) + self.assertEqual(response["accessibility"]["failure"]["code"], "runtime_error") + self.assertNotIn("text_read", response["accessibility"]) + self.assertNotIn("mutation", response["accessibility"]) + recovery.assert_not_called() + + class RuntimeActionEvidenceFailureTests(unittest.TestCase): def test_text_delivery_wire_keeps_null_and_zero_without_confirming_effect(self): for name, source_count, unknown_index in ( diff --git a/computer-use/testdata/contract/public_contract.json b/computer-use/testdata/contract/public_contract.json index c7c1cc1..571792a 100644 --- a/computer-use/testdata/contract/public_contract.json +++ b/computer-use/testdata/contract/public_contract.json @@ -47,6 +47,7 @@ "title": "Accessibility" }, "description": "Accessibility for application windows. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. activate with wN or title switches the batch target for later refs and ref-less steps. Other-window refs fail until another activate selects their window. Each step reports its window; final observation uses the last executed window. Copy rN/wN exactly; stale refs fail. keypress/type without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", + "description": "Accessibility for application windows. Linux: observe with root rN and read_text reads bounded pages of full name, value (AT-SPI Text), or description; follow next_read. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. Copy rN/wN exactly; stale refs fail. keypress/type without ref go to the focused window. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", "inputSchema": { "$defs": { "query": { @@ -240,6 +241,37 @@ "query": { "$ref": "#/$defs/query" }, + "read_text": { + "additionalProperties": false, + "description": "Linux observe only: read one live root rN without input or a tree. The ref identifies its window; omit app, scope and observe. Offsets count Unicode code points. Follow next_read to continue; pages are live reads, not a frozen document.", + "properties": { + "field": { + "description": "value reads the node's AT-SPI Text interface; unavailable text returns an error. name and description read those attributes.", + "enum": [ + "name", + "value", + "description" + ], + "type": "string" + }, + "limit": { + "description": "Maximum code points; default 4000.", + "maximum": 4000, + "minimum": 1, + "type": "integer" + }, + "start": { + "description": "Start offset, inclusive; default 0.", + "maximum": 2147483647, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "field" + ], + "type": "object" + }, "root": { "description": "observe: subtree at rN or the focused element.", "pattern": "^(focused|r[1-9][0-9]*)$", @@ -785,7 +817,7 @@ "result": { "content": [ { - "text": "{\"ok\":false,\"code\":\"invalid_argument\",\"path\":\"bogus\",\"message\":\"mode \\\"observe\\\" does not accept \\\"bogus\\\"; accepted: mode, app, window_ref, scope, observe, root\",\"hint\":\"{\\\"mode\\\":\\\"observe\\\",\\\"window_ref\\\":\\\"w1\\\"}\",\"input_emitted\":false,\"completed\":0,\"requested\":0,\"batch\":{\"rejected_before_first_action\":true},\"preflight_only\":true}", + "text": "{\"ok\":false,\"code\":\"invalid_argument\",\"path\":\"bogus\",\"message\":\"mode \\\"observe\\\" does not accept \\\"bogus\\\"; accepted: mode, app, window_ref, scope, observe, root, read_text\",\"hint\":\"{\\\"mode\\\":\\\"observe\\\",\\\"window_ref\\\":\\\"w1\\\"}\",\"input_emitted\":false,\"completed\":0,\"requested\":0,\"batch\":{\"rejected_before_first_action\":true},\"preflight_only\":true}", "type": "text" } ], diff --git a/computer-use/testdata/provider-wire/text-read.request.json b/computer-use/testdata/provider-wire/text-read.request.json new file mode 100644 index 0000000..6463dea --- /dev/null +++ b/computer-use/testdata/provider-wire/text-read.request.json @@ -0,0 +1,25 @@ +{ + "protocol": "computer-use-provider", + "schema_version": 2, + "request_id": "req-text", + "operation": "resolve_and_act", + "action": { + "protocol_version": 2, + "intent": "resolve_target", + "app": "Editor", + "target": { + "selector": "paragraph", + "windowHandle": "w2:fixture", + "providerBusName": ":1.42", + "providerObjectPath": "/org/a11y/paragraph", + "processGeneration": "pid:42:start:100", + "role": "paragraph", + "hasName": false + }, + "observation": { + "text_read": {"field": "value", "start": 1, "limit": 3}, + "include_screenshot": false, + "capture_snapshot": false + } + } +} diff --git a/computer-use/testdata/provider-wire/text-read.response.json b/computer-use/testdata/provider-wire/text-read.response.json new file mode 100644 index 0000000..fe9c407 --- /dev/null +++ b/computer-use/testdata/provider-wire/text-read.response.json @@ -0,0 +1,10 @@ +{ + "protocol": "computer-use-provider", + "schema_version": 2, + "request_id": "req-text", + "operation": "resolve_and_act", + "accessibility": { + "ok": true, + "text_read": {"field": "value", "start": 1, "end": 4, "total": 6, "text": "中🙂e"} + } +} diff --git a/docs/MCP.md b/docs/MCP.md index 8ee7bbc..7f1ad5c 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -467,8 +467,10 @@ application aN -> top-level window wN -> element rN read failure. Missing windows remain unknown. Returned refs remain usable; title-based selection, implicit focus, and window waits require a complete listing. A missing window cannot prove that a close succeeded. -- `observe` returns a window tree or a diff with `rN` refs; `find` returns - matching `rN`, and a truncated result prints the call for the next page. +- `observe` returns a window tree or a diff with `rN` refs. On Linux, + `observe` with `read_text` returns one text page instead of a tree. + `find` returns matching `rN`, and a truncated result prints the call for + the next page. - References describe current state; they are not durable selectors. Every mutation resolves its reference against the live accessibility tree again and rejects stale or cross-window targets before any input. @@ -505,6 +507,77 @@ Clipped excerpts also carry `text_truncated=true`. Text already displayed in a n or matched-text field is not repeated. Excerpts are display-only and do not change canonical text, selectors, refs, or the response budget. +### Linux text pages + +On Linux, `AX` can read a field of an existing element ref without expanding +the tree. Use `mode:"observe"`, an `rN` in `root`, and `read_text`: + +```json +{"mode":"observe","root":"r12","read_text":{"field":"value","start":0,"limit":4000}} +``` + +The Python SDK accepts the same object through +`desktop.observe(root="r12", read_text={"field": "value"})`. + +`root` must identify a node from an earlier result. The server resolves its +live identity and rejects stale refs. Omit `app`; the ref identifies its owning +application and window. `window_ref` is optional and must match the ref's owner +if supplied. Do not combine `read_text` with `scope` or the `observe` +field, even with `observe:"full"`. The `mode` remains `"observe"`. + +| Field | Source | +| --- | --- | +| `value` | The node's AT-SPI Text interface. If Text is unavailable or unreadable, the call fails explicitly; it does not substitute another property. | +| `name` | The complete accessible name, before tree clipping. | +| `description` | The complete accessible description, before tree clipping. | + +Offsets and lengths count Unicode code points, not bytes or grapheme +clusters. `start` is a nonnegative integer and defaults to `0`. `limit` is +an integer from `1` to `4000` and defaults to `4000`. Each page includes +`start` and excludes `end`. The server returns at most `limit` code points, +stopping at `total` when fewer remain. `start == total` returns a valid empty +page. `start > total`, negative offsets, and invalid limits fail explicitly. + +After the standard AX evidence, the text page starts with one JSON metadata +line, followed by a newline and the raw page text. Metadata contains `ref`, `window_ref`, +`field`, `start`, `end`, `total`, and `truncated`. When `end < total`, +`truncated` is `true` and `next_read` contains the next AX call. Otherwise, +`truncated` is `false` and there is no next page. Follow `next_read` rather +than calculating the next offset from the requested limit. + +For example, reading `A🙂BCDE` with `start:0` and `limit:4` returns: + +```text +{"ref":"r12","window_ref":"w3","field":"value","start":0,"end":4,"total":6,"truncated":true,"next_read":{"mode":"observe","window_ref":"w3","root":"r12","read_text":{"field":"value","start":4,"limit":4}}} +A🙂BC +``` + +With `COMPUTER_USE_STRUCTURED_CONTENT=1`, `structuredContent.text_page.text` +also contains the raw page text. Text pages preserve whitespace and line +breaks; they do not use the tree's preview clipping rules. + +The inline response budget can reduce the page below the requested `limit`. +Metadata and the raw text must fit together. If the budget cannot hold a +page and its continuation, the call fails explicitly. Paging does not raise +the global tree size cap. + +Each page reads the live application, not a frozen document. If the provider +detects a character-count change during one page read, that call fails. +Changes between pages and same-length edits can escape detection. Combining +pages does not guarantee a consistent snapshot. + +In Linux trees, a clipped field reports `… field length=N codepoints` and +a `read_text` call, with the field name and full length. Non-interactive +content with AT-SPI Text gets a value preview of at most 80 code points, +plus an ellipsis when clipped, and an `rN` when its visible node has a complete +provider identity. Go's 40-code-point cell values, 80-code-point +find excerpts, and existing name and description display rules still apply. + +Text pages, truncation hints, and the revised interaction guide change the +model-visible contract. They require a new evaluation attempt. Do not mix +instruction revisions within an attempt or infer performance from these +interface changes. + ### `act` steps - `act` accepts up to 64 ordered steps: actions @@ -761,7 +834,8 @@ receive synthesized keyboard input through the toggle. ### Errors -Every AX and GUI result starts with one JSON line: `ok`, and on failure +Except for successful [text pages](#linux-text-pages), AX and GUI results +start with one JSON line: `ok`, and on failure `code`, `path`, `message` and `hint` (a corrected call), then `input_emitted`, `completed` and `requested`. The public codes are: @@ -898,7 +972,7 @@ the shell or the application. | `COMPUTER_USE_ERROR_CONTEXT` | `off` stops capturing fresh UI state after a failed call | | `COMPUTER_USE_OUTPUT_MODE` | `file` writes large outputs to files and returns their paths | | `COMPUTER_USE_OUTPUT_DIR`, `COMPUTER_USE_OUTPUT_THRESHOLD` | directory (default `computer-use` in the system temporary directory) and minimum size (4000 characters) for file output | -| `COMPUTER_USE_MAX_INLINE_CHARS` | inline text budget per result, 24000 characters by default | +| `COMPUTER_USE_MAX_INLINE_CHARS` | inline text budget per result, 24000 characters by default; smaller budgets reduce `read_text` page size without raising tree caps | | `COMPUTER_USE_TRACE_FILE` | append one JSONL line per AX action | | `COMPUTER_USE_PYTHON_BINARY` | Linux: interpreter for the provider, `python3` by default | | `COMPUTER_USE_RUNTIME_TIMEOUT_SECONDS` | base provider request timeout, 30 seconds by default | diff --git a/instructions/hybrid.md b/instructions/hybrid.md index a97c50d..ab3b7ea 100644 --- a/instructions/hybrid.md +++ b/instructions/hybrid.md @@ -100,6 +100,39 @@ windows: opened w5 "Insert Sheet" (dialog, focused) - On macOS, use `open -a APP` to launch through LaunchServices. Reuse an existing application instance and verify its window before sending input. +## Read Long Text On Linux + +Tree text is a preview. A clipped field shows `… field length=N codepoints` +and a read call. Non-interactive content with AT-SPI Text has a value preview +of at most 80 code points, plus an ellipsis when clipped. When the node has +an `rN`, use that ref to read the text: + +```json +{"mode":"observe","root":"r12","read_text":{"field":"value","start":0,"limit":4000}} +``` + +Use an existing element ref for `root`. Omit `app`. An optional `window_ref` +must match the ref's owner. Keep `mode:"observe"`, but +omit the separate `observe` field and `scope`. `value` reads AT-SPI Text and +fails explicitly when Text is unavailable; it does not substitute another +property. `name` and `description` read the complete accessible properties. + +Offsets count Unicode code points, not bytes. Pages include `start` and +exclude `end`. `start` is a nonnegative integer and defaults to `0`. +`limit` is an integer from `1` to `4000` and defaults to `4000`. The server +can return fewer code points to fit the inline budget. `start == total` is +a valid empty page. Out-of-range offsets and limits fail explicitly. + +After the standard AX evidence, each text page starts with JSON containing `ref`, `window_ref`, +`field`, `start`, `end`, `total`, and `truncated`, followed by the raw text. +When more text remains, `next_read` gives the next AX call. Follow that call +rather than adding your requested limit to `start`. Structured output, when +enabled, also contains the raw text in `text_page.text`. + +Each page reads live content. A detected character-count change during the +read fails that call. Edits between pages or edits that preserve the length +can escape detection; combined pages are not a guaranteed snapshot. + ## When Something Fails Every AX and GUI error starts with a `code`, a recovery `hint`, diff --git a/sdk/python/hcu_sdk/__init__.py b/sdk/python/hcu_sdk/__init__.py index e5fa6a4..83eb367 100644 --- a/sdk/python/hcu_sdk/__init__.py +++ b/sdk/python/hcu_sdk/__init__.py @@ -434,9 +434,10 @@ def apps(self): return self.call("AX", {"mode": "apps"}) def observe(self, target=None, root=None, scope=None, observe=None, since=None, - *, window_ref=None, app=None): + *, window_ref=None, app=None, read_text=None): args = _discovery_args("observe", target, window_ref, app) - for key, value in (("root", root), ("scope", scope), ("observe", observe)): + for key, value in (("root", root), ("scope", scope), ("observe", observe), + ("read_text", read_text)): if value is not None: args[key] = value return self.call("AX", args, since=since) diff --git a/sdk/python/tests/test_sdk.py b/sdk/python/tests/test_sdk.py index 36d8192..00eddd4 100644 --- a/sdk/python/tests/test_sdk.py +++ b/sdk/python/tests/test_sdk.py @@ -130,6 +130,7 @@ def helper_calls(self): builder.observe("w1", observe="tree") builder.observe("w1", observe="full") builder.observe("w1", observe="screenshot") + builder.observe(root="r1", read_text={"field": "value", "start": 0, "limit": 4000}) builder.wait({"window": "Saved", "timeout_ms": 1}) builder.wait({"ref": "r1", "state": "checked"}, window_ref="w1") builder.wait({"query": {"role": "dialog"}, "gone": True, "timeout_ms": 1}) From fc67194baebc69903082a80788e63cef399bb868 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 09:44:31 +0000 Subject: [PATCH 13/15] fix(ax): diagnose unavailable content in window snapshots --- CHANGELOG.md | 6 + computer-use/internal/core/app_snapshot.go | 3 + .../internal/core/app_snapshot_content.go | 41 +++ .../core/app_snapshot_content_test.go | 272 ++++++++++++++++++ .../transport/contract_golden_test.go | 16 ++ .../testdata/contract/public_contract.json | 27 ++ docs/MCP.md | 16 ++ 7 files changed, 381 insertions(+) create mode 100644 computer-use/internal/core/app_snapshot_content.go create mode 100644 computer-use/internal/core/app_snapshot_content_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index da9ec61..4076249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- AX reports when a window snapshot returns only its top-level root. + The diagnostic asks for a screenshot and verification of prior action outcomes + before retrying. It does not infer a renderer crash, reload the application, + or replay an action. Reads with coverage gaps say that omitted content may + still exist; scoped reads do not trigger the window-level diagnostic. + - AX `activate` with a window ref or title can switch the target within an action batch. Later refs and ref-less steps use the selected window; refs from other windows require another explicit activation. Each step diff --git a/computer-use/internal/core/app_snapshot.go b/computer-use/internal/core/app_snapshot.go index 376c3f7..9c74ad9 100644 --- a/computer-use/internal/core/app_snapshot.go +++ b/computer-use/internal/core/app_snapshot.go @@ -289,6 +289,9 @@ func (s *AppSnapshot) renderedText() string { lines = append(lines, "windows:") lines = append(lines, renderedWindowInventory(s.Windows)...) } + if note := s.contentUnavailableNote(); note != "" { + lines = append(lines, note) + } if normalizedAXScope(s.Scope) == contract.AXScopeContent && s.scopeRoot != nil { lines = append(lines, fmt.Sprintf( "scope: content root=%s %q", diff --git a/computer-use/internal/core/app_snapshot_content.go b/computer-use/internal/core/app_snapshot_content.go new file mode 100644 index 0000000..df23f5e --- /dev/null +++ b/computer-use/internal/core/app_snapshot_content.go @@ -0,0 +1,41 @@ +package core + +import ( + "strings" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +func (snapshot *AppSnapshot) contentUnavailableNote() string { + if snapshot == nil || snapshot.WindowAbsent || + !validInternalWindowHandle(snapshot.WindowHandle) || + len(snapshot.Elements) != 1 || snapshot.deliveryOnly || snapshot.lightReadback || + snapshot.FindCandidatesOnly || len(snapshot.FindResume) != 0 || snapshot.findResume != nil || + snapshot.observationRoot != nil || snapshot.projection.Root != "" || + normalizedAXScope(snapshot.Scope) != contract.AXScopeWindow || + normalizedAXScope(snapshot.projection.Scope) != contract.AXScopeWindow { + return "" + } + coverageNote := "" + coverage := providerCoverageSummary(snapshot.Annotations) + if coverage.ProjectionHidden != 0 || coverage.ExpandableHidden != 0 || + coverage.AcquisitionIncomplete != 0 { + coverageNote = "This read has coverage gaps; omitted content may still exist. " + } + root := &snapshot.Elements[0] + if len(root.RuntimeID) > 1 || len(root.TextTruncated) != 0 || + strings.TrimSpace(root.Value) != "" || strings.TrimSpace(root.Description) != "" || + strings.TrimSpace(root.Placeholder) != "" || strings.TrimSpace(snapshot.SelectedText) != "" { + return "" + } + switch ontologyProviderRole(root) { + case "frame", "window", "dialog", "alert": + return "AX content unavailable in this snapshot: only the top-level window root was returned. " + + coverageNote + + "Visible emptiness, renderer crash, and navigation outcome are unknown. " + + "Inspect a screenshot. Verify the outcome of any prior delete or submit before repeating it. " + + "If the app supports reloading and it is safe, reload, then observe again." + default: + return "" + } +} diff --git a/computer-use/internal/core/app_snapshot_content_test.go b/computer-use/internal/core/app_snapshot_content_test.go new file mode 100644 index 0000000..8e2a561 --- /dev/null +++ b/computer-use/internal/core/app_snapshot_content_test.go @@ -0,0 +1,272 @@ +package core + +import ( + "reflect" + "strings" + "testing" + + "github.com/microsoft/Hybrid-Computer-Use/computer-use/internal/contract" +) + +func contentRootSnapshot() *AppSnapshot { + backend := &reportBackend{windows: []reportWindow{reportMain()}} + snapshot := backend.snapshot(reportMain()) + snapshot.Elements = snapshot.Elements[:1] + snapshot.TreeLines = snapshot.TreeLines[:1] + snapshot.SnapshotID = "root-only" + snapshot.projection = snapshotProjection{View: contract.TreeViewWorking, Scope: contract.AXScopeWindow} + return snapshot +} + +func TestAppSnapshotContentDiagnosisRequiresWholeWindowRoot(test *testing.T) { + for _, role := range []string{"frame", "window", "dialog", "alert"} { + test.Run(role, func(test *testing.T) { + snapshot := contentRootSnapshot() + snapshot.Elements[0].ControlType = role + if snapshot.contentUnavailableNote() == "" { + test.Fatal("complete root-only window has no content diagnostic") + } + }) + } + var absent *AppSnapshot + if absent.contentUnavailableNote() != "" { + test.Fatal("nil snapshot was diagnosed") + } + for _, scenario := range []struct { + name string + change func(*AppSnapshot) + }{ + {"static descendant", func(snapshot *AppSnapshot) { + snapshot.Elements = append(snapshot.Elements, ElementRecord{ + Index: 1, RuntimeID: []int{0, 0}, ControlType: "label", Name: "Saved successfully", + }) + }}, + {"anonymous descendant", func(snapshot *AppSnapshot) { + snapshot.Elements = append(snapshot.Elements, ElementRecord{ + Index: 1, RuntimeID: []int{0, 0}, ControlType: "panel", + }) + }}, + {"static root", func(snapshot *AppSnapshot) { snapshot.Elements[0].ControlType = "text" }}, + {"document root", func(snapshot *AppSnapshot) { snapshot.Elements[0].ControlType = "document web" }}, + {"root value", func(snapshot *AppSnapshot) { snapshot.Elements[0].Value = "Saved successfully" }}, + {"root description", func(snapshot *AppSnapshot) { snapshot.Elements[0].Description = "Status details" }}, + {"root placeholder", func(snapshot *AppSnapshot) { snapshot.Elements[0].Placeholder = "Search" }}, + {"selected text", func(snapshot *AppSnapshot) { snapshot.SelectedText = "Selected content" }}, + {"missing root", func(snapshot *AppSnapshot) { snapshot.Elements = nil }}, + {"window absent", func(snapshot *AppSnapshot) { snapshot.WindowAbsent = true }}, + {"no window identity", func(snapshot *AppSnapshot) { snapshot.WindowHandle = "" }}, + {"weak window identity", func(snapshot *AppSnapshot) { snapshot.WindowHandle = "weak:pid10/win0/frame" }}, + {"delivery only", func(snapshot *AppSnapshot) { snapshot.deliveryOnly = true }}, + {"light readback", func(snapshot *AppSnapshot) { snapshot.lightReadback = true }}, + {"find candidates", func(snapshot *AppSnapshot) { snapshot.FindCandidatesOnly = true }}, + {"find continuation", func(snapshot *AppSnapshot) { snapshot.FindResume = []int{0, 1} }}, + {"content scope", func(snapshot *AppSnapshot) { snapshot.Scope = contract.AXScopeContent }}, + {"content projection", func(snapshot *AppSnapshot) { snapshot.projection.Scope = contract.AXScopeContent }}, + {"rooted observation", func(snapshot *AppSnapshot) { snapshot.observationRoot = &snapshot.Elements[0] }}, + {"rooted projection", func(snapshot *AppSnapshot) { snapshot.projection.Root = `{"focused":true}` }}, + {"descendant-only record", func(snapshot *AppSnapshot) { snapshot.Elements[0].RuntimeID = []int{0, 1} }}, + {"truncated root text", func(snapshot *AppSnapshot) { snapshot.Elements[0].TextTruncated = []string{"name"} }}, + } { + test.Run(scenario.name, func(test *testing.T) { + snapshot := contentRootSnapshot() + scenario.change(snapshot) + if note := snapshot.contentUnavailableNote(); note != "" { + test.Fatalf("partial or content-bearing snapshot was diagnosed: %s", note) + } + snapshot.Annotations = []axProviderAnnotation{{Kind: "provider_omission", Count: 10, Reason: "filtered", ParentRuntimeID: []int{0}}} + if note := snapshot.contentUnavailableNote(); note != "" { + test.Fatalf("coverage qualification bypassed the diagnosis exclusion: %s", note) + } + }) + } +} + +func TestAppSnapshotContentDiagnosisQualifiesCoverageGaps(test *testing.T) { + for _, reason := range []string{ + "filtered", "pressure", "collapsed_menu", "provider_children_not_rendered", "modal_dialog", + "time", "chars", "budget", "depth", "render_error", "fanout", "outside_view", + "provider_probe_error", "null_reference", "filtered,depth", "unknown_reason", "", + } { + test.Run(reason, func(test *testing.T) { + snapshot := contentRootSnapshot() + snapshot.Annotations = []axProviderAnnotation{{Kind: "provider_omission", Count: 1, Reason: reason}} + note := snapshot.contentUnavailableNote() + if !strings.Contains(note, "AX content unavailable in this snapshot:") || + !strings.Contains(note, "This read has coverage gaps; omitted content may still exist.") { + test.Fatalf("root-only snapshot lost its qualified diagnosis: %s", note) + } + }) + } +} + +func TestAppSnapshotContentDiagnosisPreservesCoverage(test *testing.T) { + for _, scenario := range []struct { + reason, evidenceKey string + }{ + {"filtered", "projection_hidden_nodes"}, + {"collapsed_menu", "expandable_hidden_nodes"}, + {"provider_probe_error", "coverage_incomplete_nodes"}, + {"null_reference", "coverage_incomplete_nodes"}, + } { + test.Run(scenario.reason, func(test *testing.T) { + service := NewService() + snapshot := contentRootSnapshot() + original := []axProviderAnnotation{{ + Kind: "provider_omission", Count: 10, Reason: scenario.reason, ParentRuntimeID: []int{0}, + }} + snapshot.Annotations = append([]axProviderAnnotation(nil), original...) + if err := service.BuildAndRenderAXSnapshot(snapshot, contract.TreeViewWorking); err != nil { + test.Fatal(err) + } + providerNote := "The accessibility read omitted content." + result := service.getAppStateResult(snapshot, []string{providerNote}, false) + text := ResultText(result) + for _, required := range []string{ + "AX content unavailable in this snapshot: only the top-level window root was returned.", + "This read has coverage gaps; omitted content may still exist.", + "Visible emptiness, renderer crash, and navigation outcome are unknown.", + providerNote, + } { + if !strings.Contains(text, required) { + test.Fatalf("whole-window observation lost %q:\n%s", required, text) + } + } + if result.IsError || result.StructuredContent[scenario.evidenceKey] != 10 || + !reflect.DeepEqual(snapshot.Annotations, original) { + test.Fatalf("diagnosis changed provider coverage or result: %#v annotations=%#v", + result, snapshot.Annotations) + } + if scenario.evidenceKey == "coverage_incomplete_nodes" { + public := PrepareAXEvidence(EnforceAXResponseBudget(WithAXModeEvidence(result, contract.AXModeObserve))) + if public.StructuredContent["coverage"] != "partial" || public.StructuredContent["tree_complete"] != false { + test.Fatalf("incomplete acquisition was presented as complete: %#v", public.StructuredContent) + } + } + }) + } +} + +func TestAppSnapshotContentDiagnosisSurvivesTreeReference(test *testing.T) { + service := NewService() + snapshot := contentRootSnapshot() + if err := service.BuildAndRenderAXSnapshot(snapshot, contract.TreeViewWorking); err != nil { + test.Fatal(err) + } + first := service.getAppStateResult(snapshot, nil, false) + text := ResultText(first) + for _, required := range []string{ + "AX content unavailable in this snapshot:", + "Visible emptiness, renderer crash, and navigation outcome are unknown", + "Inspect a screenshot", "Verify the outcome of any prior delete or submit before repeating it", + "If the app supports reloading and it is safe, reload, then observe again", + } { + if !strings.Contains(text, required) { + test.Fatalf("missing diagnosis or recovery guidance %q:\n%s", required, text) + } + } + if strings.Contains(text, "coverage gaps") || strings.Contains(text, "omitted content may still exist") { + test.Fatalf("complete root-only snapshot acquired a coverage qualifier:\n%s", text) + } + if strings.Index(text, "AX content unavailable") < strings.Index(text, "window:") || + strings.Index(text, "AX content unavailable") > strings.Index(text, "ax:") { + test.Fatalf("diagnosis is not between window identity and tree:\n%s", text) + } + repeated := service.getAppStateResult(snapshot, nil, false) + if repeated.StructuredContent["tree_reference_only"] != true || + !strings.Contains(ResultText(repeated), "AX content unavailable in this snapshot:") { + test.Fatalf("tree reference lost current content diagnosis: %#v\n%s", repeated.StructuredContent, ResultText(repeated)) + } + if first.IsError || repeated.IsError { + test.Fatal("content diagnosis changed successful observation into an error") + } +} + +func TestAppSnapshotContentDiagnosisSurvivesDiff(test *testing.T) { + for _, scenario := range []struct { + name string + withContent bool + wantKind string + coverageReason string + }{ + {"unchanged root", false, "diff", ""}, + {"content disappears", true, "working_tree", ""}, + {"filtered root", false, "diff", "filtered"}, + } { + test.Run(scenario.name, func(test *testing.T) { + service := NewService() + before := contentRootSnapshot() + if scenario.withContent { + backend := &reportBackend{windows: []reportWindow{reportMain()}} + before = backend.snapshot(reportMain()) + before.projection = snapshotProjection{View: contract.TreeViewWorking, Scope: contract.AXScopeWindow} + } + if err := service.BuildAndRenderAXSnapshot(before, contract.TreeViewWorking); err != nil { + test.Fatal(err) + } + service.getAppStateResult(before, nil, false) + after := contentRootSnapshot() + if scenario.coverageReason != "" { + after.Annotations = []axProviderAnnotation{{ + Kind: "provider_omission", Count: 10, Reason: scenario.coverageReason, ParentRuntimeID: []int{0}, + }} + } + if err := service.BuildAndRenderAXSnapshot(after, contract.TreeViewWorking); err != nil { + test.Fatal(err) + } + result := service.renderAXBatchFinalObservation(after, nil, observation{mode: contract.ObserveDiff}) + if result.IsError || result.StructuredContent["final_observation_kind"] != scenario.wantKind || + !strings.Contains(ResultText(result), "AX content unavailable in this snapshot:") { + test.Fatalf("diff observation lost current content diagnosis: %#v\n%s", result.StructuredContent, ResultText(result)) + } + if strings.Contains(ResultText(result), "This read has coverage gaps;") != (scenario.coverageReason != "") { + test.Fatalf("diff changed the coverage qualification: %s", ResultText(result)) + } + }) + } +} + +func TestAppSnapshotContentDiagnosisPreservesActionUncertainty(test *testing.T) { + service := NewService() + backend := &recordingPlatformBackend{accessibilityResponse: &ProviderAccessibilityResult{ + OK: true, Snapshot: contentRootSnapshot(), + }} + service.Backend = backend + window, _ := reportObserve(test, service) + backend.accessibilityResponse = &ProviderAccessibilityResult{ + OK: true, Snapshot: contentRootSnapshot(), + Mutation: &ProviderMutationFacts{InputEventsEmitted: true, DesktopStateMutated: true}, + ActionEvidence: &ProviderActionEvidence{InputEmitted: true, Route: "synthesis"}, + } + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": window, "observe": "tree", "actions": keys("Delete"), + }) + if result.IsError { + test.Fatalf("root-only action failed: %s", ResultText(result)) + } + public := WithAXTextEvidence(PrepareAXEvidence(result)) + text := ResultText(public) + if !strings.Contains(text, `"effect_unknown":true`) || + !strings.Contains(text, "AX content unavailable in this snapshot:") { + test.Fatalf("action lost uncertainty or content diagnosis:\n%s", text) + } + if len(backend.actionRequests) != 1 || len(backend.observeRequests) != 1 { + test.Fatalf("diagnosis caused extra actions or observations: actions=%d observations=%d", + len(backend.actionRequests), len(backend.observeRequests)) + } +} + +func TestAppSnapshotContentDiagnosisDoesNotLeakFromLightReadback(test *testing.T) { + service, backend := rootOnlyService(test, false, reportMain()) + window, _ := reportObserve(test, service) + result := service.callTool(contract.PublicToolAX, map[string]any{ + "mode": "act", "window_ref": window, "observe": "tree", "actions": keys("a", "b", "c", "d"), + }) + if result.IsError || strings.Contains(ResultText(result), "AX content unavailable in this snapshot:") { + test.Fatalf("light readback produced a false diagnosis:\n%s", ResultText(result)) + } + if backend.rootOnlyReads != 2 || backend.fullReads != 2 || + len(backend.actionRequests) != 4 || len(backend.observeRequests) != 1 { + test.Fatalf("read or action counts changed: root=%d full=%d actions=%d observations=%d", + backend.rootOnlyReads, backend.fullReads, len(backend.actionRequests), len(backend.observeRequests)) + } +} diff --git a/computer-use/internal/transport/contract_golden_test.go b/computer-use/internal/transport/contract_golden_test.go index 3768be0..62f7c3b 100644 --- a/computer-use/internal/transport/contract_golden_test.go +++ b/computer-use/internal/transport/contract_golden_test.go @@ -239,6 +239,22 @@ func capturePublicContract(t *testing.T) publicContract { }) } + snapshot := transportOrderedInputTestSnapshot() + snapshot.WindowHandle = transportTestHandleReport + snapshot.Elements = snapshot.Elements[:1] + snapshot.TreeLines = snapshot.TreeLines[:1] + contentService := core.NewServiceWithBackend(&transportRecordingPlatformBackend{ + accessibilityResponse: &core.ProviderAccessibilityResult{OK: true, Snapshot: snapshot}, + }) + arguments := map[string]any{"mode": "observe", "app": "Editor", "observe": "tree"} + contract.Calls = append(contract.Calls, contractExchange{ + Request: map[string]any{"tool": "AX", "arguments": arguments}, + Response: roundTripJSON(t, handleMCPRequest(map[string]any{ + "jsonrpc": "2.0", "id": float64(100), "method": "tools/call", + "params": map[string]any{"name": "AX", "arguments": arguments}, + }, contentService)), + }) + for _, args := range [][]string{ {"--help"}, {"help", "mcp"}, diff --git a/computer-use/testdata/contract/public_contract.json b/computer-use/testdata/contract/public_contract.json index 571792a..5050d6f 100644 --- a/computer-use/testdata/contract/public_contract.json +++ b/computer-use/testdata/contract/public_contract.json @@ -1931,6 +1931,33 @@ "isError": true } } + }, + { + "request": { + "arguments": { + "app": "Editor", + "mode": "observe", + "observe": "tree" + }, + "tool": "AX" + }, + "response": { + "id": 100, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "text": "{\"ok\":true,\"input_emitted\":false}", + "type": "text" + }, + { + "text": "window: \"Document\" [ref=w1]\nAX content unavailable in this snapshot: only the top-level window root was returned. Visible emptiness, renderer crash, and navigation outcome are unknown. Inspect a screenshot. Verify the outcome of any prior delete or submit before repeating it. If the app supports reloading and it is safe, reload, then observe again.\nax:\n - window \"Document\"\n", + "type": "text" + } + ], + "isError": false + } + } } ], "cli": { diff --git a/docs/MCP.md b/docs/MCP.md index 7f1ad5c..5cdd0cd 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -988,6 +988,22 @@ loss. Tracing does not extend the timeout or change retry rules. ## Troubleshooting +When a window snapshot returns only its top-level root, AX reports +`AX content unavailable in this snapshot`. The window was observed, but AX +did not obtain its content. This does not identify a renderer crash, navigation, +or a visually empty page. When the read has coverage gaps, the diagnostic states +that omitted content may still exist. Existing coverage notes remain available. +Rooted, content-scoped, find, and intermediate lightweight reads do not trigger +this window-level diagnostic. + +Inspect a GUI screenshot and verify the outcome of any prior delete or submit +before repeating it. If the application supports reloading and it is safe, +reload and observe again. HCU does not reload or replay an action automatically. +For a persistent blank Electron window, application operators can collect +`render-process-gone`, `did-fail-load`, and navigation events to distinguish +renderer exit from loading failures. Check container shared-memory capacity +and application logs; window survival alone does not prove content recovery. + AX lists managed Linux application windows even when their process registers on the accessibility bus without exporting a tree for every window. Such windows have a `wN` marked `(no accessibility tree: use GUI)`. AX can activate, From e603dcdbfbc2713bce52730ee9c300fd51d356bd Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 09:44:31 +0000 Subject: [PATCH 14/15] docs(linux): explain Calc deferred Tab navigation Document that ordered key delivery does not guarantee completion of application navigation before following text. Explain Calc's deferred accelerator dispatch and require destination-value inspection after navigation-dependent edits. --- docs/MCP.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/MCP.md b/docs/MCP.md index 5cdd0cd..19e0ccf 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -125,6 +125,27 @@ reads the binding before the next character reuses it. A call accepts at most focused control, for example submitting a form or moving focus. Text that contains NUL or an unpaired surrogate fails before any input on this path. +### Keyboard navigation and application state + +The Linux key-event path sends keys in order. It does not wait for the +application to finish each key's deferred commands before sending the next key. +Input emission therefore does not prove that navigation finished before the +following text started an edit. + +LibreOffice Calc can receive every Tab in a burst yet put the following text +in an earlier cell. For example, typing `cancelled\t\t\t\tnote` from B2 can +put `note` in C2 even though the selection later reaches F2. The same behavior +occurs with direct XTEST input, without HCU or AT-SPI. Calc handles the first +Tab synchronously while editing, but queues later navigation commands through +its accelerator dispatcher. Following text can start an edit before those +commands run. The final selection alone does not prove which cell received +the text. + +For edits that depend on navigation, send individual `keypress` calls and +inspect the resulting application state before typing into the destination. +After committing the edit, read the destination cell's value as well as the +selection. Tab navigation does not clear the cells it passes through. + ### Accessibility on Linux Applications started by the user need no accessibility flags or From 22a95bd4648f46f6df1d99f3271b4c3276d20dd2 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Mon, 16 Nov 2026 18:00:12 +0000 Subject: [PATCH 15/15] test(contract): regenerate the public contract after the Linux AX changes --- computer-use/testdata/contract/public_contract.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/computer-use/testdata/contract/public_contract.json b/computer-use/testdata/contract/public_contract.json index 5050d6f..696862e 100644 --- a/computer-use/testdata/contract/public_contract.json +++ b/computer-use/testdata/contract/public_contract.json @@ -46,8 +46,7 @@ "openWorldHint": false, "title": "Accessibility" }, - "description": "Accessibility for application windows. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. activate with wN or title switches the batch target for later refs and ref-less steps. Other-window refs fail until another activate selects their window. Each step reports its window; final observation uses the last executed window. Copy rN/wN exactly; stale refs fail. keypress/type without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", - "description": "Accessibility for application windows. Linux: observe with root rN and read_text reads bounded pages of full name, value (AT-SPI Text), or description; follow next_read. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. Copy rN/wN exactly; stale refs fail. keypress/type without ref go to the focused window. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", + "description": "Accessibility for application windows. Linux: observe with root rN and read_text reads bounded pages of full name, value (AT-SPI Text), or description; follow next_read. Modes: apps lists confirmed windows (wN, title, app, active) and reports incomplete inventories; missing windows then remain unknown. observe returns the window tree with rN refs. find returns matching rN. act runs ordered steps, checks them before input, stops at the first failed step, and reports each target's change, windows opened/closed/focused, and a diff. activate with wN or title switches the batch target for later refs and ref-less steps. Other-window refs fail until another activate selects their window. Each step reports its window; final observation uses the last executed window. Copy rN/wN exactly; stale refs fail. keypress/type without ref use the batch target, initially the focused window unless refs or window_ref select it. select marks ref, its text substring, or ref..to_ref (paragraphs, cells) and reports the selection read back. activate/close take a title substring or wN. A wait step sends no input and fails the batch on timeout: window (title, any app), query (as find), ref with value/state/text, or text; gone inverts. settle needs no prediction: it waits until the UI changes after the step before it and stops changing; it fails only if nothing changed. After a window wait, keys go to that window, after settle to the focused one. Skip waits the report answers. Errors return code and hint. Examples:\n{\"mode\":\"act\",\"actions\":[{\"type\":\"click\",\"ref\":\"r12\"},{\"type\":\"wait\",\"window\":\"Export\"},{\"type\":\"type\",\"text\":\"q3.csv\"},{\"type\":\"keypress\",\"keys\":[\"ENTER\"]}]}\n{\"mode\":\"find\",\"window_ref\":\"w2\",\"query\":{\"role\":\"button\",\"name_contains\":\"Save\"}}", "inputSchema": { "$defs": { "query": {