Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 139 additions & 0 deletions computer-use/internal/contract/ablation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package contract

import (
"fmt"
"os"
"strings"
)

// Mechanism ablations remove one HCU mechanism each, for experiments that
// measure what the mechanism contributes. COMPUTER_USE_ABLATE is a
// comma-separated list of the names below; by default nothing is removed.
//
// A removed mechanism is absent from everything the model sees: schemas,
// tool descriptions, server instructions and error texts. Like a disabled
// channel, it is not merely rejected at call time.
//
// - batching: an AX act holds at most one input step; wait steps may
// still surround it. GUI batches are unchanged.
// - wait: AX acts take no wait steps. The tools' own stabilization after
// input is unchanged.
// - diff: an AX act returns its execution evidence and the whole current
// tree of its window, as observe "tree" does, without the effect and
// windows report, the diff, or notes that compare with an earlier read.
// observe always returns the tree, never "unchanged". The tree is read
// with the same post-action stabilization as a diff. Two things stay:
// effect_unknown in the evidence (whether input had a verified effect
// is execution evidence, not a diff) and a settle wait step's message
// (it belongs to wait).
// - guard: the target and layout guards still decide, but only log the
// input they would have stopped (COMPUTER_USE_GUARD_LOG) and let it
// through. Only the Linux provider implements this.
const (
AblationEnv = "COMPUTER_USE_ABLATE"
GuardLogEnv = "COMPUTER_USE_GUARD_LOG"
AblateBatching = "batching"
AblateWait = "wait"
AblateDiff = "diff"
AblateGuard = "guard"
)

// AblationNames lists the ablations in their canonical order.
var AblationNames = []string{AblateBatching, AblateWait, AblateDiff, AblateGuard}

// ParseAblations returns the ablations named in raw. An unknown or repeated
// name is an error: an experiment with a misspelled switch would otherwise
// run the complete HCU under an ablation label.
func ParseAblations(raw string) (map[string]bool, error) {
ablated := map[string]bool{}
if strings.TrimSpace(raw) == "" {
return ablated, nil
}
for _, part := range strings.Split(raw, ",") {
name := strings.TrimSpace(part)
if !ContainsString(AblationNames, name) {
return nil, fmt.Errorf("%s: unknown ablation %q; use a comma-separated subset of %s",
AblationEnv, name, strings.Join(AblationNames, ", "))
}
if ablated[name] {
return nil, fmt.Errorf("%s: ablation %q is listed twice", AblationEnv, name)
}
ablated[name] = true
}
return ablated, nil
}

// ValidateAblations checks COMPUTER_USE_ABLATE for a server, daemon or
// command started on goos. Entry points call it before serving anything.
func ValidateAblations(goos string, getenv func(string) string) error {
ablated, err := ParseAblations(getenv(AblationEnv))
if err != nil {
return err
}
if ablated[AblateGuard] {
if goos != "linux" {
return fmt.Errorf("%s: the guard ablation is implemented only by the Linux provider, not on %s",
AblationEnv, goos)
}
if strings.TrimSpace(getenv(GuardLogEnv)) == "" {
return fmt.Errorf("%s: the guard ablation needs %s, the file that records the input the guards would have stopped",
AblationEnv, GuardLogEnv)
}
}
return nil
}

// Ablated reports whether the named mechanism is removed. Entry points
// validate the variable at startup, so a value that does not parse here
// means the process skipped that check.
func Ablated(name string) bool {
ablated, err := ParseAblations(os.Getenv(AblationEnv))
if err != nil {
panic(err)
}
return ablated[name]
}

// SortedAblations returns the active ablations in canonical order.
func SortedAblations() []string {
names := []string{}
for _, name := range AblationNames {
if Ablated(name) {
names = append(names, name)
}
}
return names
}

// PublishedAXActionTypes is the AX action list the schema publishes and the
// decoder accepts.
func PublishedAXActionTypes() []string {
if !Ablated(AblateWait) {
return AXActionTypes
}
types := []string{}
for _, actionType := range AXActionTypes {
if actionType != "wait" {
types = append(types, actionType)
}
}
return types
}

// PublishedObserveValues is the observe list the schema publishes and the
// decoder accepts. Without diff, act takes no observe field (it always
// returns the tree), so diff and none, which only act uses, are gone too.
func PublishedObserveValues() []string {
if !Ablated(AblateDiff) {
return ObserveValues
}
return []string{ObserveTree, ObserveFull, ObserveScreenshot}
}

// AXActMaxActions is the published maximum length of an AX act.
func AXActMaxActions() int {
if Ablated(AblateBatching) && Ablated(AblateWait) {
return 1
}
return AXBatchMaxActions
}
109 changes: 109 additions & 0 deletions computer-use/internal/contract/ablation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package contract

import (
"strings"
"testing"
)

func TestParseAblations(t *testing.T) {
for raw, want := range map[string]string{
"": "",
" ": "",
"wait": "wait",
" guard , batching": "batching,guard",
"batching,wait,diff,guard": "batching,wait,diff,guard",
"diff,wait": "wait,diff",
} {
ablated, err := ParseAblations(raw)
if err != nil {
t.Fatalf("%q: %v", raw, err)
}
got := []string{}
for _, name := range AblationNames {
if ablated[name] {
got = append(got, name)
}
}
if strings.Join(got, ",") != want {
t.Fatalf("%q: got %v want %s", raw, got, want)
}
}
for _, raw := range []string{"waits", "wait,", "Wait", "wait,wait", "compact", ","} {
if _, err := ParseAblations(raw); err == nil {
t.Fatalf("%q must be rejected", raw)
}
}
}

func TestValidateAblationsAtStartup(t *testing.T) {
env := func(values map[string]string) func(string) string {
return func(key string) string { return values[key] }
}
for _, check := range []struct {
goos string
values map[string]string
fails string
}{
{"linux", map[string]string{}, ""},
{"linux", map[string]string{AblationEnv: "batching,wait,diff"}, ""},
{"darwin", map[string]string{AblationEnv: "wait"}, ""},
{"linux", map[string]string{AblationEnv: "diffs"}, "unknown ablation"},
{"linux", map[string]string{AblationEnv: "guard"}, GuardLogEnv},
{"linux", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, ""},
{"darwin", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, "only by the Linux provider"},
{"windows", map[string]string{AblationEnv: "guard", GuardLogEnv: "/tmp/g.jsonl"}, "only by the Linux provider"},
} {
err := ValidateAblations(check.goos, env(check.values))
switch {
case check.fails == "" && err != nil:
t.Fatalf("%s %v: %v", check.goos, check.values, err)
case check.fails != "" && (err == nil || !strings.Contains(err.Error(), check.fails)):
t.Fatalf("%s %v: got %v, want an error with %q", check.goos, check.values, err, check.fails)
}
}
}

func TestPublishedTextWithoutAblationIsTheReleaseText(t *testing.T) {
t.Setenv(AblationEnv, "")
if PublishedServerInstructions() != ServerInstructions {
t.Fatal("server instructions changed without an ablation")
}
if publishedAXToolDescription() != axToolDescription {
t.Fatal("AX description changed without an ablation")
}
t.Setenv(AblationEnv, "wait,diff")
if !strings.Contains(PublishedServerInstructions(), "returns controls as rN refs, and acts on them.") ||
!strings.Contains(PublishedServerInstructions(), "before input, and outside input makes their refs stale.") {
t.Fatalf("instructions: %s", PublishedServerInstructions())
}
}

func TestBatchWindowDescriptionFollowsBatchingAblation(t *testing.T) {
for mask := 0; mask < 1<<len(AblationNames); mask++ {
var names []string
batching := false
for index, name := range AblationNames {
if mask&(1<<index) != 0 {
names = append(names, name)
batching = batching || name == AblateBatching
}
}
t.Run(strings.Join(names, ","), func(t *testing.T) {
t.Setenv(AblationEnv, strings.Join(names, ","))
for _, tool := range ToolDefinitions() {
if tool.Name != PublicToolAX {
continue
}
want := 1
if batching {
want = 0
}
if got := strings.Count(tool.Description, axBatchWindowDescription); got != want {
t.Fatalf("batch window description count = %d, want %d: %s", got, want, tool.Description)
}
return
}
t.Fatal("AX tool is missing")
})
}
}
33 changes: 24 additions & 9 deletions computer-use/internal/contract/schema_validator.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"os"
"sync"

"github.com/santhosh-tekuri/jsonschema/v6"
Expand Down Expand Up @@ -36,24 +37,38 @@ type ArgumentError struct {

func (e *ArgumentError) Error() string { return e.Message }

var publicInputSchemas struct {
once sync.Once
type compiledInputSchemas struct {
schemas map[string]*jsonschema.Schema
err error
}

// publicInputSchemas holds the compiled schemas per COMPUTER_USE_ABLATE
// value, since ablations change the published schemas.
var publicInputSchemas struct {
mu sync.Mutex
byAblation map[string]*compiledInputSchemas
}

// CompiledInputSchema returns the compiled published input schema of a
// public tool, or nil for a name without one. All schemas are compiled once,
// on first use.
func CompiledInputSchema(tool string) (*jsonschema.Schema, error) {
publicInputSchemas.once.Do(func() {
publicInputSchemas.schemas, publicInputSchemas.err =
CompileInputSchemas(AllToolDefinitions())
})
if publicInputSchemas.err != nil {
return nil, publicInputSchemas.err
key := os.Getenv(AblationEnv)
publicInputSchemas.mu.Lock()
compiled := publicInputSchemas.byAblation[key]
if compiled == nil {
compiled = &compiledInputSchemas{}
compiled.schemas, compiled.err = CompileInputSchemas(AllToolDefinitions())
if publicInputSchemas.byAblation == nil {
publicInputSchemas.byAblation = map[string]*compiledInputSchemas{}
}
publicInputSchemas.byAblation[key] = compiled
}
publicInputSchemas.mu.Unlock()
if compiled.err != nil {
return nil, compiled.err
}
return publicInputSchemas.schemas[tool], nil
return compiled.schemas[tool], nil
}

func CompileInputSchemas(tools []ToolDefinition) (map[string]*jsonschema.Schema, error) {
Expand Down
3 changes: 3 additions & 0 deletions computer-use/internal/contract/tool_argument_rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ const (
ArgumentRuleTextBudget = "text_budget"
ArgumentRuleRefOwnsIdentity = "ref_owns_identity"
ArgumentRuleWaitBudget = "wait_budget"
// ArgumentRuleOneInputStep is the batching ablation's limit of one
// input step per act.
ArgumentRuleOneInputStep = "one_input_step"
)

// ArgumentRuleError is a cross-field rule violation.
Expand Down
Loading
Loading