From 63031ec5419fb5fa1252ec623ff156ad8db84f13 Mon Sep 17 00:00:00 2001 From: Haizhong Zheng Date: Sat, 21 Nov 2026 01:56:01 -0500 Subject: [PATCH 1/4] Robotics: the cap-harness engine, the benchmarks and ASPIRE no longer share code A benchmark now holds nothing of a method, and ASPIRE nothing of a benchmark. - examples/cap_harness/: the engine Robosuite and LIBERO run on (simulator service, bin/sim, neutral task functions, judge, task writer, row view, the launcher with a method folder's tasks.sh hook); RoboDojo's service builds on it. - examples/robosuite/ (was aspire_robosuite): the Robosuite tasks and an aspire/ folder. A benchmark's prepare.py writes its tasks and their reruns (rerun_of). - hermit/agent/evolver/methods/aspire/: actor.py (ASPIRE's fix-loop, search and verifier sessions over the engine's task functions, agent.params.engine; the verdict rule withholds an unverified verifier's program), rows.py (ASPIRE's verifier rows, derived before a run), direct.py (the direct-chat comparison). - The service's attempt budget is a row's max_programs; the judge scores a row's eval_seeds. A task function may be named package.module:function. - The Codex sessions' prompts and workspaces are unchanged byte for byte; ASPIRE's header names the CLI's context file (CLAUDE.md for Claude Code); Robosuite gains a Claude Code smoke config. - Tests: the engine's in examples/cap_harness/utils/tests (selected through ENGINES), ASPIRE's in tests/methods/aspire. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 + docs/framework/task.md | 7 +- docs/methods/aspire.md | 61 +- examples/README.md | 4 +- examples/aspire_robosuite/.gitignore | 1 - .../utils/agent/aspire_fields.py | 20 - examples/aspire_robosuite/utils/agent/task.py | 238 ---- .../utils/benchmark/prepare.py | 119 -- .../README.md | 122 +- .../aspire => cap_harness}/launch.sh | 45 +- examples/cap_harness/utils/agent/fields.py | 20 + .../utils/agent/sim.py | 0 .../utils/agent/sim_cli.py | 0 .../utils/agent/sim_service.py | 22 +- .../utils/agent/stage_docs.py | 2 +- examples/cap_harness/utils/agent/task.py | 156 +++ .../utils/benchmark/judge.py | 33 +- examples/cap_harness/utils/benchmark/tasks.py | 84 ++ .../utils/seg_service/requirements.txt | 0 .../utils/seg_service/service.py | 0 examples/cap_harness/utils/tests/cap_fakes.py | 215 ++++ .../utils/tests/test_cap_harness.py | 704 +++++++++++ examples/libero/.gitignore | 3 + examples/libero/README.md | 37 +- examples/libero/aspire/run.yaml | 23 +- examples/libero/aspire/run_90.yaml | 23 +- examples/libero/aspire/run_luna.yaml | 23 +- examples/libero/aspire/run_pro.yaml | 23 +- examples/libero/aspire/run_pro_long.yaml | 23 +- examples/libero/aspire/run_sample_luna.yaml | 23 +- examples/libero/aspire/smoke.yaml | 23 +- examples/libero/aspire/smoke_luna.yaml | 23 +- examples/libero/aspire/smoke_pro.yaml | 23 +- examples/libero/aspire/smoke_pro_luna.yaml | 23 +- examples/libero/aspire/tasks.sh | 6 + examples/libero/prompt_only/run.yaml | 13 +- examples/libero/prompt_only/run_pro.yaml | 13 +- examples/libero/prompt_only/smoke.yaml | 13 +- examples/libero/prompt_only/smoke_luna.yaml | 13 +- examples/libero/utils/agent/task.py | 50 +- examples/libero/utils/benchmark/prepare.py | 20 +- examples/libero/utils/launch.sh | 8 +- .../robodojo_sim/utils/agent/dojo_service.py | 8 +- examples/robosuite/.gitignore | 4 + examples/robosuite/README.md | 59 + examples/robosuite/aspire/launch.sh | 5 + .../aspire/run.yaml | 23 +- .../aspire/run_direct.yaml | 18 +- .../aspire/smoke.yaml | 23 +- examples/robosuite/aspire/smoke_claude.yaml | 58 + .../aspire/smoke_direct.yaml | 18 +- examples/robosuite/aspire/tasks.sh | 6 + .../utils/.gitignore | 0 .../utils/benchmark/.gitignore | 0 examples/robosuite/utils/benchmark/prepare.py | 47 + .../utils/benchmark/task-info/README.md | 0 hermit/agent/acting/workspace.py | 13 +- hermit/agent/evolver/methods/aspire/actor.py | 205 +++ .../agent/evolver/methods/aspire/direct.py | 72 +- .../methods/aspire}/prompts/actor_system.md | 0 .../methods/aspire}/prompts/codex_header.md | 2 +- .../aspire}/prompts/evosearch_header.md | 0 .../methods/aspire}/prompts/findings.md | 0 hermit/agent/evolver/methods/aspire/rows.py | 97 ++ tests/README.md | 4 + tests/core/test_cli_agent.py | 9 + tests/core/test_environment.py | 7 +- tests/core/test_selection.py | 4 + tests/methods/aspire/test_aspire.py | 1117 ++++------------- tests/selection.py | 13 +- 70 files changed, 2363 insertions(+), 1724 deletions(-) delete mode 100644 examples/aspire_robosuite/.gitignore delete mode 100644 examples/aspire_robosuite/utils/agent/aspire_fields.py delete mode 100644 examples/aspire_robosuite/utils/agent/task.py delete mode 100644 examples/aspire_robosuite/utils/benchmark/prepare.py rename examples/{aspire_robosuite => cap_harness}/README.md (53%) rename examples/{aspire_robosuite/aspire => cap_harness}/launch.sh (71%) create mode 100644 examples/cap_harness/utils/agent/fields.py rename examples/{aspire_robosuite => cap_harness}/utils/agent/sim.py (100%) rename examples/{aspire_robosuite => cap_harness}/utils/agent/sim_cli.py (100%) rename examples/{aspire_robosuite => cap_harness}/utils/agent/sim_service.py (95%) rename examples/{aspire_robosuite => cap_harness}/utils/agent/stage_docs.py (97%) create mode 100644 examples/cap_harness/utils/agent/task.py rename examples/{aspire_robosuite => cap_harness}/utils/benchmark/judge.py (60%) create mode 100644 examples/cap_harness/utils/benchmark/tasks.py rename examples/{aspire_robosuite => cap_harness}/utils/seg_service/requirements.txt (100%) rename examples/{aspire_robosuite => cap_harness}/utils/seg_service/service.py (100%) create mode 100644 examples/cap_harness/utils/tests/cap_fakes.py create mode 100644 examples/cap_harness/utils/tests/test_cap_harness.py create mode 100644 examples/libero/aspire/tasks.sh create mode 100644 examples/robosuite/.gitignore create mode 100644 examples/robosuite/README.md create mode 100755 examples/robosuite/aspire/launch.sh rename examples/{aspire_robosuite => robosuite}/aspire/run.yaml (70%) rename examples/{aspire_robosuite => robosuite}/aspire/run_direct.yaml (71%) rename examples/{aspire_robosuite => robosuite}/aspire/smoke.yaml (70%) create mode 100644 examples/robosuite/aspire/smoke_claude.yaml rename examples/{aspire_robosuite => robosuite}/aspire/smoke_direct.yaml (62%) create mode 100644 examples/robosuite/aspire/tasks.sh rename examples/{aspire_robosuite => robosuite}/utils/.gitignore (100%) rename examples/{aspire_robosuite => robosuite}/utils/benchmark/.gitignore (100%) create mode 100644 examples/robosuite/utils/benchmark/prepare.py rename examples/{aspire_robosuite => robosuite}/utils/benchmark/task-info/README.md (100%) create mode 100644 hermit/agent/evolver/methods/aspire/actor.py rename examples/aspire_robosuite/utils/agent/agent.py => hermit/agent/evolver/methods/aspire/direct.py (87%) rename {examples/aspire_robosuite/utils/agent => hermit/agent/evolver/methods/aspire}/prompts/actor_system.md (100%) rename {examples/aspire_robosuite/utils/agent => hermit/agent/evolver/methods/aspire}/prompts/codex_header.md (97%) rename {examples/aspire_robosuite/utils/agent => hermit/agent/evolver/methods/aspire}/prompts/evosearch_header.md (100%) rename {examples/aspire_robosuite/utils/agent => hermit/agent/evolver/methods/aspire}/prompts/findings.md (100%) create mode 100644 hermit/agent/evolver/methods/aspire/rows.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e76749..ea55697b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ Changes are grouped by software version. Entries under Unreleased describe work ### Breaking +- The robot benchmarks and ASPIRE no longer share code: a benchmark holds nothing of a method, and ASPIRE nothing of + a benchmark. The engine Robosuite and LIBERO run on (the simulator service, `bin/sim`, the neutral task functions, + the judge, the task writer, the row view) is `examples/cap_harness/`, which RoboDojo's service also builds on; + `examples/aspire_robosuite/` is `examples/robosuite/`, the benchmark's tasks and an `aspire/` folder. ASPIRE's + session (its fix-loop and verifier prompts, the evolutionary search, the facts the coordinator reads) is + `hermit/agent/evolver/methods/aspire/actor.py`, built on the engine's task functions (`agent.params.engine`), and its + direct-chat actor `direct.py` (`agent.name: hermit.agent.evolver.methods.aspire.direct:AspireActor`). A benchmark's + `prepare.py` writes its tasks and their reruns (`rerun_of`) only; ASPIRE's verifier rows are derived by `rows.py`, + which an ASPIRE folder's `tasks.sh` runs before the run, into that folder's `tasks/` and `splits/`. The service's + attempt budget is a row's `max_programs`, and the judge scores a row's `eval_seeds` (a verifier row's are its own + five); a verifier's verdict rule is ASPIRE's `collect`, which withholds the program unless `verified` (`not_scored`). + The prompts the Codex sessions see are unchanged, byte for byte; ASPIRE's header page names the CLI's context file + (`CLAUDE.md` for Claude Code), and Robosuite has a Claude Code smoke (`aspire/smoke_claude.yaml`). A task function + may be named `package.module:function`. + LIBERO's `prompt_only/` configs drop `evosearch: false`. + - The old import paths of moved modules are gone, with no forward: 0.1's, which 0.2 forwarded (`hermit.agents`, `hermit.evolver`, `hermit.llm`, `hermit.backends`, `hermit.trajectory`), and 0.2's `hermit.agent.base.direct` and `hermit.agent.base.container`, and `hermit.agent.environment` (never released). Import instead: diff --git a/docs/framework/task.md b/docs/framework/task.md index 19cb23fa..8aedae72 100644 --- a/docs/framework/task.md +++ b/docs/framework/task.md @@ -102,8 +102,9 @@ agent: `codex`, `claude-code`, `pi`, `mini-swe-agent`, `opencode`, `dsh` and `hermes` act with a coding-agent CLI in a workspace of Hermit's (or, for Codex and Claude Code, in a task's own Harbor environment: [Benchmarks](#benchmarks)): -a task gives what is its own, in configuration and, where its workspace is -built in code, in functions it names (`file.py:function`); the agent does the rest. OfficeQA's Codex runs are one such +a task gives what is its own, in configuration and, where its workspace is built in code, in functions it names +(`file.py:function`, or `package.module:function` for a method that brings its own session, as ASPIRE's +`hermit.agent.evolver.methods.aspire.actor`); the agent does the rest. OfficeQA's Codex runs are one such configuration (`examples/officeqa/utils/task.py`): ```yaml @@ -456,7 +457,7 @@ of your own is written as task directories by your `utils/benchmark/prepare.py` goes into `tests/`, uploaded after the agent ran, so the answer is never where the agent can read it; see `examples/officeqa/utils/harbor/build_tasks.py`). A task set of task directories that Hermit runs in its own sandbox or on the host, with a judge on the host (`judge.mode: host`, no `tests/`), is what the examples whose agent owns its loop -use (`examples/aspire_robosuite/`, `clbench_*`, `servelearnbench/`, `agentdyn/`, ...; [Task directories](#task-directories)). +use (`examples/robosuite/`, `libero/`, `clbench_*`, `servelearnbench/`, `agentdyn/`, ...; [Task directories](#task-directories)). A task set is checked without a model by running a command of a known answer in each task's environment and verifying it (`tests/core/test_harbor_environment.py` does this against Docker). diff --git a/docs/methods/aspire.md b/docs/methods/aspire.md index 191ad502..b577f685 100644 --- a/docs/methods/aspire.md +++ b/docs/methods/aspire.md @@ -14,16 +14,21 @@ learns only the library: the actor's harness and prompts are fixed. ```yaml template: aspire # the skill library: skills/README.md and skills/.md -data: {train: all.txt, val: verify.txt, test: rerun.txt, step_size: 1, shuffle: false} +data: # the benchmark's tasks with ASPIRE's verifiers (tasks.sh: rows.py) + {tasks: tasks, train: splits/all.txt, val: splits/verify.txt, test: splits/rerun.txt, step_size: 1, shuffle: false} agent: base: codex # the actor: ASPIRE's fix-loop subagent prompt, verbatim - params: {prompt: ../utils/agent/task.py:prompt, prepare: ../utils/agent/task.py:prepare, - answer: ../utils/agent/task.py:answer, collect: ../utils/agent/task.py:collect} + params: # ASPIRE's task functions, over the engine's + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py codex: {codex_sandbox: danger-full-access, shell_env: all, web_search: false} # the task sandbox is the boundary -environment: # the task sandbox: its code, ASPIRE's prompts and docs, the service - read: [../utils/agent, $ASPIRE_DOCS, ...] +environment: # the task sandbox: the engine's code, ASPIRE's prompts and docs + read: [../../cap_harness/utils/agent, $ASPIRE_DOCS, ...] env: [ASPIRE_SIM_URL] -judge: {path: ../utils/benchmark/judge.py} # on the host: Stage 2 +judge: {path: ../../cap_harness/utils/benchmark/judge.py} # the engine's, on the host: Stage 2 sandbox: {enabled: true, enforce: required} evolver: name: aspire # the coordinator: a Codex session with ASPIRE's guide @@ -40,16 +45,22 @@ privileged access (simulator internals, success predicates, reflection), and ref No existing template fits: `prompt-only` is one file; `continual-harness` carries a policy, sub-agents and memory ASPIRE does not have; `reharness` retrieves per instance from an embedded bank. -The example's task functions (`examples/aspire_robosuite/utils/agent/task.py`) give the actor its workspace in the -task sandbox: `bin/sim`; ASPIRE's API and artifact docs and the benchmark's page (Robosuite's, or LIBERO's); the scene-snapshot program; and a page that maps -the template's paths and commands onto the workspace. The sandbox shows nothing else of ASPIRE's checkout: not its -shipped library, worked solutions or examples. It has no GPU, so the simulator runs on the host as a service -(`utils/agent/sim_service.py`), which `bin/sim` reaches on a forwarded loopback port; the session registers there -before the agent starts (its workspace and task id: one session per rollout), and the service runs only the seeds of -the task row it reads on the host, the unseen check once and a verifier's two programs. The judge (`utils/benchmark/judge.py`, -on the host) runs the selected program once on the held-out seeds: ASPIRE's Stage 2, the coordinator's in ASPIRE. The -verifier is the same actor on the task's verifier row (`verify-`, `data.val`): ASPIRE's verifier prompt, five -seeds, two attempts, scored on its own seeds with its own verdict. +The tasks run on the [cap-harness engine](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/cap_harness), +which knows nothing of ASPIRE; ASPIRE brings its own session on it. ASPIRE's task functions +(`hermit/agent/evolver/methods/aspire/actor.py`, named by module in the configs) build on the engine's +(`agent.params.engine`): the engine gives the actor its workspace in the task sandbox (`bin/sim`; ASPIRE's API and +artifact docs and the benchmark's page, Robosuite's or LIBERO's; the scene-snapshot program), and ASPIRE adds its +assignment, the evolutionary search's page and a page that maps the template's paths and commands onto the +workspace. The sandbox shows nothing else of ASPIRE's checkout: not its shipped library, worked solutions or +examples. It has no GPU, so the simulator runs on the host as the engine's service (`utils/agent/sim_service.py`), +which `bin/sim` reaches on a forwarded loopback port; the session registers there before the agent starts (its +workspace and task id: one session per rollout), and the service runs only the seeds of the task row it reads on the +host, the unseen check once and, for a row with an attempt budget, that many programs. The engine's judge +(`utils/benchmark/judge.py`, on the host) runs the selected program once on the held-out seeds: ASPIRE's Stage 2, the +coordinator's in ASPIRE. The verifier is the same actor on the task's verifier row (`verify-`, `data.val`), +which ASPIRE derives from the benchmark's tasks before a run (`rows.py`, run by the method folder's `tasks.sh`): ASPIRE's +verifier prompt, five seeds, an attempt budget of two programs, scored on its own seeds and only with its own verdict +`verified` (ASPIRE's `collect` withholds the program otherwise). ## The step @@ -98,11 +109,12 @@ is the evaluation on the test split (`rerun-` rows, the same seeds), compa The Codex actor's settings (`agent.params`) are `aspire_root`, `aspire_docs` (the docs as the launcher stages them), `max_steps` and `evosearch` (default true; false leaves out the evolutionary search: its part of the prompt, its page in the workspace, and its -program as the scored one); its seeds are the task rows' -(`utils/benchmark/prepare.py`, which the launcher runs: `ASPIRE_EVAL_SEEDS` for the held-out ones), and the simulator's +program as the scored one) and `engine` (the engine's task functions, relative to the config); its seeds are the +task rows' (the benchmark's `utils/benchmark/prepare.py`, which the launcher runs: `ASPIRE_EVAL_SEEDS` for the +held-out ones), and the simulator's environment and workers are the simulator service's (`ASPIRE_SIM_ENV`). The direct comparison actor -(`run_direct.yaml`) reads `debug_seeds`, `workers`, `aspire_root` and `sim_env` from `agent.params`, a task row's own -seeds winning; the held-out seeds are the task rows' for both actors (the judge reads them there). The verifier's rows +(`run_direct.yaml`) reads `engine` (the engine's `sim.py`), `debug_seeds`, `workers`, `aspire_root` and `sim_env` from +`agent.params`, a task row's own seeds winning; the held-out seeds are the task rows' for both actors (the judge reads them there). The verifier's rows carry their own seeds. ## The authors' implementation against this one @@ -124,12 +136,13 @@ carry their own seeds. | Library history | `-changelog.md` beside each file | `state/changelog.md` | the adapter holds only the library | | Phase 3 | rerun of tasks below 80% whose library changed | the test split reruns every task | the evaluation measures it per task | | Models | Claude Opus 4.6 (and others) | GLM 5.3 Flash in every role | the first-evaluation row of the framework | -| Perception | SAM3 | SAM3, or any service with its wire format (`utils/seg_service/`: Grounding-DINO + SAM) | SAM3's weights are gated | +| Perception | SAM3 | SAM3, or any service with its wire format (the engine's `utils/seg_service/`: Grounding-DINO + SAM) | SAM3's weights are gated | | Evolutionary search (§2.3) | a separate evosearch subagent for tasks the fix loop leaves below 90%: K = 8 candidates per round from the top 3, stop at 80% or 5 rounds, Stage 2 on its best | the same template, verbatim, as Part 2 of the actor's session; the judge runs Stage 2 on `evosearch_best_code.py` | a decision: one session keeps the task's rollout one call | | Not ported | — | BEHAVIOR-1K, the real robot | scope | -`examples/aspire_robosuite/aspire/run_direct.yaml` replaces the Codex actor with a structured loop of `direct` calls -(`utils/agent/agent.py`), which sees a digest of each run instead of the files: a comparison, not the method. +`examples/robosuite/aspire/run_direct.yaml` replaces the Codex actor with a structured loop of `direct` calls +(`hermit/agent/evolver/methods/aspire/direct.py`, on the engine's `sim.py`), which sees a digest of each run instead of +the files: a comparison, not the method. ## What it is not @@ -140,7 +153,7 @@ Phase 3 does). ## Example -[`examples/aspire_robosuite/`](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/aspire_robosuite): the +[`examples/robosuite/aspire/`](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/robosuite/aspire): the seven Robosuite tasks of ASPIRE's harness. [`examples/libero/aspire/`](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/libero/aspire): LIBERO's five suites and LIBERO-Pro's eighty tasks on the same engine. diff --git a/examples/README.md b/examples/README.md index 8021d561..95bca7bb 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,8 +36,8 @@ a `.hermit/CARD.md`. | [`clbench_database/`](clbench_database/) | The CL-Bench database exploration stream (accuracy) | `CLBenchDatabaseAgent`: metaworld's host seat in CL-Bench's loop | [`prompt_only`](clbench_database/prompt_only/), [`mwm`](clbench_database/mwm/), [`reharness`](clbench_database/reharness/) | | [`clbench_poker/`](clbench_poker/) | The CL-Bench exploitable-poker stream (bb/hand) | `CLBenchPokerAgent` | [`prompt_only`](clbench_poker/prompt_only/), [`mwm`](clbench_poker/mwm/), [`reharness`](clbench_poker/reharness/) | | [`servelearnbench/`](servelearnbench/) | A ServeLearnBench scenario: serving stream, per-window tests | `ServeLearnBenchAgent`: the benchmark's own loop | [`prompt_only`](servelearnbench/prompt_only/), [`mwm`](servelearnbench/mwm/) | -| [`aspire_robosuite/`](aspire_robosuite/) | The Robosuite tasks of ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex with ASPIRE's fix-loop subagent and verifier prompts (`utils/agent/task.py`), GLM 5.3 Flash, ASPIRE's robot execution engine in its own venv through `bin/sim` | [`aspire`](aspire_robosuite/aspire/) | -| [`libero/`](libero/) | LIBERO's five suites and LIBERO-Pro's `_swap` and `_task` perturbations, a program per task in ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex writing the program, on the `aspire_robosuite/` engine (ASPIRE's LIBERO venv, seeded initial states), with a method's prompt or a method-neutral one | [`aspire`](libero/aspire/), [`prompt_only`](libero/prompt_only/) | +| [`robosuite/`](robosuite/) | The Robosuite tasks of ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex writing the program, GLM 5.3 Flash, on the [`cap_harness/`](cap_harness/) engine (ASPIRE's robot execution engine in its own venv through `bin/sim`), with a method's own session (ASPIRE's fix-loop subagent and verifier) | [`aspire`](robosuite/aspire/) | +| [`libero/`](libero/) | LIBERO's five suites and LIBERO-Pro's `_swap` and `_task` perturbations, a program per task in ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex writing the program, on the `cap_harness/` engine (ASPIRE's LIBERO venv, seeded initial states), with a method's own session or a method-neutral prompt | [`aspire`](libero/aspire/), [`prompt_only`](libero/prompt_only/) | | [`robodojo_sim/`](robodojo_sim/) | RoboDojo-Sim's forty-two bimanual manipulation tasks: a program that controls the robot, developed on one layout set and scored on two others, task success (a first implementation: its smoke run on the real simulator, no training run yet) | Codex writing `policy.py` (`utils/agent/task.py`), GLM 5.3 Flash, RoboDojo evaluating it as an XPolicyLab policy whose program runs in a sandbox of its own | [`prompt_only`](robodojo_sim/prompt_only/) | | [`agentdyn/`](agentdyn/) | AgentDyn's frozen benign and injected cases (Shopping, GitHub, DailyLife), clean utility and attack success rate | `bridge.py:run`: AgentDyn's native agent loop, with the defense bundle's hooks around every tool call | [`evosafe`](agentdyn/evosafe/) (EvoSafeHarness) | | [`agentdojo/`](agentdojo/) | Official banking, Slack, travel and workspace suites; frozen 57/54/406 local protocol, native utility/ASR and errors | `task.py:run`: native AgentDojo loop through the shared AgentDyn supervised runtime | [`evosafe`](agentdojo/evosafe/), [`she`](agentdojo/she/), completed Lightning/Codex reference comparisons | diff --git a/examples/aspire_robosuite/.gitignore b/examples/aspire_robosuite/.gitignore deleted file mode 100644 index a1e03960..00000000 --- a/examples/aspire_robosuite/.gitignore +++ /dev/null @@ -1 +0,0 @@ -runs/ diff --git a/examples/aspire_robosuite/utils/agent/aspire_fields.py b/examples/aspire_robosuite/utils/agent/aspire_fields.py deleted file mode 100644 index 09ac8e63..00000000 --- a/examples/aspire_robosuite/utils/agent/aspire_fields.py +++ /dev/null @@ -1,20 +0,0 @@ -"""A task of ASPIRE's engine as the actor, the simulator service and the judge read it: {id, (benchmark), suite, -task_id, the seeds, (role, max_versions, search, findings)}, the row each task was in tasks.jsonl. A task directory -(utils/benchmark/prepare.py, and LIBERO's) has every field but its id under task.toml's [metadata], in the row's -order; this puts them back, so the prompts, the service's rules and the records are what they were. LIBERO's tasks -are read so too (their configs' data.view).""" -from __future__ import annotations - -from pathlib import Path - - -def aspire_task(task: dict) -> dict: - if "instruction" not in task: # already a row of that shape - return dict(task) - return {"id": task["id"], **(task.get("metadata") or {})} - - -def task_rows(tasks: Path) -> dict[str, dict]: - """Every task's row by id: from the task directories prepare.py writes, or from a tasks.jsonl.""" - from hermit.task import Benchmark - return {tid: t.given() for tid, t in Benchmark.load(Path(tasks), view=aspire_task).tasks.items()} diff --git a/examples/aspire_robosuite/utils/agent/task.py b/examples/aspire_robosuite/utils/agent/task.py deleted file mode 100644 index d6c64610..00000000 --- a/examples/aspire_robosuite/utils/agent/task.py +++ /dev/null @@ -1,238 +0,0 @@ -"""The task functions of the Codex actor (agent.base: codex): ASPIRE's fix-loop subagent and its skill verifier, as -ASPIRE's own prompts run them. They run in the task sandbox with the session; the robot execution engine runs on the -host, as the simulator service (sim_service.py), which the session reaches only through `bin/sim`. - - prepare the workspace: bin/sim and its registration with the service (the workspace and the task's id: the - service takes the seeds and the budget from its own copy of the task row), the API and artifact docs, the scene-snapshot program, task.md; for a verifier, the - proposals under test as proposed_skills.md - prompt the adaptation page (prompts/codex_header.md), the assignment, then ASPIRE's template verbatim - answer the agent's last message (the record's output is the facts collect returns) - collect the actor's files into the log, and the facts the judge and the coordinator read: findings, skill report, - the selected program (fix_code.py, or the evolutionary search's best); the judge runs Stage 2 on it - -ASPIRE's checkout comes from agent.params.aspire_root (its prompt templates and snapshot program must be -declared readable: environment.read; its docs are read from agent.params.aspire_docs, the staged copies); the service's URL from ASPIRE_SIM_URL (environment.env). A task row carries the -seeds: development, unseen-check and held-out (utils/benchmark/prepare.py). -""" -from __future__ import annotations - -import json -import os -from pathlib import Path -import shlex -import shutil -import stat -import sys -import urllib.request - -HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE)) -import sim # noqa: E402 - -TEMPLATES = {"actor": "subagent-prompt.md", "verifier": "verifier-prompt.md"} -SKILLS = Path(".claude/skills/iterative-debugging") -EVOSEARCH = Path(".claude/skills/evosearch") -# The files of ASPIRE's checkout the task sandbox shows (environment.read): its prompts and snapshot program; not -# its shipped library, worked solutions or examples, nor its docs, which the sandbox is shown as staged -# (agent.params.aspire_docs: stage_docs.py's copies, without what names the worked programs) -ASPIRE_FILES = [ str(SKILLS / "subagent-prompt.md"), str(SKILLS / "verifier-prompt.md"), - str(SKILLS / "scripts" / "scene_snapshot.py"), str(SKILLS / "skills" / "task-exploration.md"), - str(EVOSEARCH / "subagent-prompt.md"), str(EVOSEARCH / "skills" / "evosearch-iteration.md")] -# The workspace files the log keeps (the run directories stay in the workspace) -KEEP = ("task_analysis.md", "fix_code.py", "findings.md", "findings_report.md", "skill_report.json", - "verification_report.json", "verification.md", "code", "debug/blocked") - - -def settings(task: dict, params: dict) -> dict: - """The session's assignment, from its task row (the seeds; utils/benchmark/prepare.py) and agent.params (the - checkout).""" - role = "verifier" if task.get("role") == "verifier" else "actor" # a rerun is an actor with the final library - root = params.get("aspire_root") or os.environ.get("ASPIRE_ROOT") - if not root: - raise ValueError("agent.params.aspire_root (or ASPIRE_ROOT): an ASPIRE checkout with its benchmark's venv") - benchmark = str(task.get("benchmark") or "robosuite") - return {"role": role, "root": Path(os.path.expandvars(str(root))).expanduser().resolve(), - "benchmark": benchmark, "doc": sim.profile(benchmark)["doc"], "suite": task["suite"], "task_id": int(task.get("task_id", 0)), - "dev": str(task.get("debug_seeds") or ""), "unseen": str(task.get("unseen_seeds") or ""), - "heldout": str(task.get("eval_seeds") or ""), "max_steps": params.get("max_steps")} - - -def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: - s = settings(task, params) - work = Path(workspace) - url = os.environ.get("ASPIRE_SIM_URL") - if not url: - raise ValueError("ASPIRE_SIM_URL: the simulator service's URL (launch.sh starts it; environment.env)") - # The session, registered before the agent starts: the service takes its seeds and budget from the task's row - register(url, {"workspace": str(work.resolve()), "task": task["id"], "max_steps": s["max_steps"]}) - bin_dir = work / "bin" - bin_dir.mkdir(exist_ok=True) - shutil.copy2(HERE / "sim_cli.py", bin_dir / "sim_cli.py") - (bin_dir / "sim.json").write_text(json.dumps({"workspace": str(work.resolve()), "url": url}, indent=1)) - wrapper = bin_dir / "sim" - wrapper.write_text(f"#!/bin/sh\nexec {shlex.quote(sys.executable)} \"$(dirname \"$0\")/sim_cli.py\" \"$@\"\n") - wrapper.chmod(wrapper.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - (work / "docs").mkdir(exist_ok=True) - # The repository's worked programs are not the actor's to read: the docs are the staged copies (launch.sh: - # stage_docs.py), or, where the checkout's own pages are what the sandbox shows, the same cut made here - staged = params.get("aspire_docs") - pages = {name: (Path(os.path.expandvars(str(staged))).expanduser() / name).read_text() - for name in (*sim.DOCS, s["doc"])} if staged else sim.workspace_docs(s["root"], s["benchmark"]) - for name, text in pages.items(): - (work / "docs" / name).write_text(text) - shutil.copy2(s["root"] / SKILLS / "skills" / "task-exploration.md", work / "docs" / "task-exploration.md") - if params.get("evosearch", True): # the search's procedure, for a prompt that runs it - shutil.copy2(s["root"] / EVOSEARCH / "skills" / "evosearch-iteration.md", work / "docs" / "evosearch-iteration.md") - (work / "scripts").mkdir(exist_ok=True) - shutil.copy2(s["root"] / SKILLS / "scripts" / "scene_snapshot.py", work / "scripts" / "scene_snapshot.py") - if s["role"] == "verifier": - # The coordinator stages the proposals under test as the adapter's skills/proposed.md: the verifier's - # proposed_skills.md, kept out of the library it reads (interface.py leaves the file out). - staged = sorted(work.glob("hermit/*/skills/proposed.md")) - (work / "proposed_skills.md").write_text(staged[0].read_text() if staged else "(no proposals were staged)\n") - (work / "task.md").write_text(assignment(s)) - return [] - - -def register(url: str, body: dict) -> None: - request = urllib.request.Request(url.rstrip("/") + "/register", data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(request, timeout=60) as reply: - answer = json.loads(reply.read()) - if not answer.get("ok"): - raise RuntimeError(f"the simulator service refused the session: {answer.get('error')}") - - -def task_dir(s: dict) -> str: - """The actor's $TASK_DIR as ASPIRE's template defines it, with $RUN_ROOT the workspace; the verifier's is `.`.""" - return f"{s['benchmark']}/{s['suite']}/task_{s['task_id']}" if s["role"] == "actor" else "." - - -def sim_seeds(text: str) -> list[int]: - return sim.seeds_of(text) if text else [] - - -def assignment(s: dict) -> str: - if s["role"] == "verifier": - return (f"## Your assignment\n\n- BENCHMARK: `{s['benchmark']}`\n- SUITE: `{s['suite']}`\n" - f"- TASK_ID: `{s['task_id']}`\n- GPU: handled by `bin/sim`\n- VERIFY_DIR: `.` (this workspace)\n" - f"- Development seeds: **{s['dev']}** — wherever the template says 51, 52, 53, 54, 55 (or 51-55, or " - f"seed 51), use these.\n- The proposals under test: `proposed_skills.md` in this workspace.\n") - return (f"## Task Assignment\n\nBENCHMARK: {s['benchmark']}\nSUITE: {s['suite']}\nTASK_ID: {s['task_id']}\n" - f"GPU: handled by bin/sim\nRUN_ROOT: . (this workspace)\n\n" - f"MAX_STEPS: {s['max_steps'] or 1000}\nDEV_SEEDS: {s['dev']}\nUNSEEN_SEEDS: {s['unseen'] or 'none'}\n" - f"HELDOUT_COUNT: {len(sim_seeds(s['heldout']))} (coordinator-owned; you never run them)\n") - - -def prompt(task: dict, params: dict, base: Path) -> str: - s = settings(task, params) - template = (s["root"] / SKILLS / TEMPLATES[s["role"]]).read_text() - body = template.split("", 1)[1] - body = body.split("", 1)[0].strip() - if s["role"] == "actor": - body = body.split("\n---\n", 1)[1] if body.startswith("## Task Assignment") else body # ours replaces it - header = (HERE / "prompts" / "codex_header.md").read_text().format( - role_name="fix-loop subagent" if s["role"] == "actor" else "skill verifier", - template=TEMPLATES[s["role"]], benchmark=s["benchmark"], benchmark_doc=s["doc"], - your_dir=(f"`$RUN_ROOT` is this workspace (`.`), so `$TASK_DIR` is `{task_dir(s)}`, as the template " - f"defines it." if s["role"] == "actor" else "`$VERIFY_DIR` is this workspace (`.`)."), - proposed_line=("The proposals under test are `proposed_skills.md` in this workspace (the template's " - "`$VERIFY_DIR/proposed_skills.md`); they are not in the library." if s["role"] == "verifier" - else ""), - finish_line=("only after both parts of this session: Part 1, the template below, and Part 2 after it, the " - "evolutionary search, which runs only when Part 1 leaves `fix_code.py` below 90% of the " - "development seeds. Reply with the return message of each part you ran." - if s["role"] == "actor" and params.get("evosearch", True) else - "by replying with the template's return message."), - heldout_line=("is not yours: the harness scores your selected program on the held-out seeds after you " - "finish (the template's Stage 2)." if s["role"] == "actor" else - "does not apply: your verdict is scored on the program you select, on your seeds; write it " - "as `fix_code.py` in this workspace as well.")) - text = f"{header}\n\n{assignment(s)}\n---\n\n{body}\n" - if s["role"] == "actor" and params.get("evosearch", True): - text += "\n---\n\n" + evosearch_part(s) - return text - - -def evosearch_part(s: dict) -> str: - """Part 2 of the actor's session: ASPIRE's evolutionary-search subagent (§2.3), for a task the fix loop left below - 90% of its development seeds. Its template is the fenced block of evosearch/subagent-prompt.md, verbatim after - its assignment block, which the page before it replaces.""" - template = (s["root"] / EVOSEARCH / "subagent-prompt.md").read_text() - body = template[template.index("## Task Assignment"):template.rindex("\n```")] - body = body.split("\n---\n", 1)[1].strip() - header = (HERE / "prompts" / "evosearch_header.md").read_text().format( - benchmark=s["benchmark"], suite=s["suite"], task_id=s["task_id"], dev=s["dev"], task_dir=task_dir(s), evo_dir=evo_dir(s)) - return f"{header}\n---\n\n{body}\n" - - -def evo_dir(s: dict) -> str: - """The evolutionary search's $TASK_DIR, as its template defines it, with EVOSEARCH_DIR `evosearch`.""" - return f"evosearch/{s['benchmark']}/{s['suite']}/task_{s['task_id']}" - - -def answer(last_message: str, task: dict) -> dict: - return {"answer": last_message.strip()[-4000:]} - - -def collect(task: dict, workspace: Path, log: Path, params: dict) -> dict: - """The actor's files into the log and the facts for the judge (which runs Stage 2 on `fix_code`, on the host) and - the coordinator.""" - s = settings(task, params) - work = Path(workspace) / task_dir(s) - files = Path(log) / "files" - for name in KEEP: - source = work / name - if source.is_dir(): - shutil.copytree(source, files / name, dirs_exist_ok=True) - elif source.is_file(): - files.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, files / name) - read = lambda name: (work / name).read_text(errors="replace") if (work / name).is_file() else None - facts = {"role": s["role"], "benchmark": s["benchmark"], "suite": s["suite"], "findings": read("findings.md") or read("findings_report.md"), - "skill_report": _json(read("skill_report.json")), "verification": _json(read("verification_report.json")), - "unseen": _unseen(work), "workspace": str(Path(workspace).resolve()), "max_steps": s["max_steps"]} - evo = Path(workspace) / evo_dir(s) - search = s["role"] == "actor" and params.get("evosearch", True) # a prompt that runs the search - if search and evo.is_dir(): - for name in ("evosearch_best_code.py", "findings.md", "task_analysis.md", "BLOCKED"): - if (evo / name).is_file(): - (files / "evosearch").mkdir(parents=True, exist_ok=True) - shutil.copy2(evo / name, files / "evosearch" / name) - facts["evosearch"] = {"findings": (evo / "findings.md").read_text(errors="replace") - if (evo / "findings.md").is_file() else None, - "best_code": (evo / "evosearch_best_code.py").is_file()} - # The selected program: the search's best when the search ran (it falls back to fix_code.py itself), else the fix - # loop's; with `evosearch: false`, fix_code.py alone. The judge scores it. - best = evo / "evosearch_best_code.py" - program = best if search and best.is_file() else work / "fix_code.py" - facts["scored_program"] = str(program.relative_to(workspace)) if program.is_file() else None - facts["fix_code"] = program.read_text(errors="replace") if program.is_file() else None - if params.get("prune_media", True): - prune(Path(workspace).parent) - return facts - - -def prune(run_dir: Path) -> None: - """The heavy part of a finished session's simulator runs: every run's media/ (keyframes, overlays, videos; a - session runs hundreds of episodes). The records, traces and programs stay.""" - for media in run_dir.rglob("media"): - if media.is_dir() and (media.parent / "run.json").is_file(): - shutil.rmtree(media, ignore_errors=True) - - -def _json(text): - try: - return json.loads(text) if text else None - except ValueError: - return {"unparsed": text[-4000:]} - - -def _unseen(work: Path) -> dict | None: - rows = work / "debug" / "holdout" / "results.jsonl" - if not rows.is_file(): - return None - results = [json.loads(l) for l in rows.read_text().splitlines() if l.strip()] - passed = sum(bool((sim._read_json(Path(r["run_dir"]) / "outcome.json") or {}).get("task_success")) - for r in results if r.get("run_dir")) - return {"passed": passed, "total": len(results)} diff --git a/examples/aspire_robosuite/utils/benchmark/prepare.py b/examples/aspire_robosuite/utils/benchmark/prepare.py deleted file mode 100644 index ae04112a..00000000 --- a/examples/aspire_robosuite/utils/benchmark/prepare.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Write the Robosuite tasks of ASPIRE's harness (cap_harness/robosuite/registry.py) as hermit tasks: one task -directory (Harbor's format) per task with its seeds, which the simulator service and the judge take from the task's -row (ASPIRE's Robosuite defaults: development 101-125, unseen-check 126-130, held-out 1-100). - - python prepare.py # ASPIRE's seeds - python prepare.py --preset smoke # the smoke's: three development seeds, one unseen, three held-out - python prepare.py --dev 101-103 --unseen 126 --eval 1-3 # any others - # tasks//, splits/{all,smoke,single_arm,verify,rerun,smoke_rerun,single_arm_rerun}.txt - -A task directory's task.toml holds the row's fields but its id under [metadata], in the row's order, and asks for no -network (the actor reaches only its model and the simulator service's forwarded port); it has no tests/: the judge -(judge.py) runs on the host, where the simulator is. utils/agent/aspire_fields.py reads each back as its row. - -The verifier rows (splits/verify.txt) are the Evolver's: a run gives them as `data.val`, so only `run_agent` runs -them. - -The order of splits/all.txt is the curriculum the coordinator walks (single-arm pick before stacking and -assembly, the fixed-tool wipe, then the bimanual tasks). -""" -from __future__ import annotations - -import argparse -from pathlib import Path -import shutil -import sys - -from hermit.task.directory import write_task as write_task_dir - -HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE.parent / "agent")) -from sim import seeds_of # noqa: E402 -TASKS = [ # suite, what it is (task-info only; the actor reads the language from get_task_context()) - ("cube_lifting", "Lift"), - ("cube_stack", "Stack"), - ("cube_restack", "Stack, restacked"), - ("nut_assembly", "NutAssemblySquare"), - ("spill_wipe", "Wipe, fixed tool"), - ("two_arm_lift", "TwoArmLift, bimanual"), - ("two_arm_handover", "TwoArmHandover, bimanual"), -] - - -# The seeds of a run: ASPIRE's Robosuite defaults, and the smoke's -PRESETS = {"aspire": {"dev": "101-125", "unseen": "126-130", "eval": "1-100", "verify": "101-105"}, - "smoke": {"dev": "101-103", "unseen": "126", "eval": "1-3", "verify": "101-105"}} - - -def seeds_from(presets: dict, argv=None) -> tuple[dict, str]: - """The run's seeds from the command line (a preset, each set replaceable): the three sets of a task row, and - the verifier's.""" - ap = argparse.ArgumentParser() - ap.add_argument("--preset", choices=sorted(presets), default="aspire") - ap.add_argument("--dev", help="development seeds") - ap.add_argument("--unseen", help="unseen-check seeds, run once by the actor") - ap.add_argument("--eval", help="held-out seeds, run only by the judge") - ap.add_argument("--verify", help="the verifier's seeds") - a = ap.parse_args(argv) - chosen = {k: getattr(a, k) or v for k, v in presets[a.preset].items()} - # A held-out seed a session could run would be scored as unseen: the sets must not meet - dev, unseen, held, verify = (set(seeds_of(chosen[k])) for k in ("dev", "unseen", "eval", "verify")) - for name, other in (("development", dev), ("unseen-check", unseen), ("verifier", verify)): - if held & other: - ap.error(f"the held-out seeds overlap the {name} seeds: {sorted(held & other)[:10]}") - if dev & unseen: - ap.error(f"the unseen-check seeds overlap the development seeds: {sorted(dev & unseen)[:10]}") - return {"debug_seeds": chosen["dev"], "unseen_seeds": chosen["unseen"], "eval_seeds": chosen["eval"]}, \ - chosen["verify"] - - -def instruction(row: dict) -> str: - """What a task asks, in a line (the actor's prompt is utils/agent/task.py's, built from the row).""" - what = f"{row.get('benchmark') or 'robosuite'} suite `{row['suite']}`, task id {row['task_id']}" - if row.get("role") == "verifier": - return f"Verify the proposals for {what}: solve it again from the library and the proposals alone." - if row.get("role") == "rerun": - return f"Solve {what} again with the final library." - return f"Write the program that solves {what} in ASPIRE's Code-as-Policy harness." - - -def write_task(tasks: Path, row: dict) -> Path: - """One row as a task directory: its fields but the id under [metadata], in the row's order; no network.""" - return write_task_dir(tasks / row["id"], instruction(row), metadata={k: v for k, v in row.items() if k != "id"}, - environment={"network_mode": "no-network"}) - - -def write(here: Path, tasks: list[dict], seeds: dict, verify: str, splits: dict[str, list[str]]) -> None: - """tasks/ and splits/ under `here`: each task (`id`, `suite`, `task_id`, and `benchmark` when it is not - Robosuite) with its seeds, its verifier task and its rerun task; splits/.txt for each of `splits`, with - _rerun.txt beside it (rerun.txt for `all`), and verify.txt.""" - rows = [{**t, **seeds} for t in tasks] - # The verifier of each task (ASPIRE verifier-prompt.md): a fresh actor that re-solves the task from the library - # and the proposals alone, on 5 development seeds in at most 2 attempts, with no held-out run and no findings. - verifiers = [{**t, "id": f"verify-{t['id']}", "role": "verifier", "debug_seeds": verify, "eval_seeds": "none", - "max_versions": 2, "search": "debug", "findings": False} for t in tasks] - # The rerun of each task with the final library (ASPIRE §2.4, backward transfer): the same task, the same seeds - # and a fresh actor; its own id so that the test split does not share ids with the training split. - reruns = [{**t, "id": f"rerun-{t['id']}", "role": "rerun", "findings": False, **seeds} for t in tasks] - if (here / "tasks").exists(): # this script's own output: rewritten whole - shutil.rmtree(here / "tasks") - for row in rows + verifiers + reruns: - write_task(here / "tasks", row) - out = here / "splits" - out.mkdir(exist_ok=True) - for name, ids in splits.items(): - (out / f"{name}.txt").write_text("".join(f"{i}\n" for i in ids)) - (out / ("rerun.txt" if name == "all" else f"{name}_rerun.txt")).write_text("".join(f"rerun-{i}\n" for i in ids)) - (out / "verify.txt").write_text("".join(f"verify-{t['id']}\n" for t in tasks)) - print(f"{len(rows)} tasks -> {here / 'tasks'}") - - -def main() -> None: - seeds, verify = seeds_from(PRESETS) - tasks = [{"id": f"robosuite-{suite}", "suite": suite, "task_id": 0} for suite, _ in TASKS] - ids = [t["id"] for t in tasks] - write(HERE, tasks, seeds, verify, {"all": ids, "single_arm": ids[:5], "smoke": ids[:2]}) - - -if __name__ == "__main__": - main() diff --git a/examples/aspire_robosuite/README.md b/examples/cap_harness/README.md similarity index 53% rename from examples/aspire_robosuite/README.md rename to examples/cap_harness/README.md index b52dfe34..afa1e593 100644 --- a/examples/aspire_robosuite/README.md +++ b/examples/cap_harness/README.md @@ -1,34 +1,30 @@ -# ASPIRE on Robosuite - -The Robosuite tasks of [ASPIRE](https://research.nvidia.com/labs/gear/aspire/) (Agentic Skills Discovery for -Robotics, [arXiv:2607.00272](https://arxiv.org/abs/2607.00272), [code](https://github.com/NVlabs/ASPIRE)) as a -continual-learning curriculum, run as ASPIRE runs it: ASPIRE's fix-loop subagent (the actor), its skill verifier -and its coordinator, each a Codex session given ASPIRE's own prompt. For each task the actor writes a Code-as-Policy -program (Python against ASPIRE's perception, grasping and motion API), runs it in ASPIRE's robot execution engine on -development seeds, debugs it from the per-primitive traces and RGB keyframes, and checks it once on unseen seeds; the -harness then scores it once on held-out seeds. After each task the coordinator (the -[`aspire`](../../docs/methods/aspire.md) Evolver) updates the shared skill library, admitting the actor's proposals -only if a fresh verifier re-solves the task from the library and the proposals alone, and chooses the next task. +# The cap-harness engine + +The robot execution engine of [ASPIRE](https://github.com/NVlabs/ASPIRE)'s Code-as-Policy harness (`cap-harness`), shared +by the benchmarks that run on it: [Robosuite](../robosuite/) and [LIBERO](../libero/) (and, for its simulator +service, [RoboDojo-Sim](../robodojo_sim/)). A session writes a program (Python against the harness's perception, +grasping and motion API), runs it in the engine on its task's development seeds through `bin/sim`, and the judge +scores it once on held-out seeds. The engine is a benchmark's, not a method's: what a method brings to a session +(ASPIRE's fix loop, its search, its verifier: [`aspire`](../../docs/methods/aspire.md)) is the method's own, built on +the task functions here. | Folder | What it is | |---|---| -| `utils/agent/task.py` | The Codex actor's task functions: the workspace (`bin/sim`, docs, the snapshot program) and its registration with the simulator service, the prompt (an adaptation page, the assignment, ASPIRE's template verbatim), the selected program (`collect`) | -| `utils/agent/sim_service.py` | The simulator service on the host: ASPIRE's `cap-harness` for the sandboxed actors, holding their seed rules | -| `utils/agent/stage_docs.py` | ASPIRE's docs as the actors may read them, staged outside its checkout by `launch.sh` | -| `utils/agent/sim_cli.py` | `bin/sim`: the actor's client of the simulator service | -| `utils/agent/sim.py` | The robot execution engine from the harness's side (the held-out run) and the trace digest | -| `utils/agent/prompts/` | The pages that map ASPIRE's fix-loop, verifier and evolutionary-search templates onto the workspace (and the direct actor's prompts) | -| `utils/agent/agent.py` | The structured direct-chat actor of `aspire/run_direct.yaml` (a comparison, not the method) | -| `utils/benchmark/prepare.py` | The seven Robosuite tasks with their seeds, as task directories in Harbor's format (`tasks/`, each read as its row through `utils/agent/aspire_fields.py`, the configs' `data.view`); the verifier's tasks; the reruns with the final library; `splits/` | +| `utils/agent/task.py` | The task functions every session gets: the workspace (`bin/sim`, the staged docs, the scene-exploration page and snapshot program) and its registration with the simulator service, the facts the judge reads (`collect`: `/fix_code.py` and the unseen check) | +| `utils/agent/sim_service.py` | The simulator service on the host: ASPIRE's `cap-harness` for the sandboxed sessions, holding their seed rules and a row's attempt budget (`max_programs`) | +| `utils/agent/stage_docs.py` | ASPIRE's docs as the sessions may read them, staged outside its checkout by `launch.sh` | +| `utils/agent/sim_cli.py` | `bin/sim`: a session's client of the simulator service | +| `utils/agent/sim.py` | The robot execution engine from the harness's side (the held-out run, a direct actor's runs) and the trace digest | +| `utils/agent/fields.py` | A task directory read back as its row: every config's `data.view` | +| `utils/benchmark/tasks.py` | What a benchmark's `prepare.py` writes with: its tasks and their reruns as task directories in Harbor's format, its splits, the seed presets' checks | +| `utils/benchmark/judge.py` | On the host: the scored program on the row's held-out seeds, its success rate | | `utils/seg_service/` | A SAM3-wire-compatible segmentation service from ungated models (optional) | -| `utils/benchmark/judge.py` | Stage 2 on the host: the selected program on the held-out seeds (a verifier's on its own), its success rate | -| `utils/benchmark/task-info/` | What a task and its record are, for the learner | -| `aspire/` | `run.yaml` (ASPIRE's seeds), `smoke.yaml`, the `*_direct.yaml` comparison, `launch.sh` | +| `launch.sh` | Started by a benchmark's method folder: writes the seeds into the tasks, runs the method folder's `tasks.sh` if it has one, stages the docs, starts the simulator service and runs hermit | ## Setup 1. **ASPIRE.** ASPIRE runs in its own venvs and is never imported into hermit's process (as CL-Bench's runner is - not), so it is not a dependency group of hermit's `pyproject.toml`: the example pins its commit here and + not), so it is not a dependency group of hermit's `pyproject.toml`: the engine pins its commit here and `launch.sh` names the one a run used. Clone [NVlabs/ASPIRE](https://github.com/NVlabs/ASPIRE) and bootstrap its Robosuite venv and the providers its programs call (its `docs/host-setup.md`, `docs/robosuite.md`): @@ -48,7 +44,7 @@ only if a fresh verifier re-solves the task from the library and the proposals a `CAP_HARNESS_SAM3_URL` at it): see [Segmentation without SAM3](#segmentation-without-sam3). 3. **The simulator's render device.** MuJoCo renders offscreen with EGL. Every cap-harness process runs in a - sandbox of its own (see [The actor's rules](#the-actors-rules)), which shows only the GPUs `ASPIRE_SIM_GPU` + sandbox of its own (see [A session's rules](#a-sessions-rules)), which shows only the GPUs `ASPIRE_SIM_GPU` names: the N of `/dev/nvidia` (`nvidia-smi -q` shows it as a GPU's Minor Number), e.g. `ASPIRE_SIM_GPU=8`. Without it the simulator renders in software (Mesa's llvmpipe, which the host must have), slowly. A hand-extracted NVIDIA EGL library (`__EGL_VENDOR_LIBRARY_FILENAMES`) is shown to it; other paths it needs go in @@ -62,69 +58,36 @@ only if a fresh verifier re-solves the task from the library and the proposals a MUJOCO_EGL_DEVICE_ID: "0" # inside the sandbox the GPUs ASPIRE_SIM_GPU names count from 0 ``` -4. **Codex and the sandboxes.** The actor and the coordinator are Codex CLI sessions (`codex` on `PATH`; the configs - bind them to GLM 5.3 Flash on Fireworks), each in the framework's sandbox (`sandbox.enabled: true`, `enforce: +4. **Codex and the sandboxes.** A session (and a method's learner, such as ASPIRE's coordinator) is a Codex CLI + session (`codex` on `PATH`; the configs bind them to GLM 5.3 Flash on Fireworks), each in the framework's sandbox (`sandbox.enabled: true`, `enforce: required`; bubblewrap on `PATH`, docs/guides/sandbox.md). The task sandbox has no GPU, so the simulator runs on the - host as a service (`utils/agent/sim_service.py`, started by `launch.sh`), which the actor reaches through `bin/sim` - on a forwarded loopback port; see [The actor's rules](#the-actors-rules). - -5. **Run** on the host that serves the providers: - - ```bash - export FIREWORKS_API_KEY=... - cd examples/aspire_robosuite/aspire - ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh - ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh eval --adapter final - ``` - - `./launch.sh eval --adapter final` reruns every task with the final library (the test split's `rerun-*` rows): - ASPIRE's backward transfer, compared per task with the task's first pass during training (each training step's - score in the run's `steps.jsonl`; the rerun's in the eval's results). `--adapter root` is the empty library and - `--adapter none` no library at all: the baselines on the same seeds. - - The protocol is ASPIRE's: each task learns on its development seeds and is scored once on held-out seeds; the - reruns score the same held-out seeds again, so the comparison reuses that partition (as ASPIRE's Phase 3 does). - The first evaluation ran `run.yaml` on held-out seeds 1-50 (half of ASPIRE's 1-100), and its reruns on the same - seeds. The held-out seeds are the task rows' (`prepare.py --eval`), so they are set when the launcher writes the - rows: `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh`, and `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh eval --adapter final` for the - reruns (an `agent.params.eval_seeds` override has no effect). - -## Protocol - -| | `run.yaml` | `smoke.yaml` | -|---|---|---| -| Tasks | the 7 Robosuite tasks, in the order the coordinator chooses (it starts from `splits/all.txt`'s first) | `cube_lifting`, `cube_stack` | -| Development seeds | 101-125 (ASPIRE's) | 101-103 | -| Unseen-check seeds | 126-130 (ASPIRE's), run once by the actor | 126 | -| Held-out seeds | 1-100 (ASPIRE's; `ASPIRE_EVAL_SEEDS` for others), run once by the judge | 1-3 | -| Fix loop | ASPIRE's: 3 versions per failure mode, 8 in total | the same | -| Evolutionary search | ASPIRE's, for a task the fix loop leaves below 90%: K = 8, top 3, stop at 80% or 5 rounds | the same | -| Verifier | ASPIRE's: 5 seeds (101-105), 2 attempts, verified at ≥ 4/5 | the same | -| Actor, verifier, coordinator | Codex, GLM 5.3 Flash (Fireworks), reasoning effort medium | the same | + host as a service (`utils/agent/sim_service.py`, started by `launch.sh`), which the session reaches through `bin/sim` + on a forwarded loopback port; see [A session's rules](#a-sessions-rules). ## Disk -Every actor session runs the simulator hundreds of times (a fix loop of up to 8 programs on 25 seeds, an -evolutionary search of up to 40 more), and each run keeps its record, trace and images. `collect` deletes the images +A session runs the simulator hundreds of times (ASPIRE's: a fix loop of up to 8 programs on 25 seeds, an evolutionary +search of up to 40 more), and each run keeps its record, trace and images. `collect` deletes the images (`media/`) when the session ends (`agent.params.prune_media: false` keeps them); the rest still adds up over a campaign. Set `ASPIRE_OUT` to a large disk: `launch.sh` puts the runs and the workspaces there. -## The actor's rules -The actor runs in the task sandbox, which shows it only what the config declares (`environment.read`): its own code -(`utils/agent/`), of the ASPIRE checkout the prompt templates and the snapshot program it is given, and ASPIRE's -docs as `launch.sh` stages them outside the checkout (`utils/agent/stage_docs.py`: without the lines, and for LIBERO -the section, that name ASPIRE's worked programs; the checkout's own pages are not shown). -ASPIRE's shipped skill library, its worked solutions and examples, other sessions' workspaces, the task rows and the -judge do not exist inside; its network reaches only its model and the simulator service's forwarded port. The +## A session's rules + +A session runs in the task sandbox, which shows it only what the config declares (`environment.read`): the engine's +code (`utils/agent/`), of the ASPIRE checkout the snapshot program it is given (and a method's prompt templates, which +its configs declare), and ASPIRE's docs as `launch.sh` stages them outside the checkout (`utils/agent/stage_docs.py`: +without the lines, and for LIBERO the section, that name ASPIRE's worked programs; the checkout's own pages are not +shown). ASPIRE's shipped skill library, its worked solutions and examples, other sessions' workspaces, the task rows +and the judge do not exist inside; its network reaches only its model and the simulator service's forwarded port. The simulator service is the trusted side of the seed rules: it reads the task rows on the host, `prepare` registers the session (its workspace and task id; one session per rollout directory, which the sandbox cannot create) before the -agent starts, and the service runs only the row's development and unseen-check seeds, the unseen check once and a -verifier's two programs, writing the runs into the workspace. The held-out seeds are run only by the judge, on the -host (ASPIRE's Stage 2), on the program the session selected. The actor's Codex runs with its own sandbox off -(`codex_sandbox: danger-full-access`), the task sandbox being the boundary. +agent starts, and the service runs only the row's development and unseen-check seeds, the unseen check once and, for +a row with an attempt budget (`max_programs`), that many programs, writing the runs into the workspace. The held-out +seeds are run only by the judge, on the host (ASPIRE's Stage 2), on the program the session selected. The session's +Codex runs with its own sandbox off (`codex_sandbox: danger-full-access`), the task sandbox being the boundary. -The programs the actor writes run inside cap-harness on the host, and ASPIRE's program validator is no boundary (a +The programs a session writes run inside cap-harness on the host, and ASPIRE's program validator is no boundary (a format string's attribute traversal passes it). So every cap-harness process, the service's, the judge's and the direct actor's, runs in a bubblewrap sandbox of its own (`utils/agent/sim.py`: `confine`): it reads ASPIRE's checkout but not its worked solutions, agents' skills or tests (`examples/`, `.claude/`, `.agents/`, `.codex/`, @@ -133,13 +96,14 @@ ports; one a `CAP_HARNESS_*_URL` puts on another host through an allowlist proxy GPUs `ASPIRE_SIM_GPU` names (hermit's `Sandbox(gpu=...)`). A program that escapes the validator reaches no other session, none of the judge's runs and no host file, and what it leaves in its output is read without following a link. It still runs in cap-harness's own process, so it could write its own run's outcome: the score of such a -program is not trustworthy, and the boundary is that it can affect nothing else. This sandbox is the example's own: +program is not trustworthy, and the boundary is that it can affect nothing else. This sandbox is the engine's own: it is on whatever the run's `sandbox.enabled` says, so the simulator always needs bubblewrap (the service refuses to start without it, or with an `ASPIRE_SIM_GPU` the machine lacks). Each run gets a HOME of its own, removed after it; the host's `CUDA_VISIBLE_DEVICES` is not passed in (inside, the GPUs count from 0); each confined run leaves its sandbox record under `$ASPIRE_OUT/sim-service/sandbox/` (the service's) or `$ASPIRE_OUT/sandbox/` (the judge's). + ## Segmentation without SAM3 ASPIRE's programs segment with SAM3 (`facebook/sam3`, gated). Any service with the SAM3 wire format can stand in. @@ -155,6 +119,7 @@ CUDA_VISIBLE_DEVICES=0 .venv-seg/bin/python utils/seg_service/service.py --port The models (`IDEA-Research/grounding-dino-base`, `facebook/sam-vit-huge`; `SEG_DINO_MODEL`, `SEG_SAM_MODEL`) are downloaded on first start. + ## Checking the render device Inside the simulator's sandbox EGL finds only the GPUs `ASPIRE_SIM_GPU` names, so `MUJOCO_EGL_DEVICE_ID: "0"` is the @@ -164,9 +129,10 @@ use). A host whose libEGL finds only Mesa's driver (`failed to open /dev/dri/car vendor library (`libEGL_nvidia.so.0`, the driver's `libnvidia-gl` package) and `__EGL_VENDOR_LIBRARY_FILENAMES` naming its ICD file. + ## Known host issues (Blackwell B300, sm_103) -Found while bringing the engine up on B300 nodes; none of it is in the example's code path, which takes the +Found while bringing the engine up on B300 nodes; none of it is in the engine's code path, which takes the simulator's environment and the service URLs from the host. - **NVRTC 12.8 rejects sm_103.** The cu128 PyTorch wheels' runtime compiler fails on B300 diff --git a/examples/aspire_robosuite/aspire/launch.sh b/examples/cap_harness/launch.sh similarity index 71% rename from examples/aspire_robosuite/aspire/launch.sh rename to examples/cap_harness/launch.sh index 15a70d2c..ed50b7d2 100755 --- a/examples/aspire_robosuite/aspire/launch.sh +++ b/examples/cap_harness/launch.sh @@ -1,29 +1,34 @@ #!/usr/bin/env bash -# Train, or evaluate, the ASPIRE method on Robosuite. Run it on the host that serves ASPIRE's providers. -# (Another benchmark of ASPIRE's harness starts it from its own folder, as ASPIRE_EXAMPLE: its configs, in the -# method folder ASPIRE_METHOD (default aspire), its utils/benchmark/prepare.py and tasks, its runs; ../../libero/utils/launch.sh.) +# Train, or evaluate, a method on a benchmark of the cap-harness engine (ASPIRE's harness). Run it on the host that +# serves ASPIRE's providers. A method folder of a benchmark starts it (../robosuite/aspire/launch.sh, +# ../libero/utils/launch.sh) with ASPIRE_EXAMPLE, the benchmark's folder (its utils/benchmark/prepare.py, tasks and +# task-info, its runs), and ASPIRE_METHOD, the method folder in it (its configs); from that folder: # # ASPIRE_ROOT=/path/to/ASPIRE ./launch.sh # train run.yaml # ASPIRE_CONFIG=smoke.yaml ./launch.sh # train the smoke (its seeds: ASPIRE_SEEDS=smoke) -# ./launch.sh eval --adapter final # the reruns with the final library (test split) -# ./launch.sh eval --adapter root # the same with the empty library +# ./launch.sh eval --adapter final # the reruns with the final adapter (test split) +# ./launch.sh eval --adapter root # the same with the starting one # ASPIRE_SIM_ENV=/path/to/sim_env.yaml ./launch.sh # the simulator's render environment (a YAML map) # ASPIRE_SIM_GPU=8 ./launch.sh # its GPUs, the N of /dev/nvidia (none: software) -# ASPIRE_OUT=/big/disk/aspire ./launch.sh # the runs, the sessions, Stage 2 (default ../runs) +# ASPIRE_OUT=/big/disk/aspire ./launch.sh # the runs, sessions, Stage 2 (default: runs/ beside) # ASPIRE_EVAL_SEEDS=1-50 ./launch.sh # held-out seeds other than the preset's # -# The launcher writes the seeds into the tasks (prepare.py; ASPIRE_SEEDS=smoke for the smoke's), starts the -# simulator service the actors reach through bin/sim (utils/agent/sim_service.py, on ASPIRE_SIM_PORT, default 8120; -# stopped when the run ends), and runs hermit. ASPIRE_SIM_ENV names a YAML mapping of environment variables for the -# simulator, e.g. {MUJOCO_GL: egl, PYOPENGL_PLATFORM: egl, EGL_PLATFORM: device, MUJOCO_EGL_DEVICE_ID: "0"} +# The launcher writes the seeds into the tasks (the benchmark's prepare.py; ASPIRE_SEEDS=smoke for the smoke's), +# then runs the method folder's tasks.sh when it has one (ASPIRE's adds its verifiers, and sets TASKS to the tasks its +# configs read), starts the simulator service the sessions reach through bin/sim (utils/agent/sim_service.py, on +# ASPIRE_SIM_PORT, default 8120; stopped when the run ends), and runs hermit. ASPIRE_SIM_ENV names a YAML mapping +# of environment variables for the simulator, e.g. {MUJOCO_GL: egl, PYOPENGL_PLATFORM: egl, EGL_PLATFORM: device, +# MUJOCO_EGL_DEVICE_ID: "0"} # (the default, with CUDA_VISIBLE_DEVICES="": the simulator needs no CUDA, and Robosuite asserts that the EGL # device appears in CUDA_VISIBLE_DEVICES when that is set). The model key (FIREWORKS_API_KEY) comes from the # environment; the sandboxes need bubblewrap (`bwrap`) on PATH. set -euo pipefail HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" : "${ASPIRE_ROOT:?set ASPIRE_ROOT to an ASPIRE checkout with .venv-robosuite}" -EXAMPLE="$(cd "${ASPIRE_EXAMPLE:-$HERE/..}" && pwd)" # the folder with / and utils/benchmark/ -METHOD="${ASPIRE_METHOD:-aspire}" # the folder of the configs +: "${ASPIRE_EXAMPLE:?set by a method folder of the benchmark: the benchmark folder}" +: "${ASPIRE_METHOD:?set by a method folder of the benchmark: the folder of the configs}" +EXAMPLE="$(cd "$ASPIRE_EXAMPLE" && pwd)" # the folder with /, utils/benchmark/ +METHOD="$ASPIRE_METHOD" # Absolute, from where the launcher was called: the service, the judge and the sandboxed workers each start elsewhere ASPIRE_ROOT="$(cd "$ASPIRE_ROOT" && pwd)" [[ -z "${ASPIRE_OUT:-}" ]] || { mkdir -p "$ASPIRE_OUT" && ASPIRE_OUT="$(cd "$ASPIRE_OUT" && pwd)"; } @@ -41,7 +46,7 @@ fi for url in "${CAP_HARNESS_SAM3_URL:-http://127.0.0.1:8114}" "http://127.0.0.1:8116" \ "${CAP_HARNESS_CUROBO_URL:-http://127.0.0.1:8118}"; do curl -sf -o /dev/null --max-time 3 "$url/openapi.json" \ - || { echo "ASPIRE provider at $url is not up (see ../README.md)" >&2; exit 1; } + || { echo "ASPIRE provider at $url is not up (see $HERE/README.md)" >&2; exit 1; } done # The ASPIRE commit this example was written against; another one is run, but named. PINNED=fa00f0ef2bc1d91d594b9aa502c22133c9f538f7 @@ -52,19 +57,21 @@ cd "$EXAMPLE/$METHOD" ! grep -q "ref: fireworks/" "$CONFIG" || : "${FIREWORKS_API_KEY:?set FIREWORKS_API_KEY}" # The seeds, in the tasks (task directories, each read as its row): the simulator service and the judge read them -# there. The example's utils/benchmark/prepare.py holds its benchmark's (ASPIRE's defaults and the smoke's). +# there. The benchmark's utils/benchmark/prepare.py holds its own (ASPIRE's defaults and the smoke's). PRESET="${ASPIRE_SEEDS:-$([[ "${CONFIG##*/}" == smoke* ]] && echo smoke || echo aspire)}" [[ "$PRESET" == smoke || "$PRESET" == aspire ]] || { echo "ASPIRE_SEEDS is smoke or aspire" >&2; exit 2; } SEEDS=(--preset "$PRESET") [[ -z "${ASPIRE_EVAL_SEEDS:-}" ]] || SEEDS+=(--eval "$ASPIRE_EVAL_SEEDS") (cd ../utils/benchmark && "$HERMIT" prepare.py "${SEEDS[@]}") +# The tasks the run reads: the benchmark's, or what the method folder's tasks.sh sets (run here, in that folder) +TASKS="$EXAMPLE/utils/benchmark/tasks" +[[ ! -f tasks.sh ]] || source ./tasks.sh # ASPIRE's docs as the actors may read them, outside its checkout: the task sandbox is shown these, not the # checkout's pages, which name ASPIRE's worked programs per task. export ASPIRE_DOCS="$EXAMPLE/utils/aspire-docs" -"$HERMIT" "$HERE/../utils/agent/stage_docs.py" --aspire-root "$ASPIRE_ROOT" --tasks ../utils/benchmark/tasks \ - --out "$ASPIRE_DOCS" +"$HERMIT" "$HERE/utils/agent/stage_docs.py" --aspire-root "$ASPIRE_ROOT" --tasks "$TASKS" --out "$ASPIRE_DOCS" -# Where the runs go (ASPIRE_OUT): the trainer's records, the actors' sessions (their simulator runs' records and +# Where the runs go (ASPIRE_OUT): the trainer's records, the sessions (their simulator runs' records and # traces; their images are pruned after scoring), the judge's Stage 2. Put it on a large disk. OUT="${ASPIRE_OUT:-$EXAMPLE/runs}" case "$(cd "$OUT" 2>/dev/null && pwd -P || echo "$OUT")/" in # the simulator reads the checkout, not the runs @@ -74,12 +81,12 @@ esac mkdir -p "$OUT" export ASPIRE_ROOT ASPIRE_OUT="$OUT" ASPIRE_SIM_ENV="$SIM_ENV_JSON" ASPIRE_SIM_GPU="${ASPIRE_SIM_GPU:-}" # the judge's -# The simulator service: the actors' engine, on the host; each cap-harness it starts (and the judge's) runs in a +# The simulator service: the sessions' engine, on the host; each cap-harness it starts (and the judge's) runs in a # sandbox of its own with the GPUs ASPIRE_SIM_GPU names (utils/agent/sim.py: confine). [[ -n "${ASPIRE_SIM_GPU:-}" ]] || echo "note: ASPIRE_SIM_GPU names no GPU: the simulator renders in software" >&2 PORT="${ASPIRE_SIM_PORT:-8120}" NONCE="$$-$RANDOM-$(date +%s%N)" # this launch's service answers /health with it, another run's not -"$HERMIT" "$HERE/../utils/agent/sim_service.py" --port "$PORT" --aspire-root "$ASPIRE_ROOT" --tasks ../utils/benchmark/tasks \ +"$HERMIT" "$HERE/utils/agent/sim_service.py" --port "$PORT" --aspire-root "$ASPIRE_ROOT" --tasks "$TASKS" \ --workspaces "$OUT/actor-logs" --state "$OUT/sim-service" --sim-env "$SIM_ENV_JSON" --nonce "$NONCE" \ --gpu "${ASPIRE_SIM_GPU:-}" > "$OUT/sim-service.log" 2>&1 & SERVICE=$! diff --git a/examples/cap_harness/utils/agent/fields.py b/examples/cap_harness/utils/agent/fields.py new file mode 100644 index 00000000..2424b946 --- /dev/null +++ b/examples/cap_harness/utils/agent/fields.py @@ -0,0 +1,20 @@ +"""A task of the cap-harness engine as the session's task functions, the simulator service and the judge read it: +{id, (benchmark), suite, task_id, the seeds, (max_programs), and whatever a method's own rows add}. A task directory +(a benchmark's utils/benchmark/prepare.py, through ../benchmark/tasks.py) has every field but its id under task.toml's +[metadata], in the row's order; this puts them back, so the prompts, the service's rules and the records read the row +as it was written. Every config of a benchmark on this engine names it as its data.view.""" +from __future__ import annotations + +from pathlib import Path + + +def task_row(task: dict) -> dict: + if "instruction" not in task: # already a row of that shape + return dict(task) + return {"id": task["id"], **(task.get("metadata") or {})} + + +def task_rows(tasks: Path) -> dict[str, dict]: + """Every task's row by id: from task directories, or from a tasks.jsonl.""" + from hermit.task import Benchmark + return {tid: t.given() for tid, t in Benchmark.load(Path(tasks), view=task_row).tasks.items()} diff --git a/examples/aspire_robosuite/utils/agent/sim.py b/examples/cap_harness/utils/agent/sim.py similarity index 100% rename from examples/aspire_robosuite/utils/agent/sim.py rename to examples/cap_harness/utils/agent/sim.py diff --git a/examples/aspire_robosuite/utils/agent/sim_cli.py b/examples/cap_harness/utils/agent/sim_cli.py similarity index 100% rename from examples/aspire_robosuite/utils/agent/sim_cli.py rename to examples/cap_harness/utils/agent/sim_cli.py diff --git a/examples/aspire_robosuite/utils/agent/sim_service.py b/examples/cap_harness/utils/agent/sim_service.py similarity index 95% rename from examples/aspire_robosuite/utils/agent/sim_service.py rename to examples/cap_harness/utils/agent/sim_service.py index 6b933475..30b99a7b 100644 --- a/examples/aspire_robosuite/utils/agent/sim_service.py +++ b/examples/cap_harness/utils/agent/sim_service.py @@ -1,10 +1,10 @@ -"""The robot execution engine as a host service: ASPIRE's `cap-harness` for the actors in the task sandbox. +"""The robot execution engine as a host service: ASPIRE's `cap-harness` for the sessions in the task sandbox. The task sandbox has no GPU, so the simulator runs here, on the host, beside ASPIRE's perception and planning services; a session reaches it through `bin/sim` (sim_cli.py) on a forwarded 127.0.0.1 port (ASPIRE_SIM_URL in environment.env). The service is the trusted side of the session's rules, and it trusts nothing the session sends -but the workspace and its task's id: the seeds, the role and the budget are the task row's (--tasks, read here on -the host, each task as its row: aspire_fields.py), and a rollout's directory (//-, +but the workspace and its task's id: the seeds and the budget are the task row's (--tasks, read here on +the host, each task as its row: fields.py), and a rollout's directory (//-, made by the framework outside the sandbox, which can write only inside it) registers one session, of the task its name carries: @@ -13,8 +13,8 @@ same workspace, finds its registration and what it spent as they were) POST /run {workspace, cmd: run | run-batch | validate, program, seeds, output_root, no_videos} runs cap-harness for the registered task: only its development and unseen-check seeds, the - unseen check once, a verifier's two programs (by content; a single run on the first seed is - free), the program and the output inside the workspace, the registration's max_steps + unseen check once, a row's attempt budget (`max_programs`: programs by content; a single run on the + first seed is free), the program and the output inside the workspace, the registration's max_steps GET /health The session's files are never opened by path here, since the session may have put a link in it: the program is read @@ -23,7 +23,7 @@ own directory, and the run is moved into the workspace the same way, each file replacing whatever entry has its name. A run is stopped after --run-timeout seconds, and the runs still going are stopped when the service is. -Every refusal is a 200 with {ok: false, error}. The ledger (registrations, unseen checks, a verifier's programs) is +Every refusal is a 200 with {ok: false, error}. The ledger (registrations, unseen checks, a budget's programs) is kept in --state, outside every sandbox. Started by launch.sh: python sim_service.py --port 8120 --aspire-root $ASPIRE_ROOT --tasks utils/benchmark/tasks --workspaces @@ -50,7 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import sim # noqa: E402 from sim import seeds_of # noqa: E402 -from aspire_fields import task_rows # noqa: E402 +from fields import task_rows # noqa: E402 class Ledger: @@ -171,13 +171,13 @@ def register(self, body: dict) -> dict: return {"ok": True} if any(Path(w).is_relative_to(rollout) for w in self.ledger.data): return {"ok": False, "error": "this rollout has registered its session already"} - verifier = row.get("role") == "verifier" self._check_row(row) self.ledger.data[str(workspace)] = { - "task": row["id"], "role": "verifier" if verifier else "actor", "suite": str(row["suite"]), + "task": row["id"], "suite": str(row["suite"]), "benchmark": str(row.get("benchmark") or "robosuite"), "task_id": int(row.get("task_id", 0)), "dev": seeds_of(row.get("debug_seeds")), - "unseen": seeds_of(row.get("unseen_seeds")), "max_programs": 2 if verifier else None, + "unseen": seeds_of(row.get("unseen_seeds")), + "max_programs": int(row["max_programs"]) if row.get("max_programs") else None, "max_steps": body.get("max_steps"), "unseen_used": False, "programs": []} self.ledger.save() return {"ok": True} @@ -210,7 +210,7 @@ def run(self, body: dict) -> dict: shutil.rmtree(own, ignore_errors=True) def _run(self, body, session, program, code, own_out, rollout, out, swaps) -> dict: - """The rules every engine shares (the assigned seeds, the unseen check once, a verifier's programs, the + """The rules every engine shares (the assigned seeds, the unseen check once, a budget's programs, the output moved into the workspace without following a link); what is run is the engine's (`_validate`, `_launch`: cap-harness here, another simulator in a subclass).""" if body.get("cmd") == "validate": diff --git a/examples/aspire_robosuite/utils/agent/stage_docs.py b/examples/cap_harness/utils/agent/stage_docs.py similarity index 97% rename from examples/aspire_robosuite/utils/agent/stage_docs.py rename to examples/cap_harness/utils/agent/stage_docs.py index 4c1ed997..c910ebd0 100644 --- a/examples/aspire_robosuite/utils/agent/stage_docs.py +++ b/examples/cap_harness/utils/agent/stage_docs.py @@ -13,7 +13,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import sim # noqa: E402 -from aspire_fields import task_rows # noqa: E402 +from fields import task_rows # noqa: E402 def main() -> None: diff --git a/examples/cap_harness/utils/agent/task.py b/examples/cap_harness/utils/agent/task.py new file mode 100644 index 00000000..4ce11383 --- /dev/null +++ b/examples/cap_harness/utils/agent/task.py @@ -0,0 +1,156 @@ +"""The engine's task functions for a coding-agent session (agent.base: codex, or claude): what every method's session +on a benchmark of this engine gets, and nothing of a method. They run in the task sandbox with the session; the robot +execution engine runs on the host, as the simulator service (sim_service.py), which the session reaches only through +`bin/sim`. A benchmark gives the prompt (LIBERO's ../../../libero/utils/agent/task.py); a method that brings its own +session (ASPIRE: hermit/agent/evolver/methods/aspire/actor.py) builds on these. + + prepare the workspace: bin/sim and its registration with the service (the workspace and the task's id: the + service takes the seeds and the budget from its own copy of the task row), the API and artifact docs, + the benchmark's page, the scene-exploration procedure and its snapshot program + answer the agent's last message (the record's output is the facts collect returns) + collect the session's files into the log, and the facts the judge reads: the scored program + (`/fix_code.py`) and the unseen check (`/debug/holdout`); the judge runs the program + on the held-out seeds, on the host + +The checkout of ASPIRE, whose `cap-harness` this engine is, comes from agent.params.aspire_root (its snapshot program +and exploration page must be declared readable: environment.read); its docs are read from agent.params.aspire_docs, +the staged copies; the service's URL from ASPIRE_SIM_URL (environment.env). A task row carries the seeds: +development, unseen-check and held-out (../benchmark/tasks.py). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import shutil +import stat +import sys +import urllib.request + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import sim # noqa: E402 + +SKILLS = Path(".claude/skills/iterative-debugging") +# The files of ASPIRE's checkout the task sandbox shows (environment.read) for every session: the scene-exploration +# procedure and its snapshot program; not its shipped library, worked solutions or examples, nor its docs, which the +# sandbox is shown as staged (agent.params.aspire_docs: stage_docs.py's copies, without what names the worked programs) +ASPIRE_FILES = [str(SKILLS / "scripts" / "scene_snapshot.py"), str(SKILLS / "skills" / "task-exploration.md")] +# The workspace files the log keeps (the run directories stay in the workspace) +KEEP = ("task_analysis.md", "fix_code.py", "findings.md", "code", "debug/blocked") + + +def settings(task: dict, params: dict) -> dict: + """The session's task, from its row (the seeds; ../benchmark/tasks.py) and agent.params (the checkout).""" + root = params.get("aspire_root") or os.environ.get("ASPIRE_ROOT") + if not root: + raise ValueError("agent.params.aspire_root (or ASPIRE_ROOT): an ASPIRE checkout with its benchmark's venv") + benchmark = str(task.get("benchmark") or "robosuite") + return {"root": Path(os.path.expandvars(str(root))).expanduser().resolve(), + "benchmark": benchmark, "doc": sim.profile(benchmark)["doc"], "suite": task["suite"], + "task_id": int(task.get("task_id", 0)), "dev": str(task.get("debug_seeds") or ""), + "unseen": str(task.get("unseen_seeds") or ""), "heldout": str(task.get("eval_seeds") or ""), + "max_steps": params.get("max_steps")} + + +def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: + s = settings(task, params) + work = Path(workspace) + url = os.environ.get("ASPIRE_SIM_URL") + if not url: + raise ValueError("ASPIRE_SIM_URL: the simulator service's URL (launch.sh starts it; environment.env)") + # The session, registered before the agent starts: the service takes its seeds and budget from the task's row + register(url, {"workspace": str(work.resolve()), "task": task["id"], "max_steps": s["max_steps"]}) + bin_dir = work / "bin" + bin_dir.mkdir(exist_ok=True) + shutil.copy2(HERE / "sim_cli.py", bin_dir / "sim_cli.py") + (bin_dir / "sim.json").write_text(json.dumps({"workspace": str(work.resolve()), "url": url}, indent=1)) + wrapper = bin_dir / "sim" + wrapper.write_text(f"#!/bin/sh\nexec {shlex.quote(sys.executable)} \"$(dirname \"$0\")/sim_cli.py\" \"$@\"\n") + wrapper.chmod(wrapper.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + (work / "docs").mkdir(exist_ok=True) + # The repository's worked programs are not the session's to read: the docs are the staged copies (launch.sh: + # stage_docs.py), or, where the checkout's own pages are what the sandbox shows, the same cut made here + staged = params.get("aspire_docs") + pages = {name: (Path(os.path.expandvars(str(staged))).expanduser() / name).read_text() + for name in (*sim.DOCS, s["doc"])} if staged else sim.workspace_docs(s["root"], s["benchmark"]) + for name, text in pages.items(): + (work / "docs" / name).write_text(text) + shutil.copy2(s["root"] / SKILLS / "skills" / "task-exploration.md", work / "docs" / "task-exploration.md") + (work / "scripts").mkdir(exist_ok=True) + shutil.copy2(s["root"] / SKILLS / "scripts" / "scene_snapshot.py", work / "scripts" / "scene_snapshot.py") + return [] + + +def register(url: str, body: dict) -> None: + request = urllib.request.Request(url.rstrip("/") + "/register", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=60) as reply: + answer = json.loads(reply.read()) + if not answer.get("ok"): + raise RuntimeError(f"the simulator service refused the session: {answer.get('error')}") + + +def task_dir(s: dict) -> str: + """Where the session's task files go, relative to its workspace: `//task_` (ASPIRE's + $TASK_DIR, with $RUN_ROOT the workspace). The scored program is fix_code.py there.""" + return f"{s['benchmark']}/{s['suite']}/task_{s['task_id']}" + + +def sim_seeds(text: str) -> list[int]: + return sim.seeds_of(text) + + +def answer(last_message: str, task: dict) -> dict: + return {"answer": last_message.strip()[-4000:]} + + +def collect(task: dict, workspace: Path, log: Path, params: dict) -> dict: + """The session's files into the log and the facts for the judge, which runs the scored program on the held-out + seeds, on the host.""" + s = settings(task, params) + work = Path(workspace) / task_dir(s) + facts = collect_files(task, work, log, KEEP) + program = work / "fix_code.py" + facts.update({"benchmark": s["benchmark"], "suite": s["suite"], "workspace": str(Path(workspace).resolve()), + "max_steps": s["max_steps"], + "scored_program": str(program.relative_to(workspace)) if program.is_file() else None, + "fix_code": program.read_text(errors="replace") if program.is_file() else None}) + if params.get("prune_media", True): + prune(Path(workspace).parent) + return facts + + +def collect_files(task: dict, work: Path, log: Path, keep) -> dict: + """The files of `keep` (under `work`) copied into the log's files/, and the facts every session's record has: + its notes (findings.md) and its unseen check.""" + files = Path(log) / "files" + for name in keep: + source = work / name + if source.is_dir(): + shutil.copytree(source, files / name, dirs_exist_ok=True) + elif source.is_file(): + files.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, files / name) + findings = work / "findings.md" + return {"findings": findings.read_text(errors="replace") if findings.is_file() else None, "unseen": unseen(work)} + + +def prune(run_dir: Path) -> None: + """The heavy part of a finished session's simulator runs: every run's media/ (keyframes, overlays, videos; a + session runs hundreds of episodes). The records, traces and programs stay.""" + for media in run_dir.rglob("media"): + if media.is_dir() and (media.parent / "run.json").is_file(): + shutil.rmtree(media, ignore_errors=True) + + +def unseen(work: Path) -> dict | None: + """The unseen check a session ran into `/debug/holdout`: how many of its episodes succeeded.""" + rows = work / "debug" / "holdout" / "results.jsonl" + if not rows.is_file(): + return None + results = [json.loads(l) for l in rows.read_text().splitlines() if l.strip()] + passed = sum(bool((sim._read_json(Path(r["run_dir"]) / "outcome.json") or {}).get("task_success")) + for r in results if r.get("run_dir")) + return {"passed": passed, "total": len(results)} diff --git a/examples/aspire_robosuite/utils/benchmark/judge.py b/examples/cap_harness/utils/benchmark/judge.py similarity index 60% rename from examples/aspire_robosuite/utils/benchmark/judge.py rename to examples/cap_harness/utils/benchmark/judge.py index 5b3c2f41..5e77ec1d 100644 --- a/examples/aspire_robosuite/utils/benchmark/judge.py +++ b/examples/cap_harness/utils/benchmark/judge.py @@ -1,9 +1,10 @@ -"""The coordinator's Stage 2, on the host: the session's selected program (utils/agent/task.py:collect) run once on -the seeds it is scored on, and its success rate. For an actor these are the task row's held-out seeds, which no -session can run (the simulator service refuses them); for a verifier, its own development seeds, and it counts only -with its own verdict `verified` (ASPIRE: `refuted` and `void` are never promoted) and only for one of the two programs -its attempt budget ran (the simulator service's ledger). A seed counts when ASPIRE's harness reports task_success -(the benchmark's own success check, run after the program finished), within the session's max_steps. +"""The judge, on the host: the session's scored program (the facts its task functions' `collect` returned: the +engine's utils/agent/task.py, or a method's built on it) run once on the task row's held-out seeds (`eval_seeds`), +which no session can run (the simulator service refuses them), and its success rate. A row with an attempt budget +(`max_programs`) counts only a program its budget ran (the simulator service's ledger). A seed counts when ASPIRE's +harness reports task_success (the benchmark's own success check, run after the program finished), within the +session's max_steps. A row may name its seeds for the feedback (`score_label`, default held-out); facts that +withhold their program (`not_scored`: a method's rule) score 0 with that reason. The engine's settings come from the launcher's environment: ASPIRE_ROOT (the checkout), ASPIRE_SIM_ENV (the simulator's environment, a JSON object), ASPIRE_OUT (where the runs go; the service's ledger is its sim-service/).""" @@ -21,23 +22,21 @@ def score(task: dict, output) -> dict: facts = (output.extra or {}).get("task") or {} - verifier = task.get("role") == "verifier" - seeds = sim.seeds_of(task.get("debug_seeds") if verifier else task.get("eval_seeds")) + seeds = sim.seeds_of(task.get("eval_seeds")) + what = task.get("score_label") or "held-out" # what the seeds are, in the feedback (a row's own name) unseen = facts.get("unseen") note = f"; unseen-check {unseen['passed']}/{unseen['total']}" if unseen else "" - what = "verifier" if verifier else "held-out" - if verifier and (facts.get("verification") or {}).get("verdict") != "verified": - verdict = (facts.get("verification") or {}).get("verdict") or "missing" - return {"score": 0.0, "feedback": f"verifier verdict {verdict}", "seeds": len(seeds)} + if facts.get("not_scored"): + return {"score": 0.0, "feedback": str(facts["not_scored"]), "seeds": len(seeds)} program = facts.get("fix_code") if not program or not seeds: return {"score": 0.0, "feedback": f"0/{len(seeds)} {what} seeds; not scored: " f"{'no program' if not program else 'no seeds'}{note}"} - # A Codex session's (its facts name its workspace, task.py:collect): the program must be one its budget ran. The - # direct actor runs its versions itself, at most max_versions of them. - if verifier and facts.get("workspace") and \ + # A session with an attempt budget (its facts name its workspace): the program must be one its budget ran. A + # session that runs its programs itself (no workspace: a direct actor) keeps its own count. + if task.get("max_programs") and facts.get("workspace") and \ hashlib.sha256(program.encode()).hexdigest() not in _budget(facts["workspace"]): - return {"score": 0.0, "feedback": "verifier's program is not one of the programs its attempt budget ran", + return {"score": 0.0, "feedback": "the program is not one of the programs its attempt budget ran", "seeds": len(seeds)} out = Path(os.environ.get("ASPIRE_OUT", ".")).resolve() / "stage2" / \ f"{task['id']}-{hashlib.sha256(program.encode()).hexdigest()[:12]}" @@ -57,7 +56,7 @@ def score(task: dict, output) -> dict: def _budget(workspace) -> list[str]: - """The digests of the programs a verifier's session ran within its budget (the simulator service's ledger).""" + """The digests of the programs a session ran within its budget (the simulator service's ledger).""" ledger = Path(os.environ.get("ASPIRE_OUT", ".")).resolve() / "sim-service" / "ledger.json" try: return list((json.loads(ledger.read_text()).get(str(workspace)) or {}).get("programs") or []) diff --git a/examples/cap_harness/utils/benchmark/tasks.py b/examples/cap_harness/utils/benchmark/tasks.py new file mode 100644 index 00000000..f4ce6133 --- /dev/null +++ b/examples/cap_harness/utils/benchmark/tasks.py @@ -0,0 +1,84 @@ +"""The tasks of a benchmark on the cap-harness engine, written as task directories (Harbor's format) with their seeds, +which the simulator service and the judge take from each task's row: the helpers a benchmark's own +utils/benchmark/prepare.py calls (../../../robosuite/, ../../../libero/). + +A task directory's task.toml holds the row's fields but its id under [metadata], in the row's order, and asks for no +network (the session reaches only its model and the simulator service's forwarded port); it has no tests/: the judge +(judge.py) runs on the host, where the simulator is. ../agent/fields.py reads each back as its row. + +Each task also has its rerun, `rerun-` (`rerun_of: `): the same task with the same seeds, under its own id +so that a test split of the tasks a run trained on does not share ids with its training split (a run's `launch.sh +eval`, or a continual method's evaluation of every task again with its final adapter). +""" +from __future__ import annotations + +import argparse +from pathlib import Path +import shutil +import sys + +from hermit.task.directory import write_task as write_task_dir + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "agent")) +from fields import task_rows # noqa: E402 +from sim import seeds_of # noqa: E402 + + +def seeds_from(presets: dict, argv=None) -> dict: + """The run's seeds from the command line (a preset, each set replaceable): the three sets of a task row.""" + ap = argparse.ArgumentParser() + ap.add_argument("--preset", choices=sorted(presets), default=next(iter(presets))) + ap.add_argument("--dev", help="development seeds") + ap.add_argument("--unseen", help="unseen-check seeds, run once by the session") + ap.add_argument("--eval", help="held-out seeds, run only by the judge") + a = ap.parse_args(argv) + chosen = {k: getattr(a, k) or v for k, v in presets[a.preset].items()} + # A held-out seed a session could run would be scored as unseen: the sets must not meet + dev, unseen, held = (set(seeds_of(chosen[k])) for k in ("dev", "unseen", "eval")) + for name, other in (("development", dev), ("unseen-check", unseen)): + if held & other: + ap.error(f"the held-out seeds overlap the {name} seeds: {sorted(held & other)[:10]}") + if dev & unseen: + ap.error(f"the unseen-check seeds overlap the development seeds: {sorted(dev & unseen)[:10]}") + return {"debug_seeds": chosen["dev"], "unseen_seeds": chosen["unseen"], "eval_seeds": chosen["eval"]} + + +def instruction(row: dict) -> str: + """What a task asks, in a line (a session's prompt is its config's, built from the row).""" + what = f"{row.get('benchmark') or 'robosuite'} suite `{row['suite']}`, task id {row['task_id']}" + if row.get("rerun_of"): + return f"Solve {what} again (a rerun of `{row['rerun_of']}`)." + return f"Write the program that solves {what} in ASPIRE's Code-as-Policy harness." + + +def write_task(tasks: Path, row: dict, text: str | None = None) -> Path: + """One row as a task directory: its fields but the id under [metadata], in the row's order; no network.""" + return write_task_dir(tasks / row["id"], text or instruction(row), + metadata={k: v for k, v in row.items() if k != "id"}, + environment={"network_mode": "no-network"}) + + +def rows(tasks: Path) -> dict[str, dict]: + """Every task directory under `tasks` as its row, by id (a method that adds rows of its own reads them here).""" + return task_rows(tasks) + + +def write(here: Path, tasks: list[dict], seeds: dict, splits: dict[str, list[str]]) -> None: + """tasks/ and splits/ under `here`: each task (`id`, `suite`, `task_id`, and `benchmark` when it is not + Robosuite) with its seeds, and its rerun; splits/.txt for each of `splits`, with _rerun.txt beside + it (rerun.txt for `all`).""" + rows = [{**t, **seeds} for t in tasks] + reruns = [{**t, "id": f"rerun-{t['id']}", "rerun_of": t["id"], **seeds} for t in tasks] + if (here / "tasks").exists(): # this script's own output: rewritten whole + shutil.rmtree(here / "tasks") + for row in rows + reruns: + write_task(here / "tasks", row) + out = here / "splits" + if out.exists(): + shutil.rmtree(out) + out.mkdir() + for name, ids in splits.items(): + (out / f"{name}.txt").write_text("".join(f"{i}\n" for i in ids)) + rerun = out / ("rerun.txt" if name == "all" else f"{name}_rerun.txt") + rerun.write_text("".join(f"rerun-{i}\n" for i in ids)) + print(f"{len(rows)} tasks -> {here / 'tasks'}") diff --git a/examples/aspire_robosuite/utils/seg_service/requirements.txt b/examples/cap_harness/utils/seg_service/requirements.txt similarity index 100% rename from examples/aspire_robosuite/utils/seg_service/requirements.txt rename to examples/cap_harness/utils/seg_service/requirements.txt diff --git a/examples/aspire_robosuite/utils/seg_service/service.py b/examples/cap_harness/utils/seg_service/service.py similarity index 100% rename from examples/aspire_robosuite/utils/seg_service/service.py rename to examples/cap_harness/utils/seg_service/service.py diff --git a/examples/cap_harness/utils/tests/cap_fakes.py b/examples/cap_harness/utils/tests/cap_fakes.py new file mode 100644 index 00000000..7100881d --- /dev/null +++ b/examples/cap_harness/utils/tests/cap_fakes.py @@ -0,0 +1,215 @@ +"""Stand-ins for the cap-harness engine's tests and the tests of a method on it (tests/methods/aspire/): a fake +ASPIRE checkout whose cap-harness writes a run's files without a simulator (FAKE_HARNESS), the simulator service on a +loopback port over it, a session's workspace where the framework puts it, and the task rows of each benchmark. Loaded +by path (`_load`), never imported from a conftest.""" +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +import pytest + +UTILS = Path(__file__).resolve().parents[1] # examples/cap_harness/utils +EXAMPLE = UTILS # the engine's agent/ and benchmark/ folders + + +@pytest.fixture +def tmp_path(): + """A directory outside /tmp: the template check runs under bubblewrap with an empty tmpfs at /tmp.""" + base = Path.home() / ".cache" / "hermit-tests" + base.mkdir(parents=True, exist_ok=True) + path = Path(tempfile.mkdtemp(dir=base)) + yield path + if not os.environ.get("KEEP_TEST_DIRS"): + shutil.rmtree(path, ignore_errors=True) + + +def _load(name: str, path: Path): + sys.path.insert(0, str(path.parent)) + try: + spec_ = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec_) + spec_.loader.exec_module(module) + return module + finally: + sys.path.remove(str(path.parent)) + + +def fake_aspire(tmp: Path) -> Path: + """The files of an ASPIRE checkout the task functions read: the two templates, the docs, the snapshot program.""" + root = tmp / "ASPIRE" + skills = root / ".claude" / "skills" / "iterative-debugging" + (skills / "scripts").mkdir(parents=True) + (root / "docs").mkdir() + marks = ("", + "") + (skills / "subagent-prompt.md").write_text(f"intro\n{marks[0]}\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" + f"## What You Are\nfix-loop text\n{marks[1]}\n") + (skills / "verifier-prompt.md").write_text(f"intro\n{marks[0]}\nYou are verifying. Seeds 51-55.\n{marks[1]}\n") + (skills / "scripts" / "scene_snapshot.py").write_text("result = {}\n") + for name in ("api-reference.md", "run-artifacts.md", "robosuite.md"): + (root / "docs" / name).write_text(f"# {name}\nrun examples/robosuite/cube_lifting_seed1.py\nkept\n") + (skills / "skills").mkdir() + (skills / "skills" / "task-exploration.md").write_text("# exploration\n") + evosearch = root / ".claude" / "skills" / "evosearch" + (evosearch / "skills").mkdir(parents=True) + (evosearch / "subagent-prompt.md").write_text("intro\n\n```\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" + "## EVAL SET LOCKOUT\nsearch text\n```\n") + (evosearch / "skills" / "evosearch-iteration.md").write_text("# iteration\n") + (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\nkept\n\n## Task examples\n\n| `close_microwave.py` | " + "`libero_90:33` | Close the open microwave. |\n") + for venv in (".venv-robosuite", ".venv-libero"): + harness = root / venv / "bin" / "cap-harness" + harness.parent.mkdir(parents=True) + harness.write_text(FAKE_HARNESS.format(python=sys.executable)) + harness.chmod(0o755) + return root + + +# cap-harness without a simulator: the run's files where ASPIRE writes them (run.json, results.jsonl naming the run's +# directory, outcome.json), the program and settings it was given; a program saying SLEEP runs until stopped, and its +# `# WRITE ` / `# READ ` lines are tried as a program that escaped ASPIRE's validator would (in run.json), +# and after the run `# LINK ` leaves a link in its output, `# RUNDIR ` names another run_dir and +# `# CHMOD ` changes a mode (last). +FAKE_HARNESS = """#!{python} +import argparse, json, os, pathlib, sys, time +ap = argparse.ArgumentParser() +for flag in ("--benchmark", "--suite", "--task-id", "--program", "--output-root", "--max-steps", "--seed", "--workers", + "--results-jsonl", "--log-dir", "--init-mode"): + ap.add_argument(flag) +ap.add_argument("--seeds", nargs="+") +ap.add_argument("--flat-run-dir", action="store_true") +ap.add_argument("--no-videos", action="store_true") +ap.add_argument("cmd") +a = ap.parse_args() +code = pathlib.Path(a.program).read_text() +if "SLEEP" in code: + time.sleep(60) +tried = {{}} +for line in code.splitlines(): + what, _, path = line.partition(" ")[2].partition(" ") + try: + if what == "WRITE": + pathlib.Path(path).write_text("escaped") + elif what == "READ": + pathlib.Path(path).read_text() + else: + continue + tried[path] = "done" + except OSError as exc: + tried[path] = type(exc).__name__ +out = pathlib.Path(a.output_root) +run = out / "run-1" +run.mkdir(parents=True, exist_ok=True) +(out / "run.json").write_text(json.dumps({{"program": a.program, "max_steps": a.max_steps, "code": code, + "env": sorted(os.environ), "home": os.environ.get("HOME"), + "tried": tried}})) +(run / "outcome.json").write_text(json.dumps({{"task_success": True}})) +rows = [{{"seed": int(s), "run_dir": str(run)}} for s in (a.seeds or [a.seed])] +for line in code.splitlines(): + what, _, rest = line.partition(" ")[2].partition(" ") + if what == "LINK": + name, target = rest.split() + (out / name).unlink(missing_ok=True) + (out / name).symlink_to(target) + elif what == "RUNDIR": + rows = [dict(r, run_dir=rest) for r in rows] +pathlib.Path(a.results_jsonl or out / "results.jsonl").write_text("".join(json.dumps(r) + chr(10) for r in rows)) +for line in code.splitlines(): + what, _, rest = line.partition(" ")[2].partition(" ") + if what == "CHMOD": + name, mode = rest.split() + os.chmod(out / name, int(mode, 8)) +print("ran", a.program, "into", out, "max_steps", a.max_steps) +""" + + +def _bwrap_works() -> bool: + """A bubblewrap that can make its namespaces here: every simulator run is confined (sim.confine).""" + if not shutil.which("bwrap"): + return False + return subprocess.run(["bwrap", "--unshare-user", "--unshare-net", "--ro-bind", "/", "/", "true"], + capture_output=True).returncode == 0 + + +needs_bwrap = pytest.mark.skipif(not _bwrap_works(), reason="needs a working bubblewrap: simulator runs are confined") + + +LIBERO_ROW = {"id": "libero_90-t33", "benchmark": "libero-pro", "suite": "libero_90", "task_id": 33, + "debug_seeds": "51-53", "unseen_seeds": "66", "eval_seeds": "1-3"} +ROW = {"id": "robosuite-cube_lifting", "suite": "cube_lifting", "task_id": 0, "debug_seeds": "101-103", + "unseen_seeds": "126", "eval_seeds": "1-3"} +# A row with an attempt budget: at most two programs (by content) beyond a single run on the first seed +BUDGET = {"id": "robosuite-cube_stack", "suite": "cube_stack", "task_id": 0, "debug_seeds": "101-105", + "eval_seeds": "1-3", "max_programs": 2} + + +class _Service: + """The simulator service on a free loopback port, over a fake checkout whose cap-harness writes a run's files + without a simulator (FAKE_HARNESS), serving ROW, BUDGET, LIBERO_ROW and the `rows` given.""" + + def __init__(self, tmp: Path, root: Path, rows=(), **options): + import threading + from http.server import ThreadingHTTPServer + S = _load("cap_sim_service", EXAMPLE / "agent" / "sim_service.py") + self.sessions = tmp / "sessions" + self.sessions.mkdir() + served = {r["id"]: r for r in (ROW, BUDGET, LIBERO_ROW, *rows)} + self.service = service = S.Service(root, served, [self.sessions], + S.Ledger(tmp / "ledger.json"), {}, 2, **options) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), S.handler(service)) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + + def close(self): + self.server.shutdown() + + +@pytest.fixture +def service(tmp_path, monkeypatch): + svc = _Service(tmp_path, fake_aspire(tmp_path)) + monkeypatch.setenv("ASPIRE_SIM_URL", svc.url) + yield svc + svc.close() + + +def _workspace(service, task: dict, hexa: str = "56d9fea2") -> Path: + """A session's workspace where the framework puts it: //-/workspaces/<...>/work.""" + work = service.sessions / "rollout" / f"{task['id']}-{hexa}" / "workspaces" / f"{task['id']}-da10428a" / "work" + work.mkdir(parents=True) + return work + + +def _register(url: str, body: dict) -> dict: + import urllib.request + request = urllib.request.Request(url + "/register", data=json.dumps(body).encode()) + with urllib.request.urlopen(request) as reply: + return json.loads(reply.read()) + + +def _sim(work: Path, *args): + return subprocess.run([str(work / "bin" / "sim"), *args], capture_output=True, text=True, cwd=work) + + +def _sim_module(): + sys.path.insert(0, str(EXAMPLE / "agent")) + try: + import sim + return sim + finally: + sys.path.remove(str(EXAMPLE / "agent")) + + +def _fake_gpu(tmp_path, monkeypatch) -> list[int]: + """GPU 8 as hermit's sandbox sees devices (sandbox.DEV): a machine without one can still build the command.""" + import hermit.sandbox as sandbox_module + dev = tmp_path / "dev" + dev.mkdir(exist_ok=True) + for name in ("nvidiactl", "nvidia8"): + (dev / name).touch() + monkeypatch.setattr(sandbox_module, "DEV", str(dev)) + return [8] diff --git a/examples/cap_harness/utils/tests/test_cap_harness.py b/examples/cap_harness/utils/tests/test_cap_harness.py new file mode 100644 index 00000000..61a07b36 --- /dev/null +++ b/examples/cap_harness/utils/tests/test_cap_harness.py @@ -0,0 +1,704 @@ +"""The cap-harness engine (ASPIRE's harness) shared by Robosuite and LIBERO: its task functions (the workspace and its +registration), the simulator service's rules as bin/sim meets them (a real service on a loopback port, over a fake +checkout: cap_fakes.py), the engine's sandbox, the judge (a stand-in engine), the docs it stages, and the tasks each +benchmark writes with its helpers; no simulator, no model, no network.""" +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[4] +HERE = Path(__file__).resolve().parent + + +def _fakes(): + spec_ = importlib.util.spec_from_file_location("cap_fakes", HERE / "cap_fakes.py") + module = importlib.util.module_from_spec(spec_) + spec_.loader.exec_module(module) + return module + + +F = _fakes() +tmp_path, service = F.tmp_path, F.service # fixtures +EXAMPLE, ROW, LIBERO_ROW, BUDGET = F.EXAMPLE, F.ROW, F.LIBERO_ROW, F.BUDGET +_load, fake_aspire, needs_bwrap, _Service, _workspace = F._load, F.fake_aspire, F.needs_bwrap, F._Service, F._workspace +_register, _sim, _sim_module, _fake_gpu = F._register, F._sim, F._sim_module, F._fake_gpu +LIBERO = ROOT / "examples" / "libero" +ROBOSUITE = ROOT / "examples" / "robosuite" + + +def _task_module(): + return _load("cap_task", EXAMPLE / "agent" / "task.py") + + +def test_sim_compact_makes_traces_readable(): + sim = _sim_module() + value = {"$type": "Pose", "position": {"$array_inline": [0.123456, 1.0, 2.0], "dtype": "float64", "shape": [3]}, + "obs": {"$type": "Observation", "cameras": {}}, "big": {"$array_inline": list(range(40)), "shape": [40]}} + out = sim.compact(value) + assert out == {"type": "Pose", "position": [0.1235, 1.0, 2.0], "obs": "", + "big": ""} + + +def test_the_failure_keyframe_is_the_failing_calls_own(tmp_path): + sim = _sim_module() + run = tmp_path / "run" + (run / "source").mkdir(parents=True) + trace = [{"tool": "open_gripper", "ok": True}] + [{"tool": "move_to_pose", "ok": i != 2} for i in range(4)] + (run / "source" / "program-result.json").write_text(json.dumps({"calls": trace})) + names = ["reset", "before-open-gripper", "after-open-gripper"] + for i in range(3): # the third move raised: no after-frame of its own + names += ["before-move-to-pose"] + (["after-move-to-pose"] if i < 2 else []) + for n, name in enumerate(names, 1): + d = run / "media" / "keyframes" / f"{n:06d}-{name}" + d.mkdir(parents=True) + (d / "agentview.png").write_bytes(b"png") + picked = [label for label, _ in sim.keyframes(run, limit=1)] + assert picked == [f"{names.index('before-move-to-pose', 7) + 1:06d}-before-move-to-pose"] + + +def test_prepare_registers_the_session_and_the_service_holds_its_rules(tmp_path, service): + T = _task_module() + work = _workspace(service, ROW) + params = {"aspire_root": str(tmp_path / "ASPIRE")} + T.prepare(ROW, work, params, tmp_path) + for name in ("docs/api-reference.md", "docs/task-exploration.md", "scripts/scene_snapshot.py", "bin/sim", + "bin/sim.json"): + assert (work / name).is_file(), name + assert not (work / "task.md").exists() and not (work / "docs" / "evosearch-iteration.md").exists() # a method's + assert "examples/" not in (work / "docs" / "robosuite.md").read_text() + assert "kept" in (work / "docs" / "robosuite.md").read_text() + (work / "p.py").write_text("result = {}\n") + held_out = _sim(work, "run-batch", "--seeds", "1-3", "--program", "p.py", "--output-root", "out") + assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr + outside = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", str(tmp_path / "elsewhere")) + assert outside.returncode == 2 and "inside the workspace" in outside.stderr + _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u", "--init-mode", "saved") + again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") + assert again.returncode == 2 and "run once" in again.stderr + T.prepare(ROW, work, params, tmp_path) # a retry's prepare: the registration as it was + again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") + assert again.returncode == 2 and "run once" in again.stderr # what the session spent stays spent + + +def test_a_libero_task_runs_on_its_own_venv_seeded_with_its_own_docs(tmp_path, service, monkeypatch): + T = _task_module() + params = {"aspire_root": str(tmp_path / "ASPIRE")} + s = T.settings(LIBERO_ROW, params) + assert T.task_dir(s) == "libero-pro/libero_90/task_33" and s["doc"] == "libero-pro.md" + work = _workspace(service, LIBERO_ROW) + T.prepare(LIBERO_ROW, work, params, tmp_path) + page = (work / "docs" / "libero-pro.md").read_text() + assert "kept" in page and "Task examples" not in page and "microwave" not in page # no worked solution named + assert not (work / "docs" / "robosuite.md").exists() + ran = [] + monkeypatch.setattr(service.service, "_execute", lambda cmd, env, *a, **k: ran.append((cmd, env)) or (0, "", "")) + (work / "p.py").write_text("result = {}\n") + _sim(work, "run-batch", "--seeds", "51-52", "--program", "p.py", "--output-root", "out", "--init-mode", "saved") + cmd, env = ran[-1] + assert cmd[0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "cap-harness") + option = lambda flag: cmd[cmd.index(flag) + 1] + assert (option("--benchmark"), option("--suite"), option("--task-id")) == ("libero-pro", "libero_90", "33") + assert option("--init-mode") == "seeded" # the engine's, whatever the session passed + assert env["LIBERO_CONFIG_PATH"] == str(tmp_path / "ASPIRE" / ".libero") + assert env["TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD"] == "1" + _sim(work, "validate", "--program", "p.py") + assert ran[-1][0][0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "python") + # Robosuite's runs are as they were: its venv, no init mode + robosuite = _workspace(service, ROW) + T.prepare(ROW, robosuite, params, tmp_path) + (robosuite / "p.py").write_text("result = {}\n") + _sim(robosuite, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + cmd, env = ran[-1] + assert cmd[0].endswith(".venv-robosuite/bin/cap-harness") and "--init-mode" not in cmd + assert "LIBERO_CONFIG_PATH" not in env + + +def test_the_task_sandbox_is_shown_the_staged_docs_and_never_a_page_that_lost_its_cut(tmp_path, service): + """ASPIRE's benchmark pages name its worked programs per task: an actor that could read one would be handed + a strategy for its task. The configs grant the staged copies only, and a page without its cut is refused.""" + T = _task_module() + root = tmp_path / "ASPIRE" + (tmp_path / "rows.jsonl").write_text(json.dumps(ROW) + "\n" + json.dumps(LIBERO_ROW) + "\n") + staged = tmp_path / "staged" + done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", str(root), + "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], + capture_output=True, text=True) + assert done.returncode == 0, done.stderr + assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", + "run-artifacts.md"] + assert all("examples/" not in p.read_text() and "microwave" not in p.read_text() for p in staged.iterdir()) + stage = lambda: subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", + str(root), "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], + capture_output=True, text=True) + before = {p.name: p.stat().st_ino for p in staged.iterdir()} + assert stage().returncode == 0 and {p.name: p.stat().st_ino for p in staged.iterdir()} == before # in place + (staged / "extra").mkdir() # anything else there: staged anew, without it + (staged / "notes.bin").write_bytes(b"\xff\xfe") + assert stage().returncode == 0 + assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", + "run-artifacts.md"] + (staged / "libero-pro.md").write_text("# the staged page\n") + work = _workspace(service, LIBERO_ROW) + T.prepare(LIBERO_ROW, work, {"aspire_root": str(root), "aspire_docs": str(staged)}, tmp_path) + assert (work / "docs" / "libero-pro.md").read_text() == "# the staged page\n" # the staged copy, as it is + (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\n\n## Worked programs\n| `close_microwave.py` |\n") + with pytest.raises(ValueError, match="Task examples"): + _sim_module().workspace_docs(root, "libero-pro") + # No Codex config of a benchmark on this engine grants a page of the checkout's own docs; each loads, reads its + # tasks through the engine's row view and is judged by the engine's judge + from hermit.config import Config + configs = [p for folder in (ROBOSUITE / "aspire", LIBERO / "aspire", LIBERO / "prompt_only") + for p in sorted(folder.glob("*.yaml"))] + assert len(configs) == 19 + for path in configs: + config = Config.load(path) + data = config.data + read = [str(p) for p in (data.get("environment") or {}).get("read") or []] + assert not any("/docs/" in p for p in read), path + assert config.path(data["data"]["view"].split(":")[0]) == (EXAMPLE / "agent" / "fields.py").resolve(), path + assert config.path(data["judge"]["path"]) == (EXAMPLE / "benchmark" / "judge.py").resolve(), path + if "_direct" not in path.name: + assert "$ASPIRE_DOCS" in read and data["agent"]["params"]["aspire_docs"] == "$ASPIRE_DOCS", path + assert any(config.path(p) == (EXAMPLE / "agent").resolve() for p in read if "$" not in p), path + if data["agent"]["name"] == "codex": # Codex reads AGENTS.md (the adapter's context) only up to + assert data["agent"]["params"]["project_doc_max_bytes"] >= 65536, path # this: past it, refused + if path.parent.name == "prompt_only": # no method's pages: none granted, none staged + assert not any("evosearch" in p or "-prompt.md" in p for p in read), path + + +def test_the_engine_takes_the_benchmarks_venv_and_refuses_one_it_does_not_run(tmp_path, monkeypatch): + sim = _sim_module() + root = fake_aspire(tmp_path) + engine = sim.Engine(root, benchmark="libero-pro") + assert engine.cli == root.resolve() / ".venv-libero" / "bin" / "cap-harness" + assert engine._env()["LIBERO_CONFIG_PATH"] == str(root.resolve() / ".libero") + assert sim.init_args("libero-pro") == ["--init-mode", "seeded"] and sim.init_args(None) == [] + assert sim.Engine(root).cli.parent.parent.name == ".venv-robosuite" + commands = [] # the command line of a sweep, the judge's Stage 2 included + + def recorded(cmd, **_): + commands.append(cmd) + raise RuntimeError("recorded") + + monkeypatch.setattr(sim, "confine", recorded) + for each in (engine, sim.Engine(root)): + with pytest.raises(RuntimeError, match="recorded"): + each.run(tmp_path / "p.py", suite="s", seeds=[1, 2], out=tmp_path / "o") + at = commands[0].index("--init-mode") + assert commands[0][at:at + 2] == ["--init-mode", "seeded"] and "--init-mode" not in commands[1] + shutil.rmtree(root / ".venv-libero") + with pytest.raises(FileNotFoundError, match="bootstrap_libero.sh"): + sim.Engine(root, benchmark="libero-pro") + with pytest.raises(ValueError, match="behavior"): + sim.Engine(root, benchmark="behavior") + + +def test_the_libero_tasks_are_liberos_five_suites_and_liberos_pros_eighty_with_aspires_seeds(tmp_path, monkeypatch): + """The tasks and splits prepare.py writes, read as their rows (the engine's fields.py): what the simulator service, + the judge and every method's configs read.""" + from hermit.config import Config + from hermit.task import Benchmark + P = _load("libero_prepare", LIBERO / "utils" / "benchmark" / "prepare.py") + Fd = _load("cap_fields", EXAMPLE / "agent" / "fields.py") + assert sum(P.SUITES.values()) == 130 and P.SUITES["libero_90"] == 90 + pro = [f"libero_{family}_{kind}" for family in ("object", "spatial", "goal", "10") for kind in ("swap", "task")] + assert list(P.PRO_SUITES) == pro and set(P.PRO_SUITES.values()) == {10} + monkeypatch.setattr(P, "HERE", tmp_path) + monkeypatch.setattr(sys, "argv", ["prepare.py"]) + P.main() + found = Fd.task_rows(tmp_path / "tasks") # each task directory as its row + rows = list(found.values()) + tasks = [found[i] for i in (tmp_path / "splits" / "all.txt").read_text().split()] + assert len(tasks) == 210 and len(rows) == 420 and len({r["id"] for r in rows}) == 420 # a suite and its + assert all(r["benchmark"] == "libero-pro" for r in rows) # perturbations share their task ids + assert list(found["libero_goal-t03"]) == ["id", "benchmark", "suite", "task_id", "debug_seeds", "unseen_seeds", + "eval_seeds"] # the row's keys, in its order + assert found["rerun-libero_goal-t03"] == {**found["libero_goal-t03"], "id": "rerun-libero_goal-t03", + "rerun_of": "libero_goal-t03"} # the same task, its own id + assert not any(r["id"].startswith("verify-") for r in rows) # a method's own rows are its own + assert (tmp_path / "tasks" / "libero_goal-t03" / "instruction.md").is_file() + assert all((r["debug_seeds"], r["unseen_seeds"], r["eval_seeds"]) == ("51-65", "66-80", "1-50") for r in tasks) + split = lambda name: (tmp_path / "splits" / name).read_text().split() + by = lambda *suites: [r["id"] for r in tasks if r["suite"] in suites] + assert split("all.txt") == [r["id"] for r in tasks] + assert split("libero.txt") == by("libero_object", "libero_spatial", "libero_goal", "libero_10") + assert split("libero_90.txt") == by("libero_90") and split("libero_pro.txt") == by(*pro) + assert split("libero_pro_long.txt") == by("libero_10_swap", "libero_10_task") + assert split("smoke.txt") == split("libero.txt")[:2] and split("smoke_pro.txt") == split("libero_pro.txt")[:2] + assert split("libero_sample.txt") == [f"{s}-t{n:02d}" for s in ("libero_object", "libero_spatial", "libero_goal") + for n in (0, 5)] + for path in sorted((LIBERO / "prompt_only").glob("*.yaml")): # a method that reads the tasks as written + config = Config.load(path) + data = config.data["data"] + assert config.path(data["tasks"]) == (LIBERO / "utils" / "benchmark" / "tasks").resolve(), path + monkeypatch.setitem(data, "tasks", str(tmp_path / "tasks")) # the tasks prepared here + for key in ("train", "val", "test"): + if data.get(key): + monkeypatch.setitem(data, key, str(tmp_path / "splits" / Path(data[key]).name)) + assert {t.id: t.given() for t in Benchmark.from_config(config).tasks.values()} == found, path + train, test = split(Path(data["train"]).name), split(Path(data["test"]).name) + assert train and set(train) <= {r["id"] for r in tasks}, path + assert test == [f"rerun-{i}" for i in train], path # the reruns are of the tasks it trained on + seeds = P.seeds_from(P.PRESETS, []) + assert seeds == {"debug_seeds": "51-65", "unseen_seeds": "66-80", "eval_seeds": "1-50"} + assert P.seeds_from(P.PRESETS, ["--preset", "smoke", "--eval", "1-5"])["eval_seeds"] == "1-5" + with pytest.raises(SystemExit): + P.seeds_from(P.PRESETS, ["--eval", "1-60"]) # held-out seeds a session could run + + +def test_the_neutral_libero_prompt_names_the_task_the_seeds_and_the_scored_program_and_nothing_of_a_method( + tmp_path, service): + """prompt_only/'s acting prompt and workspace: LIBERO's prompt and the engine's workspace, nothing of a method.""" + P = _load("libero_task", LIBERO / "utils" / "agent" / "task.py") + T = _task_module() + root = tmp_path / "ASPIRE" # the service fixture's ASPIRE tree + params = {"aspire_root": str(root)} # as prompt_only's configs set them + row = {**LIBERO_ROW, "suite": "libero_object_task", "task_id": 4, "id": "libero_object_task-t04"} + text = P.prompt(row, params, tmp_path) + assert "Suite `libero_object_task`, task id `4`" in text and "get_task_context()" in text + assert "**51-53**" in text and "**66**" in text and "3 held-out seeds" in text + scored = f"{T.task_dir(T.settings(row, params))}/fix_code.py" + assert f"Your program is `{scored}`" in text # the file the engine's collect scores + unseen = f"--output-root {scored.rsplit('/', 1)[0]}/debug/holdout" # where the engine's collect reads it + assert unseen in text and "docs/libero-pro.md" in text and "simulator steps" not in text + assert "at most 600 simulator steps" in P.prompt(row, {**params, "max_steps": 600}, tmp_path) + assert not any(word in text for word in ("skill", "library", "coordinator", "fix-loop", "evolutionary")) + work = _workspace(service, LIBERO_ROW) + T.prepare(LIBERO_ROW, work, params, tmp_path) + assert (work / "bin" / "sim").is_file() and (work / "docs" / "libero-pro.md").is_file() + assert (work / "docs" / "task-exploration.md").is_file() # the scene procedure stays + assert not (work / "docs" / "evosearch-iteration.md").exists() and not (work / "task.md").exists() + # The scored program is fix_code.py in the task's folder, whatever else the session left there + s = T.settings(LIBERO_ROW, params) + (work / T.task_dir(s)).mkdir(parents=True, exist_ok=True) + (work / T.task_dir(s) / "fix_code.py").write_text("# fix\n") + (work / T.task_dir(s) / "findings.md").write_text("notes\n") + (work / "evosearch").mkdir() + (work / "evosearch" / "evosearch_best_code.py").write_text("# best\n") + facts = T.collect(LIBERO_ROW, work, tmp_path / "log-a", {**params, "prune_media": False}) + assert facts["scored_program"] == f"{T.task_dir(s)}/fix_code.py" and facts["fix_code"] == "# fix\n" + assert facts["findings"] == "notes\n" and facts["unseen"] is None and "evosearch" not in facts + assert (tmp_path / "log-a" / "files" / "fix_code.py").read_text() == "# fix\n" + + +def test_a_session_cannot_forge_a_registration_for_more_seeds(tmp_path, service): + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + forged = work / "x" # a fresh directory the session can write + forged.mkdir() + (forged / "p.py").write_text("result = {}\n") + answer = _register(service.url, {"workspace": str(forged), "task": ROW["id"], "dev_seeds": [1, 2, 3]}) + assert not answer["ok"] and "registered its session already" in answer["error"] + other = _workspace(service, ROW, "0badc0de") # another rollout (the framework's): the row's seeds, + assert _register(service.url, {"workspace": str(other), "task": ROW["id"], "dev_seeds": [1, 2, 3], + "max_programs": None})["ok"] # whatever the request says + (other / "bin").mkdir() + shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", other / "bin" / "sim_cli.py") + (other / "bin" / "sim.json").write_text(json.dumps({"workspace": str(other), "url": service.url})) + (other / "p.py").write_text("result = {}\n") + held_out = subprocess.run([sys.executable, str(other / "bin" / "sim_cli.py"), "run-batch", "--seeds", "1-3", + "--program", "p.py", "--output-root", "out"], capture_output=True, text=True, cwd=other) + assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr + mismatched = _workspace(service, BUDGET, "12345678") # a rollout registers only the task its name carries + answer = _register(service.url, {"workspace": str(mismatched), "task": ROW["id"]}) + assert not answer["ok"] and "is not one of task" in answer["error"] + + +def test_a_row_with_an_attempt_budget_runs_that_many_programs(tmp_path, service): + T = _task_module() + work = _workspace(service, BUDGET) + T.prepare(BUDGET, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + codes = [] + for name in ("a.py", "a.py", "c.py"): # the same path, rewritten: a new program + (work / name).write_text(f"result = {{'v': {len(codes)}}}\n") + codes.append(_sim(work, "run-batch", "--seeds", "101-105", "--program", name, "--output-root", "out")) + assert "attempt budget" not in codes[0].stderr and "attempt budget" not in codes[1].stderr + assert codes[2].returncode == 2 and "attempt budget is 2 programs" in codes[2].stderr + + +@needs_bwrap +def test_the_service_follows_no_link_the_session_puts_in_its_workspace(tmp_path, service): + """A run is written by the service into its own directory and moved into the workspace: a link the session put + where the run's files go is replaced, not followed, and a program that is a link is not read.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + victim = tmp_path / "victim.txt" + victim.write_text("the host's own file\n") + (work / "out").mkdir() + (work / "out" / "run.json").symlink_to(victim) # where cap-harness writes + (work / "out" / "run-1").symlink_to(tmp_path) # a directory of the run's + (work / "p.py").write_text("result = {}\n") + ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + assert victim.read_text() == "the host's own file\n" and not (tmp_path / "outcome.json").exists() + assert ran.returncode == 2 and "a link in its path" in ran.stderr + (work / "out" / "run-1").unlink() + ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + assert ran.returncode == 0 and victim.read_text() == "the host's own file\n" + record = json.loads((work / "out" / "run.json").read_text()) + assert not (work / "out" / "run.json").is_symlink() and record["program"] == str(work / "p.py") + rows = (work / "out" / "results.jsonl").read_text() + assert json.loads(rows)["run_dir"] == str(work / "out" / "run-1") # the workspace's paths, not the service's + assert str(work / "p.py") in ran.stdout and str(service.service.scratch) not in ran.stdout + secret = tmp_path / "secret.py" + secret.write_text("KEY = 'host'\n") + (work / "q.py").symlink_to(secret) + refused = _sim(work, "validate", "--program", "q.py") + assert refused.returncode == 2 and "no program at q.py" in refused.stderr + assert list(service.service.scratch.iterdir()) == [] # the service's copies are gone + import urllib.request + service.service.nonce = "n-1" + with urllib.request.urlopen(service.url + "/health") as reply: # launch.sh knows its own service by it + assert json.loads(reply.read())["nonce"] == "n-1" + service.service.stop() + late = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + assert late.returncode == 1 and "stopping" in late.stderr # no run starts once the service stops + + +def test_bin_sim_quotes_the_interpreter(tmp_path, service, monkeypatch): + T = _task_module() + work = _workspace(service, ROW) + monkeypatch.setattr(T.sys, "executable", "/opt/hermit env/bin/python") + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + assert "exec '/opt/hermit env/bin/python' " in (work / "bin" / "sim").read_text() + + +@needs_bwrap +def test_a_program_that_escaped_the_validator_still_cannot_reach_the_host(tmp_path, service): + """cap-harness runs in a sandbox of its own (sim.confine): a program in it writes only its run's own output and + reads ASPIRE's checkout, not the host's files, another session's workspace or the judge's runs.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + secret, stage2 = tmp_path / "secret.txt", tmp_path / "out" / "stage2" / "robosuite-x" / "stage2.json" + secret.write_text("host secret\n") + stage2.parent.mkdir(parents=True) + stage2.write_text("{}") + other = _workspace(service, ROW, "0badc0de") + (other / "fix_code.py").write_text("result = {'theirs': 1}\n") + solution = tmp_path / "ASPIRE" / "examples" / "robosuite" / "cube_lifting_seed1.py" # a worked solution + solution.parent.mkdir(parents=True) + solution.write_text("result = {'worked': 1}\n") + history = tmp_path / "ASPIRE" / ".git" / "HEAD" # the git history holds it too + history.parent.mkdir() + history.write_text("ref: refs/heads/main\n") + native = tmp_path / "ASPIRE" / "outputs" / "run-1" / "source" / "program.py" # a run of ASPIRE's own CLI + native.parent.mkdir(parents=True) + native.write_text("result = {'native': 1}\n") + page = tmp_path / "ASPIRE" / "docs" / "libero-pro.md" # names the worked programs per task + demos = tmp_path / "ASPIRE" / "third_party" / "LIBERO-PRO" / "libero" / "datasets" / "demo.hdf5" + demos.parent.mkdir(parents=True) + demos.write_text("demonstration\n") + harness = tmp_path / "ASPIRE" / ".venv-robosuite" / "bin" / "cap-harness" + targets = {"WRITE": [tmp_path / "planted.txt", tmp_path / "ASPIRE" / "planted.py", work / "planted.txt"], + "READ": [secret, stage2, other / "fix_code.py", solution, history, native, page, demos, harness]} + (work / "p.py").write_text("result = {}\n" + "".join(f"# {w} {p}\n" for w, ps in targets.items() for p in ps)) + ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + tried = json.loads((work / "out" / "run.json").read_text())["tried"] + assert ran.returncode == 0, ran.stderr + assert not (tmp_path / "planted.txt").exists() and not (tmp_path / "ASPIRE" / "planted.py").exists() + assert tried[str(work / "planted.txt")] != "done" and not (work / "planted.txt").exists() # its workspace too + assert all(tried[str(p)] != "done" + for p in (secret, stage2, other / "fix_code.py", solution, history, native, page, demos)) + assert tried[str(harness)] == "done" # the checkout it runs from + + +@needs_bwrap +def test_the_validator_runs_on_aspires_own_venv_inside_its_sandbox(tmp_path, monkeypatch): + """ASPIRE's venv interpreter is a link to its base installation: confined, it keeps the venv (whose cap_harness + the validator imports), for the service's `bin/sim validate` and the judge's Engine alike.""" + root = fake_aspire(tmp_path) + venv = root / ".venv-robosuite" + shutil.move(str(venv / "bin" / "cap-harness"), str(tmp_path / "cap-harness")) + shutil.rmtree(venv) + subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(venv)], check=True) + shutil.move(str(tmp_path / "cap-harness"), str(venv / "bin" / "cap-harness")) + site = next(venv.glob("lib/python3*/site-packages")) + (site / "cap_harness").mkdir() + (site / "cap_harness" / "__init__.py").write_text("") + (site / "cap_harness" / "runtime.py").write_text( + "import ast\nclass _ProgramValidator(ast.NodeVisitor):\n def visit_Import(self, node):\n" + " if any(a.name != 'math' for a in node.names):\n raise ValueError('only math')\n") + assert (venv / "bin" / "python").is_symlink() + S = _load("aspire_sim_service_venv", EXAMPLE / "agent" / "sim_service.py") + service = S.Service(root, {ROW["id"]: ROW}, [tmp_path / "sessions"], S.Ledger(tmp_path / "ledger.json"), {}, 2) + work = tmp_path / "sessions" / "rollout" / f"{ROW['id']}-1a2b" / "workspaces" / "w" / "work" + work.mkdir(parents=True) + assert service.register({"workspace": str(work), "task": ROW["id"]})["ok"] + (work / "good.py").write_text("import math\nresult = {}\n") + (work / "bad.py").write_text("import os\nresult = {}\n") + good = service.run({"workspace": str(work), "cmd": "validate", "program": "good.py"}) + bad = service.run({"workspace": str(work), "cmd": "validate", "program": "bad.py"}) + assert good["code"] == 0 and "program accepted" in good["stdout"], good + assert bad["code"] != 0 and "only math" in bad["stderr"], bad + sim = _sim_module() + engine = sim.Engine(root) + assert engine.validate(work / "good.py") is None and "only math" in engine.validate(work / "bad.py") + + +@needs_bwrap +def test_the_engine_reads_no_link_and_no_run_dir_outside_its_run(tmp_path): + """The judge's and the direct actor's Engine read what a confined program could write: once it has ended, links + in its output are removed and a run_dir outside the run is not read, so neither a host file nor another run's + success reaches the records.""" + sim = _sim_module() + root = fake_aspire(tmp_path) + elsewhere = tmp_path / "another-run" / "seed_0001" + elsewhere.mkdir(parents=True) + (elsewhere / "outcome.json").write_text('{"task_success": true, "program_ok": true}') + secret = tmp_path / "secret.txt" + secret.write_text("host secret") + out = tmp_path / "stage2" / "runs" + program = tmp_path / "stage2" / "program.py" + program.parent.mkdir() + program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# RUNDIR {elsewhere}\n") + records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1, 2], out=out) + assert [r["task_success"] for r in records] == [False, False] and "not in this run's output" in records[0]["error"] + assert secret.read_text() == "host secret" and not (out / "records.json").is_symlink() + program.write_text(f"result = {{}}\n# FAIL\n# LINK run-1/outcome.json {elsewhere / 'outcome.json'}\n") + records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "again") + assert records[0]["task_success"] is False and not (tmp_path / "again" / "run-1" / "outcome.json").exists() + # a directory made unlistable keeps no link from being removed: it is made listable, or the run fails + program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# CHMOD . 0o111\n") + sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "locked") + assert secret.read_text() == "host secret" and not (tmp_path / "locked" / "records.json").is_symlink() + + +@needs_bwrap +def test_the_simulators_sandbox_shows_the_renderer_and_reaches_remote_services_by_proxy(tmp_path, monkeypatch): + """An EGL vendor library extracted by hand (its ICD and its libraries, file by file, its library found as GLVND + finds it) and ASPIRE_SIM_READ are shown; without a GPU an NVIDIA-only ICD selection is dropped; a service on + another host is reached through an allowlist proxy for that host only.""" + sim = _sim_module() + home = tmp_path / "home" # the ICD and the libraries extracted where other files are too + (home / "lib").mkdir(parents=True) + (home / "credentials").write_text("key") + icd = home / "10_nvidia.json" + icd.write_text(json.dumps({"ICD": {"library_path": str(home / "lib" / "libEGL_nvidia.so.0")}})) + for name in ("libEGL_nvidia.so.0", "libnvidia-eglcore.so.580", "notes.txt"): + (home / "lib" / name).write_text("") + extra = tmp_path / "extra" + extra.mkdir() + env = {"PATH": os.environ["PATH"], "__EGL_VENDOR_LIBRARY_FILENAMES": str(icd), + "ASPIRE_SIM_READ": str(extra), "CAP_HARNESS_SAM3_URL": "http://provider.internal:9000"} + held, args, inside = sim.confine(["true"], root=fake_aspire(tmp_path), env=env, + gpu=_fake_gpu(tmp_path, monkeypatch)) + try: + ro = set(args[i + 1] for i, a in enumerate(args) if a == "--ro-bind") + assert {str(icd), str(home / "lib" / "libEGL_nvidia.so.0"), str(home / "lib" / "libnvidia-eglcore.so.580"), + str(extra)} <= ro # the driver's files, file by file, and the explicit grant + assert not ro & {str(home), str(home / "lib"), str(home / "credentials"), str(home / "lib" / "notes.txt")} + assert inside["__EGL_VENDOR_LIBRARY_FILENAMES"] == str(icd) # with a GPU (below: gpu=[8]) it stays + assert inside["HTTPS_PROXY"].startswith("http://127.0.0.1:") and held.held[0].allow == ["provider.internal"] + finally: + held.close() + # an ICD naming its library bare: GLVND finds it on LD_LIBRARY_PATH, and so is it shown + icd.write_text(json.dumps({"ICD": {"library_path": "libEGL_nvidia.so.0"}})) + held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", gpu=_fake_gpu(tmp_path, monkeypatch), + env={**env, "LD_LIBRARY_PATH": str(home / "lib")}) + held.close() + assert str(home / "lib" / "libEGL_nvidia.so.0") in [args[i + 1] for i, a in enumerate(args) if a == "--ro-bind"] + # without a GPU an NVIDIA-only ICD selection is dropped, so EGL finds Mesa's software renderer + held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", env=env) + held.close() + assert "__EGL_VENDOR_LIBRARY_FILENAMES" not in inside + + +@needs_bwrap +def test_the_service_lets_a_run_write_only_its_output_and_delivers_no_replaced_output(tmp_path, service, monkeypatch): + """The run's own directory (its program) is read-only to it and only its output writable; an output directory + the program replaced (by a link to a host directory) is not delivered.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + (work / "p.py").write_text("result = {}\n") + sim = type(service.service).run.__globals__["sim"] + grants, real = [], sim.confine + monkeypatch.setattr(sim, "confine", lambda cmd, **k: grants.append(k) or real(cmd, **k)) + assert _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "o").returncode == 0 + own = Path(grants[-1]["readable"][0]) + assert grants[-1]["writable"] == [own / "out"] + host = tmp_path / "host-dir" + host.mkdir() + (host / "id_rsa").write_text("key") + + def replaced(cmd, env, timeout, own, out=None, gpu=()): + shutil.rmtree(out) + out.symlink_to(host) + return 0, "", "" + monkeypatch.setattr(service.service, "_execute", replaced) + ran = _sim(work, "run", "--seed", "102", "--program", "p.py", "--output-root", "o2") + assert ran.returncode == 2 and "replaced while it ran" in ran.stderr + assert (host / "id_rsa").read_text() == "key" and not (work / "o2").exists() + + +@needs_bwrap +def test_a_program_named_pyvenv_cfg_grants_nothing(tmp_path, service): + """A session names its program: one called pyvenv.cfg that says `home = ` is the run's own file, not a venv + of the sandbox's, so stays out of the simulator's sandbox.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + operator = tmp_path / "operator" + operator.mkdir() + (operator / "credentials").write_text("key") + (work / "pyvenv.cfg").write_text(f'marker = """\nhome = {operator}\n"""\nresult = {{}}\n' + f"# READ {operator / 'credentials'}\n") + ran = _sim(work, "run", "--seed", "101", "--program", "pyvenv.cfg", "--output-root", "out") + assert ran.returncode == 0, ran.stderr + assert json.loads((work / "out" / "run.json").read_text())["tried"][str(operator / "credentials")] != "done" + + +def test_the_service_refuses_to_start_on_a_gpu_the_machine_lacks(tmp_path): + """A wrong ASPIRE_SIM_GPU stops the service at its start, not each session's run after its budget is spent.""" + (tmp_path / "tasks.jsonl").write_text(json.dumps(ROW) + "\n") + done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "sim_service.py"), "--aspire-root", + str(fake_aspire(tmp_path)), "--tasks", str(tmp_path / "tasks.jsonl"), "--workspaces", + str(tmp_path / "w"), "--state", str(tmp_path / "state"), "--gpu", "999", "--port", "0"], + capture_output=True, text=True, timeout=60, env={**os.environ, "PYTHONPATH": str(ROOT)}) + assert done.returncode != 0 and ("/dev/nvidia999" in done.stderr or "bubblewrap" in done.stderr) + + +@needs_bwrap +def test_the_simulators_sandbox_reaches_only_the_services_ports(tmp_path): + """The judge runs sim.Engine in hermit's process, whose environment names its own loopback endpoints: only the + services' ports (ASPIRE's, and a CAP_HARNESS_*_URL's) are forwarded into cap-harness's sandbox.""" + sim = _sim_module() + root = fake_aspire(tmp_path) + box, args, _ = sim.confine(["true"], root=root, env={ + "PATH": os.environ["PATH"], "HA_TOOL_URL": "http://127.0.0.1:4999/mcp", + "ASPIRE_SIM_URL": "http://127.0.0.1:8120", "CAP_HARNESS_SAM3_URL": "http://127.0.0.1:9114", + "CAP_HARNESS_CUROBO_URL": "http://localhost"}) # no port: its scheme's + box.close() + forwarded = sorted(int(args[i + 1].split("=")[0]) for i, a in enumerate(args) if a == "--forward") + assert forwarded == [80, 8114, 8115, 8116, 8118, 9114] + for where in ({"hidden": [root / "runs"]}, {"writable": [root / "scratch" / "run"]}): # runs stay outside it + with pytest.raises(ValueError, match="inside ASPIRE's checkout"): + sim.confine(["true"], root=root, env={"PATH": os.environ["PATH"]}, **where) + + +@needs_bwrap +def test_a_run_takes_the_registrations_max_steps_and_stops_at_the_timeout(tmp_path, monkeypatch): + monkeypatch.setenv("TEST_PROVIDER_API_KEY", "k") # the host's secrets: not for a generated program + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "8") # the host's numbering: the sandbox's GPUs count from 0 + svc = _Service(tmp_path, fake_aspire(tmp_path), run_timeout=2) + try: + work = _workspace(svc, ROW) + assert _register(svc.url, {"workspace": str(work), "task": ROW["id"], "max_steps": 300})["ok"] + (work / "bin").mkdir() + shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", work / "bin" / "sim_cli.py") + (work / "bin" / "sim.json").write_text(json.dumps({"workspace": str(work), "url": svc.url})) + sim = lambda *args: subprocess.run([sys.executable, str(work / "bin" / "sim_cli.py"), *args], + capture_output=True, text=True, cwd=work) + (work / "p.py").write_text("result = {}\n") + ran = sim("run", "--seed", "101", "--program", "p.py", "--output-root", "o", "--max-steps", "5000") + record = json.loads((work / "o" / "run.json").read_text()) + assert ran.returncode == 0 and record["max_steps"] == "300" + assert "TEST_PROVIDER_API_KEY" not in record["env"] and "PATH" in record["env"] + assert "CUDA_VISIBLE_DEVICES" not in record["env"] + assert not Path(record["home"]).is_relative_to(work) and not Path(record["home"]).exists() # its own, gone + assert list((tmp_path / "sandbox").glob("cap-harness-*.json")) # the run's sandbox record + (work / "slow.py").write_text("SLEEP\n") + started = time.time() + slow = sim("run", "--seed", "102", "--program", "slow.py", "--output-root", "s") + assert slow.returncode == 124 and "stopped after 2 s" in slow.stderr and time.time() - started < 30 + finally: + svc.close() + + +def test_the_judge_runs_the_program_on_the_rows_held_out_seeds_within_its_budget(tmp_path, monkeypatch): + J = _load("cap_judge", EXAMPLE / "benchmark" / "judge.py") + runs, engines = [], [] + + class Engine: + def __init__(self, *a, **k): + engines.append(k) + + def validate(self, program): + assert program.is_absolute() and program.is_file() # cap-harness runs in ASPIRE's checkout + return "ValueError: only math" if "import os" in program.read_text() else None + + def run(self, program, *, suite, seeds, out, task_id=0): + runs.append(seeds) + return [{"task_success": s % 2 == 1} for s in seeds] + monkeypatch.setattr(J.sim, "Engine", Engine) + monkeypatch.setenv("ASPIRE_ROOT", str(tmp_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ASPIRE_OUT", "out") # relative, as a user may give it + out = lambda facts: SimpleNamespace(extra={"task": facts}, output=None) + actor = {"fix_code": "result = {}\n", "unseen": {"passed": 1, "total": 1}} + assert J.score(ROW, out(actor))["score"] == 2 / 3 and runs[-1] == [1, 2, 3] # the row's held-out seeds + assert J.score(ROW, out({**actor, "fix_code": "import os\n"}))["score"] == 0.0 + assert J.score(ROW, out({**actor, "max_steps": 300}))["score"] == 2 / 3 and engines[-1]["max_steps"] == 300 + assert engines[-1]["benchmark"] == "robosuite" + assert J.score(LIBERO_ROW, out(actor))["score"] == 2 / 3 and engines[-1]["benchmark"] == "libero-pro" + assert J.score(ROW, out(actor))["feedback"] == "2/3 held-out seeds; unseen-check 1/1" + # A row with an attempt budget counts only a program its budget ran (the service's ledger), under its own label + work = str(tmp_path / "w") + budget = {**BUDGET, "eval_seeds": "101-105", "score_label": "verifier"} + ran = {"fix_code": "result = {}\n", "workspace": work} + ledger = tmp_path / "out" / "sim-service" / "ledger.json" + ledger.parent.mkdir(parents=True) + ledger.write_text(json.dumps({work: {"programs": [hashlib.sha256(b"result = {}\n").hexdigest()]}})) + scored = J.score(budget, out(ran)) + assert scored["score"] == 3 / 5 and runs[-1] == [101, 102, 103, 104, 105] + assert scored["feedback"] == "3/5 verifier seeds" + third = J.score(budget, out({**ran, "fix_code": "result = {'v': 3}\n"})) # beyond its two programs + assert third["score"] == 0.0 and "attempt budget" in third["feedback"] + direct = {k: v for k, v in ran.items() if k != "workspace"} # a session that runs its own programs + assert J.score(budget, out({**direct, "fix_code": "result = {'v': 3}\n"}))["score"] == 3 / 5 + assert J.score(ROW, out({}))["score"] == 0.0 # no program: not scored + withheld = J.score(ROW, out({**actor, "not_scored": "verifier verdict void"})) # a method's rule + assert withheld == {"score": 0.0, "feedback": "verifier verdict void", "seeds": 3} + + +def test_the_robosuite_tasks_are_task_directories_read_back_as_the_rows_they_were(tmp_path, monkeypatch): + """prepare.py writes each task and its rerun as a task directory (Harbor's format, no network, judged on the + host); the engine's fields.py, every config's data.view, reads each back as exactly the row it wrote.""" + from hermit.task import Benchmark + P = _load("robosuite_prepare", ROBOSUITE / "utils" / "benchmark" / "prepare.py") + Fd = _load("cap_fields", EXAMPLE / "agent" / "fields.py") + monkeypatch.setattr(P, "HERE", tmp_path) + monkeypatch.setattr(sys, "argv", ["prepare.py", "--preset", "smoke"]) + P.main() + found = Fd.task_rows(tmp_path / "tasks") + seeds = {"debug_seeds": "101-103", "unseen_seeds": "126", "eval_seeds": "1-3"} + task = {"id": "robosuite-cube_stack", "suite": "cube_stack", "task_id": 0} + assert json.dumps(found["robosuite-cube_stack"]) == json.dumps({**task, **seeds}) # its keys in order, types + assert json.dumps(found["rerun-robosuite-cube_stack"]) == json.dumps( + {**task, "id": "rerun-robosuite-cube_stack", "rerun_of": "robosuite-cube_stack", **seeds}) + split = lambda name: (tmp_path / "splits" / name).read_text().split() + assert len(found) == 14 and split("smoke.txt") == ["robosuite-cube_lifting", "robosuite-cube_stack"] + assert split("smoke_rerun.txt") == ["rerun-robosuite-cube_lifting", "rerun-robosuite-cube_stack"] + assert split("rerun.txt") == [f"rerun-{i}" for i in split("all.txt")] and len(split("all.txt")) == 7 + loaded = Benchmark.load(tmp_path / "tasks", view=Fd.task_row).tasks["robosuite-cube_stack"] + assert loaded.private["tests"] is None and loaded.private["needs"]["network"]["agent"] == "no-network" + + +def test_prepare_refuses_held_out_seeds_a_session_could_run(): + for args in (["--eval", "1-200"], ["--unseen", "120-130"], ["--dev", "1-5"]): + done = subprocess.run([sys.executable, str(ROBOSUITE / "utils" / "benchmark" / "prepare.py"), *args], + capture_output=True, text=True) + assert done.returncode == 2 and "overlap" in done.stderr, args + + +def test_prune_drops_the_runs_images_and_keeps_their_records(tmp_path): + T = _task_module() + run = tmp_path / "run" / "work" / "debug" / "0101" / "run-x" + (run / "media" / "keyframes").mkdir(parents=True) + (run / "media" / "keyframes" / "a.png").write_bytes(b"png") + (run / "run.json").write_text('{"seed": 101}') + (tmp_path / "run" / "work" / "notes" / "media").mkdir(parents=True) # not a run's: kept + T.prune(tmp_path / "run") + assert not (run / "media").exists() and (run / "run.json").is_file() + assert (tmp_path / "run" / "work" / "notes" / "media").is_dir() diff --git a/examples/libero/.gitignore b/examples/libero/.gitignore index a1e03960..62859799 100644 --- a/examples/libero/.gitignore +++ b/examples/libero/.gitignore @@ -1 +1,4 @@ runs/ +# ASPIRE's tasks and splits, written by aspire/tasks.sh at launch +aspire/tasks/ +aspire/splits/ diff --git a/examples/libero/README.md b/examples/libero/README.md index 6b64c097..64f8f7dd 100644 --- a/examples/libero/README.md +++ b/examples/libero/README.md @@ -11,26 +11,28 @@ on held-out seeds by the task's own success check. | Folder | What it is | |---|---| -| `utils/benchmark/prepare.py` | The 210 tasks (a suite and a task id each) with their seeds, as task directories in Harbor's format (`tasks/`, read as their rows by the engine's `aspire_fields.py`), and splits; the verifier's tasks and the reruns | -| `utils/agent/` | A method-neutral acting prompt (`prompt.md`, written here: LIBERO has none) and the workspace without ASPIRE's method pages, for a method that brings no prompt of its own | +| `utils/benchmark/prepare.py` | The 210 tasks (a suite and a task id each) with their seeds, as task directories in Harbor's format (`tasks/`, each with its rerun, read as their rows by the engine's `fields.py`), and splits | +| `utils/agent/` | A method-neutral acting prompt (`prompt.md`, written here: LIBERO has none), for a method that brings no prompt of its own; the workspace is the engine's | | `utils/benchmark/task-info/` | What a task and its record are, for the learner | | `utils/launch.sh` | Starts a method folder's run: the simulator service and hermit | -| [`aspire/`](aspire/) | The ASPIRE method: its fix-loop prompts, verifier and coordinator. `run.yaml` (Object, Spatial, Goal, Long: 40 tasks), `run_90.yaml` (LIBERO-90), `run_pro.yaml` (LIBERO-Pro: 80), `run_pro_long.yaml` (LIBERO-Long-Pro), `run_luna.yaml` (`run.yaml` on Luna), `run_sample_luna.yaml` (six of the forty on Luna, for a run of a day or two), `smoke*.yaml`, `launch.sh` | +| [`aspire/`](aspire/) | The ASPIRE method (its session's task functions, verifier and coordinator are ASPIRE's own; `tasks.sh` adds its verifiers to the tasks). `run.yaml` (Object, Spatial, Goal, Long: 40 tasks), `run_90.yaml` (LIBERO-90), `run_pro.yaml` (LIBERO-Pro: 80), `run_pro_long.yaml` (LIBERO-Long-Pro), `run_luna.yaml` (`run.yaml` on Luna), `run_sample_luna.yaml` (six of the forty on Luna, for a run of a day or two), `smoke*.yaml`, `launch.sh` | | [`prompt_only/`](prompt_only/) | The prompt-only template: one `ADAPTER.md` the built-in Codex Evolver revises after each task, with the method-neutral prompt. `run.yaml`, `run_pro.yaml`, `smoke.yaml`, `smoke_luna.yaml`, `launch.sh` | The `*_luna.yaml` configs run GPT-5.6-Luna on the machine's ChatGPT login; the others GLM 5.3 Flash on Fireworks. ## The engine and the protocol -The engine is the [Robosuite example](../aspire_robosuite/)'s: its simulator service (`bin/sim` in the session's -workspace), its task functions (the workspace, the record), its judge and its sandboxes, which that README describes. +The engine is the [cap-harness engine](../cap_harness/), which Robosuite shares: its simulator service (`bin/sim` in +the session's workspace), its task functions (the workspace, the record), its judge and its sandboxes, which its +README describes. It takes the benchmark from the task row (`"benchmark": "libero-pro"`) and with it ASPIRE's settings for LIBERO (its fix-loop template's Benchmark Profiles): - `.venv-libero/bin/cap-harness`, with `LIBERO_CONFIG_PATH=$ASPIRE_ROOT/.libero` and `TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1`; - `--init-mode seeded` on every run, the session's and the judge's, so that each seed is a scene of its own; -- ASPIRE's LIBERO seeds: development 51-65, unseen-check 66-80, held-out 1-50, the verifier's 51-55; +- ASPIRE's LIBERO seeds: development 51-65, unseen-check 66-80, held-out 1-50 (and ASPIRE's verifier's, 51-55, + which its `tasks.sh` gives); - `docs/libero-pro.md` in the session's workspace without its "Task examples" section, which names ASPIRE's worked programs per task. The task sandbox is shown the staged copy only, not the checkout's page; a page without that section is refused rather than passed whole. @@ -44,17 +46,14 @@ distinct for every task: where a task's placement leaves little room, a held-out layout. A method's number is measured the same way whatever its prompt: the workspace, `bin/sim`, the seeds, the scored -program (`fix_code.py`) and the judge are the engine's. ASPIRE's configs give the session ASPIRE's own prompts (the -fix-loop subagent and the evolutionary search); `prompt_only/` gives it a method-neutral one (`utils/agent/prompt.md`: -the task, the workspace, what is scored) and `evosearch: false`, with which the engine neither stages ASPIRE's -evolutionary-search page nor scores a search's program: the scored program is `fix_code.py`. Its workspace leaves out -ASPIRE's assignment (`utils/agent/task.py:prepare`). Its task sandbox can still read the engine's folder -(`../aspire_robosuite/utils/agent`), which holds the engine's adaptations of ASPIRE's prompts, beside the code the -session runs. +program (`fix_code.py`) and the judge are the engine's, and nothing of a method is in them. ASPIRE's configs give the +session ASPIRE's own task functions (`hermit.agent.evolver.methods.aspire.actor`: the fix-loop subagent and the +evolutionary search, whose best program it scores); `prompt_only/` gives it a method-neutral prompt +(`utils/agent/prompt.md`: the task, the workspace, what is scored) and the engine's workspace and record as they are. ## Setup -As [Robosuite's](../aspire_robosuite/README.md#setup) (the providers, the render device, Codex and the sandboxes), +As [the engine's](../cap_harness/README.md#setup) (the providers, the render device, Codex and the sandboxes), with LIBERO's venv instead of Robosuite's in the ASPIRE checkout: ```bash @@ -67,7 +66,7 @@ Serve Contact-GraspNet on 8115 beside the other three (`scripts/supervise_servic contact_graspnet`); the launcher refuses to start without it, since `generate_grasps` would have no backend to call. Segmentation is ASPIRE's SAM3 service (`--providers sam3`): LIBERO's scenes hold several similar objects, which -the Robosuite example's stand-in does not tell apart. +the engine's stand-in does not tell apart. ## Run @@ -83,8 +82,8 @@ ASPIRE_SIM_GPU=0 ASPIRE_CONFIG=run_pro.yaml ./launch.sh # LIBERO-Pro ./launch.sh eval --adapter root # the same with the empty one ``` -`utils/launch.sh` starts the Robosuite example's launcher on this folder and the method folder; its variables +`utils/launch.sh` starts the engine's launcher on this folder and the method folder; its variables (`ASPIRE_SIM_ENV`, `ASPIRE_SIM_GPU`, `ASPIRE_OUT`, `ASPIRE_EVAL_SEEDS`, ...) are described there. A method on LIBERO -is a folder of configs beside these: `utils/agent/task.py` as its prompt and workspace, or its own. Run one method -folder at a time on a checkout and an `ASPIRE_OUT`: they share the task rows, the staged docs and the service's -ledger, which each launch writes. +is a folder of configs beside these: `utils/agent/task.py` as its prompt and the engine's workspace, or its own (and a +`tasks.sh` if it adds rows of its own, as ASPIRE's verifiers). Run one method folder at a time on a checkout and an +`ASPIRE_OUT`: they share the task rows, the staged docs and the service's ledger, which each launch writes. diff --git a/examples/libero/aspire/run.yaml b/examples/libero/aspire/run.yaml index 0f8450db..9db2f09b 100644 --- a/examples/libero/aspire/run.yaml +++ b/examples/libero/aspire/run.yaml @@ -3,11 +3,11 @@ # (development 51-65, unseen-check 66-80, held-out 1-50, written into the task rows by launch.sh; every run # --init-mode seeded); Codex acting and coordinating, GLM 5.3 Flash. Started by launch.sh. data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -15,10 +15,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -30,7 +31,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -40,7 +41,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_90.yaml b/examples/libero/aspire/run_90.yaml index dacaf106..1b3639a3 100644 --- a/examples/libero/aspire/run_90.yaml +++ b/examples/libero/aspire/run_90.yaml @@ -1,11 +1,11 @@ # ASPIRE on LIBERO-90: the ninety tasks of LIBERO's libero_90 suite, as run.yaml runs the other four (the same # seeds, roles and model). Started by launch.sh: ASPIRE_CONFIG=run_90.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_90.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_90_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_90.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_90_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -13,10 +13,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -28,7 +29,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -38,7 +39,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_luna.yaml b/examples/libero/aspire/run_luna.yaml index e297052a..b2edec2a 100644 --- a/examples/libero/aspire/run_luna.yaml +++ b/examples/libero/aspire/run_luna.yaml @@ -1,11 +1,11 @@ # run.yaml with the models changed and nothing else: GPT-5.6-Luna at medium effort acts and coordinates on the # machine's own ChatGPT login (`codex login`; no API key). Started by launch.sh: ASPIRE_CONFIG=run_luna.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -13,10 +13,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -28,7 +29,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -38,7 +39,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_pro.yaml b/examples/libero/aspire/run_pro.yaml index 62537701..166e6fc2 100644 --- a/examples/libero/aspire/run_pro.yaml +++ b/examples/libero/aspire/run_pro.yaml @@ -4,11 +4,11 @@ # rows by launch.sh; every run --init-mode seeded); Codex acting and coordinating, GLM 5.3 Flash. Started by # launch.sh. data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_pro.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_pro_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_pro.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_pro_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -16,10 +16,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -31,7 +32,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -41,7 +42,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_pro_long.yaml b/examples/libero/aspire/run_pro_long.yaml index 8964ba6b..f2043d26 100644 --- a/examples/libero/aspire/run_pro_long.yaml +++ b/examples/libero/aspire/run_pro_long.yaml @@ -1,11 +1,11 @@ # ASPIRE on LIBERO-Long-Pro: the twenty tasks of libero_10_swap and libero_10_task, as run_pro.yaml runs all eighty (the # same seeds, roles and model). Started by launch.sh: ASPIRE_CONFIG=run_pro_long.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_pro_long.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_pro_long_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_pro_long.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_pro_long_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -13,10 +13,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -28,7 +29,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -38,7 +39,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_sample_luna.yaml b/examples/libero/aspire/run_sample_luna.yaml index 27164d2b..c30ae837 100644 --- a/examples/libero/aspire/run_sample_luna.yaml +++ b/examples/libero/aspire/run_sample_luna.yaml @@ -3,11 +3,11 @@ # ./launch.sh. Its baseline, the same tasks with ASPIRE's empty library: # ASPIRE_CONFIG=run_sample_luna.yaml ./launch.sh eval --adapter root data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_sample.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_sample_rerun.txt # every task again with an adapter given + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_sample.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_sample_rerun.txt # every task again with an adapter given step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -15,10 +15,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -30,7 +31,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -40,7 +41,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke.yaml b/examples/libero/aspire/smoke.yaml index 5648f30e..354a3fed 100644 --- a/examples/libero/aspire/smoke.yaml +++ b/examples/libero/aspire/smoke.yaml @@ -2,11 +2,11 @@ # (launch.sh writes them into the task rows: ASPIRE_SEEDS=smoke); Codex acting (ASPIRE's fix-loop subagent and # verifier prompts) and coordinating, GLM 5.3 Flash. Started by launch.sh: ASPIRE_CONFIG=smoke.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke_luna.yaml b/examples/libero/aspire/smoke_luna.yaml index 2f746c60..ef39b42b 100644 --- a/examples/libero/aspire/smoke_luna.yaml +++ b/examples/libero/aspire/smoke_luna.yaml @@ -2,11 +2,11 @@ # machine's own ChatGPT login (`codex login`; no API key). Started by launch.sh: ASPIRE_CONFIG=smoke_luna.yaml # ./launch.sh (its seeds are the smoke's). data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke_pro.yaml b/examples/libero/aspire/smoke_pro.yaml index 638e5f0b..ae1b77fb 100644 --- a/examples/libero/aspire/smoke_pro.yaml +++ b/examples/libero/aspire/smoke_pro.yaml @@ -2,11 +2,11 @@ # (launch.sh writes them into the task rows: ASPIRE_SEEDS=smoke); Codex acting (ASPIRE's fix-loop subagent and # verifier prompts) and coordinating, GLM 5.3 Flash. Started by launch.sh: ASPIRE_CONFIG=smoke_pro.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke_pro.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_pro_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke_pro.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_pro_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke_pro_luna.yaml b/examples/libero/aspire/smoke_pro_luna.yaml index 6e1e381f..40335396 100644 --- a/examples/libero/aspire/smoke_pro_luna.yaml +++ b/examples/libero/aspire/smoke_pro_luna.yaml @@ -2,11 +2,11 @@ # machine's own ChatGPT login (`codex login`; no API key). Started by launch.sh: ASPIRE_CONFIG=smoke_pro_luna.yaml # ./launch.sh (its seeds are the smoke's). data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke_pro.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_pro_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke_pro.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_pro_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/tasks.sh b/examples/libero/aspire/tasks.sh new file mode 100644 index 00000000..eb3be258 --- /dev/null +++ b/examples/libero/aspire/tasks.sh @@ -0,0 +1,6 @@ +# Sourced by the engine's launcher (../../cap_harness/launch.sh) in this folder, after the benchmark's prepare.py: +# ASPIRE's verifiers beside LIBERO's tasks (hermit/agent/evolver/methods/aspire/rows.py, on ASPIRE's first five +# development seeds), in tasks/ and splits/ here, which this folder's configs read. +"$HERMIT" -m hermit.agent.evolver.methods.aspire.rows --benchmark ../utils/benchmark --out . --verify-seeds 51-55 \ + --engine ../../cap_harness/utils/benchmark/tasks.py +TASKS="$PWD/tasks" diff --git a/examples/libero/prompt_only/run.yaml b/examples/libero/prompt_only/run.yaml index f8668473..ebf54ea9 100644 --- a/examples/libero/prompt_only/run.yaml +++ b/examples/libero/prompt_only/run.yaml @@ -4,7 +4,7 @@ # Flash. Started by launch.sh. data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/libero.txt test: ../utils/benchmark/splits/libero_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -15,10 +15,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -31,13 +30,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/prompt_only/run_pro.yaml b/examples/libero/prompt_only/run_pro.yaml index f62382b6..b7f52fe5 100644 --- a/examples/libero/prompt_only/run_pro.yaml +++ b/examples/libero/prompt_only/run_pro.yaml @@ -3,7 +3,7 @@ # ./launch.sh data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/libero_pro.txt test: ../utils/benchmark/splits/libero_pro_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -14,10 +14,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -30,13 +29,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/prompt_only/smoke.yaml b/examples/libero/prompt_only/smoke.yaml index 83c28c19..e9863631 100644 --- a/examples/libero/prompt_only/smoke.yaml +++ b/examples/libero/prompt_only/smoke.yaml @@ -4,7 +4,7 @@ # ASPIRE_CONFIG=smoke.yaml ./launch.sh data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/smoke.txt test: ../utils/benchmark/splits/smoke_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -15,10 +15,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -31,13 +30,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/prompt_only/smoke_luna.yaml b/examples/libero/prompt_only/smoke_luna.yaml index 527e6d2c..7def1039 100644 --- a/examples/libero/prompt_only/smoke_luna.yaml +++ b/examples/libero/prompt_only/smoke_luna.yaml @@ -3,7 +3,7 @@ # ./launch.sh (its seeds are the smoke's). data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/smoke.txt test: ../utils/benchmark/splits/smoke_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -14,10 +14,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -30,13 +29,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/utils/agent/task.py b/examples/libero/utils/agent/task.py index fbdc758b..437d4034 100644 --- a/examples/libero/utils/agent/task.py +++ b/examples/libero/utils/agent/task.py @@ -1,17 +1,15 @@ -"""A method-neutral acting prompt for LIBERO, for a method that brings no acting prompt of its own (prompt-only, and -the others that learn only what the session is given as context): the task, the workspace, the simulator, the rules -and what is scored, and nothing of a method. LIBERO has no acting prompt; this one is written here. The workspace, -the simulator service, the record and the judge are the engine's (`examples/aspire_robosuite/utils/agent/task.py`), -as for ASPIRE's own prompts, so a method's number on LIBERO is measured the same way whatever its prompt; `prepare` -leaves out of the workspace what is ASPIRE's method rather than the task's. +"""LIBERO's acting prompt, for a method that brings no acting prompt of its own (prompt-only, and the others that learn +only what the session is given as context): the task, the workspace, the simulator, the rules and what is scored, +and nothing of a method. LIBERO has no acting prompt; this one is written here. The workspace, the simulator +service, the record and the judge are the cap-harness engine's (`examples/cap_harness/utils/agent/task.py`), so a +method's number on LIBERO is measured the same way whatever its prompt. agent: params: prompt: ../utils/agent/task.py:prompt - prepare: ../utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # the engine neither stages ASPIRE's search page nor scores a search's program + prepare: ../../cap_harness/utils/agent/task.py:prepare + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect """ from __future__ import annotations @@ -20,48 +18,22 @@ import sys HERE = Path(__file__).resolve().parent -ENGINE = HERE.parents[2] / "aspire_robosuite" / "utils" / "agent" -# What the engine's prepare writes that is ASPIRE's method, not the task's: ASPIRE's assignment block (its -# coordinator's terms). Its evolutionary search is left out by `evosearch: false`, which `_settings` requires, so that -# the engine's collect scores fix_code.py, the file this prompt names. -ASPIRE_ONLY = ("task.md",) +ENGINE = HERE.parents[2] / "cap_harness" / "utils" / "agent" def _engine(): """The engine's task functions, loaded from their own folder (they import sim.py beside them).""" if str(ENGINE) not in sys.path: sys.path.insert(0, str(ENGINE)) - spec = importlib.util.spec_from_file_location("aspire_robosuite_task", ENGINE / "task.py") + spec = importlib.util.spec_from_file_location("cap_harness_task", ENGINE / "task.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module -def _settings(engine, task: dict, params: dict) -> dict: - s = engine.settings(task, params) - if params.get("evosearch", True): - raise ValueError("agent.params.evosearch must be false: this prompt runs no evolutionary search, and the " - "engine's collect scores the search's program where one is left") - if s["role"] == "verifier": - raise ValueError(f"{task['id']}: a verifier row belongs to a method that verifies (ASPIRE's); " - "this prompt acts on a task") - return s - - -def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: - """The engine's workspace (bin/sim registered with the service, the docs, the scene-snapshot program), without the - search's page (`evosearch: false`) and ASPIRE_ONLY.""" - engine = _engine() - _settings(engine, task, params) - mounts = engine.prepare(task, workspace, params, base) - for name in ASPIRE_ONLY: - (Path(workspace) / name).unlink(missing_ok=True) - return mounts - - def prompt(task: dict, params: dict, base: Path) -> str: engine = _engine() - s = _settings(engine, task, params) + s = engine.settings(task, params) steps = (f" An episode runs at most {int(s['max_steps'])} simulator steps." if s["max_steps"] else "") return (HERE / "prompt.md").read_text().format( suite=s["suite"], task_id=s["task_id"], task_dir=engine.task_dir(s), dev=s["dev"], diff --git a/examples/libero/utils/benchmark/prepare.py b/examples/libero/utils/benchmark/prepare.py index 35136984..202be729 100644 --- a/examples/libero/utils/benchmark/prepare.py +++ b/examples/libero/utils/benchmark/prepare.py @@ -5,9 +5,9 @@ python prepare.py # ASPIRE's seeds python prepare.py --preset smoke # the smoke's: three development seeds, one unseen, three held-out - # tasks// (task directories), + # tasks// (task directories, and each one's rerun-/), # splits/{all,libero,libero_90,libero_pro,libero_pro_long,,smoke,smoke_pro,libero_sample}.txt, - # their *_rerun.txt (rerun.txt for all), verify.txt + # their *_rerun.txt (rerun.txt for all) LIBERO's own five suites (Liu et al., 2023): `libero_object`, `libero_spatial`, `libero_goal` and `libero_10` (LIBERO-Long), ten tasks each, which splits/libero.txt holds in that order, and `libero_90` (splits/libero_90.txt). @@ -18,8 +18,8 @@ splits/libero_pro_long.txt for LIBERO-Long-Pro (`libero_10_swap`, `libero_10_task`). A suite and its perturbations share task ids, so a row's id carries the suite. -The tasks, the verifier's and the reruns are written by the Robosuite example's prepare.py (`write`), whose engine -this benchmark runs on: one task directory each, read back as its row by its utils/agent/aspire_fields.py. +The tasks are written by the cap-harness engine's helpers (../../../cap_harness/utils/benchmark/tasks.py), the engine +this benchmark runs on: one task directory each, read back as its row by the engine's utils/agent/fields.py. """ from __future__ import annotations @@ -27,8 +27,8 @@ import sys HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE.parents[2] / "aspire_robosuite" / "utils" / "benchmark")) -from prepare import seeds_from, write # noqa: E402 +sys.path.insert(0, str(HERE.parents[2] / "cap_harness" / "utils" / "benchmark")) +from tasks import seeds_from, write # noqa: E402 BENCHMARK = "libero-pro" # cap-harness's name for its LIBERO adapter SUITES = {"libero_object": 10, "libero_spatial": 10, "libero_goal": 10, "libero_10": 10, "libero_90": 90} @@ -37,12 +37,12 @@ # A sample of the forty that a run on one GPU finishes in a day or two: task ids 0 and 5 of Object, Spatial and Goal # (Long's two-stage tasks are left out for their length) SAMPLE = [f"{suite}-t{n:02d}" for suite in ("libero_object", "libero_spatial", "libero_goal") for n in (0, 5)] -PRESETS = {"aspire": {"dev": "51-65", "unseen": "66-80", "eval": "1-50", "verify": "51-55"}, - "smoke": {"dev": "51-53", "unseen": "66", "eval": "1-3", "verify": "51-55"}} +PRESETS = {"aspire": {"dev": "51-65", "unseen": "66-80", "eval": "1-50"}, + "smoke": {"dev": "51-53", "unseen": "66", "eval": "1-3"}} def main() -> None: - seeds, verify = seeds_from(PRESETS) + seeds = seeds_from(PRESETS) # `t`: a task id is not read as the hex suffix of a rollout directory's name (the simulator service matches it) tasks = [{"id": f"{suite}-t{n:02d}", "benchmark": BENCHMARK, "suite": suite, "task_id": n} for suite, count in {**SUITES, **PRO_SUITES}.items() for n in range(count)] @@ -50,7 +50,7 @@ def main() -> None: ids = [t["id"] for t in tasks] original = of(*[s for s in SUITES if s != "libero_90"]) pro = of(*PRO_SUITES) - write(HERE, tasks, seeds, verify, + write(HERE, tasks, seeds, {"all": ids, "libero": original, "libero_90": of("libero_90"), "libero_pro": pro, "libero_pro_long": of("libero_10_swap", "libero_10_task"), **{suite: of(suite) for suite in (*SUITES, *PRO_SUITES) if suite != "libero_90"}, diff --git a/examples/libero/utils/launch.sh b/examples/libero/utils/launch.sh index 1bcd66fd..da429df6 100755 --- a/examples/libero/utils/launch.sh +++ b/examples/libero/utils/launch.sh @@ -9,8 +9,8 @@ # ./launch.sh eval --adapter final # the tasks again with the final adapter (test split) # ./launch.sh eval --adapter root # the same with the empty one # -# The engine is the Robosuite example's (../../aspire_robosuite): its launcher runs the method folder's configs, with -# the seeds of utils/benchmark/prepare.py, and takes the same variables (ASPIRE_SIM_ENV, ASPIRE_SIM_GPU, ASPIRE_OUT, +# The engine is the cap-harness engine (../../cap_harness): its launcher runs the method folder's configs, with the +# seeds of utils/benchmark/prepare.py, and takes its variables (ASPIRE_SIM_ENV, ASPIRE_SIM_GPU, ASPIRE_OUT, # ASPIRE_EVAL_SEEDS, ASPIRE_SIM_PORT, HERMIT_PYTHON; see its header). ASPIRE_ROOT needs .venv-libero. set -euo pipefail HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" @@ -19,7 +19,7 @@ METHOD="${1:?the method folder}"; shift [[ -x "$ASPIRE_ROOT/.venv-libero/bin/cap-harness" ]] \ || { echo "$ASPIRE_ROOT/.venv-libero/bin/cap-harness: run ASPIRE's scripts/bootstrap_libero.sh" >&2; exit 1; } # Contact-GraspNet, LIBERO's grasp backend (ASPIRE's SERVICE_PORTS for libero-pro), on its fixed port; the -# Robosuite launcher checks the other services +# engine's launcher checks the other services curl -sf -o /dev/null --max-time 3 "http://127.0.0.1:8115/openapi.json" \ || { echo "ASPIRE's Contact-GraspNet service at http://127.0.0.1:8115 is not up (see ../README.md)" >&2; exit 1; } -ASPIRE_EXAMPLE="$HERE/.." ASPIRE_METHOD="$METHOD" exec "$HERE/../../aspire_robosuite/aspire/launch.sh" "$@" +ASPIRE_EXAMPLE="$HERE/.." ASPIRE_METHOD="$METHOD" exec "$HERE/../../cap_harness/launch.sh" "$@" diff --git a/examples/robodojo_sim/utils/agent/dojo_service.py b/examples/robodojo_sim/utils/agent/dojo_service.py index 70313889..4e0e6157 100644 --- a/examples/robodojo_sim/utils/agent/dojo_service.py +++ b/examples/robodojo_sim/utils/agent/dojo_service.py @@ -1,6 +1,6 @@ """RoboDojo's simulator as a host service for the actors in the task sandbox: `bin/sim` (sim_cli.py) asks it to -evaluate a program on the session's development layouts. It is the ASPIRE example's simulator service -(examples/aspire_robosuite/utils/agent/sim_service.py: the registration of a session by its rollout directory and +evaluate a program on the session's development layouts. It is the cap-harness engine's simulator service +(examples/cap_harness/utils/agent/sim_service.py: the registration of a session by its rollout directory and task id, the program read and the output written without following a link the session left, the ledger) with RoboDojo's engine (dojo.Engine) in place of cap-harness: @@ -29,8 +29,8 @@ import dojo # noqa: E402 from robodojo_fields import task_rows # noqa: E402 -sys.path.append(str(HERE.parents[2] / "aspire_robosuite" / "utils" / "agent")) -import sim_service as base # noqa: E402 (the ASPIRE example's) +sys.path.append(str(HERE.parents[2] / "cap_harness" / "utils" / "agent")) +import sim_service as base # noqa: E402 (the cap-harness engine's) class Service(base.Service): diff --git a/examples/robosuite/.gitignore b/examples/robosuite/.gitignore new file mode 100644 index 00000000..62859799 --- /dev/null +++ b/examples/robosuite/.gitignore @@ -0,0 +1,4 @@ +runs/ +# ASPIRE's tasks and splits, written by aspire/tasks.sh at launch +aspire/tasks/ +aspire/splits/ diff --git a/examples/robosuite/README.md b/examples/robosuite/README.md new file mode 100644 index 00000000..af88021c --- /dev/null +++ b/examples/robosuite/README.md @@ -0,0 +1,59 @@ +# Robosuite + +The Robosuite tasks of [ASPIRE](https://research.nvidia.com/labs/gear/aspire/)'s Code-as-Policy harness (Agentic +Skills Discovery for Robotics, [arXiv:2607.00272](https://arxiv.org/abs/2607.00272), +[code](https://github.com/NVlabs/ASPIRE)) as a benchmark: for each task a session writes a program (Python against +the harness's perception, grasping and motion API), develops it on development seeds, checks it once on unseen seeds, +and is scored once on held-out seeds. The tasks run on the [cap-harness engine](../cap_harness/) (its simulator +service, `bin/sim`, its judge), which LIBERO shares; its README has the setup. + +| Folder | What it is | +|---|---| +| `utils/benchmark/prepare.py` | The seven Robosuite tasks with their seeds, as task directories in Harbor's format (`tasks/`, each with its rerun, each read as its row through the engine's `utils/agent/fields.py`, the configs' `data.view`); `splits/` | +| `utils/benchmark/task-info/` | What a task and its record are, for the learner | +| `aspire/` | ASPIRE on Robosuite: `run.yaml` (ASPIRE's seeds), `smoke.yaml`, `smoke_claude.yaml` (Claude Code acting and coordinating on a Claude subscription), the `*_direct.yaml` comparison, `launch.sh`, and `tasks.sh`, which adds ASPIRE's verifiers to the tasks | + +## ASPIRE on Robosuite + +The curriculum run as ASPIRE runs it ([`aspire`](../../docs/methods/aspire.md)): ASPIRE's fix-loop subagent (the +actor), its skill verifier and its coordinator, each a Codex session given ASPIRE's own prompt. For each task the +actor debugs its program from the per-primitive traces and RGB keyframes; the harness then scores it once on held-out +seeds. After each task the coordinator updates the shared skill library, admitting the actor's proposals only if a +fresh verifier re-solves the task from the library and the proposals alone, and chooses the next task. The actor's +task functions are ASPIRE's (`hermit/agent/evolver/methods/aspire/actor.py`, built on the engine's), and its verifier +rows are ASPIRE's too (`aspire/tasks.sh` derives them from the tasks `prepare.py` wrote). + +On the host that serves the providers, with the engine set up: + +```bash +export FIREWORKS_API_KEY=... +cd examples/robosuite/aspire +ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh +ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh eval --adapter final +``` + +`./launch.sh eval --adapter final` reruns every task with the final library (the test split's `rerun-*` rows): +ASPIRE's backward transfer, compared per task with the task's first pass during training (each training step's +score in the run's `steps.jsonl`; the rerun's in the eval's results). `--adapter root` is the empty library and +`--adapter none` no library at all: the baselines on the same seeds. + +The protocol is ASPIRE's: each task learns on its development seeds and is scored once on held-out seeds; the +reruns score the same held-out seeds again, so the comparison reuses that partition (as ASPIRE's Phase 3 does). +The first evaluation ran `run.yaml` on held-out seeds 1-50 (half of ASPIRE's 1-100), and its reruns on the same +seeds. The held-out seeds are the task rows' (`prepare.py --eval`), so they are set when the launcher writes the +rows: `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh`, and `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh eval --adapter final` for the +reruns (an `agent.params.eval_seeds` override has no effect). + +### Protocol + +| | `run.yaml` | `smoke.yaml` | +|---|---|---| +| Tasks | the 7 Robosuite tasks, in the order the coordinator chooses (it starts from `splits/all.txt`'s first) | `cube_lifting`, `cube_stack` | +| Development seeds | 101-125 (ASPIRE's) | 101-103 | +| Unseen-check seeds | 126-130 (ASPIRE's), run once by the actor | 126 | +| Held-out seeds | 1-100 (ASPIRE's; `ASPIRE_EVAL_SEEDS` for others), run once by the judge | 1-3 | +| Fix loop | ASPIRE's: 3 versions per failure mode, 8 in total | the same | +| Evolutionary search | ASPIRE's, for a task the fix loop leaves below 90%: K = 8, top 3, stop at 80% or 5 rounds | the same | +| Verifier | ASPIRE's: 5 seeds (101-105), 2 attempts, verified at ≥ 4/5 | the same | +| Actor, verifier, coordinator | Codex, GLM 5.3 Flash (Fireworks), reasoning effort medium | the same | + diff --git a/examples/robosuite/aspire/launch.sh b/examples/robosuite/aspire/launch.sh new file mode 100755 index 00000000..d3105286 --- /dev/null +++ b/examples/robosuite/aspire/launch.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Train, or evaluate, ASPIRE on Robosuite: the cap-harness engine's launcher (../../cap_harness/launch.sh, whose header +# gives the commands and variables) with this folder's configs, e.g. ASPIRE_ROOT=/path/to/ASPIRE ./launch.sh +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ASPIRE_EXAMPLE="$HERE/.." ASPIRE_METHOD=aspire exec "$HERE/../../cap_harness/launch.sh" "$@" diff --git a/examples/aspire_robosuite/aspire/run.yaml b/examples/robosuite/aspire/run.yaml similarity index 70% rename from examples/aspire_robosuite/aspire/run.yaml rename to examples/robosuite/aspire/run.yaml index 037fee61..ab4e94b8 100644 --- a/examples/aspire_robosuite/aspire/run.yaml +++ b/examples/robosuite/aspire/run.yaml @@ -2,11 +2,11 @@ # coordinator chooses (its curriculum); ASPIRE's seeds (development 101-125, unseen-check 126-130, held-out 1-100, # written into the task rows by launch.sh); Codex acting and coordinating, GLM 5.3 Flash. Started by launch.sh. data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/all.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/all.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../utils/agent/task.py:prompt - prepare: ../utils/agent/task.py:prepare - answer: ../utils/agent/task.py:answer - collect: ../utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../utils/agent + - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/aspire_robosuite/aspire/run_direct.yaml b/examples/robosuite/aspire/run_direct.yaml similarity index 71% rename from examples/aspire_robosuite/aspire/run_direct.yaml rename to examples/robosuite/aspire/run_direct.yaml index 140c3df7..70fd19cd 100644 --- a/examples/aspire_robosuite/aspire/run_direct.yaml +++ b/examples/robosuite/aspire/run_direct.yaml @@ -1,21 +1,23 @@ -# The structured direct-chat actor (utils/agent/agent.py) in place of Codex: a comparison with ASPIRE's coding agent. +# ASPIRE's structured direct-chat actor (hermit/agent/evolver/methods/aspire/direct.py) in place of Codex: a +# comparison with ASPIRE's coding agent. # ASPIRE on Robosuite: the seven Robosuite tasks of ASPIRE's harness as one curriculum, one task per step; each # task's actor debugs its program on ASPIRE's development seeds (101-125) and is scored once on held-out seeds; after # every task the coordinator repairs and grows the skill library through the verifier gate. Started by launch.sh, # inside the host that runs ASPIRE's perception, IK and planning services (see ../README.md). data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/all.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/all.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 # one task per step: the coordinator acts on each task as it finishes shuffle: false # the curriculum in the split's order task_info: ../utils/benchmark/task-info agent: - name: ../utils/agent/agent.py:AspireActor + name: hermit.agent.evolver.methods.aspire.direct:AspireActor concurrency: 1 # one simulator sweep at a time on the engine's GPU params: + engine: ../../cap_harness/utils/agent/sim.py # the engine this actor runs its programs on aspire_root: $ASPIRE_ROOT # an ASPIRE checkout with .venv-robosuite (launch.sh) sim_env: {} # the engine's environment; launch.sh fills it from ASPIRE_SIM_ENV model: fireworks/accounts/fireworks/models/glm-5p3-flash @@ -25,7 +27,7 @@ agent: evo: {k: 8, rounds: 5, threshold: 0.8, qualify_below: 0.9, parents: 3} workers: 4 judge: - path: ../utils/benchmark/judge.py + path: ../../cap_harness/utils/benchmark/judge.py template: aspire evolver: # the coordinator, as in run.yaml name: aspire diff --git a/examples/aspire_robosuite/aspire/smoke.yaml b/examples/robosuite/aspire/smoke.yaml similarity index 70% rename from examples/aspire_robosuite/aspire/smoke.yaml rename to examples/robosuite/aspire/smoke.yaml index 35cff3a4..10773d33 100644 --- a/examples/aspire_robosuite/aspire/smoke.yaml +++ b/examples/robosuite/aspire/smoke.yaml @@ -2,11 +2,11 @@ # the task rows: ASPIRE_SEEDS=smoke); Codex acting (ASPIRE's fix-loop subagent and verifier prompts) and coordinating, # GLM 5.3 Flash. Started by launch.sh: ASPIRE_CONFIG=smoke.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../utils/agent/task.py:prompt - prepare: ../utils/agent/task.py:prepare - answer: ../utils/agent/task.py:answer - collect: ../utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../utils/agent + - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/robosuite/aspire/smoke_claude.yaml b/examples/robosuite/aspire/smoke_claude.yaml new file mode 100644 index 00000000..fb59dbe8 --- /dev/null +++ b/examples/robosuite/aspire/smoke_claude.yaml @@ -0,0 +1,58 @@ +# The smoke with Claude Code: two tasks; three development seeds, one unseen-check seed, three held-out seeds +# (launch.sh writes them into the task rows: ASPIRE_SEEDS=smoke); Claude Code acting (ASPIRE's fix-loop subagent and +# verifier prompts) and coordinating, Claude Opus 5.5 on the user's Claude subscription (login: claude, its token in +# CLAUDE_CODE_OAUTH_TOKEN: docs/guides/claude-subscription.md). Started by launch.sh: +# ASPIRE_CONFIG=smoke_claude.yaml ./launch.sh +data: + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt + step_size: 1 + shuffle: false +task_info: ../utils/benchmark/task-info +agent: + base: claude + concurrency: 1 + params: + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on + aspire_root: $ASPIRE_ROOT + aspire_docs: $ASPIRE_DOCS + model: {name: claude-opus-5-5, login: claude, effort: medium} # a model name on the login + claude: + web_search: false # set explicitly: WebSearch and WebFetch denied (they run at Anthropic) + timeout_s: 5400 +environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it + read: + - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client + - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs + - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md + - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md + - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py + - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md + - $ASPIRE_ROOT/.claude/skills/evosearch/subagent-prompt.md + - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md + env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded +judge: + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds +template: aspire +evolver: + name: aspire + base: claude + model: {name: claude-opus-5-5, login: claude, effort: medium} + claude: {web_search: false} +sandbox: + enabled: true + enforce: required +trainer: + run_name: smoke-aspire-claude + epochs: 1 + call_timeout_min: 240 + budget: {hours: 8} # on the subscription: no dollar cost; the plan's own windows apply +eval: + repeats: 1 diff --git a/examples/aspire_robosuite/aspire/smoke_direct.yaml b/examples/robosuite/aspire/smoke_direct.yaml similarity index 62% rename from examples/aspire_robosuite/aspire/smoke_direct.yaml rename to examples/robosuite/aspire/smoke_direct.yaml index 8d6b7b61..549a445a 100644 --- a/examples/aspire_robosuite/aspire/smoke_direct.yaml +++ b/examples/robosuite/aspire/smoke_direct.yaml @@ -1,19 +1,21 @@ -# The structured direct-chat actor (utils/agent/agent.py) in place of Codex: a comparison with ASPIRE's coding agent. +# ASPIRE's structured direct-chat actor (hermit/agent/evolver/methods/aspire/direct.py) in place of Codex: a +# comparison with ASPIRE's coding agent. # Smoke: two tasks, three development seeds, three held-out seeds, two versions and no search; the verifier's own # rows (5 seeds, 2 attempts). Started by launch.sh: ASPIRE_CONFIG=smoke_direct.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info agent: - name: ../utils/agent/agent.py:AspireActor + name: hermit.agent.evolver.methods.aspire.direct:AspireActor concurrency: 1 params: + engine: ../../cap_harness/utils/agent/sim.py # the engine this actor runs its programs on aspire_root: $ASPIRE_ROOT sim_env: {} model: fireworks/accounts/fireworks/models/glm-5p3-flash @@ -22,7 +24,7 @@ agent: search: debug workers: 3 judge: - path: ../utils/benchmark/judge.py + path: ../../cap_harness/utils/benchmark/judge.py template: aspire evolver: # the coordinator, as in run.yaml name: aspire diff --git a/examples/robosuite/aspire/tasks.sh b/examples/robosuite/aspire/tasks.sh new file mode 100644 index 00000000..44a345c9 --- /dev/null +++ b/examples/robosuite/aspire/tasks.sh @@ -0,0 +1,6 @@ +# Sourced by the engine's launcher (../../cap_harness/launch.sh) in this folder, after the benchmark's prepare.py: +# ASPIRE's verifiers beside Robosuite's tasks (hermit/agent/evolver/methods/aspire/rows.py, on ASPIRE's first five +# development seeds), in tasks/ and splits/ here, which this folder's configs read. +"$HERMIT" -m hermit.agent.evolver.methods.aspire.rows --benchmark ../utils/benchmark --out . --verify-seeds 101-105 \ + --engine ../../cap_harness/utils/benchmark/tasks.py +TASKS="$PWD/tasks" diff --git a/examples/aspire_robosuite/utils/.gitignore b/examples/robosuite/utils/.gitignore similarity index 100% rename from examples/aspire_robosuite/utils/.gitignore rename to examples/robosuite/utils/.gitignore diff --git a/examples/aspire_robosuite/utils/benchmark/.gitignore b/examples/robosuite/utils/benchmark/.gitignore similarity index 100% rename from examples/aspire_robosuite/utils/benchmark/.gitignore rename to examples/robosuite/utils/benchmark/.gitignore diff --git a/examples/robosuite/utils/benchmark/prepare.py b/examples/robosuite/utils/benchmark/prepare.py new file mode 100644 index 00000000..cafd27f1 --- /dev/null +++ b/examples/robosuite/utils/benchmark/prepare.py @@ -0,0 +1,47 @@ +"""Write the Robosuite tasks of ASPIRE's harness (cap_harness/robosuite/registry.py) as hermit tasks: one task +directory (Harbor's format) per task with its seeds, which the simulator service and the judge take from the task's +row (ASPIRE's Robosuite defaults: development 101-125, unseen-check 126-130, held-out 1-100), through the cap-harness +engine's helpers (../../../cap_harness/utils/benchmark/tasks.py). + + python prepare.py # ASPIRE's seeds + python prepare.py --preset smoke # the smoke's: three development seeds, one unseen, three held-out + python prepare.py --dev 101-103 --unseen 126 --eval 1-3 # any others + # tasks// (and rerun-/), splits/{all,smoke,single_arm}.txt and their *_rerun.txt (rerun.txt for all) + +The order of splits/all.txt is a curriculum from simple to hard (single-arm pick before stacking and assembly, the +fixed-tool wipe, then the bimanual tasks). +""" +from __future__ import annotations + +from pathlib import Path +import sys + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parents[2] / "cap_harness" / "utils" / "benchmark")) +from tasks import seeds_from, write # noqa: E402 + +TASKS = [ # suite, what it is (task-info only; the session reads the language from get_task_context()) + ("cube_lifting", "Lift"), + ("cube_stack", "Stack"), + ("cube_restack", "Stack, restacked"), + ("nut_assembly", "NutAssemblySquare"), + ("spill_wipe", "Wipe, fixed tool"), + ("two_arm_lift", "TwoArmLift, bimanual"), + ("two_arm_handover", "TwoArmHandover, bimanual"), +] + + +# The seeds of a run: ASPIRE's Robosuite defaults, and the smoke's +PRESETS = {"aspire": {"dev": "101-125", "unseen": "126-130", "eval": "1-100"}, + "smoke": {"dev": "101-103", "unseen": "126", "eval": "1-3"}} + + +def main() -> None: + seeds = seeds_from(PRESETS) + tasks = [{"id": f"robosuite-{suite}", "suite": suite, "task_id": 0} for suite, _ in TASKS] + ids = [t["id"] for t in tasks] + write(HERE, tasks, seeds, {"all": ids, "single_arm": ids[:5], "smoke": ids[:2]}) + + +if __name__ == "__main__": + main() diff --git a/examples/aspire_robosuite/utils/benchmark/task-info/README.md b/examples/robosuite/utils/benchmark/task-info/README.md similarity index 100% rename from examples/aspire_robosuite/utils/benchmark/task-info/README.md rename to examples/robosuite/utils/benchmark/task-info/README.md diff --git a/hermit/agent/acting/workspace.py b/hermit/agent/acting/workspace.py index 106558d7..26a55d30 100644 --- a/hermit/agent/acting/workspace.py +++ b/hermit/agent/acting/workspace.py @@ -17,7 +17,8 @@ # give a reference or a login) effort: medium # Codex's model_reasoning_effort; absent: the CLI's default prompt: environment/prompt.md # the task prompt; `{field}` placeholders take the task's fields; or - # `file.py:function` (task) -> the prompt + # `file.py:function` (task) -> the prompt; every function here may + # also be `package.module:function` (a method's own) files: [environment/fixtures] # optional: copied into every workspace before the run mounts: # optional: read-only data in the workspace - {source: data/corpus, at: docs} @@ -104,9 +105,15 @@ def __missing__(self, key): return template.format_map(Fields({k: v for k, v in task.items() if not isinstance(v, (dict, list))})) +# A task function by module, as a method that brings its own task functions names them (`package.module:function`) +_MODULE_FUNCTION = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+:[A-Za-z_]\w*") + + def _task_function(value, base: Path): - """A `file.py:function` value as the function, else None.""" - return load_callable(value, Path(base)) if isinstance(value, str) and ".py:" in value else None + """A `file.py:function` or `package.module:function` value as the function, else None.""" + if not isinstance(value, str) or not (".py:" in value or _MODULE_FUNCTION.fullmatch(value)): + return None + return load_callable(value, Path(base)) def _call(fn, *args, params: dict, base: Path): diff --git a/hermit/agent/evolver/methods/aspire/actor.py b/hermit/agent/evolver/methods/aspire/actor.py new file mode 100644 index 00000000..2bd8b0bd --- /dev/null +++ b/hermit/agent/evolver/methods/aspire/actor.py @@ -0,0 +1,205 @@ +"""ASPIRE's acting sessions (agent.base: codex): its fix-loop subagent, its evolutionary search and its skill +verifier, as ASPIRE's own prompts run them, on any benchmark of the cap-harness engine. These are the task functions +an ASPIRE config names (agent.params.prompt, prepare, answer, collect); they build on the engine's own +(agent.params.engine: its utils/agent/task.py, e.g. examples/cap_harness/utils/agent/task.py), which give every +method's session its workspace, `bin/sim` and the facts the judge reads. They run in the task sandbox with the +session. + + prepare the engine's workspace, then ASPIRE's: task.md (the assignment), the search's procedure page, and for a + verifier the proposals under test as proposed_skills.md + prompt the adaptation page (prompts/codex_header.md), the assignment, then ASPIRE's template verbatim; for an + actor, Part 2: the evolutionary search (prompts/evosearch_header.md and its template) + answer the agent's last message (the record's output is the facts collect returns) + collect the engine's facts, and ASPIRE's: findings, skill report, the verifier's report, the search's; the + selected program (the search's best where it ran, else fix_code.py) is the one the judge scores, and a + verifier's program only with its own verdict `verified` + +A task's role is its row's: `verifier` for the verifier's rows, which ASPIRE derives from a benchmark's tasks +(rows.py); an actor otherwise (a rerun is an actor with the final library). ASPIRE's checkout comes from +agent.params.aspire_root (its prompt templates must be declared readable: environment.read). +""" +from __future__ import annotations + +from functools import lru_cache +import importlib.util +import json +import os +from pathlib import Path +import shutil +import sys + +HERE = Path(__file__).resolve().parent +PROMPTS = HERE / "prompts" +TEMPLATES = {"actor": "subagent-prompt.md", "verifier": "verifier-prompt.md"} +SKILLS = Path(".claude/skills/iterative-debugging") +EVOSEARCH = Path(".claude/skills/evosearch") +# The files of ASPIRE's checkout its sessions read beyond the engine's (environment.read): its prompt templates and +# the search's procedure +ASPIRE_FILES = [str(SKILLS / "subagent-prompt.md"), str(SKILLS / "verifier-prompt.md"), + str(EVOSEARCH / "subagent-prompt.md"), str(EVOSEARCH / "skills" / "evosearch-iteration.md")] +# ASPIRE's files the log keeps beyond the engine's +KEEP = ("findings_report.md", "skill_report.json", "verification_report.json", "verification.md") + + +@lru_cache(maxsize=None) +def _load(path: str): + module_path = Path(path) + if str(module_path.parent) not in sys.path: + sys.path.insert(0, str(module_path.parent)) + spec = importlib.util.spec_from_file_location(f"_cap_engine_{abs(hash(path))}", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def engine(params: dict, base: Path): + """The engine's task functions: agent.params.engine, a path relative to the config's folder.""" + path = params.get("engine") + if not path: + raise ValueError("agent.params.engine: the engine's task functions (e.g. " + "../../cap_harness/utils/agent/task.py)") + return _load(str((Path(base) / os.path.expandvars(str(path))).resolve())) + + +def settings(task: dict, params: dict, base: Path) -> dict: + """The engine's settings of the task, with ASPIRE's role.""" + role = "verifier" if task.get("role") == "verifier" else "actor" # a rerun is an actor with the final library + return {**engine(params, base).settings(task, params), "role": role} + + +def task_dir(s: dict, eng) -> str: + """The actor's $TASK_DIR as ASPIRE's template defines it, with $RUN_ROOT the workspace; the verifier's is `.`.""" + return eng.task_dir(s) if s["role"] == "actor" else "." + + +def evo_dir(s: dict) -> str: + """The evolutionary search's $TASK_DIR, as its template defines it, with EVOSEARCH_DIR `evosearch`.""" + return f"evosearch/{s['benchmark']}/{s['suite']}/task_{s['task_id']}" + + +def assignment(s: dict, eng) -> str: + if s["role"] == "verifier": + return (f"## Your assignment\n\n- BENCHMARK: `{s['benchmark']}`\n- SUITE: `{s['suite']}`\n" + f"- TASK_ID: `{s['task_id']}`\n- GPU: handled by `bin/sim`\n- VERIFY_DIR: `.` (this workspace)\n" + f"- Development seeds: **{s['dev']}** — wherever the template says 51, 52, 53, 54, 55 (or 51-55, or " + f"seed 51), use these.\n- The proposals under test: `proposed_skills.md` in this workspace.\n") + return (f"## Task Assignment\n\nBENCHMARK: {s['benchmark']}\nSUITE: {s['suite']}\nTASK_ID: {s['task_id']}\n" + f"GPU: handled by bin/sim\nRUN_ROOT: . (this workspace)\n\n" + f"MAX_STEPS: {s['max_steps'] or 1000}\nDEV_SEEDS: {s['dev']}\nUNSEEN_SEEDS: {s['unseen'] or 'none'}\n" + f"HELDOUT_COUNT: {len(eng.sim_seeds(s['heldout']))} (coordinator-owned; you never run them)\n") + + +def prompt(task: dict, params: dict, base: Path) -> str: + eng = engine(params, base) + s = settings(task, params, base) + template = (s["root"] / SKILLS / TEMPLATES[s["role"]]).read_text() + body = template.split("", 1)[1] + body = body.split("", 1)[0].strip() + if s["role"] == "actor": + body = body.split("\n---\n", 1)[1] if body.startswith("## Task Assignment") else body # ours replaces it + header = (PROMPTS / "codex_header.md").read_text().format( + role_name="fix-loop subagent" if s["role"] == "actor" else "skill verifier", + template=TEMPLATES[s["role"]], benchmark=s["benchmark"], benchmark_doc=s["doc"], + context_file=context_file(params), + your_dir=(f"`$RUN_ROOT` is this workspace (`.`), so `$TASK_DIR` is `{task_dir(s, eng)}`, as the template " + f"defines it." if s["role"] == "actor" else "`$VERIFY_DIR` is this workspace (`.`)."), + proposed_line=("The proposals under test are `proposed_skills.md` in this workspace (the template's " + "`$VERIFY_DIR/proposed_skills.md`); they are not in the library." if s["role"] == "verifier" + else ""), + finish_line=("only after both parts of this session: Part 1, the template below, and Part 2 after it, the " + "evolutionary search, which runs only when Part 1 leaves `fix_code.py` below 90% of the " + "development seeds. Reply with the return message of each part you ran." + if s["role"] == "actor" and params.get("evosearch", True) else + "by replying with the template's return message."), + heldout_line=("is not yours: the harness scores your selected program on the held-out seeds after you " + "finish (the template's Stage 2)." if s["role"] == "actor" else + "does not apply: your verdict is scored on the program you select, on your seeds; write it " + "as `fix_code.py` in this workspace as well.")) + text = f"{header}\n\n{assignment(s, eng)}\n---\n\n{body}\n" + if s["role"] == "actor" and params.get("evosearch", True): + text += "\n---\n\n" + evosearch_part(s, eng) + return text + + +def context_file(params: dict) -> str: + """The file the session's CLI reads the adapter's context from: Claude Code's CLAUDE.md, Codex's AGENTS.md.""" + return "CLAUDE.md" if str(params.get("backend") or "codex") in ("claude", "claude-code") else "AGENTS.md" + + +def evosearch_part(s: dict, eng) -> str: + """Part 2 of the actor's session: ASPIRE's evolutionary-search subagent (§2.3), for a task the fix loop left below + 90% of its development seeds. Its template is the fenced block of evosearch/subagent-prompt.md, verbatim after + its assignment block, which the page before it replaces.""" + template = (s["root"] / EVOSEARCH / "subagent-prompt.md").read_text() + body = template[template.index("## Task Assignment"):template.rindex("\n```")] + body = body.split("\n---\n", 1)[1].strip() + header = (PROMPTS / "evosearch_header.md").read_text().format( + benchmark=s["benchmark"], suite=s["suite"], task_id=s["task_id"], dev=s["dev"], task_dir=task_dir(s, eng), + evo_dir=evo_dir(s)) + return f"{header}\n---\n\n{body}\n" + + +def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: + eng = engine(params, base) + s = settings(task, params, base) + mounts = eng.prepare(task, workspace, params, base) + work = Path(workspace) + if params.get("evosearch", True): # the search's procedure, for a prompt that runs it + shutil.copy2(s["root"] / EVOSEARCH / "skills" / "evosearch-iteration.md", + work / "docs" / "evosearch-iteration.md") + if s["role"] == "verifier": + # The coordinator stages the proposals under test as the adapter's skills/proposed.md: the verifier's + # proposed_skills.md, kept out of the library it reads (interface.py leaves the file out). + staged = sorted(work.glob("hermit/*/skills/proposed.md")) + (work / "proposed_skills.md").write_text(staged[0].read_text() if staged else "(no proposals were staged)\n") + (work / "task.md").write_text(assignment(s, eng)) + return mounts + + +def answer(last_message: str, task: dict) -> dict: + return {"answer": last_message.strip()[-4000:]} + + +def collect(task: dict, workspace: Path, log: Path, params: dict, base: Path) -> dict: + """The engine's files and facts, and ASPIRE's: what the judge and the coordinator read.""" + eng = engine(params, base) + s = settings(task, params, base) + work = Path(workspace) / task_dir(s, eng) + facts = eng.collect_files(task, work, log, (*eng.KEEP, *KEEP)) + files = Path(log) / "files" + read = lambda name: (work / name).read_text(errors="replace") if (work / name).is_file() else None + facts = {"role": s["role"], "benchmark": s["benchmark"], "suite": s["suite"], + "findings": facts["findings"] or read("findings_report.md"), + "skill_report": _json(read("skill_report.json")), "verification": _json(read("verification_report.json")), + "unseen": facts["unseen"], "workspace": str(Path(workspace).resolve()), "max_steps": s["max_steps"]} + evo = Path(workspace) / evo_dir(s) + search = s["role"] == "actor" and params.get("evosearch", True) # a prompt that runs the search + if search and evo.is_dir(): + for name in ("evosearch_best_code.py", "findings.md", "task_analysis.md", "BLOCKED"): + if (evo / name).is_file(): + (files / "evosearch").mkdir(parents=True, exist_ok=True) + shutil.copy2(evo / name, files / "evosearch" / name) + facts["evosearch"] = {"findings": (evo / "findings.md").read_text(errors="replace") + if (evo / "findings.md").is_file() else None, + "best_code": (evo / "evosearch_best_code.py").is_file()} + # The selected program: the search's best when the search ran (it falls back to fix_code.py itself), else the fix + # loop's; with `evosearch: false`, fix_code.py alone. The judge scores it. + best = evo / "evosearch_best_code.py" + program = best if search and best.is_file() else work / "fix_code.py" + facts["scored_program"] = str(program.relative_to(workspace)) if program.is_file() else None + facts["fix_code"] = program.read_text(errors="replace") if program.is_file() else None + # A verifier's program counts only with its own verdict `verified` (ASPIRE: `refuted` and `void` are never + # promoted): without it, nothing is scored. + if s["role"] == "verifier" and (facts["verification"] or {}).get("verdict") != "verified": + facts["not_scored"] = f"verifier verdict {(facts['verification'] or {}).get('verdict') or 'missing'}" + facts["scored_program"] = facts["fix_code"] = None + if params.get("prune_media", True): + eng.prune(Path(workspace).parent) + return facts + + +def _json(text): + try: + return json.loads(text) if text else None + except ValueError: + return {"unparsed": text[-4000:]} diff --git a/examples/aspire_robosuite/utils/agent/agent.py b/hermit/agent/evolver/methods/aspire/direct.py similarity index 87% rename from examples/aspire_robosuite/utils/agent/agent.py rename to hermit/agent/evolver/methods/aspire/direct.py index b942cd73..9f647f62 100644 --- a/examples/aspire_robosuite/utils/agent/agent.py +++ b/hermit/agent/evolver/methods/aspire/direct.py @@ -1,17 +1,19 @@ -"""ASPIRE's actor on one Robosuite task: write a Code-as-Policy program, run it in the robot execution engine on the -task's development seeds, diagnose from the per-primitive traces and keyframes, repair, and keep the best version +"""ASPIRE's actor on one task of the cap-harness engine, in direct chat: write a Code-as-Policy program, run it in +the robot execution engine on the task's development seeds, diagnose from the per-primitive traces and keyframes, +repair, and keep the best version (ASPIRE §2.1, the iterative-debugging loop of NVlabs/ASPIRE .claude/skills/iterative-debugging/subagent-prompt.md); optionally continue with the evolutionary search over the program (ASPIRE §2.3, Algorithm 1; .claude/skills/evosearch/); then write the findings the coordinator distills skills from. The judge runs the -selected program on the held-out seeds (utils/benchmark/judge.py). +selected program on the held-out seeds (the engine's utils/benchmark/judge.py). The adapter is the shared skill library: its context (ADAPTER.md, and the skills it exports) is the "Shared skill library" section of the actor's instructions, the only thing that carries from one task to the next. adapter=None runs without one. Every model call is one Direct call (Layer 1, docs/framework/agent-layers.md); the loop is this file's. -The robot execution engine is ASPIRE's harness in its own venv (sim.py), started in this process: this actor needs -the GPU, so its configs run it on the host (environment.runner: host), a comparison, not the method. +The robot execution engine is ASPIRE's harness in its own venv (the engine's sim.py, agent.params.engine: e.g. +../../cap_harness/utils/agent/sim.py), started in this process: this actor needs the GPU, so its configs run it on +the host (environment.runner: host), a comparison, not the method. Output: the development rate of the selected program. Log: a compact directory (every program version, each sweep's records and per-seed digests, the keyframes the model was shown, findings.md, skill_report.json, the transcript); ASPIRE's full @@ -20,6 +22,7 @@ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor +import importlib.util import json import os from pathlib import Path @@ -30,12 +33,9 @@ import time from hermit.agent.acting import Agent -from hermit.task import RunOutput +from hermit.common import RunOutput HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE)) -import sim # noqa: E402 -from sim import seeds_of # noqa: E402,F401 DEFAULT_MODEL = "fireworks/accounts/fireworks/models/glm-5p3-flash" PROMPTS = HERE / "prompts" @@ -75,10 +75,25 @@ def json_of(reply: str): return None +def engine_module(path: Path): + """The engine's sim.py (agent.params.engine), loaded from its own folder once.""" + path = Path(path).resolve() + name = f"_cap_sim_{abs(hash(str(path)))}" + if name not in sys.modules: + if str(path.parent) not in sys.path: + sys.path.insert(0, str(path.parent)) + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return sys.modules[name] + + class AspireActor(Agent): - """`agent.name: ../utils/agent/agent.py:AspireActor` + """`agent.name: hermit.agent.evolver.methods.aspire.direct:AspireActor` - params: aspire_root (an ASPIRE checkout with .venv-robosuite), sim_env (the engine's environment), model, + params: engine (the engine's sim.py, relative to the config), aspire_root (an ASPIRE checkout with + .venv-robosuite), sim_env (the engine's environment), model, effort, max_tokens, debug_seeds (a task row's own seeds win; its held-out seeds are the row's, for the judge), max_versions, search (debug | evo), evo {k, rounds, threshold, qualify_below, parents}, workers, max_steps, failed_shown, keyframes, evidence_chars, @@ -133,18 +148,23 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput: root = p.get("aspire_root") or os.environ.get("ASPIRE_ROOT") if not root: raise ValueError("agent.params.aspire_root (or ASPIRE_ROOT): an ASPIRE checkout") + if not p.get("engine"): + raise ValueError("agent.params.engine: the engine's sim.py (e.g. ../../cap_harness/utils/agent/sim.py)") + sim = engine_module(self.path(os.path.expandvars(str(p["engine"])))) engine = sim.Engine(self.path(os.path.expandvars(root)), env=p.get("sim_env"), workers=int(p.get("workers", 4)), max_steps=p.get("max_steps"), timeout_s=float(p.get("sweep_timeout_s", 3600))) # A task row's own settings win over agent.params: the verifier's rows carry its seeds and budget. p = {**p, **{k: task[k] for k in ROW_KEYS if k in task}} - debug = seeds_of(p.get("debug_seeds") or "101-125") + if task.get("rerun_of"): # a rerun with the final library (ASPIRE §2.4): no findings to distill + p["findings"] = False + debug = sim.seeds_of(p.get("debug_seeds") or "101-125") if not debug: raise ValueError(f"{task['id']}: no development seeds") stamp = time.strftime("%Y%m%dT%H%M%S") + f"-{time.time_ns() % 10**6:06d}" log = self.log_dir() / f"{task['id']}-{stamp}" scratch = self.path(p.get("scratch") or (self.log_dir() / "scratch")) / f"{task['id']}-{stamp}" - run = _Run(self, engine, task, log, scratch, debug, self._skills(adapter, task), p) + run = _Run(self, sim, engine, task, log, scratch, debug, self._skills(adapter, task), p) try: run.explore() @@ -164,12 +184,14 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput: rate = best.passed / len(debug) # the development rate; the judge runs the held-out seeds (Stage 2) role = "verifier" if p.get("role") == "verifier" else "actor" - # the facts in the Codex actor's schema (task.py:collect), which the judge and the coordinator read; a - # verifier's verdict is ASPIRE's rule on its own seeds (>= 4 of 5) + # the facts in the Codex actor's schema (actor.py:collect), which the judge and the coordinator read; a + # verifier's verdict is ASPIRE's rule on its own seeds (>= 4 of 5), and its program counts only when verified facts = {"role": role, "suite": task["suite"], "findings": findings, "skill_report": report, "fix_code": best.source, "unseen": None, "max_steps": p.get("max_steps"), **({"verification": {"verdict": "verified" if best.passed >= 0.8 * len(debug) else "refuted"}} if role == "verifier" else {})} + if role == "verifier" and facts["verification"]["verdict"] != "verified": + facts.update(not_scored=f"verifier verdict {facts['verification']['verdict']}", fix_code=None) extra = {"task": facts, "selected": best.name, "debug_passed": best.passed, "debug_total": len(debug), "versions": [v.summary() for v in run.versions], "usage": run.totals} return RunOutput(output=rate, log=log, wall_s=time.time() - started, **run.spent(), @@ -213,9 +235,9 @@ def summary(self) -> dict: class _Run: """The state of one task's debugging: the versions, the transcript, the log directory.""" - def __init__(self, actor: AspireActor, engine: sim.Engine, task: dict, log: Path, scratch: Path, + def __init__(self, actor: AspireActor, sim, engine, task: dict, log: Path, scratch: Path, debug: list[int], skills: str, params: dict): - self.actor, self.engine, self.task, self.log, self.scratch = actor, engine, task, log, scratch + self.actor, self.sim, self.engine, self.task, self.log, self.scratch = actor, sim, engine, task, log, scratch self.debug, self.skills = debug, skills self.suite, self.task_id = task["suite"], int(task.get("task_id", 0)) self.versions: list[_Version] = [] @@ -279,7 +301,7 @@ def sweep(self, name: str, source: str, seeds: list[int], *, parent=None, note=" out = self.log / "sweeps" / name out.mkdir(parents=True, exist_ok=True) (out / "records.json").write_text(json.dumps(records, indent=1)) - (out / "digest.txt").write_text("\n\n".join(sim.digest(r) for r in records)) + (out / "digest.txt").write_text("\n\n".join(self.sim.digest(r) for r in records)) return _Version(name, source, records, parent, note) def evidence(self, version: _Version) -> list[dict]: @@ -292,15 +314,15 @@ def evidence(self, version: _Version) -> list[dict]: parts: list[dict] = [] budget = int(self.params.get("evidence_chars", 40000)) # per version shown: the digests share it for record in shown: - text.append(_clip_middle(sim.digest(record), max(2000, budget // max(1, len(shown))))) + text.append(_clip_middle(self.sim.digest(record), max(2000, budget // max(1, len(shown))))) n_frames = int(self.params.get("keyframes", 2)) for record in shown[:2]: - for label, png in sim.keyframes(record.get("run_dir"), limit=n_frames): + for label, png in self.sim.keyframes(record.get("run_dir"), limit=n_frames): kept = self.log / "keyframes" / version.name / f"seed{record['seed']}-{label}.png" kept.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(png, kept) parts.append({"type": "text", "text": f"keyframe {version.name} seed {record['seed']} {label}:"}) - parts.append(sim.image_part(kept)) + parts.append(self.sim.image_part(kept)) return [{"type": "text", "text": "\n\n".join(text)}, *parts] # --- Stage 0: explore once ---------------------------------------------------------------------------------- @@ -308,14 +330,14 @@ def explore(self): probe = (Path(self.engine.root) / ".claude/skills/iterative-debugging/scripts/scene_snapshot.py").read_text() version = self.sweep("probe", probe, self.debug[:1]) record = version.records[0] - self.scene = sim.digest(record, call_chars=1500) + self.scene = self.sim.digest(record, call_chars=1500) for camera in ("agentview", "robot0_eye_in_hand"): - for label, png in sim.keyframes(record.get("run_dir"), camera=camera, limit=1): + for label, png in self.sim.keyframes(record.get("run_dir"), camera=camera, limit=1): kept = self.log / "keyframes" / "probe" / f"{camera}-{label}.png" kept.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(png, kept) self.scene_images += [{"type": "text", "text": f"initial scene, camera {camera}:"}, - sim.image_part(kept)] + self.sim.image_part(kept)] def _task_text(self) -> str: return (f"Task: robosuite suite `{self.suite}`, task id {self.task_id}. Development seeds: {self.debug}.\n\n" @@ -401,7 +423,7 @@ def evolve(self, start: _Version, k: int = 8, rounds: int = 5, threshold: float def findings(self, best: _Version): template = (PROMPTS / "findings.md").read_text().split("-->", 1)[1].strip() history = self.changelog() + "\n\n" + "\n\n".join( - f"{v.name} failures:\n" + "\n".join(sim.digest(r, call_chars=200) for r in v.failed()[:2]) + f"{v.name} failures:\n" + "\n".join(self.sim.digest(r, call_chars=200) for r in v.failed()[:2]) for v in self.versions if v.failed()) prompt = template.format(selected=best.name, debug_passed=best.passed, debug_total=len(best.records), suite=self.suite, history=history[-60000:], program=best.source) diff --git a/examples/aspire_robosuite/utils/agent/prompts/actor_system.md b/hermit/agent/evolver/methods/aspire/prompts/actor_system.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/actor_system.md rename to hermit/agent/evolver/methods/aspire/prompts/actor_system.md diff --git a/examples/aspire_robosuite/utils/agent/prompts/codex_header.md b/hermit/agent/evolver/methods/aspire/prompts/codex_header.md similarity index 97% rename from examples/aspire_robosuite/utils/agent/prompts/codex_header.md rename to hermit/agent/evolver/methods/aspire/prompts/codex_header.md index 4c10951d..64b2afdc 100644 --- a/examples/aspire_robosuite/utils/agent/prompts/codex_header.md +++ b/hermit/agent/evolver/methods/aspire/prompts/codex_header.md @@ -17,7 +17,7 @@ things in it map onto your workspace as follows. Where the template and this pag `bin/sim` asks the harness's simulator to run them, with the benchmark ({benchmark}), suite, task id, init mode, GPU, render settings, `--flat-run-dir` and the results file (`/results.jsonl`, with the logs under `/logs/`); the runs appear in your workspace. It accepts only your assigned seeds, and the unseen-check seeds once. Perception, IK and planning services are already running; you need not check them. -- **The shared skill library** is the "Shared skill library" section of your AGENTS.md, and the same files are under +- **The shared skill library** is the "Shared skill library" section of your {context_file}, and the same files are under `hermit/aspire/skills/` in this workspace. Wherever the template says `.claude/skills/iterative-debugging/skills/` (or `skills/.md`), it means these, except `skills/task-exploration.md`, which is `docs/task-exploration.md`. {proposed_line} diff --git a/examples/aspire_robosuite/utils/agent/prompts/evosearch_header.md b/hermit/agent/evolver/methods/aspire/prompts/evosearch_header.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/evosearch_header.md rename to hermit/agent/evolver/methods/aspire/prompts/evosearch_header.md diff --git a/examples/aspire_robosuite/utils/agent/prompts/findings.md b/hermit/agent/evolver/methods/aspire/prompts/findings.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/findings.md rename to hermit/agent/evolver/methods/aspire/prompts/findings.md diff --git a/hermit/agent/evolver/methods/aspire/rows.py b/hermit/agent/evolver/methods/aspire/rows.py new file mode 100644 index 00000000..9626f915 --- /dev/null +++ b/hermit/agent/evolver/methods/aspire/rows.py @@ -0,0 +1,97 @@ +"""ASPIRE's own task rows beside a benchmark's: for every task, its verifier. A benchmark of the cap-harness engine +writes only its tasks and their reruns (its utils/benchmark/prepare.py); an ASPIRE run reads them and these, which +this writes into a folder of its own (the ASPIRE configs' `tasks/` and `splits/`), from the benchmark's prepared +tasks and splits, with the engine's task writer (`--engine`: its utils/benchmark/tasks.py, which reads the tasks back +as rows and writes a row as a task directory; agents never import the task layer). The ASPIRE folder's tasks.sh runs +it after the benchmark's prepare.py (the engine's launch.sh): + + python -m hermit.agent.evolver.methods.aspire.rows --benchmark ../utils/benchmark --out . --verify-seeds 101-105 \ + --engine ../../cap_harness/utils/benchmark/tasks.py + +- `verify-`: the verifier of each task (ASPIRE verifier-prompt.md): a fresh session that re-solves the task from + the library and the proposals alone, on its own development seeds (`--verify-seeds`, ASPIRE's first five) in at + most two programs (the simulator service's attempt budget, `max_programs`), with no findings; the judge scores its + program on those seeds. Only the coordinator runs them (the configs' `data.val`, through run_agent). + +The benchmark's tasks and splits are copied as they are (its reruns are ASPIRE's Phase 3 rerun with the final +library, §2.4), and splits/verify.txt holds every verifier. +""" +from __future__ import annotations + +import argparse +from functools import lru_cache +import importlib.util +from pathlib import Path +import re +import shutil +import sys + +SEEDS = ("debug_seeds", "unseen_seeds", "eval_seeds") + + +def seeds_of(value) -> set[int]: + """The seeds a set names: `101-125`, `1,3,5`, or none.""" + out: set[int] = set() + for part in re.split(r"[,\s]+", str(value or "").strip()): + if not part or part == "none": + continue + low, _, high = part.partition("-") + out.update(range(int(low), int(high or low) + 1)) + return out + + +@lru_cache(maxsize=None) +def engine(path: str): + """The engine's task writer (its utils/benchmark/tasks.py), loaded from its own folder once.""" + module_path = Path(path).resolve() + if str(module_path.parent) not in sys.path: + sys.path.insert(0, str(module_path.parent)) + spec = importlib.util.spec_from_file_location(f"_cap_tasks_{abs(hash(str(module_path)))}", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def what(row: dict) -> str: + return f"{row.get('benchmark') or 'robosuite'} suite `{row['suite']}`, task id {row['task_id']}" + + +def verifier(row: dict, seeds: str) -> dict: + base = {k: v for k, v in row.items() if k not in SEEDS} + return {**base, "id": f"verify-{row['id']}", "role": "verifier", "debug_seeds": seeds, "eval_seeds": seeds, + "score_label": "verifier", "max_programs": 2, "max_versions": 2, "search": "debug", "findings": False} + + +def write(benchmark: Path, out: Path, verify_seeds: str, engine_tasks: Path) -> None: + eng = engine(str(engine_tasks)) + rows = eng.rows(benchmark / "tasks") + order = (benchmark / "splits" / "all.txt").read_text().split() # the benchmark's own order of its tasks + tasks = {tid: rows[tid] for tid in order} + for row in tasks.values(): + clash = seeds_of(verify_seeds) & seeds_of(row.get("eval_seeds")) + if clash: + raise ValueError(f"{row['id']}: the verifier's seeds overlap the held-out seeds: {sorted(clash)[:10]}") + for name in ("tasks", "splits"): # this script's own output: rewritten whole + if (out / name).exists(): + shutil.rmtree(out / name) + shutil.copytree(benchmark / "tasks", out / "tasks") + for row in tasks.values(): + text = f"Verify the proposals for {what(row)}: solve it again from the library and the proposals alone." + eng.write_task(out / "tasks", verifier(row, verify_seeds), text) + shutil.copytree(benchmark / "splits", out / "splits") + (out / "splits" / "verify.txt").write_text("".join(f"verify-{i}\n" for i in tasks)) + print(f"{len(tasks)} verifiers -> {out / 'tasks'}") + + +def main(argv=None) -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--benchmark", type=Path, required=True, help="the benchmark's utils/benchmark (tasks/, splits/)") + ap.add_argument("--out", type=Path, required=True, help="where tasks/ and splits/ go") + ap.add_argument("--verify-seeds", required=True, help="the verifier's development seeds (ASPIRE's first five)") + ap.add_argument("--engine", type=Path, required=True, help="the engine's task writer (utils/benchmark/tasks.py)") + a = ap.parse_args(argv) + write(a.benchmark, a.out, a.verify_seeds, a.engine) + + +if __name__ == "__main__": + main() diff --git a/tests/README.md b/tests/README.md index c4fc2885..4f45e3ba 100644 --- a/tests/README.md +++ b/tests/README.md @@ -33,6 +33,10 @@ selected with the benchmark’s `evosafe` example (plus `core` and the existing installed instead; model responses come from a local HTTP server. The Docker hook check also skips without an engine and the `python:3.12-slim` image. +The cap-harness engine that Robosuite and LIBERO share has its tests in `examples/cap_harness/utils/tests/` (its +stand-ins, `cap_fakes.py`, are ASPIRE's tests' too); a change to the engine runs the groups of the methods with an +example on one of its benchmarks (`ENGINES` in `selection.py`), and `core`. + CI (`.github/workflows/tests.yml`) runs `--changed` on a pull request and the whole suite on main, daily and on demand. diff --git a/tests/core/test_cli_agent.py b/tests/core/test_cli_agent.py index 27c7e6ad..b12321b9 100644 --- a/tests/core/test_cli_agent.py +++ b/tests/core/test_cli_agent.py @@ -158,6 +158,15 @@ def test_a_run_with_the_tasks_hooks_records_the_hosts_log(fake): assert [s.kind for s in steps] == ["reasoning", "command", "command", "message"] +def test_a_task_function_may_be_named_by_its_module_and_a_prompt_stays_text(tmp_path): + """A method that brings its own task functions names them by module (`package.module:function`, ASPIRE's + hermit.agent.evolver.methods.aspire.actor); a prompt written as text is never read as one.""" + assert cli._task_function("os.path:basename", tmp_path)("a/b.txt") == "b.txt" + for text in ("Answer: yes", "Solve it:now", "see notes.md:3", "json:loads"): + assert cli._task_function(text, tmp_path) is None, text + assert cli._prompt({"prompt": "Solve it:now"}, tmp_path, {}) == "Solve it:now" + + def test_the_answer_pattern_and_why_there_is_none(fake): tmp_path = fake params = {"web_search": False, "prompt": "go", "answer": "(.*?)", "workspaces": "ws"} diff --git a/tests/core/test_environment.py b/tests/core/test_environment.py index 89064276..3bc6aa90 100644 --- a/tests/core/test_environment.py +++ b/tests/core/test_environment.py @@ -350,7 +350,12 @@ def test_no_shipped_config_runs_a_method_on_an_agent_that_lacks_what_it_needs(): supports = supports_of(name, path.parent, agent.get("params"), may_import=False) if supports is None and ":" in name: file, _, symbol = name.partition(":") - supports = _declared_supports((path.parent / file).resolve(), symbol) + if file.endswith(".py"): + source = (path.parent / file).resolve() + else: # `package.module:Class`: its source, not imported + import importlib.util + source = Path(importlib.util.find_spec(file).origin) + supports = _declared_supports(source, symbol) assert supports is not None and needs <= set(supports), f"{path.relative_to(root)}: {name} lacks {needs}" checked += 1 assert checked >= 10 # MWM, ASPIRE, ReHarness and AHE's examples diff --git a/tests/core/test_selection.py b/tests/core/test_selection.py index 941e695b..c48640c1 100644 --- a/tests/core/test_selection.py +++ b/tests/core/test_selection.py @@ -61,6 +61,10 @@ def test_a_benchmarks_shared_files_run_the_methods_with_an_example_there(): "ace", "ahe", "continual_harness", "harness2", "meta_harness_adapter", "opro", "optimizers", "reharness", "rrsi_adapter", "core"} assert groups_for("examples/servelearnbench/utils/x.py") == {"mwm", "core"} + # an engine several benchmarks run on: the methods with an example under any of them + assert groups_for("examples/cap_harness/utils/agent/sim.py") == {"aspire", "core"} + assert groups_for("examples/libero/utils/agent/task.py") == {"aspire", "core"} + assert groups_for("examples/robosuite/aspire/run.yaml") == {"aspire", "core"} groups, why = select(["hermit/cli.py", "examples/officeqa/rrsi_adapter/README.md", "docs/index.md"]) assert groups == {"core", "rrsi_adapter"} and why == {"core": "hermit/cli.py", "rrsi_adapter": "examples/officeqa/rrsi_adapter/README.md"} diff --git a/tests/methods/aspire/test_aspire.py b/tests/methods/aspire/test_aspire.py index 516e9cf6..cea73988 100644 --- a/tests/methods/aspire/test_aspire.py +++ b/tests/methods/aspire/test_aspire.py @@ -1,29 +1,58 @@ """ASPIRE: the aspire template (the skill library, its reader and its check), the coordinator's hook into the -Trainer (next_tasks), and the example's pieces: the Codex task functions (prompt assembly, the workspace and its -registration), the simulator service's rules as bin/sim meets them (a real service on a loopback port, over a fake -checkout), the judge's Stage 2 (a stand-in engine), and pure helpers; no simulator, no model, no network.""" -import hashlib +Trainer (next_tasks), ASPIRE's acting sessions on the cap-harness engine (actor.py: the prompt, the workspace beyond the +engine's, the facts the coordinator and the judge read), its verifier rows (rows.py), its direct-chat actor +(direct.py) and its configs on Robosuite and LIBERO. The engine is the stand-in of its own tests +(examples/cap_harness/utils/tests/cap_fakes.py): no simulator, no model, no network.""" import importlib.util import json import os from pathlib import Path import shutil -import subprocess import sys -import tempfile -import time from types import SimpleNamespace import pytest from hermit.agent.evolver.base import StepResult +from hermit.agent.evolver.methods.aspire import actor as A +from hermit.agent.evolver.methods.aspire import direct as D from hermit.agent.evolver.methods.aspire import evolver as E +from hermit.agent.evolver.methods.aspire import rows as Rw from hermit.spec import check_spec ROOT = Path(__file__).resolve().parents[3] TEMPLATE = ROOT / "hermit" / "adapter" / "templates" / "aspire" -EXAMPLE = ROOT / "examples" / "aspire_robosuite" / "utils" -LIBERO = ROOT / "examples" / "libero" # LIBERO on the same engine: its tasks and its methods' configs +ENGINE = ROOT / "examples" / "cap_harness" / "utils" +ROBOSUITE = ROOT / "examples" / "robosuite" +LIBERO = ROOT / "examples" / "libero" + + +def _fakes(): + spec_ = importlib.util.spec_from_file_location("cap_fakes", ENGINE / "tests" / "cap_fakes.py") + module = importlib.util.module_from_spec(spec_) + spec_.loader.exec_module(module) + return module + + +F = _fakes() +tmp_path = F.tmp_path # a fixture: a directory outside /tmp, for the template check's sandbox +_load, fake_aspire, _Service, _workspace, _sim = F._load, F.fake_aspire, F._Service, F._workspace, F._sim +ROW, LIBERO_ROW = F.ROW, F.LIBERO_ROW +# ASPIRE's verifier of ROW, as rows.py derives it +VERIFY = Rw.verifier(ROW, "101-105") +# The engine as ASPIRE's configs name it (agent.params.engine), relative to the config's folder +PARAMS = {"engine": "../../cap_harness/utils/agent/task.py"} +BASE = ROBOSUITE / "aspire" + + +@pytest.fixture +def service(tmp_path, monkeypatch): + """The engine's simulator service, serving ASPIRE's verifier row besides the benchmarks' rows.""" + svc = _Service(tmp_path, fake_aspire(tmp_path), rows=(VERIFY,)) + monkeypatch.setenv("ASPIRE_SIM_URL", svc.url) + yield svc + svc.close() + ENTRY = """## Re-localize after the approach **Failure signature.** The gripper closes on air: the cube moved when the arm passed over it. @@ -40,17 +69,6 @@ """ -@pytest.fixture -def tmp_path(): - """A directory outside /tmp: the template check runs under bubblewrap with an empty tmpfs at /tmp.""" - base = Path.home() / ".cache" / "hermit-tests" - base.mkdir(parents=True, exist_ok=True) - path = Path(tempfile.mkdtemp(dir=base)) - yield path - if not os.environ.get("KEEP_TEST_DIRS"): - shutil.rmtree(path, ignore_errors=True) - - def seed(tmp: Path) -> Path: adapter = tmp / "adapter" shutil.copytree(TEMPLATE, adapter, ignore=shutil.ignore_patterns("__pycache__")) @@ -62,8 +80,6 @@ def spec(path) -> dict: return {"passed": r.passed, "issues": [{"path": i.path, "problem": i.problem} for i in r.issues]} -# ---------- the template ---------- - def test_template_root_passes_its_own_check(): assert check_spec(TEMPLATE, TEMPLATE).passed @@ -140,8 +156,6 @@ def test_reader_leaves_out_the_staged_proposals(tmp_path): assert "Re-localize" not in module.library()["text"] -# ---------- the coordinator ---------- - def test_next_tasks_reads_only_a_regular_file(tmp_path): """output/next_tasks.json is the learner's, read on the host: a link or a pipe in its place names no task.""" path = tmp_path / "next_tasks.json" @@ -155,6 +169,7 @@ def test_next_tasks_reads_only_a_regular_file(tmp_path): os.mkfifo(path) assert E.next_tasks(path) is None # returns at once, not blocked on the pipe + def test_the_coordinator_chooses_tasks_and_returns_its_named_next_task(tmp_path, monkeypatch): from hermit.agent.evolver.agent import AgentEvolver out = tmp_path / "output" @@ -179,33 +194,21 @@ def test_the_coordinator_prompt_carries_the_guides_sections(): assert placeholder in text -# ---------- the example's actor helpers ---------- - -def _actor_module(): - sys.path.insert(0, str(EXAMPLE / "agent")) - try: - spec_ = importlib.util.spec_from_file_location("aspire_actor", EXAMPLE / "agent" / "agent.py") - module = importlib.util.module_from_spec(spec_) - spec_.loader.exec_module(module) - return module - finally: - sys.path.remove(str(EXAMPLE / "agent")) - - def test_actor_reads_seeds_and_the_outermost_program_fence(): - A = _actor_module() - assert A.seeds_of("101-103 7") == [101, 102, 103, 7] - assert A.seeds_of([1, "3-4"]) == [1, 3, 4] + sim = D.engine_module(ENGINE / "agent" / "sim.py") # the engine the direct actor runs its programs on + assert sim.seeds_of("101-103 7") == [101, 102, 103, 7] + assert sim.seeds_of([1, "3-4"]) == [1, 3, 4] + assert D.engine_module(ENGINE / "agent" / "sim.py") is sim # loaded once reply = "diagnosis\n```python\ns = '''\n```python\nx\n```\n'''\nresult = {}\n```\ntrailing" - assert A.program_of(reply).endswith("result = {}\n") - assert A.program_of("no program") is None - assert A.json_of('text ```json\n{"a": 1}\n```') == {"a": 1} + assert D.program_of(reply).endswith("result = {}\n") + assert D.program_of("no program") is None + assert D.json_of('text ```json\n{"a": 1}\n```') == {"a": 1} def test_the_direct_actor_keeps_its_models_settings_and_an_unknown_cost_unknown(monkeypatch): import threading from hermit.agent.model import llm - A = _actor_module() + A = D specs = [] monkeypatch.setattr(llm.ChatClient, "from_spec", classmethod(lambda cls, spec, **k: specs.append(spec))) monkeypatch.setattr("hermit.agent.base.interface.make", lambda *a, **k: object()) @@ -223,7 +226,8 @@ def test_the_direct_actor_keeps_its_models_settings_and_an_unknown_cost_unknown( def test_a_failed_direct_task_keeps_what_it_spent(tmp_path, monkeypatch): - A = _actor_module() + A = D + sim = D.engine_module(ENGINE / "agent" / "sim.py") class Engine: def __init__(self, root, **k): @@ -235,9 +239,12 @@ def __init__(self, root, **k): def explore(run): # one paid call, then an outage past the retries run.account([SimpleNamespace(input_tokens=10, output_tokens=5, cost_usd=0.5)], "explore", "r", "", "stop") raise RuntimeError("model unavailable: 503") - monkeypatch.setattr(A.sim, "Engine", Engine) + monkeypatch.setattr(sim, "Engine", Engine) monkeypatch.setattr(A._Run, "explore", explore) - actor = A.AspireActor({"aspire_root": str(tmp_path), "scratch": str(tmp_path / "scratch")}, base=tmp_path) + actor = A.AspireActor({"aspire_root": str(tmp_path), "scratch": str(tmp_path / "scratch"), + "engine": str(ENGINE / "agent" / "sim.py")}, base=tmp_path) + with pytest.raises(ValueError, match="agent.params.engine"): # it names the engine it runs on + A.AspireActor({"aspire_root": str(tmp_path)}, base=tmp_path).run(dict(ROW), None) monkeypatch.setattr(actor, "log_dir", lambda: tmp_path / "logs") monkeypatch.setattr(actor, "session", lambda adapter: None) out = actor.run(dict(ROW), None) @@ -249,887 +256,189 @@ def explore(run): # one paid call, the run_one(dict(ROW), None, lambda task, adapter: actor.run(task, adapter), lambda task, out: {"score": 0}) -def test_sim_compact_makes_traces_readable(): - sim = _sim_module() - value = {"$type": "Pose", "position": {"$array_inline": [0.123456, 1.0, 2.0], "dtype": "float64", "shape": [3]}, - "obs": {"$type": "Observation", "cameras": {}}, "big": {"$array_inline": list(range(40)), "shape": [40]}} - out = sim.compact(value) - assert out == {"type": "Pose", "position": [0.1235, 1.0, 2.0], "obs": "", - "big": ""} - - -def test_the_failure_keyframe_is_the_failing_calls_own(tmp_path): - sim = _sim_module() - run = tmp_path / "run" - (run / "source").mkdir(parents=True) - trace = [{"tool": "open_gripper", "ok": True}] + [{"tool": "move_to_pose", "ok": i != 2} for i in range(4)] - (run / "source" / "program-result.json").write_text(json.dumps({"calls": trace})) - names = ["reset", "before-open-gripper", "after-open-gripper"] - for i in range(3): # the third move raised: no after-frame of its own - names += ["before-move-to-pose"] + (["after-move-to-pose"] if i < 2 else []) - for n, name in enumerate(names, 1): - d = run / "media" / "keyframes" / f"{n:06d}-{name}" - d.mkdir(parents=True) - (d / "agentview.png").write_bytes(b"png") - picked = [label for label, _ in sim.keyframes(run, limit=1)] - assert picked == [f"{names.index('before-move-to-pose', 7) + 1:06d}-before-move-to-pose"] - - -# ---------- the Codex actor's task functions ---------- - -def _load(name: str, path: Path): - sys.path.insert(0, str(path.parent)) - try: - spec_ = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec_) - spec_.loader.exec_module(module) - return module - finally: - sys.path.remove(str(path.parent)) - - -def _task_module(): - return _load("aspire_task", EXAMPLE / "agent" / "task.py") - - -def fake_aspire(tmp: Path) -> Path: - """The files of an ASPIRE checkout the task functions read: the two templates, the docs, the snapshot program.""" - root = tmp / "ASPIRE" - skills = root / ".claude" / "skills" / "iterative-debugging" - (skills / "scripts").mkdir(parents=True) - (root / "docs").mkdir() - marks = ("", - "") - (skills / "subagent-prompt.md").write_text(f"intro\n{marks[0]}\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" - f"## What You Are\nfix-loop text\n{marks[1]}\n") - (skills / "verifier-prompt.md").write_text(f"intro\n{marks[0]}\nYou are verifying. Seeds 51-55.\n{marks[1]}\n") - (skills / "scripts" / "scene_snapshot.py").write_text("result = {}\n") - for name in ("api-reference.md", "run-artifacts.md", "robosuite.md"): - (root / "docs" / name).write_text(f"# {name}\nrun examples/robosuite/cube_lifting_seed1.py\nkept\n") - (skills / "skills").mkdir() - (skills / "skills" / "task-exploration.md").write_text("# exploration\n") - evosearch = root / ".claude" / "skills" / "evosearch" - (evosearch / "skills").mkdir(parents=True) - (evosearch / "subagent-prompt.md").write_text("intro\n\n```\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" - "## EVAL SET LOCKOUT\nsearch text\n```\n") - (evosearch / "skills" / "evosearch-iteration.md").write_text("# iteration\n") - (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\nkept\n\n## Task examples\n\n| `close_microwave.py` | " - "`libero_90:33` | Close the open microwave. |\n") - for venv in (".venv-robosuite", ".venv-libero"): - harness = root / venv / "bin" / "cap-harness" - harness.parent.mkdir(parents=True) - harness.write_text(FAKE_HARNESS.format(python=sys.executable)) - harness.chmod(0o755) - return root - - -# cap-harness without a simulator: the run's files where ASPIRE writes them (run.json, results.jsonl naming the run's -# directory, outcome.json), the program and settings it was given; a program saying SLEEP runs until stopped, and its -# `# WRITE ` / `# READ ` lines are tried as a program that escaped ASPIRE's validator would (in run.json), -# and after the run `# LINK ` leaves a link in its output, `# RUNDIR ` names another run_dir and -# `# CHMOD ` changes a mode (last). -FAKE_HARNESS = """#!{python} -import argparse, json, os, pathlib, sys, time -ap = argparse.ArgumentParser() -for flag in ("--benchmark", "--suite", "--task-id", "--program", "--output-root", "--max-steps", "--seed", "--workers", - "--results-jsonl", "--log-dir", "--init-mode"): - ap.add_argument(flag) -ap.add_argument("--seeds", nargs="+") -ap.add_argument("--flat-run-dir", action="store_true") -ap.add_argument("--no-videos", action="store_true") -ap.add_argument("cmd") -a = ap.parse_args() -code = pathlib.Path(a.program).read_text() -if "SLEEP" in code: - time.sleep(60) -tried = {{}} -for line in code.splitlines(): - what, _, path = line.partition(" ")[2].partition(" ") - try: - if what == "WRITE": - pathlib.Path(path).write_text("escaped") - elif what == "READ": - pathlib.Path(path).read_text() - else: - continue - tried[path] = "done" - except OSError as exc: - tried[path] = type(exc).__name__ -out = pathlib.Path(a.output_root) -run = out / "run-1" -run.mkdir(parents=True, exist_ok=True) -(out / "run.json").write_text(json.dumps({{"program": a.program, "max_steps": a.max_steps, "code": code, - "env": sorted(os.environ), "home": os.environ.get("HOME"), - "tried": tried}})) -(run / "outcome.json").write_text(json.dumps({{"task_success": True}})) -rows = [{{"seed": int(s), "run_dir": str(run)}} for s in (a.seeds or [a.seed])] -for line in code.splitlines(): - what, _, rest = line.partition(" ")[2].partition(" ") - if what == "LINK": - name, target = rest.split() - (out / name).unlink(missing_ok=True) - (out / name).symlink_to(target) - elif what == "RUNDIR": - rows = [dict(r, run_dir=rest) for r in rows] -pathlib.Path(a.results_jsonl or out / "results.jsonl").write_text("".join(json.dumps(r) + chr(10) for r in rows)) -for line in code.splitlines(): - what, _, rest = line.partition(" ")[2].partition(" ") - if what == "CHMOD": - name, mode = rest.split() - os.chmod(out / name, int(mode, 8)) -print("ran", a.program, "into", out, "max_steps", a.max_steps) -""" - - -def _bwrap_works() -> bool: - """A bubblewrap that can make its namespaces here: every simulator run is confined (sim.confine).""" - if not shutil.which("bwrap"): - return False - return subprocess.run(["bwrap", "--unshare-user", "--unshare-net", "--ro-bind", "/", "/", "true"], - capture_output=True).returncode == 0 - - -needs_bwrap = pytest.mark.skipif(not _bwrap_works(), reason="needs a working bubblewrap: simulator runs are confined") - - -LIBERO_ROW = {"id": "libero_90-t33", "benchmark": "libero-pro", "suite": "libero_90", "task_id": 33, - "debug_seeds": "51-53", "unseen_seeds": "66", "eval_seeds": "1-3"} -ROW = {"id": "robosuite-cube_lifting", "suite": "cube_lifting", "task_id": 0, "debug_seeds": "101-103", - "unseen_seeds": "126", "eval_seeds": "1-3"} -VERIFY = {"id": "verify-robosuite-cube_lifting", "suite": "cube_lifting", "task_id": 0, "role": "verifier", - "debug_seeds": "101-105", "eval_seeds": "none"} - - def test_the_actor_prompt_is_the_header_the_assignment_and_aspires_template(tmp_path): - T = _task_module() - params = {"aspire_root": str(fake_aspire(tmp_path))} - text = T.prompt(ROW, params, tmp_path) + T = A + params = {**PARAMS, "aspire_root": str(fake_aspire(tmp_path))} + text = T.prompt(ROW, params, BASE) assert "fix-loop subagent" in text and "`$TASK_DIR` is `robosuite/cube_lifting/task_0`" in text assert "DEV_SEEDS: 101-103" in text and "UNSEEN_SEEDS: 126" in text and "HELDOUT_COUNT: 3" in text assert "## What You Are\nfix-loop text" in text and "BENCHMARK: " not in text # ours replaces theirs # Part 2, the evolutionary search, follows the fix loop with its own page and its template after its assignment assert text.index("fix-loop text") < text.index("# Part 2: the evolutionary search") < text.index("search text") assert "`TASK_DIR` is `evosearch/robosuite/cube_lifting/task_0`" in text - assert "# Part 2" not in T.prompt(ROW, {**params, "evosearch": False}, tmp_path) - verifier = T.prompt(VERIFY, params, tmp_path) + assert "# Part 2" not in T.prompt(ROW, {**params, "evosearch": False}, BASE) + verifier = T.prompt(VERIFY, params, BASE) assert "skill verifier" in verifier and "Development seeds: **101-105**" in verifier assert "You are verifying. Seeds 51-55." in verifier and "# Part 2" not in verifier - - -class _Service: - """The simulator service on a free loopback port, over a fake checkout whose cap-harness writes a run's files - without a simulator (FAKE_HARNESS).""" - - def __init__(self, tmp: Path, root: Path, **options): - import threading - from http.server import ThreadingHTTPServer - S = _load("aspire_sim_service", EXAMPLE / "agent" / "sim_service.py") - self.sessions = tmp / "sessions" - self.sessions.mkdir() - self.service = service = S.Service(root, {r["id"]: r for r in (ROW, VERIFY, LIBERO_ROW)}, [self.sessions], - S.Ledger(tmp / "ledger.json"), {}, 2, **options) - self.server = ThreadingHTTPServer(("127.0.0.1", 0), S.handler(service)) - threading.Thread(target=self.server.serve_forever, daemon=True).start() - self.url = f"http://127.0.0.1:{self.server.server_address[1]}" - - def close(self): - self.server.shutdown() - - -@pytest.fixture -def service(tmp_path, monkeypatch): - svc = _Service(tmp_path, fake_aspire(tmp_path)) - monkeypatch.setenv("ASPIRE_SIM_URL", svc.url) - yield svc - svc.close() - - -def _workspace(service, task: dict, hexa: str = "56d9fea2") -> Path: - """A session's workspace where the framework puts it: //-/workspaces/<...>/work.""" - work = service.sessions / "rollout" / f"{task['id']}-{hexa}" / "workspaces" / f"{task['id']}-da10428a" / "work" - (work / "hermit" / "aspire" / "skills").mkdir(parents=True) - return work - - -def _register(url: str, body: dict) -> dict: - import urllib.request - request = urllib.request.Request(url + "/register", data=json.dumps(body).encode()) - with urllib.request.urlopen(request) as reply: - return json.loads(reply.read()) - - -def _sim(work: Path, *args): - return subprocess.run([str(work / "bin" / "sim"), *args], capture_output=True, text=True, cwd=work) - - -def test_prepare_registers_the_session_and_the_service_holds_its_rules(tmp_path, service): - T = _task_module() + libero = A.prompt(LIBERO_ROW, params, BASE) # LIBERO's tasks on the same engine + assert "BENCHMARK: libero-pro" in libero and "`$TASK_DIR` is `libero-pro/libero_90/task_33`" in libero + assert "the benchmark (libero-pro)" in libero and "`docs/libero-pro.md`" in libero and "robosuite" not in libero + assert "`TASK_DIR` is `evosearch/libero-pro/libero_90/task_33`" in libero and "DEV_SEEDS: 51-53" in libero + with pytest.raises(ValueError, match="agent.params.engine"): + A.prompt(ROW, {"aspire_root": params["aspire_root"]}, BASE) + # The library is in the context file of the session's CLI: Codex's AGENTS.md, Claude Code's CLAUDE.md + assert "section of your AGENTS.md" in text and "CLAUDE.md" not in text + claude = A.prompt(ROW, {**params, "backend": "claude"}, BASE) + assert "section of your CLAUDE.md" in claude and claude.replace("CLAUDE.md", "AGENTS.md") == text + + +def test_aspires_workspace_is_the_engines_with_its_assignment_its_search_page_and_the_proposals(tmp_path, service): + params = {**PARAMS, "aspire_root": str(tmp_path / "ASPIRE")} work = _workspace(service, ROW) - params = {"aspire_root": str(tmp_path / "ASPIRE")} - T.prepare(ROW, work, params, tmp_path) - for name in ("task.md", "docs/api-reference.md", "docs/evosearch-iteration.md", "docs/task-exploration.md", + A.prepare(ROW, work, params, BASE) + for name in ("task.md", "docs/evosearch-iteration.md", "docs/api-reference.md", "docs/task-exploration.md", "scripts/scene_snapshot.py", "bin/sim", "bin/sim.json"): - assert (work / name).is_file(), name - assert "examples/" not in (work / "docs" / "robosuite.md").read_text() - assert "kept" in (work / "docs" / "robosuite.md").read_text() - (work / "p.py").write_text("result = {}\n") - held_out = _sim(work, "run-batch", "--seeds", "1-3", "--program", "p.py", "--output-root", "out") - assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr - outside = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", str(tmp_path / "elsewhere")) - assert outside.returncode == 2 and "inside the workspace" in outside.stderr - _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u", "--init-mode", "saved") - again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") - assert again.returncode == 2 and "run once" in again.stderr - T.prepare(ROW, work, params, tmp_path) # a retry's prepare: the registration as it was - again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") - assert again.returncode == 2 and "run once" in again.stderr # what the session spent stays spent - - -def test_a_libero_task_runs_on_its_own_venv_seeded_with_its_own_docs(tmp_path, service, monkeypatch): - T = _task_module() - params = {"aspire_root": str(tmp_path / "ASPIRE")} - text = T.prompt(LIBERO_ROW, params, tmp_path) - assert "BENCHMARK: libero-pro" in text and "`$TASK_DIR` is `libero-pro/libero_90/task_33`" in text - assert "the benchmark (libero-pro)" in text and "`docs/libero-pro.md`" in text and "robosuite" not in text - assert "`TASK_DIR` is `evosearch/libero-pro/libero_90/task_33`" in text and "DEV_SEEDS: 51-53" in text - work = _workspace(service, LIBERO_ROW) - T.prepare(LIBERO_ROW, work, params, tmp_path) - page = (work / "docs" / "libero-pro.md").read_text() - assert "kept" in page and "Task examples" not in page and "microwave" not in page # no worked solution named - assert not (work / "docs" / "robosuite.md").exists() - ran = [] - monkeypatch.setattr(service.service, "_execute", lambda cmd, env, *a, **k: ran.append((cmd, env)) or (0, "", "")) - (work / "p.py").write_text("result = {}\n") - _sim(work, "run-batch", "--seeds", "51-52", "--program", "p.py", "--output-root", "out", "--init-mode", "saved") - cmd, env = ran[-1] - assert cmd[0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "cap-harness") - option = lambda flag: cmd[cmd.index(flag) + 1] - assert (option("--benchmark"), option("--suite"), option("--task-id")) == ("libero-pro", "libero_90", "33") - assert option("--init-mode") == "seeded" # the engine's, whatever the session passed - assert env["LIBERO_CONFIG_PATH"] == str(tmp_path / "ASPIRE" / ".libero") - assert env["TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD"] == "1" - _sim(work, "validate", "--program", "p.py") - assert ran[-1][0][0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "python") - # Robosuite's runs are as they were: its venv, no init mode - robosuite = _workspace(service, ROW) - T.prepare(ROW, robosuite, params, tmp_path) - (robosuite / "p.py").write_text("result = {}\n") - _sim(robosuite, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - cmd, env = ran[-1] - assert cmd[0].endswith(".venv-robosuite/bin/cap-harness") and "--init-mode" not in cmd - assert "LIBERO_CONFIG_PATH" not in env - - -def test_the_task_sandbox_is_shown_the_staged_docs_and_never_a_page_that_lost_its_cut(tmp_path, service): - """ASPIRE's benchmark pages name its worked programs per task: an actor that could read one would be handed - a strategy for its task. The configs grant the staged copies only, and a page without its cut is refused.""" - T = _task_module() - root = tmp_path / "ASPIRE" - (tmp_path / "rows.jsonl").write_text(json.dumps(ROW) + "\n" + json.dumps(LIBERO_ROW) + "\n") - staged = tmp_path / "staged" - done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", str(root), - "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], - capture_output=True, text=True) - assert done.returncode == 0, done.stderr - assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", - "run-artifacts.md"] - assert all("examples/" not in p.read_text() and "microwave" not in p.read_text() for p in staged.iterdir()) - stage = lambda: subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", - str(root), "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], - capture_output=True, text=True) - before = {p.name: p.stat().st_ino for p in staged.iterdir()} - assert stage().returncode == 0 and {p.name: p.stat().st_ino for p in staged.iterdir()} == before # in place - (staged / "extra").mkdir() # anything else there: staged anew, without it - (staged / "notes.bin").write_bytes(b"\xff\xfe") - assert stage().returncode == 0 - assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", - "run-artifacts.md"] - (staged / "libero-pro.md").write_text("# the staged page\n") - work = _workspace(service, LIBERO_ROW) - T.prepare(LIBERO_ROW, work, {"aspire_root": str(root), "aspire_docs": str(staged)}, tmp_path) - assert (work / "docs" / "libero-pro.md").read_text() == "# the staged page\n" # the staged copy, as it is - (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\n\n## Worked programs\n| `close_microwave.py` |\n") - with pytest.raises(ValueError, match="Task examples"): - _sim_module().workspace_docs(root, "libero-pro") - # No Codex config of either example grants a page of the checkout's own docs, and each loads - from hermit.config import Config - configs = [p for folder in (EXAMPLE.parent / "aspire", LIBERO / "aspire", LIBERO / "prompt_only") - for p in sorted(folder.glob("*.yaml"))] - assert len(configs) == 18 - for path in configs: - data = Config.load(path).data - read = [str(p) for p in (data.get("environment") or {}).get("read") or []] - assert not any("/docs/" in p for p in read), path - if "_direct" not in path.name: - assert "$ASPIRE_DOCS" in read and data["agent"]["params"]["aspire_docs"] == "$ASPIRE_DOCS", path - # Codex reads AGENTS.md (the adapter's context) only up to this: past it every session is refused - assert data["agent"]["params"]["project_doc_max_bytes"] >= 65536, path - if path.parent.name == "prompt_only": # no search: none staged, granted or scored - assert data["agent"]["params"]["evosearch"] is False and not any("evosearch" in p for p in read), path - if path.is_relative_to(EXAMPLE.parent): - for key in ("train", "val", "test"): # the splits prepare.py writes - name = Path(data["data"][key]).name - assert name in {"all.txt", "rerun.txt", "verify.txt", "smoke.txt", "smoke_rerun.txt"}, (path, name) - - -def test_the_engine_takes_the_benchmarks_venv_and_refuses_one_it_does_not_run(tmp_path, monkeypatch): - sim = _sim_module() - root = fake_aspire(tmp_path) - engine = sim.Engine(root, benchmark="libero-pro") - assert engine.cli == root.resolve() / ".venv-libero" / "bin" / "cap-harness" - assert engine._env()["LIBERO_CONFIG_PATH"] == str(root.resolve() / ".libero") - assert sim.init_args("libero-pro") == ["--init-mode", "seeded"] and sim.init_args(None) == [] - assert sim.Engine(root).cli.parent.parent.name == ".venv-robosuite" - commands = [] # the command line of a sweep, the judge's Stage 2 included - - def recorded(cmd, **_): - commands.append(cmd) - raise RuntimeError("recorded") - - monkeypatch.setattr(sim, "confine", recorded) - for each in (engine, sim.Engine(root)): - with pytest.raises(RuntimeError, match="recorded"): - each.run(tmp_path / "p.py", suite="s", seeds=[1, 2], out=tmp_path / "o") - at = commands[0].index("--init-mode") - assert commands[0][at:at + 2] == ["--init-mode", "seeded"] and "--init-mode" not in commands[1] - shutil.rmtree(root / ".venv-libero") - with pytest.raises(FileNotFoundError, match="bootstrap_libero.sh"): - sim.Engine(root, benchmark="libero-pro") - with pytest.raises(ValueError, match="behavior"): - sim.Engine(root, benchmark="behavior") - - -def test_the_libero_tasks_are_liberos_five_suites_and_liberos_pros_eighty_with_aspires_seeds(tmp_path, monkeypatch): - """The tasks and splits prepare.py writes, read as their rows (aspire_fields.py): what the simulator service, the - judge and every method's configs read.""" - from hermit.config import Config - from hermit.task import Benchmark - P = _load("libero_prepare", LIBERO / "utils" / "benchmark" / "prepare.py") - F = _load("aspire_fields", EXAMPLE / "agent" / "aspire_fields.py") - assert sum(P.SUITES.values()) == 130 and P.SUITES["libero_90"] == 90 - pro = [f"libero_{family}_{kind}" for family in ("object", "spatial", "goal", "10") for kind in ("swap", "task")] - assert list(P.PRO_SUITES) == pro and set(P.PRO_SUITES.values()) == {10} - monkeypatch.setattr(P, "HERE", tmp_path) - monkeypatch.setattr(sys, "argv", ["prepare.py"]) - P.main() - found = F.task_rows(tmp_path / "tasks") # each task directory as its row - rows = list(found.values()) - tasks = [found[i] for i in (tmp_path / "splits" / "all.txt").read_text().split()] - assert len(tasks) == 210 and len(rows) == 630 and len({r["id"] for r in rows}) == 630 # a suite and its - assert all(r["benchmark"] == "libero-pro" for r in rows) # perturbations share their task ids - assert list(found["libero_goal-t03"]) == ["id", "benchmark", "suite", "task_id", "debug_seeds", "unseen_seeds", - "eval_seeds"] # the row's keys, in its order - assert found["verify-libero_goal-t03"]["findings"] is False and found["rerun-libero_goal-t03"]["role"] == "rerun" - assert (tmp_path / "tasks" / "libero_goal-t03" / "instruction.md").is_file() - assert all((r["debug_seeds"], r["unseen_seeds"], r["eval_seeds"]) == ("51-65", "66-80", "1-50") for r in tasks) - split = lambda name: (tmp_path / "splits" / name).read_text().split() - by = lambda *suites: [r["id"] for r in tasks if r["suite"] in suites] - assert split("all.txt") == [r["id"] for r in tasks] - assert split("libero.txt") == by("libero_object", "libero_spatial", "libero_goal", "libero_10") - assert split("libero_90.txt") == by("libero_90") and split("libero_pro.txt") == by(*pro) - assert split("libero_pro_long.txt") == by("libero_10_swap", "libero_10_task") - assert split("smoke.txt") == split("libero.txt")[:2] and split("smoke_pro.txt") == split("libero_pro.txt")[:2] - assert split("libero_sample.txt") == [f"{s}-t{n:02d}" for s in ("libero_object", "libero_spatial", "libero_goal") - for n in (0, 5)] - for path in sorted((LIBERO / "aspire").glob("*.yaml")) + sorted((LIBERO / "prompt_only").glob("*.yaml")): - config = Config.load(path) - data = config.data["data"] - assert config.path(data["tasks"]) == (LIBERO / "utils" / "benchmark" / "tasks").resolve(), path - assert config.path(data["view"].split(":")[0]) == (EXAMPLE / "agent" / "aspire_fields.py").resolve(), path - monkeypatch.setitem(data, "tasks", str(tmp_path / "tasks")) # the tasks prepared here - for key in ("train", "val", "test"): - if data.get(key): - monkeypatch.setitem(data, key, str(tmp_path / "splits" / Path(data[key]).name)) - assert {t.id: t.given() for t in Benchmark.from_config(config).tasks.values()} == found, path - train, test = split(Path(data["train"]).name), split(Path(data["test"]).name) - assert train and set(train) <= {r["id"] for r in tasks}, path - assert test == [f"rerun-{i}" for i in train], path # the reruns are of the tasks it trained on - if data.get("val"): # the coordinator's verifiers (ASPIRE) - assert set(split(Path(data["val"]).name)) == {f"verify-{r['id']}" for r in tasks}, path - seeds, verify = P.seeds_from(P.PRESETS, []) - assert seeds == {"debug_seeds": "51-65", "unseen_seeds": "66-80", "eval_seeds": "1-50"} and verify == "51-55" - assert P.seeds_from(P.PRESETS, ["--preset", "smoke", "--eval", "1-5"])[0]["eval_seeds"] == "1-5" - with pytest.raises(SystemExit): - P.seeds_from(P.PRESETS, ["--eval", "1-60"]) # held-out seeds a session could run - - -def test_the_neutral_libero_prompt_names_the_task_the_seeds_and_the_scored_program_and_nothing_of_a_method( - tmp_path, service): - """prompt_only/'s acting prompt and workspace: the engine's, without ASPIRE's fix-loop, library or search.""" - P = _load("libero_task", LIBERO / "utils" / "agent" / "task.py") - T = _task_module() - root = tmp_path / "ASPIRE" # the service fixture's ASPIRE tree - params = {"aspire_root": str(root), "evosearch": False} # as prompt_only's configs set them - row = {**LIBERO_ROW, "suite": "libero_object_task", "task_id": 4, "id": "libero_object_task-t04"} - text = P.prompt(row, params, tmp_path) - assert "Suite `libero_object_task`, task id `4`" in text and "get_task_context()" in text - assert "**51-53**" in text and "**66**" in text and "3 held-out seeds" in text - scored = f"{T.task_dir(T.settings(row, params))}/fix_code.py" - assert f"Your program is `{scored}`" in text # the file the engine's collect scores - unseen = f"--output-root {scored.rsplit('/', 1)[0]}/debug/holdout" # where the engine's collect reads it - assert unseen in text and "docs/libero-pro.md" in text and "simulator steps" not in text - assert "at most 600 simulator steps" in P.prompt(row, {**params, "max_steps": 600}, tmp_path) - assert not any(word in text for word in ("skill", "library", "coordinator", "fix-loop", "evolutionary")) - with pytest.raises(ValueError, match="verifier"): - P.prompt({**row, "role": "verifier", "id": "verify-x"}, params, tmp_path) - for call in (lambda: P.prompt(row, {"aspire_root": str(root)}, tmp_path), # without evosearch: false the - lambda: P.prepare(LIBERO_ROW, tmp_path / "w", {"aspire_root": str(root)}, tmp_path)): # engine - with pytest.raises(ValueError, match="evosearch must be false"): # would score a search's - call() # program - work = _workspace(service, LIBERO_ROW) - P.prepare(LIBERO_ROW, work, params, tmp_path) - assert (work / "bin" / "sim").is_file() and (work / "docs" / "libero-pro.md").is_file() - assert (work / "docs" / "task-exploration.md").is_file() # the scene procedure stays - assert not (work / "docs" / "evosearch-iteration.md").exists() and not (work / "task.md").exists() - # The scored program is fix_code.py even where a search's best is left; ASPIRE's own configs score the search's - s = T.settings(LIBERO_ROW, params) - (work / T.task_dir(s)).mkdir(parents=True, exist_ok=True) - (work / T.task_dir(s) / "fix_code.py").write_text("# fix\n") - (work / T.evo_dir(s)).mkdir(parents=True) - (work / T.evo_dir(s) / "evosearch_best_code.py").write_text("# best\n") - neutral = T.collect(LIBERO_ROW, work, tmp_path / "log-a", {**params, "prune_media": False}) - assert neutral["scored_program"].endswith("/fix_code.py") and neutral["fix_code"] == "# fix\n" - assert "evosearch" not in neutral - aspire = T.collect(LIBERO_ROW, work, tmp_path / "log-b", {"aspire_root": str(root), "prune_media": False}) - assert aspire["scored_program"].endswith("/evosearch_best_code.py") and aspire["fix_code"] == "# best\n" - - -def test_a_session_cannot_forge_a_registration_for_more_seeds(tmp_path, service): - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - forged = work / "x" # a fresh directory the session can write - forged.mkdir() - (forged / "p.py").write_text("result = {}\n") - answer = _register(service.url, {"workspace": str(forged), "task": ROW["id"], "dev_seeds": [1, 2, 3]}) - assert not answer["ok"] and "registered its session already" in answer["error"] - other = _workspace(service, ROW, "0badc0de") # another rollout (the framework's): the row's seeds, - assert _register(service.url, {"workspace": str(other), "task": ROW["id"], "dev_seeds": [1, 2, 3], - "max_programs": None})["ok"] # whatever the request says - (other / "bin").mkdir() - shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", other / "bin" / "sim_cli.py") - (other / "bin" / "sim.json").write_text(json.dumps({"workspace": str(other), "url": service.url})) - (other / "p.py").write_text("result = {}\n") - held_out = subprocess.run([sys.executable, str(other / "bin" / "sim_cli.py"), "run-batch", "--seeds", "1-3", - "--program", "p.py", "--output-root", "out"], capture_output=True, text=True, cwd=other) - assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr - mismatched = _workspace(service, VERIFY, "12345678") # a rollout registers only the task its name carries - answer = _register(service.url, {"workspace": str(mismatched), "task": ROW["id"]}) - assert not answer["ok"] and "is not one of task" in answer["error"] - - -def test_a_verifier_may_run_two_programs_and_sees_the_staged_proposals(tmp_path, service): - T = _task_module() - work = _workspace(service, VERIFY) - (work / "hermit" / "aspire" / "skills" / "proposed.md").write_text(ENTRY) - T.prepare(VERIFY, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - assert (work / "proposed_skills.md").read_text() == ENTRY + assert (work / name).is_file(), name # ASPIRE's pages beside the engine's workspace + assert (work / "task.md").read_text() == A.assignment(A.settings(ROW, params, BASE), A.engine(params, BASE)) + plain = _workspace(service, ROW, "0badc0de") + A.prepare(ROW, plain, {**params, "evosearch": False}, BASE) + assert not (plain / "docs" / "evosearch-iteration.md").exists() + # The verifier sees the proposals the coordinator staged in its copy of the library, and runs two programs + verifier = _workspace(service, VERIFY) + (verifier / "hermit" / "aspire" / "skills").mkdir(parents=True) + (verifier / "hermit" / "aspire" / "skills" / "proposed.md").write_text(ENTRY) + A.prepare(VERIFY, verifier, params, BASE) + assert (verifier / "proposed_skills.md").read_text() == ENTRY codes = [] for name in ("a.py", "a.py", "c.py"): # the same path, rewritten: a new program - (work / name).write_text(f"result = {{'v': {len(codes)}}}\n") - codes.append(_sim(work, "run-batch", "--seeds", "101-105", "--program", name, "--output-root", "out")) + (verifier / name).write_text(f"result = {{'v': {len(codes)}}}\n") + codes.append(_sim(verifier, "run-batch", "--seeds", "101-105", "--program", name, "--output-root", "out")) assert "attempt budget" not in codes[0].stderr and "attempt budget" not in codes[1].stderr assert codes[2].returncode == 2 and "attempt budget is 2 programs" in codes[2].stderr -@needs_bwrap -def test_the_service_follows_no_link_the_session_puts_in_its_workspace(tmp_path, service): - """A run is written by the service into its own directory and moved into the workspace: a link the session put - where the run's files go is replaced, not followed, and a program that is a link is not read.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - victim = tmp_path / "victim.txt" - victim.write_text("the host's own file\n") - (work / "out").mkdir() - (work / "out" / "run.json").symlink_to(victim) # where cap-harness writes - (work / "out" / "run-1").symlink_to(tmp_path) # a directory of the run's - (work / "p.py").write_text("result = {}\n") - ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - assert victim.read_text() == "the host's own file\n" and not (tmp_path / "outcome.json").exists() - assert ran.returncode == 2 and "a link in its path" in ran.stderr - (work / "out" / "run-1").unlink() - ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - assert ran.returncode == 0 and victim.read_text() == "the host's own file\n" - record = json.loads((work / "out" / "run.json").read_text()) - assert not (work / "out" / "run.json").is_symlink() and record["program"] == str(work / "p.py") - rows = (work / "out" / "results.jsonl").read_text() - assert json.loads(rows)["run_dir"] == str(work / "out" / "run-1") # the workspace's paths, not the service's - assert str(work / "p.py") in ran.stdout and str(service.service.scratch) not in ran.stdout - secret = tmp_path / "secret.py" - secret.write_text("KEY = 'host'\n") - (work / "q.py").symlink_to(secret) - refused = _sim(work, "validate", "--program", "q.py") - assert refused.returncode == 2 and "no program at q.py" in refused.stderr - assert list(service.service.scratch.iterdir()) == [] # the service's copies are gone - import urllib.request - service.service.nonce = "n-1" - with urllib.request.urlopen(service.url + "/health") as reply: # launch.sh knows its own service by it - assert json.loads(reply.read())["nonce"] == "n-1" - service.service.stop() - late = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - assert late.returncode == 1 and "stopping" in late.stderr # no run starts once the service stops - - -def test_bin_sim_quotes_the_interpreter(tmp_path, service, monkeypatch): - T = _task_module() - work = _workspace(service, ROW) - monkeypatch.setattr(T.sys, "executable", "/opt/hermit env/bin/python") - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - assert "exec '/opt/hermit env/bin/python' " in (work / "bin" / "sim").read_text() - - -@needs_bwrap -def test_a_program_that_escaped_the_validator_still_cannot_reach_the_host(tmp_path, service): - """cap-harness runs in a sandbox of its own (sim.confine): a program in it writes only its run's own output and - reads ASPIRE's checkout, not the host's files, another session's workspace or the judge's runs.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - secret, stage2 = tmp_path / "secret.txt", tmp_path / "out" / "stage2" / "robosuite-x" / "stage2.json" - secret.write_text("host secret\n") - stage2.parent.mkdir(parents=True) - stage2.write_text("{}") - other = _workspace(service, ROW, "0badc0de") - (other / "fix_code.py").write_text("result = {'theirs': 1}\n") - solution = tmp_path / "ASPIRE" / "examples" / "robosuite" / "cube_lifting_seed1.py" # a worked solution - solution.parent.mkdir(parents=True) - solution.write_text("result = {'worked': 1}\n") - history = tmp_path / "ASPIRE" / ".git" / "HEAD" # the git history holds it too - history.parent.mkdir() - history.write_text("ref: refs/heads/main\n") - native = tmp_path / "ASPIRE" / "outputs" / "run-1" / "source" / "program.py" # a run of ASPIRE's own CLI - native.parent.mkdir(parents=True) - native.write_text("result = {'native': 1}\n") - page = tmp_path / "ASPIRE" / "docs" / "libero-pro.md" # names the worked programs per task - demos = tmp_path / "ASPIRE" / "third_party" / "LIBERO-PRO" / "libero" / "datasets" / "demo.hdf5" - demos.parent.mkdir(parents=True) - demos.write_text("demonstration\n") - harness = tmp_path / "ASPIRE" / ".venv-robosuite" / "bin" / "cap-harness" - targets = {"WRITE": [tmp_path / "planted.txt", tmp_path / "ASPIRE" / "planted.py", work / "planted.txt"], - "READ": [secret, stage2, other / "fix_code.py", solution, history, native, page, demos, harness]} - (work / "p.py").write_text("result = {}\n" + "".join(f"# {w} {p}\n" for w, ps in targets.items() for p in ps)) - ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - tried = json.loads((work / "out" / "run.json").read_text())["tried"] - assert ran.returncode == 0, ran.stderr - assert not (tmp_path / "planted.txt").exists() and not (tmp_path / "ASPIRE" / "planted.py").exists() - assert tried[str(work / "planted.txt")] != "done" and not (work / "planted.txt").exists() # its workspace too - assert all(tried[str(p)] != "done" - for p in (secret, stage2, other / "fix_code.py", solution, history, native, page, demos)) - assert tried[str(harness)] == "done" # the checkout it runs from - - -@needs_bwrap -def test_the_validator_runs_on_aspires_own_venv_inside_its_sandbox(tmp_path, monkeypatch): - """ASPIRE's venv interpreter is a link to its base installation: confined, it keeps the venv (whose cap_harness - the validator imports), for the service's `bin/sim validate` and the judge's Engine alike.""" - root = fake_aspire(tmp_path) - venv = root / ".venv-robosuite" - shutil.move(str(venv / "bin" / "cap-harness"), str(tmp_path / "cap-harness")) - shutil.rmtree(venv) - subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(venv)], check=True) - shutil.move(str(tmp_path / "cap-harness"), str(venv / "bin" / "cap-harness")) - site = next(venv.glob("lib/python3*/site-packages")) - (site / "cap_harness").mkdir() - (site / "cap_harness" / "__init__.py").write_text("") - (site / "cap_harness" / "runtime.py").write_text( - "import ast\nclass _ProgramValidator(ast.NodeVisitor):\n def visit_Import(self, node):\n" - " if any(a.name != 'math' for a in node.names):\n raise ValueError('only math')\n") - assert (venv / "bin" / "python").is_symlink() - S = _load("aspire_sim_service_venv", EXAMPLE / "agent" / "sim_service.py") - service = S.Service(root, {ROW["id"]: ROW}, [tmp_path / "sessions"], S.Ledger(tmp_path / "ledger.json"), {}, 2) - work = tmp_path / "sessions" / "rollout" / f"{ROW['id']}-1a2b" / "workspaces" / "w" / "work" - work.mkdir(parents=True) - assert service.register({"workspace": str(work), "task": ROW["id"]})["ok"] - (work / "good.py").write_text("import math\nresult = {}\n") - (work / "bad.py").write_text("import os\nresult = {}\n") - good = service.run({"workspace": str(work), "cmd": "validate", "program": "good.py"}) - bad = service.run({"workspace": str(work), "cmd": "validate", "program": "bad.py"}) - assert good["code"] == 0 and "program accepted" in good["stdout"], good - assert bad["code"] != 0 and "only math" in bad["stderr"], bad - sim = _sim_module() - engine = sim.Engine(root) - assert engine.validate(work / "good.py") is None and "only math" in engine.validate(work / "bad.py") - - -def _sim_module(): - sys.path.insert(0, str(EXAMPLE / "agent")) - try: - import sim - return sim - finally: - sys.path.remove(str(EXAMPLE / "agent")) - - -@needs_bwrap -def test_the_engine_reads_no_link_and_no_run_dir_outside_its_run(tmp_path): - """The judge's and the direct actor's Engine read what a confined program could write: once it has ended, links - in its output are removed and a run_dir outside the run is not read, so neither a host file nor another run's - success reaches the records.""" - sim = _sim_module() +def test_aspires_facts_score_the_searchs_best_and_a_verifiers_program_only_when_verified(tmp_path): root = fake_aspire(tmp_path) - elsewhere = tmp_path / "another-run" / "seed_0001" - elsewhere.mkdir(parents=True) - (elsewhere / "outcome.json").write_text('{"task_success": true, "program_ok": true}') - secret = tmp_path / "secret.txt" - secret.write_text("host secret") - out = tmp_path / "stage2" / "runs" - program = tmp_path / "stage2" / "program.py" - program.parent.mkdir() - program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# RUNDIR {elsewhere}\n") - records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1, 2], out=out) - assert [r["task_success"] for r in records] == [False, False] and "not in this run's output" in records[0]["error"] - assert secret.read_text() == "host secret" and not (out / "records.json").is_symlink() - program.write_text(f"result = {{}}\n# FAIL\n# LINK run-1/outcome.json {elsewhere / 'outcome.json'}\n") - records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "again") - assert records[0]["task_success"] is False and not (tmp_path / "again" / "run-1" / "outcome.json").exists() - # a directory made unlistable keeps no link from being removed: it is made listable, or the run fails - program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# CHMOD . 0o111\n") - sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "locked") - assert secret.read_text() == "host secret" and not (tmp_path / "locked" / "records.json").is_symlink() - - -def _fake_gpu(tmp_path, monkeypatch) -> list[int]: - """GPU 8 as hermit's sandbox sees devices (sandbox.DEV): a machine without one can still build the command.""" - import hermit.sandbox as sandbox_module - dev = tmp_path / "dev" - dev.mkdir(exist_ok=True) - for name in ("nvidiactl", "nvidia8"): - (dev / name).touch() - monkeypatch.setattr(sandbox_module, "DEV", str(dev)) - return [8] - - -@needs_bwrap -def test_the_simulators_sandbox_shows_the_renderer_and_reaches_remote_services_by_proxy(tmp_path, monkeypatch): - """An EGL vendor library extracted by hand (its ICD and its libraries, file by file, its library found as GLVND - finds it) and ASPIRE_SIM_READ are shown; without a GPU an NVIDIA-only ICD selection is dropped; a service on - another host is reached through an allowlist proxy for that host only.""" - sim = _sim_module() - home = tmp_path / "home" # the ICD and the libraries extracted where other files are too - (home / "lib").mkdir(parents=True) - (home / "credentials").write_text("key") - icd = home / "10_nvidia.json" - icd.write_text(json.dumps({"ICD": {"library_path": str(home / "lib" / "libEGL_nvidia.so.0")}})) - for name in ("libEGL_nvidia.so.0", "libnvidia-eglcore.so.580", "notes.txt"): - (home / "lib" / name).write_text("") - extra = tmp_path / "extra" - extra.mkdir() - env = {"PATH": os.environ["PATH"], "__EGL_VENDOR_LIBRARY_FILENAMES": str(icd), - "ASPIRE_SIM_READ": str(extra), "CAP_HARNESS_SAM3_URL": "http://provider.internal:9000"} - held, args, inside = sim.confine(["true"], root=fake_aspire(tmp_path), env=env, - gpu=_fake_gpu(tmp_path, monkeypatch)) - try: - ro = set(args[i + 1] for i, a in enumerate(args) if a == "--ro-bind") - assert {str(icd), str(home / "lib" / "libEGL_nvidia.so.0"), str(home / "lib" / "libnvidia-eglcore.so.580"), - str(extra)} <= ro # the driver's files, file by file, and the explicit grant - assert not ro & {str(home), str(home / "lib"), str(home / "credentials"), str(home / "lib" / "notes.txt")} - assert inside["__EGL_VENDOR_LIBRARY_FILENAMES"] == str(icd) # with a GPU (below: gpu=[8]) it stays - assert inside["HTTPS_PROXY"].startswith("http://127.0.0.1:") and held.held[0].allow == ["provider.internal"] - finally: - held.close() - # an ICD naming its library bare: GLVND finds it on LD_LIBRARY_PATH, and so is it shown - icd.write_text(json.dumps({"ICD": {"library_path": "libEGL_nvidia.so.0"}})) - held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", gpu=_fake_gpu(tmp_path, monkeypatch), - env={**env, "LD_LIBRARY_PATH": str(home / "lib")}) - held.close() - assert str(home / "lib" / "libEGL_nvidia.so.0") in [args[i + 1] for i, a in enumerate(args) if a == "--ro-bind"] - # without a GPU an NVIDIA-only ICD selection is dropped, so EGL finds Mesa's software renderer - held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", env=env) - held.close() - assert "__EGL_VENDOR_LIBRARY_FILENAMES" not in inside - - -@needs_bwrap -def test_the_service_lets_a_run_write_only_its_output_and_delivers_no_replaced_output(tmp_path, service, monkeypatch): - """The run's own directory (its program) is read-only to it and only its output writable; an output directory - the program replaced (by a link to a host directory) is not delivered.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - (work / "p.py").write_text("result = {}\n") - sim = type(service.service).run.__globals__["sim"] - grants, real = [], sim.confine - monkeypatch.setattr(sim, "confine", lambda cmd, **k: grants.append(k) or real(cmd, **k)) - assert _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "o").returncode == 0 - own = Path(grants[-1]["readable"][0]) - assert grants[-1]["writable"] == [own / "out"] - host = tmp_path / "host-dir" - host.mkdir() - (host / "id_rsa").write_text("key") - - def replaced(cmd, env, timeout, own, out=None, gpu=()): - shutil.rmtree(out) - out.symlink_to(host) - return 0, "", "" - monkeypatch.setattr(service.service, "_execute", replaced) - ran = _sim(work, "run", "--seed", "102", "--program", "p.py", "--output-root", "o2") - assert ran.returncode == 2 and "replaced while it ran" in ran.stderr - assert (host / "id_rsa").read_text() == "key" and not (work / "o2").exists() - - -@needs_bwrap -def test_a_program_named_pyvenv_cfg_grants_nothing(tmp_path, service): - """A session names its program: one called pyvenv.cfg that says `home = ` is the run's own file, not a venv - of the sandbox's, so stays out of the simulator's sandbox.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - operator = tmp_path / "operator" - operator.mkdir() - (operator / "credentials").write_text("key") - (work / "pyvenv.cfg").write_text(f'marker = """\nhome = {operator}\n"""\nresult = {{}}\n' - f"# READ {operator / 'credentials'}\n") - ran = _sim(work, "run", "--seed", "101", "--program", "pyvenv.cfg", "--output-root", "out") - assert ran.returncode == 0, ran.stderr - assert json.loads((work / "out" / "run.json").read_text())["tried"][str(operator / "credentials")] != "done" - - -def test_the_service_refuses_to_start_on_a_gpu_the_machine_lacks(tmp_path): - """A wrong ASPIRE_SIM_GPU stops the service at its start, not each session's run after its budget is spent.""" - (tmp_path / "tasks.jsonl").write_text(json.dumps(ROW) + "\n") - done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "sim_service.py"), "--aspire-root", - str(fake_aspire(tmp_path)), "--tasks", str(tmp_path / "tasks.jsonl"), "--workspaces", - str(tmp_path / "w"), "--state", str(tmp_path / "state"), "--gpu", "999", "--port", "0"], - capture_output=True, text=True, timeout=60, env={**os.environ, "PYTHONPATH": str(ROOT)}) - assert done.returncode != 0 and ("/dev/nvidia999" in done.stderr or "bubblewrap" in done.stderr) - - -@needs_bwrap -def test_the_simulators_sandbox_reaches_only_the_services_ports(tmp_path): - """The judge runs sim.Engine in hermit's process, whose environment names its own loopback endpoints: only the - services' ports (ASPIRE's, and a CAP_HARNESS_*_URL's) are forwarded into cap-harness's sandbox.""" - sim = _sim_module() - root = fake_aspire(tmp_path) - box, args, _ = sim.confine(["true"], root=root, env={ - "PATH": os.environ["PATH"], "HA_TOOL_URL": "http://127.0.0.1:4999/mcp", - "ASPIRE_SIM_URL": "http://127.0.0.1:8120", "CAP_HARNESS_SAM3_URL": "http://127.0.0.1:9114", - "CAP_HARNESS_CUROBO_URL": "http://localhost"}) # no port: its scheme's - box.close() - forwarded = sorted(int(args[i + 1].split("=")[0]) for i, a in enumerate(args) if a == "--forward") - assert forwarded == [80, 8114, 8115, 8116, 8118, 9114] - for where in ({"hidden": [root / "runs"]}, {"writable": [root / "scratch" / "run"]}): # runs stay outside it - with pytest.raises(ValueError, match="inside ASPIRE's checkout"): - sim.confine(["true"], root=root, env={"PATH": os.environ["PATH"]}, **where) - - -@needs_bwrap -def test_a_run_takes_the_registrations_max_steps_and_stops_at_the_timeout(tmp_path, monkeypatch): - monkeypatch.setenv("TEST_PROVIDER_API_KEY", "k") # the host's secrets: not for a generated program - monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "8") # the host's numbering: the sandbox's GPUs count from 0 - svc = _Service(tmp_path, fake_aspire(tmp_path), run_timeout=2) - try: - work = _workspace(svc, ROW) - assert _register(svc.url, {"workspace": str(work), "task": ROW["id"], "max_steps": 300})["ok"] - (work / "bin").mkdir() - shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", work / "bin" / "sim_cli.py") - (work / "bin" / "sim.json").write_text(json.dumps({"workspace": str(work), "url": svc.url})) - sim = lambda *args: subprocess.run([sys.executable, str(work / "bin" / "sim_cli.py"), *args], - capture_output=True, text=True, cwd=work) - (work / "p.py").write_text("result = {}\n") - ran = sim("run", "--seed", "101", "--program", "p.py", "--output-root", "o", "--max-steps", "5000") - record = json.loads((work / "o" / "run.json").read_text()) - assert ran.returncode == 0 and record["max_steps"] == "300" - assert "TEST_PROVIDER_API_KEY" not in record["env"] and "PATH" in record["env"] - assert "CUDA_VISIBLE_DEVICES" not in record["env"] - assert not Path(record["home"]).is_relative_to(work) and not Path(record["home"]).exists() # its own, gone - assert list((tmp_path / "sandbox").glob("cap-harness-*.json")) # the run's sandbox record - (work / "slow.py").write_text("SLEEP\n") - started = time.time() - slow = sim("run", "--seed", "102", "--program", "slow.py", "--output-root", "s") - assert slow.returncode == 124 and "stopped after 2 s" in slow.stderr and time.time() - started < 30 - finally: - svc.close() - - -def test_the_judge_runs_stage_two_on_the_rows_seeds_and_a_verifier_only_when_verified(tmp_path, monkeypatch): - J = _load("aspire_judge", EXAMPLE / "benchmark" / "judge.py") - runs, engines = [], [] + params = {**PARAMS, "aspire_root": str(root), "prune_media": False} + eng = A.engine(params, BASE) + work = tmp_path / "work" + s = A.settings(ROW, params, BASE) + (work / A.task_dir(s, eng)).mkdir(parents=True) + (work / A.task_dir(s, eng) / "fix_code.py").write_text("# fix\n") + (work / A.task_dir(s, eng) / "skill_report.json").write_text('{"graded": []}') + (work / A.evo_dir(s)).mkdir(parents=True) + (work / A.evo_dir(s) / "evosearch_best_code.py").write_text("# best\n") + facts = A.collect(ROW, work, tmp_path / "log-a", params, BASE) + assert facts["scored_program"].endswith("/evosearch_best_code.py") and facts["fix_code"] == "# best\n" + assert facts["role"] == "actor" and facts["skill_report"] == {"graded": []} and facts["evosearch"]["best_code"] + assert (tmp_path / "log-a" / "files" / "skill_report.json").is_file() + plain = A.collect(ROW, work, tmp_path / "log-b", {**params, "evosearch": False}, BASE) + assert plain["fix_code"] == "# fix\n" and "evosearch" not in plain + # A verifier's program counts only with its own verdict `verified`; without it nothing is scored + vwork = tmp_path / "verify" + vwork.mkdir() + (vwork / "fix_code.py").write_text("# v\n") + (vwork / "verification_report.json").write_text('{"verdict": "refuted"}') + refuted = A.collect(VERIFY, vwork, tmp_path / "log-c", params, BASE) + assert refuted["fix_code"] is None and refuted["not_scored"] == "verifier verdict refuted" + (vwork / "verification_report.json").write_text('{"verdict": "verified"}') + verified = A.collect(VERIFY, vwork, tmp_path / "log-d", params, BASE) + assert verified["fix_code"] == "# v\n" and "not_scored" not in verified and verified["role"] == "verifier" + + +def test_the_direct_verifiers_program_counts_only_when_verified(tmp_path, monkeypatch): + sim = D.engine_module(ENGINE / "agent" / "sim.py") class Engine: - def __init__(self, *a, **k): - engines.append(k) - - def validate(self, program): - assert program.is_absolute() and program.is_file() # cap-harness runs in ASPIRE's checkout - return "ValueError: only math" if "import os" in program.read_text() else None - - def run(self, program, *, suite, seeds, out, task_id=0): - runs.append(seeds) - return [{"task_success": s % 2 == 1} for s in seeds] - monkeypatch.setattr(J.sim, "Engine", Engine) - monkeypatch.setenv("ASPIRE_ROOT", str(tmp_path)) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ASPIRE_OUT", "out") # relative, as a user may give it - out = lambda facts: SimpleNamespace(extra={"task": facts}, output=None) - actor = {"role": "actor", "fix_code": "result = {}\n", "unseen": {"passed": 1, "total": 1}} - assert J.score(ROW, out(actor))["score"] == 2 / 3 and runs[-1] == [1, 2, 3] # the row's held-out seeds - assert J.score(ROW, out({**actor, "fix_code": "import os\n"}))["score"] == 0.0 - assert J.score(ROW, out({**actor, "max_steps": 300}))["score"] == 2 / 3 and engines[-1]["max_steps"] == 300 - assert engines[-1]["benchmark"] == "robosuite" - assert J.score(LIBERO_ROW, out(actor))["score"] == 2 / 3 and engines[-1]["benchmark"] == "libero-pro" - work = str(tmp_path / "w") - verified = {"role": "verifier", "fix_code": "result = {}\n", "verification": {"verdict": "verified"}, - "workspace": work} - ledger = tmp_path / "out" / "sim-service" / "ledger.json" - ledger.parent.mkdir(parents=True) - ledger.write_text(json.dumps({work: {"programs": [hashlib.sha256(b"result = {}\n").hexdigest()]}})) - assert J.score(VERIFY, out(verified))["score"] == 3 / 5 and runs[-1] == [101, 102, 103, 104, 105] - assert J.score(VERIFY, out({**verified, "verification": {"verdict": "void"}}))["score"] == 0.0 - third = J.score(VERIFY, out({**verified, "fix_code": "result = {'v': 3}\n"})) # beyond its two programs - assert third["score"] == 0.0 and "attempt budget" in third["feedback"] - direct = {k: v for k, v in verified.items() if k != "workspace"} # the direct actor's: it runs its own versions - assert J.score(VERIFY, out({**direct, "fix_code": "result = {'v': 3}\n"}))["score"] == 3 / 5 - assert J.score(ROW, out({"role": "actor"}))["score"] == 0.0 # no program: not scored - - -def test_the_robosuite_tasks_are_task_directories_read_back_as_the_rows_they_were(tmp_path, monkeypatch): - """prepare.py writes each task, verifier and rerun as a task directory (Harbor's format, no network, judged on the - host); aspire_fields.py, every config's data.view, reads each back as exactly the row tasks.jsonl held.""" - from hermit.config import Config - from hermit.task import Benchmark - P = _load("robosuite_prepare", EXAMPLE / "benchmark" / "prepare.py") - F = _load("aspire_fields", EXAMPLE / "agent" / "aspire_fields.py") - monkeypatch.setattr(P, "HERE", tmp_path) + def __init__(self, root, **k): + self.root = Path(root) + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "api-reference.md").write_text("# api\n") + passes = {"n": 0} + + def debug_loop(run, max_versions): + records = [{"task_success": i < passes["n"], "program_ok": True} for i in range(len(run.debug))] + version = D._Version("v00", "result = {}\n", records) + run.versions.append(version) + return version + monkeypatch.setattr(sim, "Engine", Engine) + monkeypatch.setattr(D._Run, "explore", lambda run: None) + monkeypatch.setattr(D._Run, "debug_loop", debug_loop) + actor = D.AspireActor({"aspire_root": str(tmp_path), "scratch": str(tmp_path / "scratch"), + "engine": str(ENGINE / "agent" / "sim.py")}, base=tmp_path) + monkeypatch.setattr(actor, "log_dir", lambda: tmp_path / "logs") + monkeypatch.setattr(actor, "session", lambda adapter: None) + out = actor.run(dict(VERIFY), None) # 0 of 5: refuted, its program withheld + assert out.extra["task"]["not_scored"] == "verifier verdict refuted" and out.extra["task"]["fix_code"] is None + passes["n"] = 4 + out = actor.run(dict(VERIFY), None) # 4 of 5: verified + assert out.extra["task"]["verification"]["verdict"] == "verified" and out.extra["task"]["fix_code"] + assert "not_scored" not in out.extra["task"] + + +def test_aspires_verifiers_are_derived_from_a_benchmarks_tasks(tmp_path, monkeypatch): + """rows.py: every task of the benchmark (its splits/all.txt, in that order) gets its verifier beside it; the + benchmark's tasks, reruns and splits are copied as they are.""" + P = _load("robosuite_prepare", ROBOSUITE / "utils" / "benchmark" / "prepare.py") + monkeypatch.setattr(P, "HERE", tmp_path / "bench") monkeypatch.setattr(sys, "argv", ["prepare.py", "--preset", "smoke"]) + (tmp_path / "bench").mkdir() P.main() - found = F.task_rows(tmp_path / "tasks") - seeds = {"debug_seeds": "101-103", "unseen_seeds": "126", "eval_seeds": "1-3"} + out = tmp_path / "aspire" + writer = ENGINE / "benchmark" / "tasks.py" # the engine's task writer + Rw.main(["--benchmark", str(tmp_path / "bench"), "--out", str(out), "--verify-seeds", "101-105", + "--engine", str(writer)]) + Fd = _load("cap_fields", ENGINE / "agent" / "fields.py") + found = Fd.task_rows(out / "tasks") task = {"id": "robosuite-cube_stack", "suite": "cube_stack", "task_id": 0} - assert json.dumps(found["robosuite-cube_stack"]) == json.dumps({**task, **seeds}) # its keys in order, types assert json.dumps(found["verify-robosuite-cube_stack"]) == json.dumps( {**task, "id": "verify-robosuite-cube_stack", "role": "verifier", "debug_seeds": "101-105", - "eval_seeds": "none", "max_versions": 2, "search": "debug", "findings": False}) - assert json.dumps(found["rerun-robosuite-cube_stack"]) == json.dumps( - {**task, "id": "rerun-robosuite-cube_stack", "role": "rerun", "findings": False, **seeds}) - assert len(found) == 21 and (tmp_path / "splits" / "smoke.txt").read_text().split() == \ - ["robosuite-cube_lifting", "robosuite-cube_stack"] - loaded = Benchmark.load(tmp_path / "tasks", view=F.aspire_task).tasks["robosuite-cube_stack"] - assert loaded.private["tests"] is None and loaded.private["needs"]["network"]["agent"] == "no-network" - for path in sorted((EXAMPLE.parent / "aspire").glob("*.yaml")): + "eval_seeds": "101-105", "score_label": "verifier", "max_programs": 2, "max_versions": 2, + "search": "debug", "findings": False}) # its keys in order: the row the service reads + bench = Fd.task_rows(tmp_path / "bench" / "tasks") + assert {k: v for k, v in found.items() if not k.startswith("verify-")} == bench + assert len(found) == 21 and (out / "tasks" / "verify-robosuite-cube_stack" / "instruction.md").is_file() + order = (tmp_path / "bench" / "splits" / "all.txt").read_text().split() + assert (out / "splits" / "verify.txt").read_text().split() == [f"verify-{i}" for i in order] + for split in (tmp_path / "bench" / "splits").iterdir(): + assert (out / "splits" / split.name).read_text() == split.read_text() + Rw.write(tmp_path / "bench", out, "101-105", writer) # again: rewritten whole, the same + assert Fd.task_rows(out / "tasks") == found + with pytest.raises(ValueError, match="overlap the held-out seeds"): + Rw.write(tmp_path / "bench", out, "1-5", writer) + + +def test_the_aspire_configs_read_its_verifiers_and_name_its_session_and_the_engine(tmp_path, monkeypatch): + """Every ASPIRE config on Robosuite and LIBERO: its tasks are the benchmark's with ASPIRE's verifiers (tasks.sh), + the coordinator's are the verifier rows, the test split the reruns of what it trained on, and the session's task + functions are ASPIRE's over the engine's.""" + from hermit.config import Config + from hermit.task import Benchmark + from hermit.agent.acting import workspace as cli + configs = sorted((ROBOSUITE / "aspire").glob("*.yaml")) + sorted((LIBERO / "aspire").glob("*.yaml")) + assert len(configs) == 15 + prepared = {} + for bench, seeds in ((ROBOSUITE, "101-105"), (LIBERO, "51-55")): + P = _load(f"prepare_{bench.name}", bench / "utils" / "benchmark" / "prepare.py") + monkeypatch.setattr(P, "HERE", tmp_path / bench.name) + monkeypatch.setattr(sys, "argv", ["prepare.py", "--preset", "smoke"]) + (tmp_path / bench.name).mkdir() + P.main() + hook = (bench / "aspire" / "tasks.sh").read_text() + assert f"--verify-seeds {seeds}" in hook and "--engine ../../cap_harness/utils/benchmark/tasks.py" in hook + Rw.write(tmp_path / bench.name, tmp_path / bench.name / "aspire", seeds, ENGINE / "benchmark" / "tasks.py") + prepared[bench] = tmp_path / bench.name / "aspire" + for path in configs: config = Config.load(path) - data = config.data["data"] - assert config.path(data["tasks"]) == (EXAMPLE / "benchmark" / "tasks").resolve(), path - monkeypatch.setitem(data, "tasks", str(tmp_path / "tasks")) + data, params = config.data["data"], config.data["agent"]["params"] + here = prepared[path.parents[1]] + assert config.path(data["tasks"]) == (path.parent / "tasks").resolve(), path for key in ("train", "val", "test"): - monkeypatch.setitem(data, key, str(tmp_path / "splits" / Path(data[key]).name)) - assert {t.id: t.given() for t in Benchmark.from_config(config).tasks.values()} == found, path - - -def test_prepare_refuses_held_out_seeds_a_session_could_run(): - for args in (["--eval", "1-200"], ["--unseen", "120-130"], ["--eval", "1-100", "--verify", "50-54"]): - done = subprocess.run([sys.executable, str(EXAMPLE / "benchmark" / "prepare.py"), *args], capture_output=True, - text=True) - assert done.returncode == 2 and "overlap" in done.stderr, args - - -def test_prune_drops_the_runs_images_and_keeps_their_records(tmp_path): - T = _task_module() - run = tmp_path / "run" / "work" / "debug" / "0101" / "run-x" - (run / "media" / "keyframes").mkdir(parents=True) - (run / "media" / "keyframes" / "a.png").write_bytes(b"png") - (run / "run.json").write_text('{"seed": 101}') - (tmp_path / "run" / "work" / "notes" / "media").mkdir(parents=True) # not a run's: kept - T.prune(tmp_path / "run") - assert not (run / "media").exists() and (run / "run.json").is_file() - assert (tmp_path / "run" / "work" / "notes" / "media").is_dir() + assert config.path(data[key]).parent == (path.parent / "splits").resolve(), (path, key) + monkeypatch.setitem(data, key, str(here / "splits" / Path(data[key]).name)) + monkeypatch.setitem(data, "tasks", str(here / "tasks")) + tasks = Benchmark.from_config(config) + split = lambda key: (here / "splits" / Path(data[key]).name).read_text().split() + assert split("val") == [i for i in (here / "splits" / "verify.txt").read_text().split()], path + assert split("test") == [f"rerun-{i}" for i in split("train")], path + assert all(tasks.tasks[i].given()["role"] == "verifier" for i in split("val")), path + if "_direct" in path.name: + assert config.data["agent"]["name"] == "hermit.agent.evolver.methods.aspire.direct:AspireActor", path + assert config.path(params["engine"]) == (ENGINE / "agent" / "sim.py").resolve(), path + continue + assert config.path(params["engine"]) == (ENGINE / "agent" / "task.py").resolve(), path + for name in ("prompt", "prepare", "answer", "collect"): + assert cli._task_function(params[name], config.base) is getattr(A, name), (path, name) + read = [str(p) for p in (config.data.get("environment") or {}).get("read") or []] + for name in A.ASPIRE_FILES: # its templates and the search's page, declared + assert f"$ASPIRE_ROOT/{name}" in read, (path, name) diff --git a/tests/selection.py b/tests/selection.py index 27adaccc..37f93bde 100644 --- a/tests/selection.py +++ b/tests/selection.py @@ -26,8 +26,7 @@ def fnmatch(path: str, pattern: str) -> bool: # group -> the paths (patterns, as `fnmatch` above reads them) that are the method's own METHODS = { - "aspire": ["hermit/agent/evolver/methods/aspire/*", "hermit/adapter/templates/aspire/*", - "examples/aspire_robosuite/*", "examples/libero/*"], + "aspire": ["hermit/agent/evolver/methods/aspire/*", "hermit/adapter/templates/aspire/*", "examples/*/aspire/*"], "she": ["hermit/agent/evolver/methods/she/*", "hermit/adapter/templates/she/*", "examples/*/she/*"], "ace": ["hermit/agent/evolver/methods/ace/*", "hermit/adapter/templates/ace/*", "examples/*/ace/*"], "ahe": ["hermit/agent/evolver/methods/ahe_adapter/*", "hermit/adapter/templates/ahe/*", @@ -56,6 +55,9 @@ def fnmatch(path: str, pattern: str) -> bool: "she": {"evosafe"}, # AgentDyn runner tests freeze SHE configs "evosafe": {"she"}, # SHE reuses the CLI roles and verdict contract "harness2": {"continual_harness"}} # Continual Harness deploys with the harness2_codex template +# an engine several benchmarks run on (examples//) -> those benchmarks: a change to it runs the groups of every +# method with an example under one of them +ENGINES = {"cap_harness": ("robosuite", "libero", "robodojo_sim")} # what every method is built on: a change here runs every group (tests/core/test_selection.py checks that every # framework module these import, or a method's tests import, is here too) SHARED = ["hermit/agent/__init__.py", "hermit/agent/base/*", "hermit/environment/*", @@ -93,9 +95,10 @@ def groups_for(path: str) -> set[str] | None: if path.startswith("examples/") and path.count("/") <= 2 and path.endswith("README.md"): return set() # the examples' and a benchmark's description if path.startswith("examples/"): - # a benchmark's shared files (utils/, data): every method with an example under that benchmark - bench = path.split("/")[1] - users = {name for name, patterns in METHODS.items() for p in patterns + # a benchmark's shared files (utils/, data): every method with an example under that benchmark (an engine's: + # under any benchmark on it) + benches = ENGINES.get(path.split("/")[1], (path.split("/")[1],)) + users = {name for name, patterns in METHODS.items() for p in patterns for bench in benches if p.startswith("examples/*/") and _has_example(bench, p.split("/")[2])} return users | {"core"} if any(fnmatch(path, p) for p in CORE): From 40b1b2145939992e66b27725d8b25dca708102bd Mon Sep 17 00:00:00 2001 From: Haizhong Zheng Date: Sat, 21 Nov 2026 02:26:44 -0500 Subject: [PATCH 2/4] Review fixes: name Claude Code beside Codex in the docs, RoboDojo's base service, the direct actor's scope; Claude's Bash timeout - The engine and Robosuite READMEs and actor.py said every session is Codex; they now name Claude Code too (smoke_claude.yaml). - RoboDojo's README and two comments still called its base "the ASPIRE example's service"; it is the cap-harness engine's. - direct.py's docstring said any cap-harness task; its prompts and runs are Robosuite's. - smoke_claude.yaml raises Claude Code's Bash timeout (2 min by default, 10 at most) through the session's environment, since a bin/sim run-batch can take longer. Co-Authored-By: Claude Opus 5.5 --- examples/cap_harness/README.md | 20 ++++++++++--------- examples/robodojo_sim/README.md | 2 +- .../robodojo_sim/utils/agent/dojo_service.py | 2 +- .../robodojo_sim/utils/tests/test_robodojo.py | 2 +- examples/robosuite/README.md | 13 ++++++------ examples/robosuite/aspire/smoke_claude.yaml | 3 +++ hermit/agent/evolver/methods/aspire/actor.py | 2 +- hermit/agent/evolver/methods/aspire/direct.py | 6 +++--- 8 files changed, 28 insertions(+), 22 deletions(-) diff --git a/examples/cap_harness/README.md b/examples/cap_harness/README.md index afa1e593..00e8f3ad 100644 --- a/examples/cap_harness/README.md +++ b/examples/cap_harness/README.md @@ -1,7 +1,7 @@ # The cap-harness engine -The robot execution engine of [ASPIRE](https://github.com/NVlabs/ASPIRE)'s Code-as-Policy harness (`cap-harness`), shared -by the benchmarks that run on it: [Robosuite](../robosuite/) and [LIBERO](../libero/) (and, for its simulator +The robot execution engine of [ASPIRE](https://github.com/NVlabs/ASPIRE)'s Code-as-Policy harness (`cap-harness`), +shared by the benchmarks that run on it: [Robosuite](../robosuite/) and [LIBERO](../libero/) (and, for its simulator service, [RoboDojo-Sim](../robodojo_sim/)). A session writes a program (Python against the harness's perception, grasping and motion API), runs it in the engine on its task's development seeds through `bin/sim`, and the judge scores it once on held-out seeds. The engine is a benchmark's, not a method's: what a method brings to a session @@ -58,11 +58,13 @@ the task functions here. MUJOCO_EGL_DEVICE_ID: "0" # inside the sandbox the GPUs ASPIRE_SIM_GPU names count from 0 ``` -4. **Codex and the sandboxes.** A session (and a method's learner, such as ASPIRE's coordinator) is a Codex CLI - session (`codex` on `PATH`; the configs bind them to GLM 5.3 Flash on Fireworks), each in the framework's sandbox (`sandbox.enabled: true`, `enforce: - required`; bubblewrap on `PATH`, docs/guides/sandbox.md). The task sandbox has no GPU, so the simulator runs on the - host as a service (`utils/agent/sim_service.py`, started by `launch.sh`), which the session reaches through `bin/sim` - on a forwarded loopback port; see [A session's rules](#a-sessions-rules). +4. **The CLI and the sandboxes.** A session (and a method's learner, such as ASPIRE's coordinator) is a Codex CLI + session (`codex` on `PATH`; most configs bind them to GLM 5.3 Flash on Fireworks) or a Claude Code one (`claude` on + `PATH`, on a Claude subscription: Robosuite's `aspire/smoke_claude.yaml`), each in the framework's sandbox + (`sandbox.enabled: true`, `enforce: required`; bubblewrap on `PATH`, docs/guides/sandbox.md). The task sandbox has + no GPU, so the simulator runs on the host as a service (`utils/agent/sim_service.py`, started by `launch.sh`), + which the session reaches through `bin/sim` on a forwarded loopback port; see [A session's + rules](#a-sessions-rules). ## Disk @@ -84,8 +86,8 @@ simulator service is the trusted side of the seed rules: it reads the task rows session (its workspace and task id; one session per rollout directory, which the sandbox cannot create) before the agent starts, and the service runs only the row's development and unseen-check seeds, the unseen check once and, for a row with an attempt budget (`max_programs`), that many programs, writing the runs into the workspace. The held-out -seeds are run only by the judge, on the host (ASPIRE's Stage 2), on the program the session selected. The session's -Codex runs with its own sandbox off (`codex_sandbox: danger-full-access`), the task sandbox being the boundary. +seeds are run only by the judge, on the host (ASPIRE's Stage 2), on the program the session selected. A Codex +session runs with its own sandbox off (`codex_sandbox: danger-full-access`), the task sandbox being the boundary. The programs a session writes run inside cap-harness on the host, and ASPIRE's program validator is no boundary (a format string's attribute traversal passes it). So every cap-harness process, the service's, the judge's and the diff --git a/examples/robodojo_sim/README.md b/examples/robodojo_sim/README.md index 22a571bc..22ac6381 100644 --- a/examples/robodojo_sim/README.md +++ b/examples/robodojo_sim/README.md @@ -25,7 +25,7 @@ what is left is listed under [Not verified](#not-verified). | `utils/policy/install.py` | Copies the policy into the RoboDojo checkout (`XPolicyLab/policy/hermit_program`) | | `utils/agent/dojo.py` | The engine: one evaluation of a program (`scripts/robodojo.sh eval`), its result per layout | | `utils/agent/split_eval.sh` | One evaluation with the simulator in RoboDojo's container (`ROBODOJO_IMAGE`) | -| `utils/agent/dojo_service.py`, `sim_cli.py` | The simulator service on the host and `bin/sim`, the actor's client of it (the ASPIRE example's service with this engine) | +| `utils/agent/dojo_service.py`, `sim_cli.py` | The simulator service on the host and `bin/sim`, the actor's client of it (the cap-harness engine's service, `../cap_harness/utils/agent/sim_service.py`, with this engine) | | `utils/agent/task.py`, `prompts/`, `docs/interface.md` | The Codex actor's task functions, its prompt and the interface page it is given | | `utils/benchmark/prepare.py` | The forty-two tasks with their development and held-out layouts, as task directories in Harbor's format (`tasks/`, each read as its row through `utils/agent/robodojo_fields.py`, the configs' `data.view`), and `splits/` | | `utils/benchmark/judge.py` | The program on the held-out layout sets, on the host | diff --git a/examples/robodojo_sim/utils/agent/dojo_service.py b/examples/robodojo_sim/utils/agent/dojo_service.py index 4e0e6157..c5df9749 100644 --- a/examples/robodojo_sim/utils/agent/dojo_service.py +++ b/examples/robodojo_sim/utils/agent/dojo_service.py @@ -81,7 +81,7 @@ def _launch(self, body, session, program: Path, own_out: Path, seeds: list[int]) if len(records) < len(seeds): lines.append(f"{len(seeds) - len(records)} layout(s) have no result (the evaluation ended early)") lines.append(f"{passed}/{len(records)} layouts succeeded") - # As the ASPIRE service reports a run it stopped: 124, and why + # As the cap-harness service reports a run it stopped: 124, and why summary = own_out / "records.json" # dojo.Engine.run's account of the run run = json.loads(summary.read_text()) if summary.is_file() else {} if run.get("timed_out"): diff --git a/examples/robodojo_sim/utils/tests/test_robodojo.py b/examples/robodojo_sim/utils/tests/test_robodojo.py index 93f45260..1649c1f2 100644 --- a/examples/robodojo_sim/utils/tests/test_robodojo.py +++ b/examples/robodojo_sim/utils/tests/test_robodojo.py @@ -522,7 +522,7 @@ def run(self, program, *, task, layout_seed, layouts, out, private=None): assert done["ok"] and "1/2 layouts succeeded" in done["stdout"] and ran == [("stack_bowls", 2, 2)] assert "not assigned to you" in request([0, 1, 2, 3])["error"] # more than its development layouts assert "from the first" in request([1, 2])["stderr"] and len(ran) == 1 - # A run past its timeout is said, as the ASPIRE service says it (124); after stop() no simulator starts + # A run past its timeout is said, as the cap-harness service says it (124); after stop() no simulator starts timed_out = {"value": True} def summarized(program, *, task, layout_seed, layouts, out, private=None): diff --git a/examples/robosuite/README.md b/examples/robosuite/README.md index af88021c..c7a3b782 100644 --- a/examples/robosuite/README.md +++ b/examples/robosuite/README.md @@ -16,12 +16,13 @@ service, `bin/sim`, its judge), which LIBERO shares; its README has the setup. ## ASPIRE on Robosuite The curriculum run as ASPIRE runs it ([`aspire`](../../docs/methods/aspire.md)): ASPIRE's fix-loop subagent (the -actor), its skill verifier and its coordinator, each a Codex session given ASPIRE's own prompt. For each task the -actor debugs its program from the per-primitive traces and RGB keyframes; the harness then scores it once on held-out -seeds. After each task the coordinator updates the shared skill library, admitting the actor's proposals only if a -fresh verifier re-solves the task from the library and the proposals alone, and chooses the next task. The actor's -task functions are ASPIRE's (`hermit/agent/evolver/methods/aspire/actor.py`, built on the engine's), and its verifier -rows are ASPIRE's too (`aspire/tasks.sh` derives them from the tasks `prepare.py` wrote). +actor), its skill verifier and its coordinator, each a Codex session (or Claude Code, `smoke_claude.yaml`) given +ASPIRE's own prompt. For each task the actor debugs its program from the per-primitive traces and RGB keyframes; the +harness then scores it once on held-out seeds. After each task the coordinator updates the shared skill library, +admitting the actor's proposals only if a fresh verifier re-solves the task from the library and the proposals alone, +and chooses the next task. The actor's task functions are ASPIRE's (`hermit/agent/evolver/methods/aspire/actor.py`, +built on the engine's), and its verifier rows are ASPIRE's too (`aspire/tasks.sh` derives them from the tasks +`prepare.py` wrote). On the host that serves the providers, with the engine set up: diff --git a/examples/robosuite/aspire/smoke_claude.yaml b/examples/robosuite/aspire/smoke_claude.yaml index fb59dbe8..ea3ccd8d 100644 --- a/examples/robosuite/aspire/smoke_claude.yaml +++ b/examples/robosuite/aspire/smoke_claude.yaml @@ -27,6 +27,9 @@ agent: claude: web_search: false # set explicitly: WebSearch and WebFetch denied (they run at Anthropic) timeout_s: 5400 + env: # Claude Code's Bash stops a command at 2 min (10 at most); a run-batch may take longer + BASH_DEFAULT_TIMEOUT_MS: "1800000" + BASH_MAX_TIMEOUT_MS: "3600000" environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client diff --git a/hermit/agent/evolver/methods/aspire/actor.py b/hermit/agent/evolver/methods/aspire/actor.py index 2bd8b0bd..13d09960 100644 --- a/hermit/agent/evolver/methods/aspire/actor.py +++ b/hermit/agent/evolver/methods/aspire/actor.py @@ -1,4 +1,4 @@ -"""ASPIRE's acting sessions (agent.base: codex): its fix-loop subagent, its evolutionary search and its skill +"""ASPIRE's acting sessions (agent.base: codex or claude): its fix-loop subagent, its evolutionary search and its skill verifier, as ASPIRE's own prompts run them, on any benchmark of the cap-harness engine. These are the task functions an ASPIRE config names (agent.params.prompt, prepare, answer, collect); they build on the engine's own (agent.params.engine: its utils/agent/task.py, e.g. examples/cap_harness/utils/agent/task.py), which give every diff --git a/hermit/agent/evolver/methods/aspire/direct.py b/hermit/agent/evolver/methods/aspire/direct.py index 9f647f62..d64afccf 100644 --- a/hermit/agent/evolver/methods/aspire/direct.py +++ b/hermit/agent/evolver/methods/aspire/direct.py @@ -1,6 +1,6 @@ -"""ASPIRE's actor on one task of the cap-harness engine, in direct chat: write a Code-as-Policy program, run it in -the robot execution engine on the task's development seeds, diagnose from the per-primitive traces and keyframes, -repair, and keep the best version +"""ASPIRE's actor on one Robosuite task of the cap-harness engine, in direct chat (its prompts and its runs are +Robosuite's): write a Code-as-Policy program, run it in the robot execution engine on the task's development seeds, +diagnose from the per-primitive traces and keyframes, repair, and keep the best version (ASPIRE §2.1, the iterative-debugging loop of NVlabs/ASPIRE .claude/skills/iterative-debugging/subagent-prompt.md); optionally continue with the evolutionary search over the program (ASPIRE §2.3, Algorithm 1; .claude/skills/evosearch/); then write the findings the coordinator distills skills from. The judge runs the From 5100ea945997c12909214560459ca72897b6c1c0 Mon Sep 17 00:00:00 2001 From: Haizhong Zheng Date: Sat, 21 Nov 2026 03:00:24 -0500 Subject: [PATCH 3/4] Drop the Robosuite Claude Code smoke config until it can run as committed smoke_claude.yaml (task sandbox, login: claude) needs CLAUDE_CODE_OAUTH_TOKEN, which the machine the PR was validated on does not have, so it never ran in the form it was committed in. ASPIRE's header still names the CLI's context file (CLAUDE.md for Claude Code); a Claude Code config can follow once it can run. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- examples/cap_harness/README.md | 10 +-- .../utils/tests/test_cap_harness.py | 2 +- examples/robosuite/README.md | 4 +- examples/robosuite/aspire/smoke_claude.yaml | 61 ------------------- tests/methods/aspire/test_aspire.py | 2 +- 6 files changed, 10 insertions(+), 72 deletions(-) delete mode 100644 examples/robosuite/aspire/smoke_claude.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ea55697b..57b44f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,7 @@ Changes are grouped by software version. Entries under Unreleased describe work attempt budget is a row's `max_programs`, and the judge scores a row's `eval_seeds` (a verifier row's are its own five); a verifier's verdict rule is ASPIRE's `collect`, which withholds the program unless `verified` (`not_scored`). The prompts the Codex sessions see are unchanged, byte for byte; ASPIRE's header page names the CLI's context file - (`CLAUDE.md` for Claude Code), and Robosuite has a Claude Code smoke (`aspire/smoke_claude.yaml`). A task function - may be named `package.module:function`. + (`CLAUDE.md` for Claude Code). A task function may be named `package.module:function`. LIBERO's `prompt_only/` configs drop `evosearch: false`. - The old import paths of moved modules are gone, with no forward: 0.1's, which 0.2 forwarded (`hermit.agents`, diff --git a/examples/cap_harness/README.md b/examples/cap_harness/README.md index 00e8f3ad..45341b85 100644 --- a/examples/cap_harness/README.md +++ b/examples/cap_harness/README.md @@ -60,11 +60,11 @@ the task functions here. 4. **The CLI and the sandboxes.** A session (and a method's learner, such as ASPIRE's coordinator) is a Codex CLI session (`codex` on `PATH`; most configs bind them to GLM 5.3 Flash on Fireworks) or a Claude Code one (`claude` on - `PATH`, on a Claude subscription: Robosuite's `aspire/smoke_claude.yaml`), each in the framework's sandbox - (`sandbox.enabled: true`, `enforce: required`; bubblewrap on `PATH`, docs/guides/sandbox.md). The task sandbox has - no GPU, so the simulator runs on the host as a service (`utils/agent/sim_service.py`, started by `launch.sh`), - which the session reaches through `bin/sim` on a forwarded loopback port; see [A session's - rules](#a-sessions-rules). + `PATH`, `agent.base: claude`; on a Claude subscription, `login: claude`: docs/guides/claude-subscription.md), each + in the framework's sandbox (`sandbox.enabled: true`, `enforce: required`; bubblewrap on `PATH`, + docs/guides/sandbox.md). The task sandbox has no GPU, so the simulator runs on the host as a service + (`utils/agent/sim_service.py`, started by `launch.sh`), which the session reaches through `bin/sim` on a forwarded + loopback port; see [A session's rules](#a-sessions-rules). ## Disk diff --git a/examples/cap_harness/utils/tests/test_cap_harness.py b/examples/cap_harness/utils/tests/test_cap_harness.py index 61a07b36..d2c89f80 100644 --- a/examples/cap_harness/utils/tests/test_cap_harness.py +++ b/examples/cap_harness/utils/tests/test_cap_harness.py @@ -158,7 +158,7 @@ def test_the_task_sandbox_is_shown_the_staged_docs_and_never_a_page_that_lost_it from hermit.config import Config configs = [p for folder in (ROBOSUITE / "aspire", LIBERO / "aspire", LIBERO / "prompt_only") for p in sorted(folder.glob("*.yaml"))] - assert len(configs) == 19 + assert len(configs) == 18 for path in configs: config = Config.load(path) data = config.data diff --git a/examples/robosuite/README.md b/examples/robosuite/README.md index c7a3b782..2d48a625 100644 --- a/examples/robosuite/README.md +++ b/examples/robosuite/README.md @@ -11,12 +11,12 @@ service, `bin/sim`, its judge), which LIBERO shares; its README has the setup. |---|---| | `utils/benchmark/prepare.py` | The seven Robosuite tasks with their seeds, as task directories in Harbor's format (`tasks/`, each with its rerun, each read as its row through the engine's `utils/agent/fields.py`, the configs' `data.view`); `splits/` | | `utils/benchmark/task-info/` | What a task and its record are, for the learner | -| `aspire/` | ASPIRE on Robosuite: `run.yaml` (ASPIRE's seeds), `smoke.yaml`, `smoke_claude.yaml` (Claude Code acting and coordinating on a Claude subscription), the `*_direct.yaml` comparison, `launch.sh`, and `tasks.sh`, which adds ASPIRE's verifiers to the tasks | +| `aspire/` | ASPIRE on Robosuite: `run.yaml` (ASPIRE's seeds), `smoke.yaml`, the `*_direct.yaml` comparison, `launch.sh`, and `tasks.sh`, which adds ASPIRE's verifiers to the tasks | ## ASPIRE on Robosuite The curriculum run as ASPIRE runs it ([`aspire`](../../docs/methods/aspire.md)): ASPIRE's fix-loop subagent (the -actor), its skill verifier and its coordinator, each a Codex session (or Claude Code, `smoke_claude.yaml`) given +actor), its skill verifier and its coordinator, each a Codex session (or Claude Code: `agent.base: claude`) given ASPIRE's own prompt. For each task the actor debugs its program from the per-primitive traces and RGB keyframes; the harness then scores it once on held-out seeds. After each task the coordinator updates the shared skill library, admitting the actor's proposals only if a fresh verifier re-solves the task from the library and the proposals alone, diff --git a/examples/robosuite/aspire/smoke_claude.yaml b/examples/robosuite/aspire/smoke_claude.yaml deleted file mode 100644 index ea3ccd8d..00000000 --- a/examples/robosuite/aspire/smoke_claude.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# The smoke with Claude Code: two tasks; three development seeds, one unseen-check seed, three held-out seeds -# (launch.sh writes them into the task rows: ASPIRE_SEEDS=smoke); Claude Code acting (ASPIRE's fix-loop subagent and -# verifier prompts) and coordinating, Claude Opus 5.5 on the user's Claude subscription (login: claude, its token in -# CLAUDE_CODE_OAUTH_TOKEN: docs/guides/claude-subscription.md). Started by launch.sh: -# ASPIRE_CONFIG=smoke_claude.yaml ./launch.sh -data: - tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) - view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was - train: splits/smoke.txt - val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: splits/smoke_rerun.txt - step_size: 1 - shuffle: false -task_info: ../utils/benchmark/task-info -agent: - base: claude - concurrency: 1 - params: - prompt: hermit.agent.evolver.methods.aspire.actor:prompt - prepare: hermit.agent.evolver.methods.aspire.actor:prepare - answer: hermit.agent.evolver.methods.aspire.actor:answer - collect: hermit.agent.evolver.methods.aspire.actor:collect - engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on - aspire_root: $ASPIRE_ROOT - aspire_docs: $ASPIRE_DOCS - model: {name: claude-opus-5-5, login: claude, effort: medium} # a model name on the login - claude: - web_search: false # set explicitly: WebSearch and WebFetch denied (they run at Anthropic) - timeout_s: 5400 - env: # Claude Code's Bash stops a command at 2 min (10 at most); a run-batch may take longer - BASH_DEFAULT_TIMEOUT_MS: "1800000" - BASH_MAX_TIMEOUT_MS: "3600000" -environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it - read: - - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client - - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md - - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md - - $ASPIRE_ROOT/.claude/skills/evosearch/subagent-prompt.md - - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md - env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded -judge: - path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds -template: aspire -evolver: - name: aspire - base: claude - model: {name: claude-opus-5-5, login: claude, effort: medium} - claude: {web_search: false} -sandbox: - enabled: true - enforce: required -trainer: - run_name: smoke-aspire-claude - epochs: 1 - call_timeout_min: 240 - budget: {hours: 8} # on the subscription: no dollar cost; the plan's own windows apply -eval: - repeats: 1 diff --git a/tests/methods/aspire/test_aspire.py b/tests/methods/aspire/test_aspire.py index cea73988..4d86ca16 100644 --- a/tests/methods/aspire/test_aspire.py +++ b/tests/methods/aspire/test_aspire.py @@ -406,7 +406,7 @@ def test_the_aspire_configs_read_its_verifiers_and_name_its_session_and_the_engi from hermit.task import Benchmark from hermit.agent.acting import workspace as cli configs = sorted((ROBOSUITE / "aspire").glob("*.yaml")) + sorted((LIBERO / "aspire").glob("*.yaml")) - assert len(configs) == 15 + assert len(configs) == 14 prepared = {} for bench, seeds in ((ROBOSUITE, "101-105"), (LIBERO, "51-55")): P = _load(f"prepare_{bench.name}", bench / "utils" / "benchmark" / "prepare.py") From 428e2b035a769081c04e5613bf265edf5c92d377 Mon Sep 17 00:00:00 2001 From: Haizhong Zheng Date: Sat, 21 Nov 2026 03:18:39 -0500 Subject: [PATCH 4/4] Keep ASPIRE's header page as it was: no runtime text changes in this PR The {context_file} line existed only for a Claude Code session, and this PR no longer ships a Claude Code config. ASPIRE's prompt pages are now byte-identical to main's; the Claude Code wording can come with a Claude Code config. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +-- examples/robosuite/README.md | 2 +- hermit/agent/evolver/methods/aspire/actor.py | 8 +------- .../agent/evolver/methods/aspire/prompts/codex_header.md | 2 +- tests/methods/aspire/test_aspire.py | 4 ---- 5 files changed, 4 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57b44f1e..898b6236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,7 @@ Changes are grouped by software version. Entries under Unreleased describe work which an ASPIRE folder's `tasks.sh` runs before the run, into that folder's `tasks/` and `splits/`. The service's attempt budget is a row's `max_programs`, and the judge scores a row's `eval_seeds` (a verifier row's are its own five); a verifier's verdict rule is ASPIRE's `collect`, which withholds the program unless `verified` (`not_scored`). - The prompts the Codex sessions see are unchanged, byte for byte; ASPIRE's header page names the CLI's context file - (`CLAUDE.md` for Claude Code). A task function may be named `package.module:function`. + The prompts the sessions see are unchanged, byte for byte. A task function may be named `package.module:function`. LIBERO's `prompt_only/` configs drop `evosearch: false`. - The old import paths of moved modules are gone, with no forward: 0.1's, which 0.2 forwarded (`hermit.agents`, diff --git a/examples/robosuite/README.md b/examples/robosuite/README.md index 2d48a625..d5bc3c51 100644 --- a/examples/robosuite/README.md +++ b/examples/robosuite/README.md @@ -16,7 +16,7 @@ service, `bin/sim`, its judge), which LIBERO shares; its README has the setup. ## ASPIRE on Robosuite The curriculum run as ASPIRE runs it ([`aspire`](../../docs/methods/aspire.md)): ASPIRE's fix-loop subagent (the -actor), its skill verifier and its coordinator, each a Codex session (or Claude Code: `agent.base: claude`) given +actor), its skill verifier and its coordinator, each a Codex session given ASPIRE's own prompt. For each task the actor debugs its program from the per-primitive traces and RGB keyframes; the harness then scores it once on held-out seeds. After each task the coordinator updates the shared skill library, admitting the actor's proposals only if a fresh verifier re-solves the task from the library and the proposals alone, diff --git a/hermit/agent/evolver/methods/aspire/actor.py b/hermit/agent/evolver/methods/aspire/actor.py index 13d09960..667a9ba5 100644 --- a/hermit/agent/evolver/methods/aspire/actor.py +++ b/hermit/agent/evolver/methods/aspire/actor.py @@ -1,4 +1,4 @@ -"""ASPIRE's acting sessions (agent.base: codex or claude): its fix-loop subagent, its evolutionary search and its skill +"""ASPIRE's acting sessions (agent.base: codex): its fix-loop subagent, its evolutionary search and its skill verifier, as ASPIRE's own prompts run them, on any benchmark of the cap-harness engine. These are the task functions an ASPIRE config names (agent.params.prompt, prepare, answer, collect); they build on the engine's own (agent.params.engine: its utils/agent/task.py, e.g. examples/cap_harness/utils/agent/task.py), which give every @@ -100,7 +100,6 @@ def prompt(task: dict, params: dict, base: Path) -> str: header = (PROMPTS / "codex_header.md").read_text().format( role_name="fix-loop subagent" if s["role"] == "actor" else "skill verifier", template=TEMPLATES[s["role"]], benchmark=s["benchmark"], benchmark_doc=s["doc"], - context_file=context_file(params), your_dir=(f"`$RUN_ROOT` is this workspace (`.`), so `$TASK_DIR` is `{task_dir(s, eng)}`, as the template " f"defines it." if s["role"] == "actor" else "`$VERIFY_DIR` is this workspace (`.`)."), proposed_line=("The proposals under test are `proposed_skills.md` in this workspace (the template's " @@ -121,11 +120,6 @@ def prompt(task: dict, params: dict, base: Path) -> str: return text -def context_file(params: dict) -> str: - """The file the session's CLI reads the adapter's context from: Claude Code's CLAUDE.md, Codex's AGENTS.md.""" - return "CLAUDE.md" if str(params.get("backend") or "codex") in ("claude", "claude-code") else "AGENTS.md" - - def evosearch_part(s: dict, eng) -> str: """Part 2 of the actor's session: ASPIRE's evolutionary-search subagent (§2.3), for a task the fix loop left below 90% of its development seeds. Its template is the fenced block of evosearch/subagent-prompt.md, verbatim after diff --git a/hermit/agent/evolver/methods/aspire/prompts/codex_header.md b/hermit/agent/evolver/methods/aspire/prompts/codex_header.md index 64b2afdc..4c10951d 100644 --- a/hermit/agent/evolver/methods/aspire/prompts/codex_header.md +++ b/hermit/agent/evolver/methods/aspire/prompts/codex_header.md @@ -17,7 +17,7 @@ things in it map onto your workspace as follows. Where the template and this pag `bin/sim` asks the harness's simulator to run them, with the benchmark ({benchmark}), suite, task id, init mode, GPU, render settings, `--flat-run-dir` and the results file (`/results.jsonl`, with the logs under `/logs/`); the runs appear in your workspace. It accepts only your assigned seeds, and the unseen-check seeds once. Perception, IK and planning services are already running; you need not check them. -- **The shared skill library** is the "Shared skill library" section of your {context_file}, and the same files are under +- **The shared skill library** is the "Shared skill library" section of your AGENTS.md, and the same files are under `hermit/aspire/skills/` in this workspace. Wherever the template says `.claude/skills/iterative-debugging/skills/` (or `skills/.md`), it means these, except `skills/task-exploration.md`, which is `docs/task-exploration.md`. {proposed_line} diff --git a/tests/methods/aspire/test_aspire.py b/tests/methods/aspire/test_aspire.py index 4d86ca16..20515f4b 100644 --- a/tests/methods/aspire/test_aspire.py +++ b/tests/methods/aspire/test_aspire.py @@ -276,10 +276,6 @@ def test_the_actor_prompt_is_the_header_the_assignment_and_aspires_template(tmp_ assert "`TASK_DIR` is `evosearch/libero-pro/libero_90/task_33`" in libero and "DEV_SEEDS: 51-53" in libero with pytest.raises(ValueError, match="agent.params.engine"): A.prompt(ROW, {"aspire_root": params["aspire_root"]}, BASE) - # The library is in the context file of the session's CLI: Codex's AGENTS.md, Claude Code's CLAUDE.md - assert "section of your AGENTS.md" in text and "CLAUDE.md" not in text - claude = A.prompt(ROW, {**params, "backend": "claude"}, BASE) - assert "section of your CLAUDE.md" in claude and claude.replace("CLAUDE.md", "AGENTS.md") == text def test_aspires_workspace_is_the_engines_with_its_assignment_its_search_page_and_the_proposals(tmp_path, service):