diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e76749..898b6236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ Changes are grouped by software version. Entries under Unreleased describe work ### Breaking +- The robot benchmarks and ASPIRE no longer share code: a benchmark holds nothing of a method, and ASPIRE nothing of + a benchmark. The engine Robosuite and LIBERO run on (the simulator service, `bin/sim`, the neutral task functions, + the judge, the task writer, the row view) is `examples/cap_harness/`, which RoboDojo's service also builds on; + `examples/aspire_robosuite/` is `examples/robosuite/`, the benchmark's tasks and an `aspire/` folder. ASPIRE's + session (its fix-loop and verifier prompts, the evolutionary search, the facts the coordinator reads) is + `hermit/agent/evolver/methods/aspire/actor.py`, built on the engine's task functions (`agent.params.engine`), and its + direct-chat actor `direct.py` (`agent.name: hermit.agent.evolver.methods.aspire.direct:AspireActor`). A benchmark's + `prepare.py` writes its tasks and their reruns (`rerun_of`) only; ASPIRE's verifier rows are derived by `rows.py`, + which an ASPIRE folder's `tasks.sh` runs before the run, into that folder's `tasks/` and `splits/`. The service's + attempt budget is a row's `max_programs`, and the judge scores a row's `eval_seeds` (a verifier row's are its own + five); a verifier's verdict rule is ASPIRE's `collect`, which withholds the program unless `verified` (`not_scored`). + The prompts the sessions see are unchanged, byte for byte. A task function may be named `package.module:function`. + LIBERO's `prompt_only/` configs drop `evosearch: false`. + - The old import paths of moved modules are gone, with no forward: 0.1's, which 0.2 forwarded (`hermit.agents`, `hermit.evolver`, `hermit.llm`, `hermit.backends`, `hermit.trajectory`), and 0.2's `hermit.agent.base.direct` and `hermit.agent.base.container`, and `hermit.agent.environment` (never released). Import instead: diff --git a/docs/framework/task.md b/docs/framework/task.md index 19cb23fa..8aedae72 100644 --- a/docs/framework/task.md +++ b/docs/framework/task.md @@ -102,8 +102,9 @@ agent: `codex`, `claude-code`, `pi`, `mini-swe-agent`, `opencode`, `dsh` and `hermes` act with a coding-agent CLI in a workspace of Hermit's (or, for Codex and Claude Code, in a task's own Harbor environment: [Benchmarks](#benchmarks)): -a task gives what is its own, in configuration and, where its workspace is -built in code, in functions it names (`file.py:function`); the agent does the rest. OfficeQA's Codex runs are one such +a task gives what is its own, in configuration and, where its workspace is built in code, in functions it names +(`file.py:function`, or `package.module:function` for a method that brings its own session, as ASPIRE's +`hermit.agent.evolver.methods.aspire.actor`); the agent does the rest. OfficeQA's Codex runs are one such configuration (`examples/officeqa/utils/task.py`): ```yaml @@ -456,7 +457,7 @@ of your own is written as task directories by your `utils/benchmark/prepare.py` goes into `tests/`, uploaded after the agent ran, so the answer is never where the agent can read it; see `examples/officeqa/utils/harbor/build_tasks.py`). A task set of task directories that Hermit runs in its own sandbox or on the host, with a judge on the host (`judge.mode: host`, no `tests/`), is what the examples whose agent owns its loop -use (`examples/aspire_robosuite/`, `clbench_*`, `servelearnbench/`, `agentdyn/`, ...; [Task directories](#task-directories)). +use (`examples/robosuite/`, `libero/`, `clbench_*`, `servelearnbench/`, `agentdyn/`, ...; [Task directories](#task-directories)). A task set is checked without a model by running a command of a known answer in each task's environment and verifying it (`tests/core/test_harbor_environment.py` does this against Docker). diff --git a/docs/methods/aspire.md b/docs/methods/aspire.md index 191ad502..b577f685 100644 --- a/docs/methods/aspire.md +++ b/docs/methods/aspire.md @@ -14,16 +14,21 @@ learns only the library: the actor's harness and prompts are fixed. ```yaml template: aspire # the skill library: skills/README.md and skills/.md -data: {train: all.txt, val: verify.txt, test: rerun.txt, step_size: 1, shuffle: false} +data: # the benchmark's tasks with ASPIRE's verifiers (tasks.sh: rows.py) + {tasks: tasks, train: splits/all.txt, val: splits/verify.txt, test: splits/rerun.txt, step_size: 1, shuffle: false} agent: base: codex # the actor: ASPIRE's fix-loop subagent prompt, verbatim - params: {prompt: ../utils/agent/task.py:prompt, prepare: ../utils/agent/task.py:prepare, - answer: ../utils/agent/task.py:answer, collect: ../utils/agent/task.py:collect} + params: # ASPIRE's task functions, over the engine's + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py codex: {codex_sandbox: danger-full-access, shell_env: all, web_search: false} # the task sandbox is the boundary -environment: # the task sandbox: its code, ASPIRE's prompts and docs, the service - read: [../utils/agent, $ASPIRE_DOCS, ...] +environment: # the task sandbox: the engine's code, ASPIRE's prompts and docs + read: [../../cap_harness/utils/agent, $ASPIRE_DOCS, ...] env: [ASPIRE_SIM_URL] -judge: {path: ../utils/benchmark/judge.py} # on the host: Stage 2 +judge: {path: ../../cap_harness/utils/benchmark/judge.py} # the engine's, on the host: Stage 2 sandbox: {enabled: true, enforce: required} evolver: name: aspire # the coordinator: a Codex session with ASPIRE's guide @@ -40,16 +45,22 @@ privileged access (simulator internals, success predicates, reflection), and ref No existing template fits: `prompt-only` is one file; `continual-harness` carries a policy, sub-agents and memory ASPIRE does not have; `reharness` retrieves per instance from an embedded bank. -The example's task functions (`examples/aspire_robosuite/utils/agent/task.py`) give the actor its workspace in the -task sandbox: `bin/sim`; ASPIRE's API and artifact docs and the benchmark's page (Robosuite's, or LIBERO's); the scene-snapshot program; and a page that maps -the template's paths and commands onto the workspace. The sandbox shows nothing else of ASPIRE's checkout: not its -shipped library, worked solutions or examples. It has no GPU, so the simulator runs on the host as a service -(`utils/agent/sim_service.py`), which `bin/sim` reaches on a forwarded loopback port; the session registers there -before the agent starts (its workspace and task id: one session per rollout), and the service runs only the seeds of -the task row it reads on the host, the unseen check once and a verifier's two programs. The judge (`utils/benchmark/judge.py`, -on the host) runs the selected program once on the held-out seeds: ASPIRE's Stage 2, the coordinator's in ASPIRE. The -verifier is the same actor on the task's verifier row (`verify-`, `data.val`): ASPIRE's verifier prompt, five -seeds, two attempts, scored on its own seeds with its own verdict. +The tasks run on the [cap-harness engine](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/cap_harness), +which knows nothing of ASPIRE; ASPIRE brings its own session on it. ASPIRE's task functions +(`hermit/agent/evolver/methods/aspire/actor.py`, named by module in the configs) build on the engine's +(`agent.params.engine`): the engine gives the actor its workspace in the task sandbox (`bin/sim`; ASPIRE's API and +artifact docs and the benchmark's page, Robosuite's or LIBERO's; the scene-snapshot program), and ASPIRE adds its +assignment, the evolutionary search's page and a page that maps the template's paths and commands onto the +workspace. The sandbox shows nothing else of ASPIRE's checkout: not its shipped library, worked solutions or +examples. It has no GPU, so the simulator runs on the host as the engine's service (`utils/agent/sim_service.py`), +which `bin/sim` reaches on a forwarded loopback port; the session registers there before the agent starts (its +workspace and task id: one session per rollout), and the service runs only the seeds of the task row it reads on the +host, the unseen check once and, for a row with an attempt budget, that many programs. The engine's judge +(`utils/benchmark/judge.py`, on the host) runs the selected program once on the held-out seeds: ASPIRE's Stage 2, the +coordinator's in ASPIRE. The verifier is the same actor on the task's verifier row (`verify-`, `data.val`), +which ASPIRE derives from the benchmark's tasks before a run (`rows.py`, run by the method folder's `tasks.sh`): ASPIRE's +verifier prompt, five seeds, an attempt budget of two programs, scored on its own seeds and only with its own verdict +`verified` (ASPIRE's `collect` withholds the program otherwise). ## The step @@ -98,11 +109,12 @@ is the evaluation on the test split (`rerun-` rows, the same seeds), compa The Codex actor's settings (`agent.params`) are `aspire_root`, `aspire_docs` (the docs as the launcher stages them), `max_steps` and `evosearch` (default true; false leaves out the evolutionary search: its part of the prompt, its page in the workspace, and its -program as the scored one); its seeds are the task rows' -(`utils/benchmark/prepare.py`, which the launcher runs: `ASPIRE_EVAL_SEEDS` for the held-out ones), and the simulator's +program as the scored one) and `engine` (the engine's task functions, relative to the config); its seeds are the +task rows' (the benchmark's `utils/benchmark/prepare.py`, which the launcher runs: `ASPIRE_EVAL_SEEDS` for the +held-out ones), and the simulator's environment and workers are the simulator service's (`ASPIRE_SIM_ENV`). The direct comparison actor -(`run_direct.yaml`) reads `debug_seeds`, `workers`, `aspire_root` and `sim_env` from `agent.params`, a task row's own -seeds winning; the held-out seeds are the task rows' for both actors (the judge reads them there). The verifier's rows +(`run_direct.yaml`) reads `engine` (the engine's `sim.py`), `debug_seeds`, `workers`, `aspire_root` and `sim_env` from +`agent.params`, a task row's own seeds winning; the held-out seeds are the task rows' for both actors (the judge reads them there). The verifier's rows carry their own seeds. ## The authors' implementation against this one @@ -124,12 +136,13 @@ carry their own seeds. | Library history | `-changelog.md` beside each file | `state/changelog.md` | the adapter holds only the library | | Phase 3 | rerun of tasks below 80% whose library changed | the test split reruns every task | the evaluation measures it per task | | Models | Claude Opus 4.6 (and others) | GLM 5.3 Flash in every role | the first-evaluation row of the framework | -| Perception | SAM3 | SAM3, or any service with its wire format (`utils/seg_service/`: Grounding-DINO + SAM) | SAM3's weights are gated | +| Perception | SAM3 | SAM3, or any service with its wire format (the engine's `utils/seg_service/`: Grounding-DINO + SAM) | SAM3's weights are gated | | Evolutionary search (§2.3) | a separate evosearch subagent for tasks the fix loop leaves below 90%: K = 8 candidates per round from the top 3, stop at 80% or 5 rounds, Stage 2 on its best | the same template, verbatim, as Part 2 of the actor's session; the judge runs Stage 2 on `evosearch_best_code.py` | a decision: one session keeps the task's rollout one call | | Not ported | — | BEHAVIOR-1K, the real robot | scope | -`examples/aspire_robosuite/aspire/run_direct.yaml` replaces the Codex actor with a structured loop of `direct` calls -(`utils/agent/agent.py`), which sees a digest of each run instead of the files: a comparison, not the method. +`examples/robosuite/aspire/run_direct.yaml` replaces the Codex actor with a structured loop of `direct` calls +(`hermit/agent/evolver/methods/aspire/direct.py`, on the engine's `sim.py`), which sees a digest of each run instead of +the files: a comparison, not the method. ## What it is not @@ -140,7 +153,7 @@ Phase 3 does). ## Example -[`examples/aspire_robosuite/`](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/aspire_robosuite): the +[`examples/robosuite/aspire/`](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/robosuite/aspire): the seven Robosuite tasks of ASPIRE's harness. [`examples/libero/aspire/`](https://github.com/Infini-AI-Lab/hermit/tree/main/examples/libero/aspire): LIBERO's five suites and LIBERO-Pro's eighty tasks on the same engine. diff --git a/examples/README.md b/examples/README.md index 8021d561..95bca7bb 100644 --- a/examples/README.md +++ b/examples/README.md @@ -36,8 +36,8 @@ a `.hermit/CARD.md`. | [`clbench_database/`](clbench_database/) | The CL-Bench database exploration stream (accuracy) | `CLBenchDatabaseAgent`: metaworld's host seat in CL-Bench's loop | [`prompt_only`](clbench_database/prompt_only/), [`mwm`](clbench_database/mwm/), [`reharness`](clbench_database/reharness/) | | [`clbench_poker/`](clbench_poker/) | The CL-Bench exploitable-poker stream (bb/hand) | `CLBenchPokerAgent` | [`prompt_only`](clbench_poker/prompt_only/), [`mwm`](clbench_poker/mwm/), [`reharness`](clbench_poker/reharness/) | | [`servelearnbench/`](servelearnbench/) | A ServeLearnBench scenario: serving stream, per-window tests | `ServeLearnBenchAgent`: the benchmark's own loop | [`prompt_only`](servelearnbench/prompt_only/), [`mwm`](servelearnbench/mwm/) | -| [`aspire_robosuite/`](aspire_robosuite/) | The Robosuite tasks of ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex with ASPIRE's fix-loop subagent and verifier prompts (`utils/agent/task.py`), GLM 5.3 Flash, ASPIRE's robot execution engine in its own venv through `bin/sim` | [`aspire`](aspire_robosuite/aspire/) | -| [`libero/`](libero/) | LIBERO's five suites and LIBERO-Pro's `_swap` and `_task` perturbations, a program per task in ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex writing the program, on the `aspire_robosuite/` engine (ASPIRE's LIBERO venv, seeded initial states), with a method's prompt or a method-neutral one | [`aspire`](libero/aspire/), [`prompt_only`](libero/prompt_only/) | +| [`robosuite/`](robosuite/) | The Robosuite tasks of ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex writing the program, GLM 5.3 Flash, on the [`cap_harness/`](cap_harness/) engine (ASPIRE's robot execution engine in its own venv through `bin/sim`), with a method's own session (ASPIRE's fix-loop subagent and verifier) | [`aspire`](robosuite/aspire/) | +| [`libero/`](libero/) | LIBERO's five suites and LIBERO-Pro's `_swap` and `_task` perturbations, a program per task in ASPIRE's Code-as-Policy harness: development and held-out seeds, task success | Codex writing the program, on the `cap_harness/` engine (ASPIRE's LIBERO venv, seeded initial states), with a method's own session or a method-neutral prompt | [`aspire`](libero/aspire/), [`prompt_only`](libero/prompt_only/) | | [`robodojo_sim/`](robodojo_sim/) | RoboDojo-Sim's forty-two bimanual manipulation tasks: a program that controls the robot, developed on one layout set and scored on two others, task success (a first implementation: its smoke run on the real simulator, no training run yet) | Codex writing `policy.py` (`utils/agent/task.py`), GLM 5.3 Flash, RoboDojo evaluating it as an XPolicyLab policy whose program runs in a sandbox of its own | [`prompt_only`](robodojo_sim/prompt_only/) | | [`agentdyn/`](agentdyn/) | AgentDyn's frozen benign and injected cases (Shopping, GitHub, DailyLife), clean utility and attack success rate | `bridge.py:run`: AgentDyn's native agent loop, with the defense bundle's hooks around every tool call | [`evosafe`](agentdyn/evosafe/) (EvoSafeHarness) | | [`agentdojo/`](agentdojo/) | Official banking, Slack, travel and workspace suites; frozen 57/54/406 local protocol, native utility/ASR and errors | `task.py:run`: native AgentDojo loop through the shared AgentDyn supervised runtime | [`evosafe`](agentdojo/evosafe/), [`she`](agentdojo/she/), completed Lightning/Codex reference comparisons | diff --git a/examples/aspire_robosuite/.gitignore b/examples/aspire_robosuite/.gitignore deleted file mode 100644 index a1e03960..00000000 --- a/examples/aspire_robosuite/.gitignore +++ /dev/null @@ -1 +0,0 @@ -runs/ diff --git a/examples/aspire_robosuite/utils/agent/aspire_fields.py b/examples/aspire_robosuite/utils/agent/aspire_fields.py deleted file mode 100644 index 09ac8e63..00000000 --- a/examples/aspire_robosuite/utils/agent/aspire_fields.py +++ /dev/null @@ -1,20 +0,0 @@ -"""A task of ASPIRE's engine as the actor, the simulator service and the judge read it: {id, (benchmark), suite, -task_id, the seeds, (role, max_versions, search, findings)}, the row each task was in tasks.jsonl. A task directory -(utils/benchmark/prepare.py, and LIBERO's) has every field but its id under task.toml's [metadata], in the row's -order; this puts them back, so the prompts, the service's rules and the records are what they were. LIBERO's tasks -are read so too (their configs' data.view).""" -from __future__ import annotations - -from pathlib import Path - - -def aspire_task(task: dict) -> dict: - if "instruction" not in task: # already a row of that shape - return dict(task) - return {"id": task["id"], **(task.get("metadata") or {})} - - -def task_rows(tasks: Path) -> dict[str, dict]: - """Every task's row by id: from the task directories prepare.py writes, or from a tasks.jsonl.""" - from hermit.task import Benchmark - return {tid: t.given() for tid, t in Benchmark.load(Path(tasks), view=aspire_task).tasks.items()} diff --git a/examples/aspire_robosuite/utils/agent/task.py b/examples/aspire_robosuite/utils/agent/task.py deleted file mode 100644 index d6c64610..00000000 --- a/examples/aspire_robosuite/utils/agent/task.py +++ /dev/null @@ -1,238 +0,0 @@ -"""The task functions of the Codex actor (agent.base: codex): ASPIRE's fix-loop subagent and its skill verifier, as -ASPIRE's own prompts run them. They run in the task sandbox with the session; the robot execution engine runs on the -host, as the simulator service (sim_service.py), which the session reaches only through `bin/sim`. - - prepare the workspace: bin/sim and its registration with the service (the workspace and the task's id: the - service takes the seeds and the budget from its own copy of the task row), the API and artifact docs, the scene-snapshot program, task.md; for a verifier, the - proposals under test as proposed_skills.md - prompt the adaptation page (prompts/codex_header.md), the assignment, then ASPIRE's template verbatim - answer the agent's last message (the record's output is the facts collect returns) - collect the actor's files into the log, and the facts the judge and the coordinator read: findings, skill report, - the selected program (fix_code.py, or the evolutionary search's best); the judge runs Stage 2 on it - -ASPIRE's checkout comes from agent.params.aspire_root (its prompt templates and snapshot program must be -declared readable: environment.read; its docs are read from agent.params.aspire_docs, the staged copies); the service's URL from ASPIRE_SIM_URL (environment.env). A task row carries the -seeds: development, unseen-check and held-out (utils/benchmark/prepare.py). -""" -from __future__ import annotations - -import json -import os -from pathlib import Path -import shlex -import shutil -import stat -import sys -import urllib.request - -HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE)) -import sim # noqa: E402 - -TEMPLATES = {"actor": "subagent-prompt.md", "verifier": "verifier-prompt.md"} -SKILLS = Path(".claude/skills/iterative-debugging") -EVOSEARCH = Path(".claude/skills/evosearch") -# The files of ASPIRE's checkout the task sandbox shows (environment.read): its prompts and snapshot program; not -# its shipped library, worked solutions or examples, nor its docs, which the sandbox is shown as staged -# (agent.params.aspire_docs: stage_docs.py's copies, without what names the worked programs) -ASPIRE_FILES = [ str(SKILLS / "subagent-prompt.md"), str(SKILLS / "verifier-prompt.md"), - str(SKILLS / "scripts" / "scene_snapshot.py"), str(SKILLS / "skills" / "task-exploration.md"), - str(EVOSEARCH / "subagent-prompt.md"), str(EVOSEARCH / "skills" / "evosearch-iteration.md")] -# The workspace files the log keeps (the run directories stay in the workspace) -KEEP = ("task_analysis.md", "fix_code.py", "findings.md", "findings_report.md", "skill_report.json", - "verification_report.json", "verification.md", "code", "debug/blocked") - - -def settings(task: dict, params: dict) -> dict: - """The session's assignment, from its task row (the seeds; utils/benchmark/prepare.py) and agent.params (the - checkout).""" - role = "verifier" if task.get("role") == "verifier" else "actor" # a rerun is an actor with the final library - root = params.get("aspire_root") or os.environ.get("ASPIRE_ROOT") - if not root: - raise ValueError("agent.params.aspire_root (or ASPIRE_ROOT): an ASPIRE checkout with its benchmark's venv") - benchmark = str(task.get("benchmark") or "robosuite") - return {"role": role, "root": Path(os.path.expandvars(str(root))).expanduser().resolve(), - "benchmark": benchmark, "doc": sim.profile(benchmark)["doc"], "suite": task["suite"], "task_id": int(task.get("task_id", 0)), - "dev": str(task.get("debug_seeds") or ""), "unseen": str(task.get("unseen_seeds") or ""), - "heldout": str(task.get("eval_seeds") or ""), "max_steps": params.get("max_steps")} - - -def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: - s = settings(task, params) - work = Path(workspace) - url = os.environ.get("ASPIRE_SIM_URL") - if not url: - raise ValueError("ASPIRE_SIM_URL: the simulator service's URL (launch.sh starts it; environment.env)") - # The session, registered before the agent starts: the service takes its seeds and budget from the task's row - register(url, {"workspace": str(work.resolve()), "task": task["id"], "max_steps": s["max_steps"]}) - bin_dir = work / "bin" - bin_dir.mkdir(exist_ok=True) - shutil.copy2(HERE / "sim_cli.py", bin_dir / "sim_cli.py") - (bin_dir / "sim.json").write_text(json.dumps({"workspace": str(work.resolve()), "url": url}, indent=1)) - wrapper = bin_dir / "sim" - wrapper.write_text(f"#!/bin/sh\nexec {shlex.quote(sys.executable)} \"$(dirname \"$0\")/sim_cli.py\" \"$@\"\n") - wrapper.chmod(wrapper.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - (work / "docs").mkdir(exist_ok=True) - # The repository's worked programs are not the actor's to read: the docs are the staged copies (launch.sh: - # stage_docs.py), or, where the checkout's own pages are what the sandbox shows, the same cut made here - staged = params.get("aspire_docs") - pages = {name: (Path(os.path.expandvars(str(staged))).expanduser() / name).read_text() - for name in (*sim.DOCS, s["doc"])} if staged else sim.workspace_docs(s["root"], s["benchmark"]) - for name, text in pages.items(): - (work / "docs" / name).write_text(text) - shutil.copy2(s["root"] / SKILLS / "skills" / "task-exploration.md", work / "docs" / "task-exploration.md") - if params.get("evosearch", True): # the search's procedure, for a prompt that runs it - shutil.copy2(s["root"] / EVOSEARCH / "skills" / "evosearch-iteration.md", work / "docs" / "evosearch-iteration.md") - (work / "scripts").mkdir(exist_ok=True) - shutil.copy2(s["root"] / SKILLS / "scripts" / "scene_snapshot.py", work / "scripts" / "scene_snapshot.py") - if s["role"] == "verifier": - # The coordinator stages the proposals under test as the adapter's skills/proposed.md: the verifier's - # proposed_skills.md, kept out of the library it reads (interface.py leaves the file out). - staged = sorted(work.glob("hermit/*/skills/proposed.md")) - (work / "proposed_skills.md").write_text(staged[0].read_text() if staged else "(no proposals were staged)\n") - (work / "task.md").write_text(assignment(s)) - return [] - - -def register(url: str, body: dict) -> None: - request = urllib.request.Request(url.rstrip("/") + "/register", data=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(request, timeout=60) as reply: - answer = json.loads(reply.read()) - if not answer.get("ok"): - raise RuntimeError(f"the simulator service refused the session: {answer.get('error')}") - - -def task_dir(s: dict) -> str: - """The actor's $TASK_DIR as ASPIRE's template defines it, with $RUN_ROOT the workspace; the verifier's is `.`.""" - return f"{s['benchmark']}/{s['suite']}/task_{s['task_id']}" if s["role"] == "actor" else "." - - -def sim_seeds(text: str) -> list[int]: - return sim.seeds_of(text) if text else [] - - -def assignment(s: dict) -> str: - if s["role"] == "verifier": - return (f"## Your assignment\n\n- BENCHMARK: `{s['benchmark']}`\n- SUITE: `{s['suite']}`\n" - f"- TASK_ID: `{s['task_id']}`\n- GPU: handled by `bin/sim`\n- VERIFY_DIR: `.` (this workspace)\n" - f"- Development seeds: **{s['dev']}** — wherever the template says 51, 52, 53, 54, 55 (or 51-55, or " - f"seed 51), use these.\n- The proposals under test: `proposed_skills.md` in this workspace.\n") - return (f"## Task Assignment\n\nBENCHMARK: {s['benchmark']}\nSUITE: {s['suite']}\nTASK_ID: {s['task_id']}\n" - f"GPU: handled by bin/sim\nRUN_ROOT: . (this workspace)\n\n" - f"MAX_STEPS: {s['max_steps'] or 1000}\nDEV_SEEDS: {s['dev']}\nUNSEEN_SEEDS: {s['unseen'] or 'none'}\n" - f"HELDOUT_COUNT: {len(sim_seeds(s['heldout']))} (coordinator-owned; you never run them)\n") - - -def prompt(task: dict, params: dict, base: Path) -> str: - s = settings(task, params) - template = (s["root"] / SKILLS / TEMPLATES[s["role"]]).read_text() - body = template.split("", 1)[1] - body = body.split("", 1)[0].strip() - if s["role"] == "actor": - body = body.split("\n---\n", 1)[1] if body.startswith("## Task Assignment") else body # ours replaces it - header = (HERE / "prompts" / "codex_header.md").read_text().format( - role_name="fix-loop subagent" if s["role"] == "actor" else "skill verifier", - template=TEMPLATES[s["role"]], benchmark=s["benchmark"], benchmark_doc=s["doc"], - your_dir=(f"`$RUN_ROOT` is this workspace (`.`), so `$TASK_DIR` is `{task_dir(s)}`, as the template " - f"defines it." if s["role"] == "actor" else "`$VERIFY_DIR` is this workspace (`.`)."), - proposed_line=("The proposals under test are `proposed_skills.md` in this workspace (the template's " - "`$VERIFY_DIR/proposed_skills.md`); they are not in the library." if s["role"] == "verifier" - else ""), - finish_line=("only after both parts of this session: Part 1, the template below, and Part 2 after it, the " - "evolutionary search, which runs only when Part 1 leaves `fix_code.py` below 90% of the " - "development seeds. Reply with the return message of each part you ran." - if s["role"] == "actor" and params.get("evosearch", True) else - "by replying with the template's return message."), - heldout_line=("is not yours: the harness scores your selected program on the held-out seeds after you " - "finish (the template's Stage 2)." if s["role"] == "actor" else - "does not apply: your verdict is scored on the program you select, on your seeds; write it " - "as `fix_code.py` in this workspace as well.")) - text = f"{header}\n\n{assignment(s)}\n---\n\n{body}\n" - if s["role"] == "actor" and params.get("evosearch", True): - text += "\n---\n\n" + evosearch_part(s) - return text - - -def evosearch_part(s: dict) -> str: - """Part 2 of the actor's session: ASPIRE's evolutionary-search subagent (§2.3), for a task the fix loop left below - 90% of its development seeds. Its template is the fenced block of evosearch/subagent-prompt.md, verbatim after - its assignment block, which the page before it replaces.""" - template = (s["root"] / EVOSEARCH / "subagent-prompt.md").read_text() - body = template[template.index("## Task Assignment"):template.rindex("\n```")] - body = body.split("\n---\n", 1)[1].strip() - header = (HERE / "prompts" / "evosearch_header.md").read_text().format( - benchmark=s["benchmark"], suite=s["suite"], task_id=s["task_id"], dev=s["dev"], task_dir=task_dir(s), evo_dir=evo_dir(s)) - return f"{header}\n---\n\n{body}\n" - - -def evo_dir(s: dict) -> str: - """The evolutionary search's $TASK_DIR, as its template defines it, with EVOSEARCH_DIR `evosearch`.""" - return f"evosearch/{s['benchmark']}/{s['suite']}/task_{s['task_id']}" - - -def answer(last_message: str, task: dict) -> dict: - return {"answer": last_message.strip()[-4000:]} - - -def collect(task: dict, workspace: Path, log: Path, params: dict) -> dict: - """The actor's files into the log and the facts for the judge (which runs Stage 2 on `fix_code`, on the host) and - the coordinator.""" - s = settings(task, params) - work = Path(workspace) / task_dir(s) - files = Path(log) / "files" - for name in KEEP: - source = work / name - if source.is_dir(): - shutil.copytree(source, files / name, dirs_exist_ok=True) - elif source.is_file(): - files.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, files / name) - read = lambda name: (work / name).read_text(errors="replace") if (work / name).is_file() else None - facts = {"role": s["role"], "benchmark": s["benchmark"], "suite": s["suite"], "findings": read("findings.md") or read("findings_report.md"), - "skill_report": _json(read("skill_report.json")), "verification": _json(read("verification_report.json")), - "unseen": _unseen(work), "workspace": str(Path(workspace).resolve()), "max_steps": s["max_steps"]} - evo = Path(workspace) / evo_dir(s) - search = s["role"] == "actor" and params.get("evosearch", True) # a prompt that runs the search - if search and evo.is_dir(): - for name in ("evosearch_best_code.py", "findings.md", "task_analysis.md", "BLOCKED"): - if (evo / name).is_file(): - (files / "evosearch").mkdir(parents=True, exist_ok=True) - shutil.copy2(evo / name, files / "evosearch" / name) - facts["evosearch"] = {"findings": (evo / "findings.md").read_text(errors="replace") - if (evo / "findings.md").is_file() else None, - "best_code": (evo / "evosearch_best_code.py").is_file()} - # The selected program: the search's best when the search ran (it falls back to fix_code.py itself), else the fix - # loop's; with `evosearch: false`, fix_code.py alone. The judge scores it. - best = evo / "evosearch_best_code.py" - program = best if search and best.is_file() else work / "fix_code.py" - facts["scored_program"] = str(program.relative_to(workspace)) if program.is_file() else None - facts["fix_code"] = program.read_text(errors="replace") if program.is_file() else None - if params.get("prune_media", True): - prune(Path(workspace).parent) - return facts - - -def prune(run_dir: Path) -> None: - """The heavy part of a finished session's simulator runs: every run's media/ (keyframes, overlays, videos; a - session runs hundreds of episodes). The records, traces and programs stay.""" - for media in run_dir.rglob("media"): - if media.is_dir() and (media.parent / "run.json").is_file(): - shutil.rmtree(media, ignore_errors=True) - - -def _json(text): - try: - return json.loads(text) if text else None - except ValueError: - return {"unparsed": text[-4000:]} - - -def _unseen(work: Path) -> dict | None: - rows = work / "debug" / "holdout" / "results.jsonl" - if not rows.is_file(): - return None - results = [json.loads(l) for l in rows.read_text().splitlines() if l.strip()] - passed = sum(bool((sim._read_json(Path(r["run_dir"]) / "outcome.json") or {}).get("task_success")) - for r in results if r.get("run_dir")) - return {"passed": passed, "total": len(results)} diff --git a/examples/aspire_robosuite/utils/benchmark/prepare.py b/examples/aspire_robosuite/utils/benchmark/prepare.py deleted file mode 100644 index ae04112a..00000000 --- a/examples/aspire_robosuite/utils/benchmark/prepare.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Write the Robosuite tasks of ASPIRE's harness (cap_harness/robosuite/registry.py) as hermit tasks: one task -directory (Harbor's format) per task with its seeds, which the simulator service and the judge take from the task's -row (ASPIRE's Robosuite defaults: development 101-125, unseen-check 126-130, held-out 1-100). - - python prepare.py # ASPIRE's seeds - python prepare.py --preset smoke # the smoke's: three development seeds, one unseen, three held-out - python prepare.py --dev 101-103 --unseen 126 --eval 1-3 # any others - # tasks//, splits/{all,smoke,single_arm,verify,rerun,smoke_rerun,single_arm_rerun}.txt - -A task directory's task.toml holds the row's fields but its id under [metadata], in the row's order, and asks for no -network (the actor reaches only its model and the simulator service's forwarded port); it has no tests/: the judge -(judge.py) runs on the host, where the simulator is. utils/agent/aspire_fields.py reads each back as its row. - -The verifier rows (splits/verify.txt) are the Evolver's: a run gives them as `data.val`, so only `run_agent` runs -them. - -The order of splits/all.txt is the curriculum the coordinator walks (single-arm pick before stacking and -assembly, the fixed-tool wipe, then the bimanual tasks). -""" -from __future__ import annotations - -import argparse -from pathlib import Path -import shutil -import sys - -from hermit.task.directory import write_task as write_task_dir - -HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE.parent / "agent")) -from sim import seeds_of # noqa: E402 -TASKS = [ # suite, what it is (task-info only; the actor reads the language from get_task_context()) - ("cube_lifting", "Lift"), - ("cube_stack", "Stack"), - ("cube_restack", "Stack, restacked"), - ("nut_assembly", "NutAssemblySquare"), - ("spill_wipe", "Wipe, fixed tool"), - ("two_arm_lift", "TwoArmLift, bimanual"), - ("two_arm_handover", "TwoArmHandover, bimanual"), -] - - -# The seeds of a run: ASPIRE's Robosuite defaults, and the smoke's -PRESETS = {"aspire": {"dev": "101-125", "unseen": "126-130", "eval": "1-100", "verify": "101-105"}, - "smoke": {"dev": "101-103", "unseen": "126", "eval": "1-3", "verify": "101-105"}} - - -def seeds_from(presets: dict, argv=None) -> tuple[dict, str]: - """The run's seeds from the command line (a preset, each set replaceable): the three sets of a task row, and - the verifier's.""" - ap = argparse.ArgumentParser() - ap.add_argument("--preset", choices=sorted(presets), default="aspire") - ap.add_argument("--dev", help="development seeds") - ap.add_argument("--unseen", help="unseen-check seeds, run once by the actor") - ap.add_argument("--eval", help="held-out seeds, run only by the judge") - ap.add_argument("--verify", help="the verifier's seeds") - a = ap.parse_args(argv) - chosen = {k: getattr(a, k) or v for k, v in presets[a.preset].items()} - # A held-out seed a session could run would be scored as unseen: the sets must not meet - dev, unseen, held, verify = (set(seeds_of(chosen[k])) for k in ("dev", "unseen", "eval", "verify")) - for name, other in (("development", dev), ("unseen-check", unseen), ("verifier", verify)): - if held & other: - ap.error(f"the held-out seeds overlap the {name} seeds: {sorted(held & other)[:10]}") - if dev & unseen: - ap.error(f"the unseen-check seeds overlap the development seeds: {sorted(dev & unseen)[:10]}") - return {"debug_seeds": chosen["dev"], "unseen_seeds": chosen["unseen"], "eval_seeds": chosen["eval"]}, \ - chosen["verify"] - - -def instruction(row: dict) -> str: - """What a task asks, in a line (the actor's prompt is utils/agent/task.py's, built from the row).""" - what = f"{row.get('benchmark') or 'robosuite'} suite `{row['suite']}`, task id {row['task_id']}" - if row.get("role") == "verifier": - return f"Verify the proposals for {what}: solve it again from the library and the proposals alone." - if row.get("role") == "rerun": - return f"Solve {what} again with the final library." - return f"Write the program that solves {what} in ASPIRE's Code-as-Policy harness." - - -def write_task(tasks: Path, row: dict) -> Path: - """One row as a task directory: its fields but the id under [metadata], in the row's order; no network.""" - return write_task_dir(tasks / row["id"], instruction(row), metadata={k: v for k, v in row.items() if k != "id"}, - environment={"network_mode": "no-network"}) - - -def write(here: Path, tasks: list[dict], seeds: dict, verify: str, splits: dict[str, list[str]]) -> None: - """tasks/ and splits/ under `here`: each task (`id`, `suite`, `task_id`, and `benchmark` when it is not - Robosuite) with its seeds, its verifier task and its rerun task; splits/.txt for each of `splits`, with - _rerun.txt beside it (rerun.txt for `all`), and verify.txt.""" - rows = [{**t, **seeds} for t in tasks] - # The verifier of each task (ASPIRE verifier-prompt.md): a fresh actor that re-solves the task from the library - # and the proposals alone, on 5 development seeds in at most 2 attempts, with no held-out run and no findings. - verifiers = [{**t, "id": f"verify-{t['id']}", "role": "verifier", "debug_seeds": verify, "eval_seeds": "none", - "max_versions": 2, "search": "debug", "findings": False} for t in tasks] - # The rerun of each task with the final library (ASPIRE §2.4, backward transfer): the same task, the same seeds - # and a fresh actor; its own id so that the test split does not share ids with the training split. - reruns = [{**t, "id": f"rerun-{t['id']}", "role": "rerun", "findings": False, **seeds} for t in tasks] - if (here / "tasks").exists(): # this script's own output: rewritten whole - shutil.rmtree(here / "tasks") - for row in rows + verifiers + reruns: - write_task(here / "tasks", row) - out = here / "splits" - out.mkdir(exist_ok=True) - for name, ids in splits.items(): - (out / f"{name}.txt").write_text("".join(f"{i}\n" for i in ids)) - (out / ("rerun.txt" if name == "all" else f"{name}_rerun.txt")).write_text("".join(f"rerun-{i}\n" for i in ids)) - (out / "verify.txt").write_text("".join(f"verify-{t['id']}\n" for t in tasks)) - print(f"{len(rows)} tasks -> {here / 'tasks'}") - - -def main() -> None: - seeds, verify = seeds_from(PRESETS) - tasks = [{"id": f"robosuite-{suite}", "suite": suite, "task_id": 0} for suite, _ in TASKS] - ids = [t["id"] for t in tasks] - write(HERE, tasks, seeds, verify, {"all": ids, "single_arm": ids[:5], "smoke": ids[:2]}) - - -if __name__ == "__main__": - main() diff --git a/examples/aspire_robosuite/README.md b/examples/cap_harness/README.md similarity index 51% rename from examples/aspire_robosuite/README.md rename to examples/cap_harness/README.md index b52dfe34..45341b85 100644 --- a/examples/aspire_robosuite/README.md +++ b/examples/cap_harness/README.md @@ -1,34 +1,30 @@ -# ASPIRE on Robosuite - -The Robosuite tasks of [ASPIRE](https://research.nvidia.com/labs/gear/aspire/) (Agentic Skills Discovery for -Robotics, [arXiv:2607.00272](https://arxiv.org/abs/2607.00272), [code](https://github.com/NVlabs/ASPIRE)) as a -continual-learning curriculum, run as ASPIRE runs it: ASPIRE's fix-loop subagent (the actor), its skill verifier -and its coordinator, each a Codex session given ASPIRE's own prompt. For each task the actor writes a Code-as-Policy -program (Python against ASPIRE's perception, grasping and motion API), runs it in ASPIRE's robot execution engine on -development seeds, debugs it from the per-primitive traces and RGB keyframes, and checks it once on unseen seeds; the -harness then scores it once on held-out seeds. After each task the coordinator (the -[`aspire`](../../docs/methods/aspire.md) Evolver) updates the shared skill library, admitting the actor's proposals -only if a fresh verifier re-solves the task from the library and the proposals alone, and chooses the next task. +# The cap-harness engine + +The robot execution engine of [ASPIRE](https://github.com/NVlabs/ASPIRE)'s Code-as-Policy harness (`cap-harness`), +shared by the benchmarks that run on it: [Robosuite](../robosuite/) and [LIBERO](../libero/) (and, for its simulator +service, [RoboDojo-Sim](../robodojo_sim/)). A session writes a program (Python against the harness's perception, +grasping and motion API), runs it in the engine on its task's development seeds through `bin/sim`, and the judge +scores it once on held-out seeds. The engine is a benchmark's, not a method's: what a method brings to a session +(ASPIRE's fix loop, its search, its verifier: [`aspire`](../../docs/methods/aspire.md)) is the method's own, built on +the task functions here. | Folder | What it is | |---|---| -| `utils/agent/task.py` | The Codex actor's task functions: the workspace (`bin/sim`, docs, the snapshot program) and its registration with the simulator service, the prompt (an adaptation page, the assignment, ASPIRE's template verbatim), the selected program (`collect`) | -| `utils/agent/sim_service.py` | The simulator service on the host: ASPIRE's `cap-harness` for the sandboxed actors, holding their seed rules | -| `utils/agent/stage_docs.py` | ASPIRE's docs as the actors may read them, staged outside its checkout by `launch.sh` | -| `utils/agent/sim_cli.py` | `bin/sim`: the actor's client of the simulator service | -| `utils/agent/sim.py` | The robot execution engine from the harness's side (the held-out run) and the trace digest | -| `utils/agent/prompts/` | The pages that map ASPIRE's fix-loop, verifier and evolutionary-search templates onto the workspace (and the direct actor's prompts) | -| `utils/agent/agent.py` | The structured direct-chat actor of `aspire/run_direct.yaml` (a comparison, not the method) | -| `utils/benchmark/prepare.py` | The seven Robosuite tasks with their seeds, as task directories in Harbor's format (`tasks/`, each read as its row through `utils/agent/aspire_fields.py`, the configs' `data.view`); the verifier's tasks; the reruns with the final library; `splits/` | +| `utils/agent/task.py` | The task functions every session gets: the workspace (`bin/sim`, the staged docs, the scene-exploration page and snapshot program) and its registration with the simulator service, the facts the judge reads (`collect`: `/fix_code.py` and the unseen check) | +| `utils/agent/sim_service.py` | The simulator service on the host: ASPIRE's `cap-harness` for the sandboxed sessions, holding their seed rules and a row's attempt budget (`max_programs`) | +| `utils/agent/stage_docs.py` | ASPIRE's docs as the sessions may read them, staged outside its checkout by `launch.sh` | +| `utils/agent/sim_cli.py` | `bin/sim`: a session's client of the simulator service | +| `utils/agent/sim.py` | The robot execution engine from the harness's side (the held-out run, a direct actor's runs) and the trace digest | +| `utils/agent/fields.py` | A task directory read back as its row: every config's `data.view` | +| `utils/benchmark/tasks.py` | What a benchmark's `prepare.py` writes with: its tasks and their reruns as task directories in Harbor's format, its splits, the seed presets' checks | +| `utils/benchmark/judge.py` | On the host: the scored program on the row's held-out seeds, its success rate | | `utils/seg_service/` | A SAM3-wire-compatible segmentation service from ungated models (optional) | -| `utils/benchmark/judge.py` | Stage 2 on the host: the selected program on the held-out seeds (a verifier's on its own), its success rate | -| `utils/benchmark/task-info/` | What a task and its record are, for the learner | -| `aspire/` | `run.yaml` (ASPIRE's seeds), `smoke.yaml`, the `*_direct.yaml` comparison, `launch.sh` | +| `launch.sh` | Started by a benchmark's method folder: writes the seeds into the tasks, runs the method folder's `tasks.sh` if it has one, stages the docs, starts the simulator service and runs hermit | ## Setup 1. **ASPIRE.** ASPIRE runs in its own venvs and is never imported into hermit's process (as CL-Bench's runner is - not), so it is not a dependency group of hermit's `pyproject.toml`: the example pins its commit here and + not), so it is not a dependency group of hermit's `pyproject.toml`: the engine pins its commit here and `launch.sh` names the one a run used. Clone [NVlabs/ASPIRE](https://github.com/NVlabs/ASPIRE) and bootstrap its Robosuite venv and the providers its programs call (its `docs/host-setup.md`, `docs/robosuite.md`): @@ -48,7 +44,7 @@ only if a fresh verifier re-solves the task from the library and the proposals a `CAP_HARNESS_SAM3_URL` at it): see [Segmentation without SAM3](#segmentation-without-sam3). 3. **The simulator's render device.** MuJoCo renders offscreen with EGL. Every cap-harness process runs in a - sandbox of its own (see [The actor's rules](#the-actors-rules)), which shows only the GPUs `ASPIRE_SIM_GPU` + sandbox of its own (see [A session's rules](#a-sessions-rules)), which shows only the GPUs `ASPIRE_SIM_GPU` names: the N of `/dev/nvidia` (`nvidia-smi -q` shows it as a GPU's Minor Number), e.g. `ASPIRE_SIM_GPU=8`. Without it the simulator renders in software (Mesa's llvmpipe, which the host must have), slowly. A hand-extracted NVIDIA EGL library (`__EGL_VENDOR_LIBRARY_FILENAMES`) is shown to it; other paths it needs go in @@ -62,69 +58,38 @@ only if a fresh verifier re-solves the task from the library and the proposals a MUJOCO_EGL_DEVICE_ID: "0" # inside the sandbox the GPUs ASPIRE_SIM_GPU names count from 0 ``` -4. **Codex and the sandboxes.** The actor and the coordinator are Codex CLI sessions (`codex` on `PATH`; the configs - bind them to GLM 5.3 Flash on Fireworks), each in the framework's sandbox (`sandbox.enabled: true`, `enforce: - required`; bubblewrap on `PATH`, docs/guides/sandbox.md). The task sandbox has no GPU, so the simulator runs on the - host as a service (`utils/agent/sim_service.py`, started by `launch.sh`), which the actor reaches through `bin/sim` - on a forwarded loopback port; see [The actor's rules](#the-actors-rules). - -5. **Run** on the host that serves the providers: - - ```bash - export FIREWORKS_API_KEY=... - cd examples/aspire_robosuite/aspire - ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh - ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh eval --adapter final - ``` - - `./launch.sh eval --adapter final` reruns every task with the final library (the test split's `rerun-*` rows): - ASPIRE's backward transfer, compared per task with the task's first pass during training (each training step's - score in the run's `steps.jsonl`; the rerun's in the eval's results). `--adapter root` is the empty library and - `--adapter none` no library at all: the baselines on the same seeds. - - The protocol is ASPIRE's: each task learns on its development seeds and is scored once on held-out seeds; the - reruns score the same held-out seeds again, so the comparison reuses that partition (as ASPIRE's Phase 3 does). - The first evaluation ran `run.yaml` on held-out seeds 1-50 (half of ASPIRE's 1-100), and its reruns on the same - seeds. The held-out seeds are the task rows' (`prepare.py --eval`), so they are set when the launcher writes the - rows: `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh`, and `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh eval --adapter final` for the - reruns (an `agent.params.eval_seeds` override has no effect). - -## Protocol - -| | `run.yaml` | `smoke.yaml` | -|---|---|---| -| Tasks | the 7 Robosuite tasks, in the order the coordinator chooses (it starts from `splits/all.txt`'s first) | `cube_lifting`, `cube_stack` | -| Development seeds | 101-125 (ASPIRE's) | 101-103 | -| Unseen-check seeds | 126-130 (ASPIRE's), run once by the actor | 126 | -| Held-out seeds | 1-100 (ASPIRE's; `ASPIRE_EVAL_SEEDS` for others), run once by the judge | 1-3 | -| Fix loop | ASPIRE's: 3 versions per failure mode, 8 in total | the same | -| Evolutionary search | ASPIRE's, for a task the fix loop leaves below 90%: K = 8, top 3, stop at 80% or 5 rounds | the same | -| Verifier | ASPIRE's: 5 seeds (101-105), 2 attempts, verified at ≥ 4/5 | the same | -| Actor, verifier, coordinator | Codex, GLM 5.3 Flash (Fireworks), reasoning effort medium | the same | +4. **The CLI and the sandboxes.** A session (and a method's learner, such as ASPIRE's coordinator) is a Codex CLI + session (`codex` on `PATH`; most configs bind them to GLM 5.3 Flash on Fireworks) or a Claude Code one (`claude` on + `PATH`, `agent.base: claude`; on a Claude subscription, `login: claude`: docs/guides/claude-subscription.md), each + in the framework's sandbox (`sandbox.enabled: true`, `enforce: required`; bubblewrap on `PATH`, + docs/guides/sandbox.md). The task sandbox has no GPU, so the simulator runs on the host as a service + (`utils/agent/sim_service.py`, started by `launch.sh`), which the session reaches through `bin/sim` on a forwarded + loopback port; see [A session's rules](#a-sessions-rules). ## Disk -Every actor session runs the simulator hundreds of times (a fix loop of up to 8 programs on 25 seeds, an -evolutionary search of up to 40 more), and each run keeps its record, trace and images. `collect` deletes the images +A session runs the simulator hundreds of times (ASPIRE's: a fix loop of up to 8 programs on 25 seeds, an evolutionary +search of up to 40 more), and each run keeps its record, trace and images. `collect` deletes the images (`media/`) when the session ends (`agent.params.prune_media: false` keeps them); the rest still adds up over a campaign. Set `ASPIRE_OUT` to a large disk: `launch.sh` puts the runs and the workspaces there. -## The actor's rules -The actor runs in the task sandbox, which shows it only what the config declares (`environment.read`): its own code -(`utils/agent/`), of the ASPIRE checkout the prompt templates and the snapshot program it is given, and ASPIRE's -docs as `launch.sh` stages them outside the checkout (`utils/agent/stage_docs.py`: without the lines, and for LIBERO -the section, that name ASPIRE's worked programs; the checkout's own pages are not shown). -ASPIRE's shipped skill library, its worked solutions and examples, other sessions' workspaces, the task rows and the -judge do not exist inside; its network reaches only its model and the simulator service's forwarded port. The +## A session's rules + +A session runs in the task sandbox, which shows it only what the config declares (`environment.read`): the engine's +code (`utils/agent/`), of the ASPIRE checkout the snapshot program it is given (and a method's prompt templates, which +its configs declare), and ASPIRE's docs as `launch.sh` stages them outside the checkout (`utils/agent/stage_docs.py`: +without the lines, and for LIBERO the section, that name ASPIRE's worked programs; the checkout's own pages are not +shown). ASPIRE's shipped skill library, its worked solutions and examples, other sessions' workspaces, the task rows +and the judge do not exist inside; its network reaches only its model and the simulator service's forwarded port. The simulator service is the trusted side of the seed rules: it reads the task rows on the host, `prepare` registers the session (its workspace and task id; one session per rollout directory, which the sandbox cannot create) before the -agent starts, and the service runs only the row's development and unseen-check seeds, the unseen check once and a -verifier's two programs, writing the runs into the workspace. The held-out seeds are run only by the judge, on the -host (ASPIRE's Stage 2), on the program the session selected. The actor's Codex runs with its own sandbox off -(`codex_sandbox: danger-full-access`), the task sandbox being the boundary. +agent starts, and the service runs only the row's development and unseen-check seeds, the unseen check once and, for +a row with an attempt budget (`max_programs`), that many programs, writing the runs into the workspace. The held-out +seeds are run only by the judge, on the host (ASPIRE's Stage 2), on the program the session selected. A Codex +session runs with its own sandbox off (`codex_sandbox: danger-full-access`), the task sandbox being the boundary. -The programs the actor writes run inside cap-harness on the host, and ASPIRE's program validator is no boundary (a +The programs a session writes run inside cap-harness on the host, and ASPIRE's program validator is no boundary (a format string's attribute traversal passes it). So every cap-harness process, the service's, the judge's and the direct actor's, runs in a bubblewrap sandbox of its own (`utils/agent/sim.py`: `confine`): it reads ASPIRE's checkout but not its worked solutions, agents' skills or tests (`examples/`, `.claude/`, `.agents/`, `.codex/`, @@ -133,13 +98,14 @@ ports; one a `CAP_HARNESS_*_URL` puts on another host through an allowlist proxy GPUs `ASPIRE_SIM_GPU` names (hermit's `Sandbox(gpu=...)`). A program that escapes the validator reaches no other session, none of the judge's runs and no host file, and what it leaves in its output is read without following a link. It still runs in cap-harness's own process, so it could write its own run's outcome: the score of such a -program is not trustworthy, and the boundary is that it can affect nothing else. This sandbox is the example's own: +program is not trustworthy, and the boundary is that it can affect nothing else. This sandbox is the engine's own: it is on whatever the run's `sandbox.enabled` says, so the simulator always needs bubblewrap (the service refuses to start without it, or with an `ASPIRE_SIM_GPU` the machine lacks). Each run gets a HOME of its own, removed after it; the host's `CUDA_VISIBLE_DEVICES` is not passed in (inside, the GPUs count from 0); each confined run leaves its sandbox record under `$ASPIRE_OUT/sim-service/sandbox/` (the service's) or `$ASPIRE_OUT/sandbox/` (the judge's). + ## Segmentation without SAM3 ASPIRE's programs segment with SAM3 (`facebook/sam3`, gated). Any service with the SAM3 wire format can stand in. @@ -155,6 +121,7 @@ CUDA_VISIBLE_DEVICES=0 .venv-seg/bin/python utils/seg_service/service.py --port The models (`IDEA-Research/grounding-dino-base`, `facebook/sam-vit-huge`; `SEG_DINO_MODEL`, `SEG_SAM_MODEL`) are downloaded on first start. + ## Checking the render device Inside the simulator's sandbox EGL finds only the GPUs `ASPIRE_SIM_GPU` names, so `MUJOCO_EGL_DEVICE_ID: "0"` is the @@ -164,9 +131,10 @@ use). A host whose libEGL finds only Mesa's driver (`failed to open /dev/dri/car vendor library (`libEGL_nvidia.so.0`, the driver's `libnvidia-gl` package) and `__EGL_VENDOR_LIBRARY_FILENAMES` naming its ICD file. + ## Known host issues (Blackwell B300, sm_103) -Found while bringing the engine up on B300 nodes; none of it is in the example's code path, which takes the +Found while bringing the engine up on B300 nodes; none of it is in the engine's code path, which takes the simulator's environment and the service URLs from the host. - **NVRTC 12.8 rejects sm_103.** The cu128 PyTorch wheels' runtime compiler fails on B300 diff --git a/examples/aspire_robosuite/aspire/launch.sh b/examples/cap_harness/launch.sh similarity index 71% rename from examples/aspire_robosuite/aspire/launch.sh rename to examples/cap_harness/launch.sh index 15a70d2c..ed50b7d2 100755 --- a/examples/aspire_robosuite/aspire/launch.sh +++ b/examples/cap_harness/launch.sh @@ -1,29 +1,34 @@ #!/usr/bin/env bash -# Train, or evaluate, the ASPIRE method on Robosuite. Run it on the host that serves ASPIRE's providers. -# (Another benchmark of ASPIRE's harness starts it from its own folder, as ASPIRE_EXAMPLE: its configs, in the -# method folder ASPIRE_METHOD (default aspire), its utils/benchmark/prepare.py and tasks, its runs; ../../libero/utils/launch.sh.) +# Train, or evaluate, a method on a benchmark of the cap-harness engine (ASPIRE's harness). Run it on the host that +# serves ASPIRE's providers. A method folder of a benchmark starts it (../robosuite/aspire/launch.sh, +# ../libero/utils/launch.sh) with ASPIRE_EXAMPLE, the benchmark's folder (its utils/benchmark/prepare.py, tasks and +# task-info, its runs), and ASPIRE_METHOD, the method folder in it (its configs); from that folder: # # ASPIRE_ROOT=/path/to/ASPIRE ./launch.sh # train run.yaml # ASPIRE_CONFIG=smoke.yaml ./launch.sh # train the smoke (its seeds: ASPIRE_SEEDS=smoke) -# ./launch.sh eval --adapter final # the reruns with the final library (test split) -# ./launch.sh eval --adapter root # the same with the empty library +# ./launch.sh eval --adapter final # the reruns with the final adapter (test split) +# ./launch.sh eval --adapter root # the same with the starting one # ASPIRE_SIM_ENV=/path/to/sim_env.yaml ./launch.sh # the simulator's render environment (a YAML map) # ASPIRE_SIM_GPU=8 ./launch.sh # its GPUs, the N of /dev/nvidia (none: software) -# ASPIRE_OUT=/big/disk/aspire ./launch.sh # the runs, the sessions, Stage 2 (default ../runs) +# ASPIRE_OUT=/big/disk/aspire ./launch.sh # the runs, sessions, Stage 2 (default: runs/ beside) # ASPIRE_EVAL_SEEDS=1-50 ./launch.sh # held-out seeds other than the preset's # -# The launcher writes the seeds into the tasks (prepare.py; ASPIRE_SEEDS=smoke for the smoke's), starts the -# simulator service the actors reach through bin/sim (utils/agent/sim_service.py, on ASPIRE_SIM_PORT, default 8120; -# stopped when the run ends), and runs hermit. ASPIRE_SIM_ENV names a YAML mapping of environment variables for the -# simulator, e.g. {MUJOCO_GL: egl, PYOPENGL_PLATFORM: egl, EGL_PLATFORM: device, MUJOCO_EGL_DEVICE_ID: "0"} +# The launcher writes the seeds into the tasks (the benchmark's prepare.py; ASPIRE_SEEDS=smoke for the smoke's), +# then runs the method folder's tasks.sh when it has one (ASPIRE's adds its verifiers, and sets TASKS to the tasks its +# configs read), starts the simulator service the sessions reach through bin/sim (utils/agent/sim_service.py, on +# ASPIRE_SIM_PORT, default 8120; stopped when the run ends), and runs hermit. ASPIRE_SIM_ENV names a YAML mapping +# of environment variables for the simulator, e.g. {MUJOCO_GL: egl, PYOPENGL_PLATFORM: egl, EGL_PLATFORM: device, +# MUJOCO_EGL_DEVICE_ID: "0"} # (the default, with CUDA_VISIBLE_DEVICES="": the simulator needs no CUDA, and Robosuite asserts that the EGL # device appears in CUDA_VISIBLE_DEVICES when that is set). The model key (FIREWORKS_API_KEY) comes from the # environment; the sandboxes need bubblewrap (`bwrap`) on PATH. set -euo pipefail HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" : "${ASPIRE_ROOT:?set ASPIRE_ROOT to an ASPIRE checkout with .venv-robosuite}" -EXAMPLE="$(cd "${ASPIRE_EXAMPLE:-$HERE/..}" && pwd)" # the folder with / and utils/benchmark/ -METHOD="${ASPIRE_METHOD:-aspire}" # the folder of the configs +: "${ASPIRE_EXAMPLE:?set by a method folder of the benchmark: the benchmark folder}" +: "${ASPIRE_METHOD:?set by a method folder of the benchmark: the folder of the configs}" +EXAMPLE="$(cd "$ASPIRE_EXAMPLE" && pwd)" # the folder with /, utils/benchmark/ +METHOD="$ASPIRE_METHOD" # Absolute, from where the launcher was called: the service, the judge and the sandboxed workers each start elsewhere ASPIRE_ROOT="$(cd "$ASPIRE_ROOT" && pwd)" [[ -z "${ASPIRE_OUT:-}" ]] || { mkdir -p "$ASPIRE_OUT" && ASPIRE_OUT="$(cd "$ASPIRE_OUT" && pwd)"; } @@ -41,7 +46,7 @@ fi for url in "${CAP_HARNESS_SAM3_URL:-http://127.0.0.1:8114}" "http://127.0.0.1:8116" \ "${CAP_HARNESS_CUROBO_URL:-http://127.0.0.1:8118}"; do curl -sf -o /dev/null --max-time 3 "$url/openapi.json" \ - || { echo "ASPIRE provider at $url is not up (see ../README.md)" >&2; exit 1; } + || { echo "ASPIRE provider at $url is not up (see $HERE/README.md)" >&2; exit 1; } done # The ASPIRE commit this example was written against; another one is run, but named. PINNED=fa00f0ef2bc1d91d594b9aa502c22133c9f538f7 @@ -52,19 +57,21 @@ cd "$EXAMPLE/$METHOD" ! grep -q "ref: fireworks/" "$CONFIG" || : "${FIREWORKS_API_KEY:?set FIREWORKS_API_KEY}" # The seeds, in the tasks (task directories, each read as its row): the simulator service and the judge read them -# there. The example's utils/benchmark/prepare.py holds its benchmark's (ASPIRE's defaults and the smoke's). +# there. The benchmark's utils/benchmark/prepare.py holds its own (ASPIRE's defaults and the smoke's). PRESET="${ASPIRE_SEEDS:-$([[ "${CONFIG##*/}" == smoke* ]] && echo smoke || echo aspire)}" [[ "$PRESET" == smoke || "$PRESET" == aspire ]] || { echo "ASPIRE_SEEDS is smoke or aspire" >&2; exit 2; } SEEDS=(--preset "$PRESET") [[ -z "${ASPIRE_EVAL_SEEDS:-}" ]] || SEEDS+=(--eval "$ASPIRE_EVAL_SEEDS") (cd ../utils/benchmark && "$HERMIT" prepare.py "${SEEDS[@]}") +# The tasks the run reads: the benchmark's, or what the method folder's tasks.sh sets (run here, in that folder) +TASKS="$EXAMPLE/utils/benchmark/tasks" +[[ ! -f tasks.sh ]] || source ./tasks.sh # ASPIRE's docs as the actors may read them, outside its checkout: the task sandbox is shown these, not the # checkout's pages, which name ASPIRE's worked programs per task. export ASPIRE_DOCS="$EXAMPLE/utils/aspire-docs" -"$HERMIT" "$HERE/../utils/agent/stage_docs.py" --aspire-root "$ASPIRE_ROOT" --tasks ../utils/benchmark/tasks \ - --out "$ASPIRE_DOCS" +"$HERMIT" "$HERE/utils/agent/stage_docs.py" --aspire-root "$ASPIRE_ROOT" --tasks "$TASKS" --out "$ASPIRE_DOCS" -# Where the runs go (ASPIRE_OUT): the trainer's records, the actors' sessions (their simulator runs' records and +# Where the runs go (ASPIRE_OUT): the trainer's records, the sessions (their simulator runs' records and # traces; their images are pruned after scoring), the judge's Stage 2. Put it on a large disk. OUT="${ASPIRE_OUT:-$EXAMPLE/runs}" case "$(cd "$OUT" 2>/dev/null && pwd -P || echo "$OUT")/" in # the simulator reads the checkout, not the runs @@ -74,12 +81,12 @@ esac mkdir -p "$OUT" export ASPIRE_ROOT ASPIRE_OUT="$OUT" ASPIRE_SIM_ENV="$SIM_ENV_JSON" ASPIRE_SIM_GPU="${ASPIRE_SIM_GPU:-}" # the judge's -# The simulator service: the actors' engine, on the host; each cap-harness it starts (and the judge's) runs in a +# The simulator service: the sessions' engine, on the host; each cap-harness it starts (and the judge's) runs in a # sandbox of its own with the GPUs ASPIRE_SIM_GPU names (utils/agent/sim.py: confine). [[ -n "${ASPIRE_SIM_GPU:-}" ]] || echo "note: ASPIRE_SIM_GPU names no GPU: the simulator renders in software" >&2 PORT="${ASPIRE_SIM_PORT:-8120}" NONCE="$$-$RANDOM-$(date +%s%N)" # this launch's service answers /health with it, another run's not -"$HERMIT" "$HERE/../utils/agent/sim_service.py" --port "$PORT" --aspire-root "$ASPIRE_ROOT" --tasks ../utils/benchmark/tasks \ +"$HERMIT" "$HERE/utils/agent/sim_service.py" --port "$PORT" --aspire-root "$ASPIRE_ROOT" --tasks "$TASKS" \ --workspaces "$OUT/actor-logs" --state "$OUT/sim-service" --sim-env "$SIM_ENV_JSON" --nonce "$NONCE" \ --gpu "${ASPIRE_SIM_GPU:-}" > "$OUT/sim-service.log" 2>&1 & SERVICE=$! diff --git a/examples/cap_harness/utils/agent/fields.py b/examples/cap_harness/utils/agent/fields.py new file mode 100644 index 00000000..2424b946 --- /dev/null +++ b/examples/cap_harness/utils/agent/fields.py @@ -0,0 +1,20 @@ +"""A task of the cap-harness engine as the session's task functions, the simulator service and the judge read it: +{id, (benchmark), suite, task_id, the seeds, (max_programs), and whatever a method's own rows add}. A task directory +(a benchmark's utils/benchmark/prepare.py, through ../benchmark/tasks.py) has every field but its id under task.toml's +[metadata], in the row's order; this puts them back, so the prompts, the service's rules and the records read the row +as it was written. Every config of a benchmark on this engine names it as its data.view.""" +from __future__ import annotations + +from pathlib import Path + + +def task_row(task: dict) -> dict: + if "instruction" not in task: # already a row of that shape + return dict(task) + return {"id": task["id"], **(task.get("metadata") or {})} + + +def task_rows(tasks: Path) -> dict[str, dict]: + """Every task's row by id: from task directories, or from a tasks.jsonl.""" + from hermit.task import Benchmark + return {tid: t.given() for tid, t in Benchmark.load(Path(tasks), view=task_row).tasks.items()} diff --git a/examples/aspire_robosuite/utils/agent/sim.py b/examples/cap_harness/utils/agent/sim.py similarity index 100% rename from examples/aspire_robosuite/utils/agent/sim.py rename to examples/cap_harness/utils/agent/sim.py diff --git a/examples/aspire_robosuite/utils/agent/sim_cli.py b/examples/cap_harness/utils/agent/sim_cli.py similarity index 100% rename from examples/aspire_robosuite/utils/agent/sim_cli.py rename to examples/cap_harness/utils/agent/sim_cli.py diff --git a/examples/aspire_robosuite/utils/agent/sim_service.py b/examples/cap_harness/utils/agent/sim_service.py similarity index 95% rename from examples/aspire_robosuite/utils/agent/sim_service.py rename to examples/cap_harness/utils/agent/sim_service.py index 6b933475..30b99a7b 100644 --- a/examples/aspire_robosuite/utils/agent/sim_service.py +++ b/examples/cap_harness/utils/agent/sim_service.py @@ -1,10 +1,10 @@ -"""The robot execution engine as a host service: ASPIRE's `cap-harness` for the actors in the task sandbox. +"""The robot execution engine as a host service: ASPIRE's `cap-harness` for the sessions in the task sandbox. The task sandbox has no GPU, so the simulator runs here, on the host, beside ASPIRE's perception and planning services; a session reaches it through `bin/sim` (sim_cli.py) on a forwarded 127.0.0.1 port (ASPIRE_SIM_URL in environment.env). The service is the trusted side of the session's rules, and it trusts nothing the session sends -but the workspace and its task's id: the seeds, the role and the budget are the task row's (--tasks, read here on -the host, each task as its row: aspire_fields.py), and a rollout's directory (//-, +but the workspace and its task's id: the seeds and the budget are the task row's (--tasks, read here on +the host, each task as its row: fields.py), and a rollout's directory (//-, made by the framework outside the sandbox, which can write only inside it) registers one session, of the task its name carries: @@ -13,8 +13,8 @@ same workspace, finds its registration and what it spent as they were) POST /run {workspace, cmd: run | run-batch | validate, program, seeds, output_root, no_videos} runs cap-harness for the registered task: only its development and unseen-check seeds, the - unseen check once, a verifier's two programs (by content; a single run on the first seed is - free), the program and the output inside the workspace, the registration's max_steps + unseen check once, a row's attempt budget (`max_programs`: programs by content; a single run on the + first seed is free), the program and the output inside the workspace, the registration's max_steps GET /health The session's files are never opened by path here, since the session may have put a link in it: the program is read @@ -23,7 +23,7 @@ own directory, and the run is moved into the workspace the same way, each file replacing whatever entry has its name. A run is stopped after --run-timeout seconds, and the runs still going are stopped when the service is. -Every refusal is a 200 with {ok: false, error}. The ledger (registrations, unseen checks, a verifier's programs) is +Every refusal is a 200 with {ok: false, error}. The ledger (registrations, unseen checks, a budget's programs) is kept in --state, outside every sandbox. Started by launch.sh: python sim_service.py --port 8120 --aspire-root $ASPIRE_ROOT --tasks utils/benchmark/tasks --workspaces @@ -50,7 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import sim # noqa: E402 from sim import seeds_of # noqa: E402 -from aspire_fields import task_rows # noqa: E402 +from fields import task_rows # noqa: E402 class Ledger: @@ -171,13 +171,13 @@ def register(self, body: dict) -> dict: return {"ok": True} if any(Path(w).is_relative_to(rollout) for w in self.ledger.data): return {"ok": False, "error": "this rollout has registered its session already"} - verifier = row.get("role") == "verifier" self._check_row(row) self.ledger.data[str(workspace)] = { - "task": row["id"], "role": "verifier" if verifier else "actor", "suite": str(row["suite"]), + "task": row["id"], "suite": str(row["suite"]), "benchmark": str(row.get("benchmark") or "robosuite"), "task_id": int(row.get("task_id", 0)), "dev": seeds_of(row.get("debug_seeds")), - "unseen": seeds_of(row.get("unseen_seeds")), "max_programs": 2 if verifier else None, + "unseen": seeds_of(row.get("unseen_seeds")), + "max_programs": int(row["max_programs"]) if row.get("max_programs") else None, "max_steps": body.get("max_steps"), "unseen_used": False, "programs": []} self.ledger.save() return {"ok": True} @@ -210,7 +210,7 @@ def run(self, body: dict) -> dict: shutil.rmtree(own, ignore_errors=True) def _run(self, body, session, program, code, own_out, rollout, out, swaps) -> dict: - """The rules every engine shares (the assigned seeds, the unseen check once, a verifier's programs, the + """The rules every engine shares (the assigned seeds, the unseen check once, a budget's programs, the output moved into the workspace without following a link); what is run is the engine's (`_validate`, `_launch`: cap-harness here, another simulator in a subclass).""" if body.get("cmd") == "validate": diff --git a/examples/aspire_robosuite/utils/agent/stage_docs.py b/examples/cap_harness/utils/agent/stage_docs.py similarity index 97% rename from examples/aspire_robosuite/utils/agent/stage_docs.py rename to examples/cap_harness/utils/agent/stage_docs.py index 4c1ed997..c910ebd0 100644 --- a/examples/aspire_robosuite/utils/agent/stage_docs.py +++ b/examples/cap_harness/utils/agent/stage_docs.py @@ -13,7 +13,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import sim # noqa: E402 -from aspire_fields import task_rows # noqa: E402 +from fields import task_rows # noqa: E402 def main() -> None: diff --git a/examples/cap_harness/utils/agent/task.py b/examples/cap_harness/utils/agent/task.py new file mode 100644 index 00000000..4ce11383 --- /dev/null +++ b/examples/cap_harness/utils/agent/task.py @@ -0,0 +1,156 @@ +"""The engine's task functions for a coding-agent session (agent.base: codex, or claude): what every method's session +on a benchmark of this engine gets, and nothing of a method. They run in the task sandbox with the session; the robot +execution engine runs on the host, as the simulator service (sim_service.py), which the session reaches only through +`bin/sim`. A benchmark gives the prompt (LIBERO's ../../../libero/utils/agent/task.py); a method that brings its own +session (ASPIRE: hermit/agent/evolver/methods/aspire/actor.py) builds on these. + + prepare the workspace: bin/sim and its registration with the service (the workspace and the task's id: the + service takes the seeds and the budget from its own copy of the task row), the API and artifact docs, + the benchmark's page, the scene-exploration procedure and its snapshot program + answer the agent's last message (the record's output is the facts collect returns) + collect the session's files into the log, and the facts the judge reads: the scored program + (`/fix_code.py`) and the unseen check (`/debug/holdout`); the judge runs the program + on the held-out seeds, on the host + +The checkout of ASPIRE, whose `cap-harness` this engine is, comes from agent.params.aspire_root (its snapshot program +and exploration page must be declared readable: environment.read); its docs are read from agent.params.aspire_docs, +the staged copies; the service's URL from ASPIRE_SIM_URL (environment.env). A task row carries the seeds: +development, unseen-check and held-out (../benchmark/tasks.py). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import shutil +import stat +import sys +import urllib.request + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import sim # noqa: E402 + +SKILLS = Path(".claude/skills/iterative-debugging") +# The files of ASPIRE's checkout the task sandbox shows (environment.read) for every session: the scene-exploration +# procedure and its snapshot program; not its shipped library, worked solutions or examples, nor its docs, which the +# sandbox is shown as staged (agent.params.aspire_docs: stage_docs.py's copies, without what names the worked programs) +ASPIRE_FILES = [str(SKILLS / "scripts" / "scene_snapshot.py"), str(SKILLS / "skills" / "task-exploration.md")] +# The workspace files the log keeps (the run directories stay in the workspace) +KEEP = ("task_analysis.md", "fix_code.py", "findings.md", "code", "debug/blocked") + + +def settings(task: dict, params: dict) -> dict: + """The session's task, from its row (the seeds; ../benchmark/tasks.py) and agent.params (the checkout).""" + root = params.get("aspire_root") or os.environ.get("ASPIRE_ROOT") + if not root: + raise ValueError("agent.params.aspire_root (or ASPIRE_ROOT): an ASPIRE checkout with its benchmark's venv") + benchmark = str(task.get("benchmark") or "robosuite") + return {"root": Path(os.path.expandvars(str(root))).expanduser().resolve(), + "benchmark": benchmark, "doc": sim.profile(benchmark)["doc"], "suite": task["suite"], + "task_id": int(task.get("task_id", 0)), "dev": str(task.get("debug_seeds") or ""), + "unseen": str(task.get("unseen_seeds") or ""), "heldout": str(task.get("eval_seeds") or ""), + "max_steps": params.get("max_steps")} + + +def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: + s = settings(task, params) + work = Path(workspace) + url = os.environ.get("ASPIRE_SIM_URL") + if not url: + raise ValueError("ASPIRE_SIM_URL: the simulator service's URL (launch.sh starts it; environment.env)") + # The session, registered before the agent starts: the service takes its seeds and budget from the task's row + register(url, {"workspace": str(work.resolve()), "task": task["id"], "max_steps": s["max_steps"]}) + bin_dir = work / "bin" + bin_dir.mkdir(exist_ok=True) + shutil.copy2(HERE / "sim_cli.py", bin_dir / "sim_cli.py") + (bin_dir / "sim.json").write_text(json.dumps({"workspace": str(work.resolve()), "url": url}, indent=1)) + wrapper = bin_dir / "sim" + wrapper.write_text(f"#!/bin/sh\nexec {shlex.quote(sys.executable)} \"$(dirname \"$0\")/sim_cli.py\" \"$@\"\n") + wrapper.chmod(wrapper.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + (work / "docs").mkdir(exist_ok=True) + # The repository's worked programs are not the session's to read: the docs are the staged copies (launch.sh: + # stage_docs.py), or, where the checkout's own pages are what the sandbox shows, the same cut made here + staged = params.get("aspire_docs") + pages = {name: (Path(os.path.expandvars(str(staged))).expanduser() / name).read_text() + for name in (*sim.DOCS, s["doc"])} if staged else sim.workspace_docs(s["root"], s["benchmark"]) + for name, text in pages.items(): + (work / "docs" / name).write_text(text) + shutil.copy2(s["root"] / SKILLS / "skills" / "task-exploration.md", work / "docs" / "task-exploration.md") + (work / "scripts").mkdir(exist_ok=True) + shutil.copy2(s["root"] / SKILLS / "scripts" / "scene_snapshot.py", work / "scripts" / "scene_snapshot.py") + return [] + + +def register(url: str, body: dict) -> None: + request = urllib.request.Request(url.rstrip("/") + "/register", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=60) as reply: + answer = json.loads(reply.read()) + if not answer.get("ok"): + raise RuntimeError(f"the simulator service refused the session: {answer.get('error')}") + + +def task_dir(s: dict) -> str: + """Where the session's task files go, relative to its workspace: `//task_` (ASPIRE's + $TASK_DIR, with $RUN_ROOT the workspace). The scored program is fix_code.py there.""" + return f"{s['benchmark']}/{s['suite']}/task_{s['task_id']}" + + +def sim_seeds(text: str) -> list[int]: + return sim.seeds_of(text) + + +def answer(last_message: str, task: dict) -> dict: + return {"answer": last_message.strip()[-4000:]} + + +def collect(task: dict, workspace: Path, log: Path, params: dict) -> dict: + """The session's files into the log and the facts for the judge, which runs the scored program on the held-out + seeds, on the host.""" + s = settings(task, params) + work = Path(workspace) / task_dir(s) + facts = collect_files(task, work, log, KEEP) + program = work / "fix_code.py" + facts.update({"benchmark": s["benchmark"], "suite": s["suite"], "workspace": str(Path(workspace).resolve()), + "max_steps": s["max_steps"], + "scored_program": str(program.relative_to(workspace)) if program.is_file() else None, + "fix_code": program.read_text(errors="replace") if program.is_file() else None}) + if params.get("prune_media", True): + prune(Path(workspace).parent) + return facts + + +def collect_files(task: dict, work: Path, log: Path, keep) -> dict: + """The files of `keep` (under `work`) copied into the log's files/, and the facts every session's record has: + its notes (findings.md) and its unseen check.""" + files = Path(log) / "files" + for name in keep: + source = work / name + if source.is_dir(): + shutil.copytree(source, files / name, dirs_exist_ok=True) + elif source.is_file(): + files.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, files / name) + findings = work / "findings.md" + return {"findings": findings.read_text(errors="replace") if findings.is_file() else None, "unseen": unseen(work)} + + +def prune(run_dir: Path) -> None: + """The heavy part of a finished session's simulator runs: every run's media/ (keyframes, overlays, videos; a + session runs hundreds of episodes). The records, traces and programs stay.""" + for media in run_dir.rglob("media"): + if media.is_dir() and (media.parent / "run.json").is_file(): + shutil.rmtree(media, ignore_errors=True) + + +def unseen(work: Path) -> dict | None: + """The unseen check a session ran into `/debug/holdout`: how many of its episodes succeeded.""" + rows = work / "debug" / "holdout" / "results.jsonl" + if not rows.is_file(): + return None + results = [json.loads(l) for l in rows.read_text().splitlines() if l.strip()] + passed = sum(bool((sim._read_json(Path(r["run_dir"]) / "outcome.json") or {}).get("task_success")) + for r in results if r.get("run_dir")) + return {"passed": passed, "total": len(results)} diff --git a/examples/aspire_robosuite/utils/benchmark/judge.py b/examples/cap_harness/utils/benchmark/judge.py similarity index 60% rename from examples/aspire_robosuite/utils/benchmark/judge.py rename to examples/cap_harness/utils/benchmark/judge.py index 5b3c2f41..5e77ec1d 100644 --- a/examples/aspire_robosuite/utils/benchmark/judge.py +++ b/examples/cap_harness/utils/benchmark/judge.py @@ -1,9 +1,10 @@ -"""The coordinator's Stage 2, on the host: the session's selected program (utils/agent/task.py:collect) run once on -the seeds it is scored on, and its success rate. For an actor these are the task row's held-out seeds, which no -session can run (the simulator service refuses them); for a verifier, its own development seeds, and it counts only -with its own verdict `verified` (ASPIRE: `refuted` and `void` are never promoted) and only for one of the two programs -its attempt budget ran (the simulator service's ledger). A seed counts when ASPIRE's harness reports task_success -(the benchmark's own success check, run after the program finished), within the session's max_steps. +"""The judge, on the host: the session's scored program (the facts its task functions' `collect` returned: the +engine's utils/agent/task.py, or a method's built on it) run once on the task row's held-out seeds (`eval_seeds`), +which no session can run (the simulator service refuses them), and its success rate. A row with an attempt budget +(`max_programs`) counts only a program its budget ran (the simulator service's ledger). A seed counts when ASPIRE's +harness reports task_success (the benchmark's own success check, run after the program finished), within the +session's max_steps. A row may name its seeds for the feedback (`score_label`, default held-out); facts that +withhold their program (`not_scored`: a method's rule) score 0 with that reason. The engine's settings come from the launcher's environment: ASPIRE_ROOT (the checkout), ASPIRE_SIM_ENV (the simulator's environment, a JSON object), ASPIRE_OUT (where the runs go; the service's ledger is its sim-service/).""" @@ -21,23 +22,21 @@ def score(task: dict, output) -> dict: facts = (output.extra or {}).get("task") or {} - verifier = task.get("role") == "verifier" - seeds = sim.seeds_of(task.get("debug_seeds") if verifier else task.get("eval_seeds")) + seeds = sim.seeds_of(task.get("eval_seeds")) + what = task.get("score_label") or "held-out" # what the seeds are, in the feedback (a row's own name) unseen = facts.get("unseen") note = f"; unseen-check {unseen['passed']}/{unseen['total']}" if unseen else "" - what = "verifier" if verifier else "held-out" - if verifier and (facts.get("verification") or {}).get("verdict") != "verified": - verdict = (facts.get("verification") or {}).get("verdict") or "missing" - return {"score": 0.0, "feedback": f"verifier verdict {verdict}", "seeds": len(seeds)} + if facts.get("not_scored"): + return {"score": 0.0, "feedback": str(facts["not_scored"]), "seeds": len(seeds)} program = facts.get("fix_code") if not program or not seeds: return {"score": 0.0, "feedback": f"0/{len(seeds)} {what} seeds; not scored: " f"{'no program' if not program else 'no seeds'}{note}"} - # A Codex session's (its facts name its workspace, task.py:collect): the program must be one its budget ran. The - # direct actor runs its versions itself, at most max_versions of them. - if verifier and facts.get("workspace") and \ + # A session with an attempt budget (its facts name its workspace): the program must be one its budget ran. A + # session that runs its programs itself (no workspace: a direct actor) keeps its own count. + if task.get("max_programs") and facts.get("workspace") and \ hashlib.sha256(program.encode()).hexdigest() not in _budget(facts["workspace"]): - return {"score": 0.0, "feedback": "verifier's program is not one of the programs its attempt budget ran", + return {"score": 0.0, "feedback": "the program is not one of the programs its attempt budget ran", "seeds": len(seeds)} out = Path(os.environ.get("ASPIRE_OUT", ".")).resolve() / "stage2" / \ f"{task['id']}-{hashlib.sha256(program.encode()).hexdigest()[:12]}" @@ -57,7 +56,7 @@ def score(task: dict, output) -> dict: def _budget(workspace) -> list[str]: - """The digests of the programs a verifier's session ran within its budget (the simulator service's ledger).""" + """The digests of the programs a session ran within its budget (the simulator service's ledger).""" ledger = Path(os.environ.get("ASPIRE_OUT", ".")).resolve() / "sim-service" / "ledger.json" try: return list((json.loads(ledger.read_text()).get(str(workspace)) or {}).get("programs") or []) diff --git a/examples/cap_harness/utils/benchmark/tasks.py b/examples/cap_harness/utils/benchmark/tasks.py new file mode 100644 index 00000000..f4ce6133 --- /dev/null +++ b/examples/cap_harness/utils/benchmark/tasks.py @@ -0,0 +1,84 @@ +"""The tasks of a benchmark on the cap-harness engine, written as task directories (Harbor's format) with their seeds, +which the simulator service and the judge take from each task's row: the helpers a benchmark's own +utils/benchmark/prepare.py calls (../../../robosuite/, ../../../libero/). + +A task directory's task.toml holds the row's fields but its id under [metadata], in the row's order, and asks for no +network (the session reaches only its model and the simulator service's forwarded port); it has no tests/: the judge +(judge.py) runs on the host, where the simulator is. ../agent/fields.py reads each back as its row. + +Each task also has its rerun, `rerun-` (`rerun_of: `): the same task with the same seeds, under its own id +so that a test split of the tasks a run trained on does not share ids with its training split (a run's `launch.sh +eval`, or a continual method's evaluation of every task again with its final adapter). +""" +from __future__ import annotations + +import argparse +from pathlib import Path +import shutil +import sys + +from hermit.task.directory import write_task as write_task_dir + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "agent")) +from fields import task_rows # noqa: E402 +from sim import seeds_of # noqa: E402 + + +def seeds_from(presets: dict, argv=None) -> dict: + """The run's seeds from the command line (a preset, each set replaceable): the three sets of a task row.""" + ap = argparse.ArgumentParser() + ap.add_argument("--preset", choices=sorted(presets), default=next(iter(presets))) + ap.add_argument("--dev", help="development seeds") + ap.add_argument("--unseen", help="unseen-check seeds, run once by the session") + ap.add_argument("--eval", help="held-out seeds, run only by the judge") + a = ap.parse_args(argv) + chosen = {k: getattr(a, k) or v for k, v in presets[a.preset].items()} + # A held-out seed a session could run would be scored as unseen: the sets must not meet + dev, unseen, held = (set(seeds_of(chosen[k])) for k in ("dev", "unseen", "eval")) + for name, other in (("development", dev), ("unseen-check", unseen)): + if held & other: + ap.error(f"the held-out seeds overlap the {name} seeds: {sorted(held & other)[:10]}") + if dev & unseen: + ap.error(f"the unseen-check seeds overlap the development seeds: {sorted(dev & unseen)[:10]}") + return {"debug_seeds": chosen["dev"], "unseen_seeds": chosen["unseen"], "eval_seeds": chosen["eval"]} + + +def instruction(row: dict) -> str: + """What a task asks, in a line (a session's prompt is its config's, built from the row).""" + what = f"{row.get('benchmark') or 'robosuite'} suite `{row['suite']}`, task id {row['task_id']}" + if row.get("rerun_of"): + return f"Solve {what} again (a rerun of `{row['rerun_of']}`)." + return f"Write the program that solves {what} in ASPIRE's Code-as-Policy harness." + + +def write_task(tasks: Path, row: dict, text: str | None = None) -> Path: + """One row as a task directory: its fields but the id under [metadata], in the row's order; no network.""" + return write_task_dir(tasks / row["id"], text or instruction(row), + metadata={k: v for k, v in row.items() if k != "id"}, + environment={"network_mode": "no-network"}) + + +def rows(tasks: Path) -> dict[str, dict]: + """Every task directory under `tasks` as its row, by id (a method that adds rows of its own reads them here).""" + return task_rows(tasks) + + +def write(here: Path, tasks: list[dict], seeds: dict, splits: dict[str, list[str]]) -> None: + """tasks/ and splits/ under `here`: each task (`id`, `suite`, `task_id`, and `benchmark` when it is not + Robosuite) with its seeds, and its rerun; splits/.txt for each of `splits`, with _rerun.txt beside + it (rerun.txt for `all`).""" + rows = [{**t, **seeds} for t in tasks] + reruns = [{**t, "id": f"rerun-{t['id']}", "rerun_of": t["id"], **seeds} for t in tasks] + if (here / "tasks").exists(): # this script's own output: rewritten whole + shutil.rmtree(here / "tasks") + for row in rows + reruns: + write_task(here / "tasks", row) + out = here / "splits" + if out.exists(): + shutil.rmtree(out) + out.mkdir() + for name, ids in splits.items(): + (out / f"{name}.txt").write_text("".join(f"{i}\n" for i in ids)) + rerun = out / ("rerun.txt" if name == "all" else f"{name}_rerun.txt") + rerun.write_text("".join(f"rerun-{i}\n" for i in ids)) + print(f"{len(rows)} tasks -> {here / 'tasks'}") diff --git a/examples/aspire_robosuite/utils/seg_service/requirements.txt b/examples/cap_harness/utils/seg_service/requirements.txt similarity index 100% rename from examples/aspire_robosuite/utils/seg_service/requirements.txt rename to examples/cap_harness/utils/seg_service/requirements.txt diff --git a/examples/aspire_robosuite/utils/seg_service/service.py b/examples/cap_harness/utils/seg_service/service.py similarity index 100% rename from examples/aspire_robosuite/utils/seg_service/service.py rename to examples/cap_harness/utils/seg_service/service.py diff --git a/examples/cap_harness/utils/tests/cap_fakes.py b/examples/cap_harness/utils/tests/cap_fakes.py new file mode 100644 index 00000000..7100881d --- /dev/null +++ b/examples/cap_harness/utils/tests/cap_fakes.py @@ -0,0 +1,215 @@ +"""Stand-ins for the cap-harness engine's tests and the tests of a method on it (tests/methods/aspire/): a fake +ASPIRE checkout whose cap-harness writes a run's files without a simulator (FAKE_HARNESS), the simulator service on a +loopback port over it, a session's workspace where the framework puts it, and the task rows of each benchmark. Loaded +by path (`_load`), never imported from a conftest.""" +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +import pytest + +UTILS = Path(__file__).resolve().parents[1] # examples/cap_harness/utils +EXAMPLE = UTILS # the engine's agent/ and benchmark/ folders + + +@pytest.fixture +def tmp_path(): + """A directory outside /tmp: the template check runs under bubblewrap with an empty tmpfs at /tmp.""" + base = Path.home() / ".cache" / "hermit-tests" + base.mkdir(parents=True, exist_ok=True) + path = Path(tempfile.mkdtemp(dir=base)) + yield path + if not os.environ.get("KEEP_TEST_DIRS"): + shutil.rmtree(path, ignore_errors=True) + + +def _load(name: str, path: Path): + sys.path.insert(0, str(path.parent)) + try: + spec_ = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec_) + spec_.loader.exec_module(module) + return module + finally: + sys.path.remove(str(path.parent)) + + +def fake_aspire(tmp: Path) -> Path: + """The files of an ASPIRE checkout the task functions read: the two templates, the docs, the snapshot program.""" + root = tmp / "ASPIRE" + skills = root / ".claude" / "skills" / "iterative-debugging" + (skills / "scripts").mkdir(parents=True) + (root / "docs").mkdir() + marks = ("", + "") + (skills / "subagent-prompt.md").write_text(f"intro\n{marks[0]}\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" + f"## What You Are\nfix-loop text\n{marks[1]}\n") + (skills / "verifier-prompt.md").write_text(f"intro\n{marks[0]}\nYou are verifying. Seeds 51-55.\n{marks[1]}\n") + (skills / "scripts" / "scene_snapshot.py").write_text("result = {}\n") + for name in ("api-reference.md", "run-artifacts.md", "robosuite.md"): + (root / "docs" / name).write_text(f"# {name}\nrun examples/robosuite/cube_lifting_seed1.py\nkept\n") + (skills / "skills").mkdir() + (skills / "skills" / "task-exploration.md").write_text("# exploration\n") + evosearch = root / ".claude" / "skills" / "evosearch" + (evosearch / "skills").mkdir(parents=True) + (evosearch / "subagent-prompt.md").write_text("intro\n\n```\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" + "## EVAL SET LOCKOUT\nsearch text\n```\n") + (evosearch / "skills" / "evosearch-iteration.md").write_text("# iteration\n") + (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\nkept\n\n## Task examples\n\n| `close_microwave.py` | " + "`libero_90:33` | Close the open microwave. |\n") + for venv in (".venv-robosuite", ".venv-libero"): + harness = root / venv / "bin" / "cap-harness" + harness.parent.mkdir(parents=True) + harness.write_text(FAKE_HARNESS.format(python=sys.executable)) + harness.chmod(0o755) + return root + + +# cap-harness without a simulator: the run's files where ASPIRE writes them (run.json, results.jsonl naming the run's +# directory, outcome.json), the program and settings it was given; a program saying SLEEP runs until stopped, and its +# `# WRITE ` / `# READ ` lines are tried as a program that escaped ASPIRE's validator would (in run.json), +# and after the run `# LINK ` leaves a link in its output, `# RUNDIR ` names another run_dir and +# `# CHMOD ` changes a mode (last). +FAKE_HARNESS = """#!{python} +import argparse, json, os, pathlib, sys, time +ap = argparse.ArgumentParser() +for flag in ("--benchmark", "--suite", "--task-id", "--program", "--output-root", "--max-steps", "--seed", "--workers", + "--results-jsonl", "--log-dir", "--init-mode"): + ap.add_argument(flag) +ap.add_argument("--seeds", nargs="+") +ap.add_argument("--flat-run-dir", action="store_true") +ap.add_argument("--no-videos", action="store_true") +ap.add_argument("cmd") +a = ap.parse_args() +code = pathlib.Path(a.program).read_text() +if "SLEEP" in code: + time.sleep(60) +tried = {{}} +for line in code.splitlines(): + what, _, path = line.partition(" ")[2].partition(" ") + try: + if what == "WRITE": + pathlib.Path(path).write_text("escaped") + elif what == "READ": + pathlib.Path(path).read_text() + else: + continue + tried[path] = "done" + except OSError as exc: + tried[path] = type(exc).__name__ +out = pathlib.Path(a.output_root) +run = out / "run-1" +run.mkdir(parents=True, exist_ok=True) +(out / "run.json").write_text(json.dumps({{"program": a.program, "max_steps": a.max_steps, "code": code, + "env": sorted(os.environ), "home": os.environ.get("HOME"), + "tried": tried}})) +(run / "outcome.json").write_text(json.dumps({{"task_success": True}})) +rows = [{{"seed": int(s), "run_dir": str(run)}} for s in (a.seeds or [a.seed])] +for line in code.splitlines(): + what, _, rest = line.partition(" ")[2].partition(" ") + if what == "LINK": + name, target = rest.split() + (out / name).unlink(missing_ok=True) + (out / name).symlink_to(target) + elif what == "RUNDIR": + rows = [dict(r, run_dir=rest) for r in rows] +pathlib.Path(a.results_jsonl or out / "results.jsonl").write_text("".join(json.dumps(r) + chr(10) for r in rows)) +for line in code.splitlines(): + what, _, rest = line.partition(" ")[2].partition(" ") + if what == "CHMOD": + name, mode = rest.split() + os.chmod(out / name, int(mode, 8)) +print("ran", a.program, "into", out, "max_steps", a.max_steps) +""" + + +def _bwrap_works() -> bool: + """A bubblewrap that can make its namespaces here: every simulator run is confined (sim.confine).""" + if not shutil.which("bwrap"): + return False + return subprocess.run(["bwrap", "--unshare-user", "--unshare-net", "--ro-bind", "/", "/", "true"], + capture_output=True).returncode == 0 + + +needs_bwrap = pytest.mark.skipif(not _bwrap_works(), reason="needs a working bubblewrap: simulator runs are confined") + + +LIBERO_ROW = {"id": "libero_90-t33", "benchmark": "libero-pro", "suite": "libero_90", "task_id": 33, + "debug_seeds": "51-53", "unseen_seeds": "66", "eval_seeds": "1-3"} +ROW = {"id": "robosuite-cube_lifting", "suite": "cube_lifting", "task_id": 0, "debug_seeds": "101-103", + "unseen_seeds": "126", "eval_seeds": "1-3"} +# A row with an attempt budget: at most two programs (by content) beyond a single run on the first seed +BUDGET = {"id": "robosuite-cube_stack", "suite": "cube_stack", "task_id": 0, "debug_seeds": "101-105", + "eval_seeds": "1-3", "max_programs": 2} + + +class _Service: + """The simulator service on a free loopback port, over a fake checkout whose cap-harness writes a run's files + without a simulator (FAKE_HARNESS), serving ROW, BUDGET, LIBERO_ROW and the `rows` given.""" + + def __init__(self, tmp: Path, root: Path, rows=(), **options): + import threading + from http.server import ThreadingHTTPServer + S = _load("cap_sim_service", EXAMPLE / "agent" / "sim_service.py") + self.sessions = tmp / "sessions" + self.sessions.mkdir() + served = {r["id"]: r for r in (ROW, BUDGET, LIBERO_ROW, *rows)} + self.service = service = S.Service(root, served, [self.sessions], + S.Ledger(tmp / "ledger.json"), {}, 2, **options) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), S.handler(service)) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + + def close(self): + self.server.shutdown() + + +@pytest.fixture +def service(tmp_path, monkeypatch): + svc = _Service(tmp_path, fake_aspire(tmp_path)) + monkeypatch.setenv("ASPIRE_SIM_URL", svc.url) + yield svc + svc.close() + + +def _workspace(service, task: dict, hexa: str = "56d9fea2") -> Path: + """A session's workspace where the framework puts it: //-/workspaces/<...>/work.""" + work = service.sessions / "rollout" / f"{task['id']}-{hexa}" / "workspaces" / f"{task['id']}-da10428a" / "work" + work.mkdir(parents=True) + return work + + +def _register(url: str, body: dict) -> dict: + import urllib.request + request = urllib.request.Request(url + "/register", data=json.dumps(body).encode()) + with urllib.request.urlopen(request) as reply: + return json.loads(reply.read()) + + +def _sim(work: Path, *args): + return subprocess.run([str(work / "bin" / "sim"), *args], capture_output=True, text=True, cwd=work) + + +def _sim_module(): + sys.path.insert(0, str(EXAMPLE / "agent")) + try: + import sim + return sim + finally: + sys.path.remove(str(EXAMPLE / "agent")) + + +def _fake_gpu(tmp_path, monkeypatch) -> list[int]: + """GPU 8 as hermit's sandbox sees devices (sandbox.DEV): a machine without one can still build the command.""" + import hermit.sandbox as sandbox_module + dev = tmp_path / "dev" + dev.mkdir(exist_ok=True) + for name in ("nvidiactl", "nvidia8"): + (dev / name).touch() + monkeypatch.setattr(sandbox_module, "DEV", str(dev)) + return [8] diff --git a/examples/cap_harness/utils/tests/test_cap_harness.py b/examples/cap_harness/utils/tests/test_cap_harness.py new file mode 100644 index 00000000..d2c89f80 --- /dev/null +++ b/examples/cap_harness/utils/tests/test_cap_harness.py @@ -0,0 +1,704 @@ +"""The cap-harness engine (ASPIRE's harness) shared by Robosuite and LIBERO: its task functions (the workspace and its +registration), the simulator service's rules as bin/sim meets them (a real service on a loopback port, over a fake +checkout: cap_fakes.py), the engine's sandbox, the judge (a stand-in engine), the docs it stages, and the tasks each +benchmark writes with its helpers; no simulator, no model, no network.""" +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[4] +HERE = Path(__file__).resolve().parent + + +def _fakes(): + spec_ = importlib.util.spec_from_file_location("cap_fakes", HERE / "cap_fakes.py") + module = importlib.util.module_from_spec(spec_) + spec_.loader.exec_module(module) + return module + + +F = _fakes() +tmp_path, service = F.tmp_path, F.service # fixtures +EXAMPLE, ROW, LIBERO_ROW, BUDGET = F.EXAMPLE, F.ROW, F.LIBERO_ROW, F.BUDGET +_load, fake_aspire, needs_bwrap, _Service, _workspace = F._load, F.fake_aspire, F.needs_bwrap, F._Service, F._workspace +_register, _sim, _sim_module, _fake_gpu = F._register, F._sim, F._sim_module, F._fake_gpu +LIBERO = ROOT / "examples" / "libero" +ROBOSUITE = ROOT / "examples" / "robosuite" + + +def _task_module(): + return _load("cap_task", EXAMPLE / "agent" / "task.py") + + +def test_sim_compact_makes_traces_readable(): + sim = _sim_module() + value = {"$type": "Pose", "position": {"$array_inline": [0.123456, 1.0, 2.0], "dtype": "float64", "shape": [3]}, + "obs": {"$type": "Observation", "cameras": {}}, "big": {"$array_inline": list(range(40)), "shape": [40]}} + out = sim.compact(value) + assert out == {"type": "Pose", "position": [0.1235, 1.0, 2.0], "obs": "", + "big": ""} + + +def test_the_failure_keyframe_is_the_failing_calls_own(tmp_path): + sim = _sim_module() + run = tmp_path / "run" + (run / "source").mkdir(parents=True) + trace = [{"tool": "open_gripper", "ok": True}] + [{"tool": "move_to_pose", "ok": i != 2} for i in range(4)] + (run / "source" / "program-result.json").write_text(json.dumps({"calls": trace})) + names = ["reset", "before-open-gripper", "after-open-gripper"] + for i in range(3): # the third move raised: no after-frame of its own + names += ["before-move-to-pose"] + (["after-move-to-pose"] if i < 2 else []) + for n, name in enumerate(names, 1): + d = run / "media" / "keyframes" / f"{n:06d}-{name}" + d.mkdir(parents=True) + (d / "agentview.png").write_bytes(b"png") + picked = [label for label, _ in sim.keyframes(run, limit=1)] + assert picked == [f"{names.index('before-move-to-pose', 7) + 1:06d}-before-move-to-pose"] + + +def test_prepare_registers_the_session_and_the_service_holds_its_rules(tmp_path, service): + T = _task_module() + work = _workspace(service, ROW) + params = {"aspire_root": str(tmp_path / "ASPIRE")} + T.prepare(ROW, work, params, tmp_path) + for name in ("docs/api-reference.md", "docs/task-exploration.md", "scripts/scene_snapshot.py", "bin/sim", + "bin/sim.json"): + assert (work / name).is_file(), name + assert not (work / "task.md").exists() and not (work / "docs" / "evosearch-iteration.md").exists() # a method's + assert "examples/" not in (work / "docs" / "robosuite.md").read_text() + assert "kept" in (work / "docs" / "robosuite.md").read_text() + (work / "p.py").write_text("result = {}\n") + held_out = _sim(work, "run-batch", "--seeds", "1-3", "--program", "p.py", "--output-root", "out") + assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr + outside = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", str(tmp_path / "elsewhere")) + assert outside.returncode == 2 and "inside the workspace" in outside.stderr + _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u", "--init-mode", "saved") + again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") + assert again.returncode == 2 and "run once" in again.stderr + T.prepare(ROW, work, params, tmp_path) # a retry's prepare: the registration as it was + again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") + assert again.returncode == 2 and "run once" in again.stderr # what the session spent stays spent + + +def test_a_libero_task_runs_on_its_own_venv_seeded_with_its_own_docs(tmp_path, service, monkeypatch): + T = _task_module() + params = {"aspire_root": str(tmp_path / "ASPIRE")} + s = T.settings(LIBERO_ROW, params) + assert T.task_dir(s) == "libero-pro/libero_90/task_33" and s["doc"] == "libero-pro.md" + work = _workspace(service, LIBERO_ROW) + T.prepare(LIBERO_ROW, work, params, tmp_path) + page = (work / "docs" / "libero-pro.md").read_text() + assert "kept" in page and "Task examples" not in page and "microwave" not in page # no worked solution named + assert not (work / "docs" / "robosuite.md").exists() + ran = [] + monkeypatch.setattr(service.service, "_execute", lambda cmd, env, *a, **k: ran.append((cmd, env)) or (0, "", "")) + (work / "p.py").write_text("result = {}\n") + _sim(work, "run-batch", "--seeds", "51-52", "--program", "p.py", "--output-root", "out", "--init-mode", "saved") + cmd, env = ran[-1] + assert cmd[0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "cap-harness") + option = lambda flag: cmd[cmd.index(flag) + 1] + assert (option("--benchmark"), option("--suite"), option("--task-id")) == ("libero-pro", "libero_90", "33") + assert option("--init-mode") == "seeded" # the engine's, whatever the session passed + assert env["LIBERO_CONFIG_PATH"] == str(tmp_path / "ASPIRE" / ".libero") + assert env["TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD"] == "1" + _sim(work, "validate", "--program", "p.py") + assert ran[-1][0][0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "python") + # Robosuite's runs are as they were: its venv, no init mode + robosuite = _workspace(service, ROW) + T.prepare(ROW, robosuite, params, tmp_path) + (robosuite / "p.py").write_text("result = {}\n") + _sim(robosuite, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + cmd, env = ran[-1] + assert cmd[0].endswith(".venv-robosuite/bin/cap-harness") and "--init-mode" not in cmd + assert "LIBERO_CONFIG_PATH" not in env + + +def test_the_task_sandbox_is_shown_the_staged_docs_and_never_a_page_that_lost_its_cut(tmp_path, service): + """ASPIRE's benchmark pages name its worked programs per task: an actor that could read one would be handed + a strategy for its task. The configs grant the staged copies only, and a page without its cut is refused.""" + T = _task_module() + root = tmp_path / "ASPIRE" + (tmp_path / "rows.jsonl").write_text(json.dumps(ROW) + "\n" + json.dumps(LIBERO_ROW) + "\n") + staged = tmp_path / "staged" + done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", str(root), + "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], + capture_output=True, text=True) + assert done.returncode == 0, done.stderr + assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", + "run-artifacts.md"] + assert all("examples/" not in p.read_text() and "microwave" not in p.read_text() for p in staged.iterdir()) + stage = lambda: subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", + str(root), "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], + capture_output=True, text=True) + before = {p.name: p.stat().st_ino for p in staged.iterdir()} + assert stage().returncode == 0 and {p.name: p.stat().st_ino for p in staged.iterdir()} == before # in place + (staged / "extra").mkdir() # anything else there: staged anew, without it + (staged / "notes.bin").write_bytes(b"\xff\xfe") + assert stage().returncode == 0 + assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", + "run-artifacts.md"] + (staged / "libero-pro.md").write_text("# the staged page\n") + work = _workspace(service, LIBERO_ROW) + T.prepare(LIBERO_ROW, work, {"aspire_root": str(root), "aspire_docs": str(staged)}, tmp_path) + assert (work / "docs" / "libero-pro.md").read_text() == "# the staged page\n" # the staged copy, as it is + (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\n\n## Worked programs\n| `close_microwave.py` |\n") + with pytest.raises(ValueError, match="Task examples"): + _sim_module().workspace_docs(root, "libero-pro") + # No Codex config of a benchmark on this engine grants a page of the checkout's own docs; each loads, reads its + # tasks through the engine's row view and is judged by the engine's judge + from hermit.config import Config + configs = [p for folder in (ROBOSUITE / "aspire", LIBERO / "aspire", LIBERO / "prompt_only") + for p in sorted(folder.glob("*.yaml"))] + assert len(configs) == 18 + for path in configs: + config = Config.load(path) + data = config.data + read = [str(p) for p in (data.get("environment") or {}).get("read") or []] + assert not any("/docs/" in p for p in read), path + assert config.path(data["data"]["view"].split(":")[0]) == (EXAMPLE / "agent" / "fields.py").resolve(), path + assert config.path(data["judge"]["path"]) == (EXAMPLE / "benchmark" / "judge.py").resolve(), path + if "_direct" not in path.name: + assert "$ASPIRE_DOCS" in read and data["agent"]["params"]["aspire_docs"] == "$ASPIRE_DOCS", path + assert any(config.path(p) == (EXAMPLE / "agent").resolve() for p in read if "$" not in p), path + if data["agent"]["name"] == "codex": # Codex reads AGENTS.md (the adapter's context) only up to + assert data["agent"]["params"]["project_doc_max_bytes"] >= 65536, path # this: past it, refused + if path.parent.name == "prompt_only": # no method's pages: none granted, none staged + assert not any("evosearch" in p or "-prompt.md" in p for p in read), path + + +def test_the_engine_takes_the_benchmarks_venv_and_refuses_one_it_does_not_run(tmp_path, monkeypatch): + sim = _sim_module() + root = fake_aspire(tmp_path) + engine = sim.Engine(root, benchmark="libero-pro") + assert engine.cli == root.resolve() / ".venv-libero" / "bin" / "cap-harness" + assert engine._env()["LIBERO_CONFIG_PATH"] == str(root.resolve() / ".libero") + assert sim.init_args("libero-pro") == ["--init-mode", "seeded"] and sim.init_args(None) == [] + assert sim.Engine(root).cli.parent.parent.name == ".venv-robosuite" + commands = [] # the command line of a sweep, the judge's Stage 2 included + + def recorded(cmd, **_): + commands.append(cmd) + raise RuntimeError("recorded") + + monkeypatch.setattr(sim, "confine", recorded) + for each in (engine, sim.Engine(root)): + with pytest.raises(RuntimeError, match="recorded"): + each.run(tmp_path / "p.py", suite="s", seeds=[1, 2], out=tmp_path / "o") + at = commands[0].index("--init-mode") + assert commands[0][at:at + 2] == ["--init-mode", "seeded"] and "--init-mode" not in commands[1] + shutil.rmtree(root / ".venv-libero") + with pytest.raises(FileNotFoundError, match="bootstrap_libero.sh"): + sim.Engine(root, benchmark="libero-pro") + with pytest.raises(ValueError, match="behavior"): + sim.Engine(root, benchmark="behavior") + + +def test_the_libero_tasks_are_liberos_five_suites_and_liberos_pros_eighty_with_aspires_seeds(tmp_path, monkeypatch): + """The tasks and splits prepare.py writes, read as their rows (the engine's fields.py): what the simulator service, + the judge and every method's configs read.""" + from hermit.config import Config + from hermit.task import Benchmark + P = _load("libero_prepare", LIBERO / "utils" / "benchmark" / "prepare.py") + Fd = _load("cap_fields", EXAMPLE / "agent" / "fields.py") + assert sum(P.SUITES.values()) == 130 and P.SUITES["libero_90"] == 90 + pro = [f"libero_{family}_{kind}" for family in ("object", "spatial", "goal", "10") for kind in ("swap", "task")] + assert list(P.PRO_SUITES) == pro and set(P.PRO_SUITES.values()) == {10} + monkeypatch.setattr(P, "HERE", tmp_path) + monkeypatch.setattr(sys, "argv", ["prepare.py"]) + P.main() + found = Fd.task_rows(tmp_path / "tasks") # each task directory as its row + rows = list(found.values()) + tasks = [found[i] for i in (tmp_path / "splits" / "all.txt").read_text().split()] + assert len(tasks) == 210 and len(rows) == 420 and len({r["id"] for r in rows}) == 420 # a suite and its + assert all(r["benchmark"] == "libero-pro" for r in rows) # perturbations share their task ids + assert list(found["libero_goal-t03"]) == ["id", "benchmark", "suite", "task_id", "debug_seeds", "unseen_seeds", + "eval_seeds"] # the row's keys, in its order + assert found["rerun-libero_goal-t03"] == {**found["libero_goal-t03"], "id": "rerun-libero_goal-t03", + "rerun_of": "libero_goal-t03"} # the same task, its own id + assert not any(r["id"].startswith("verify-") for r in rows) # a method's own rows are its own + assert (tmp_path / "tasks" / "libero_goal-t03" / "instruction.md").is_file() + assert all((r["debug_seeds"], r["unseen_seeds"], r["eval_seeds"]) == ("51-65", "66-80", "1-50") for r in tasks) + split = lambda name: (tmp_path / "splits" / name).read_text().split() + by = lambda *suites: [r["id"] for r in tasks if r["suite"] in suites] + assert split("all.txt") == [r["id"] for r in tasks] + assert split("libero.txt") == by("libero_object", "libero_spatial", "libero_goal", "libero_10") + assert split("libero_90.txt") == by("libero_90") and split("libero_pro.txt") == by(*pro) + assert split("libero_pro_long.txt") == by("libero_10_swap", "libero_10_task") + assert split("smoke.txt") == split("libero.txt")[:2] and split("smoke_pro.txt") == split("libero_pro.txt")[:2] + assert split("libero_sample.txt") == [f"{s}-t{n:02d}" for s in ("libero_object", "libero_spatial", "libero_goal") + for n in (0, 5)] + for path in sorted((LIBERO / "prompt_only").glob("*.yaml")): # a method that reads the tasks as written + config = Config.load(path) + data = config.data["data"] + assert config.path(data["tasks"]) == (LIBERO / "utils" / "benchmark" / "tasks").resolve(), path + monkeypatch.setitem(data, "tasks", str(tmp_path / "tasks")) # the tasks prepared here + for key in ("train", "val", "test"): + if data.get(key): + monkeypatch.setitem(data, key, str(tmp_path / "splits" / Path(data[key]).name)) + assert {t.id: t.given() for t in Benchmark.from_config(config).tasks.values()} == found, path + train, test = split(Path(data["train"]).name), split(Path(data["test"]).name) + assert train and set(train) <= {r["id"] for r in tasks}, path + assert test == [f"rerun-{i}" for i in train], path # the reruns are of the tasks it trained on + seeds = P.seeds_from(P.PRESETS, []) + assert seeds == {"debug_seeds": "51-65", "unseen_seeds": "66-80", "eval_seeds": "1-50"} + assert P.seeds_from(P.PRESETS, ["--preset", "smoke", "--eval", "1-5"])["eval_seeds"] == "1-5" + with pytest.raises(SystemExit): + P.seeds_from(P.PRESETS, ["--eval", "1-60"]) # held-out seeds a session could run + + +def test_the_neutral_libero_prompt_names_the_task_the_seeds_and_the_scored_program_and_nothing_of_a_method( + tmp_path, service): + """prompt_only/'s acting prompt and workspace: LIBERO's prompt and the engine's workspace, nothing of a method.""" + P = _load("libero_task", LIBERO / "utils" / "agent" / "task.py") + T = _task_module() + root = tmp_path / "ASPIRE" # the service fixture's ASPIRE tree + params = {"aspire_root": str(root)} # as prompt_only's configs set them + row = {**LIBERO_ROW, "suite": "libero_object_task", "task_id": 4, "id": "libero_object_task-t04"} + text = P.prompt(row, params, tmp_path) + assert "Suite `libero_object_task`, task id `4`" in text and "get_task_context()" in text + assert "**51-53**" in text and "**66**" in text and "3 held-out seeds" in text + scored = f"{T.task_dir(T.settings(row, params))}/fix_code.py" + assert f"Your program is `{scored}`" in text # the file the engine's collect scores + unseen = f"--output-root {scored.rsplit('/', 1)[0]}/debug/holdout" # where the engine's collect reads it + assert unseen in text and "docs/libero-pro.md" in text and "simulator steps" not in text + assert "at most 600 simulator steps" in P.prompt(row, {**params, "max_steps": 600}, tmp_path) + assert not any(word in text for word in ("skill", "library", "coordinator", "fix-loop", "evolutionary")) + work = _workspace(service, LIBERO_ROW) + T.prepare(LIBERO_ROW, work, params, tmp_path) + assert (work / "bin" / "sim").is_file() and (work / "docs" / "libero-pro.md").is_file() + assert (work / "docs" / "task-exploration.md").is_file() # the scene procedure stays + assert not (work / "docs" / "evosearch-iteration.md").exists() and not (work / "task.md").exists() + # The scored program is fix_code.py in the task's folder, whatever else the session left there + s = T.settings(LIBERO_ROW, params) + (work / T.task_dir(s)).mkdir(parents=True, exist_ok=True) + (work / T.task_dir(s) / "fix_code.py").write_text("# fix\n") + (work / T.task_dir(s) / "findings.md").write_text("notes\n") + (work / "evosearch").mkdir() + (work / "evosearch" / "evosearch_best_code.py").write_text("# best\n") + facts = T.collect(LIBERO_ROW, work, tmp_path / "log-a", {**params, "prune_media": False}) + assert facts["scored_program"] == f"{T.task_dir(s)}/fix_code.py" and facts["fix_code"] == "# fix\n" + assert facts["findings"] == "notes\n" and facts["unseen"] is None and "evosearch" not in facts + assert (tmp_path / "log-a" / "files" / "fix_code.py").read_text() == "# fix\n" + + +def test_a_session_cannot_forge_a_registration_for_more_seeds(tmp_path, service): + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + forged = work / "x" # a fresh directory the session can write + forged.mkdir() + (forged / "p.py").write_text("result = {}\n") + answer = _register(service.url, {"workspace": str(forged), "task": ROW["id"], "dev_seeds": [1, 2, 3]}) + assert not answer["ok"] and "registered its session already" in answer["error"] + other = _workspace(service, ROW, "0badc0de") # another rollout (the framework's): the row's seeds, + assert _register(service.url, {"workspace": str(other), "task": ROW["id"], "dev_seeds": [1, 2, 3], + "max_programs": None})["ok"] # whatever the request says + (other / "bin").mkdir() + shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", other / "bin" / "sim_cli.py") + (other / "bin" / "sim.json").write_text(json.dumps({"workspace": str(other), "url": service.url})) + (other / "p.py").write_text("result = {}\n") + held_out = subprocess.run([sys.executable, str(other / "bin" / "sim_cli.py"), "run-batch", "--seeds", "1-3", + "--program", "p.py", "--output-root", "out"], capture_output=True, text=True, cwd=other) + assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr + mismatched = _workspace(service, BUDGET, "12345678") # a rollout registers only the task its name carries + answer = _register(service.url, {"workspace": str(mismatched), "task": ROW["id"]}) + assert not answer["ok"] and "is not one of task" in answer["error"] + + +def test_a_row_with_an_attempt_budget_runs_that_many_programs(tmp_path, service): + T = _task_module() + work = _workspace(service, BUDGET) + T.prepare(BUDGET, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + codes = [] + for name in ("a.py", "a.py", "c.py"): # the same path, rewritten: a new program + (work / name).write_text(f"result = {{'v': {len(codes)}}}\n") + codes.append(_sim(work, "run-batch", "--seeds", "101-105", "--program", name, "--output-root", "out")) + assert "attempt budget" not in codes[0].stderr and "attempt budget" not in codes[1].stderr + assert codes[2].returncode == 2 and "attempt budget is 2 programs" in codes[2].stderr + + +@needs_bwrap +def test_the_service_follows_no_link_the_session_puts_in_its_workspace(tmp_path, service): + """A run is written by the service into its own directory and moved into the workspace: a link the session put + where the run's files go is replaced, not followed, and a program that is a link is not read.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + victim = tmp_path / "victim.txt" + victim.write_text("the host's own file\n") + (work / "out").mkdir() + (work / "out" / "run.json").symlink_to(victim) # where cap-harness writes + (work / "out" / "run-1").symlink_to(tmp_path) # a directory of the run's + (work / "p.py").write_text("result = {}\n") + ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + assert victim.read_text() == "the host's own file\n" and not (tmp_path / "outcome.json").exists() + assert ran.returncode == 2 and "a link in its path" in ran.stderr + (work / "out" / "run-1").unlink() + ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + assert ran.returncode == 0 and victim.read_text() == "the host's own file\n" + record = json.loads((work / "out" / "run.json").read_text()) + assert not (work / "out" / "run.json").is_symlink() and record["program"] == str(work / "p.py") + rows = (work / "out" / "results.jsonl").read_text() + assert json.loads(rows)["run_dir"] == str(work / "out" / "run-1") # the workspace's paths, not the service's + assert str(work / "p.py") in ran.stdout and str(service.service.scratch) not in ran.stdout + secret = tmp_path / "secret.py" + secret.write_text("KEY = 'host'\n") + (work / "q.py").symlink_to(secret) + refused = _sim(work, "validate", "--program", "q.py") + assert refused.returncode == 2 and "no program at q.py" in refused.stderr + assert list(service.service.scratch.iterdir()) == [] # the service's copies are gone + import urllib.request + service.service.nonce = "n-1" + with urllib.request.urlopen(service.url + "/health") as reply: # launch.sh knows its own service by it + assert json.loads(reply.read())["nonce"] == "n-1" + service.service.stop() + late = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + assert late.returncode == 1 and "stopping" in late.stderr # no run starts once the service stops + + +def test_bin_sim_quotes_the_interpreter(tmp_path, service, monkeypatch): + T = _task_module() + work = _workspace(service, ROW) + monkeypatch.setattr(T.sys, "executable", "/opt/hermit env/bin/python") + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + assert "exec '/opt/hermit env/bin/python' " in (work / "bin" / "sim").read_text() + + +@needs_bwrap +def test_a_program_that_escaped_the_validator_still_cannot_reach_the_host(tmp_path, service): + """cap-harness runs in a sandbox of its own (sim.confine): a program in it writes only its run's own output and + reads ASPIRE's checkout, not the host's files, another session's workspace or the judge's runs.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + secret, stage2 = tmp_path / "secret.txt", tmp_path / "out" / "stage2" / "robosuite-x" / "stage2.json" + secret.write_text("host secret\n") + stage2.parent.mkdir(parents=True) + stage2.write_text("{}") + other = _workspace(service, ROW, "0badc0de") + (other / "fix_code.py").write_text("result = {'theirs': 1}\n") + solution = tmp_path / "ASPIRE" / "examples" / "robosuite" / "cube_lifting_seed1.py" # a worked solution + solution.parent.mkdir(parents=True) + solution.write_text("result = {'worked': 1}\n") + history = tmp_path / "ASPIRE" / ".git" / "HEAD" # the git history holds it too + history.parent.mkdir() + history.write_text("ref: refs/heads/main\n") + native = tmp_path / "ASPIRE" / "outputs" / "run-1" / "source" / "program.py" # a run of ASPIRE's own CLI + native.parent.mkdir(parents=True) + native.write_text("result = {'native': 1}\n") + page = tmp_path / "ASPIRE" / "docs" / "libero-pro.md" # names the worked programs per task + demos = tmp_path / "ASPIRE" / "third_party" / "LIBERO-PRO" / "libero" / "datasets" / "demo.hdf5" + demos.parent.mkdir(parents=True) + demos.write_text("demonstration\n") + harness = tmp_path / "ASPIRE" / ".venv-robosuite" / "bin" / "cap-harness" + targets = {"WRITE": [tmp_path / "planted.txt", tmp_path / "ASPIRE" / "planted.py", work / "planted.txt"], + "READ": [secret, stage2, other / "fix_code.py", solution, history, native, page, demos, harness]} + (work / "p.py").write_text("result = {}\n" + "".join(f"# {w} {p}\n" for w, ps in targets.items() for p in ps)) + ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") + tried = json.loads((work / "out" / "run.json").read_text())["tried"] + assert ran.returncode == 0, ran.stderr + assert not (tmp_path / "planted.txt").exists() and not (tmp_path / "ASPIRE" / "planted.py").exists() + assert tried[str(work / "planted.txt")] != "done" and not (work / "planted.txt").exists() # its workspace too + assert all(tried[str(p)] != "done" + for p in (secret, stage2, other / "fix_code.py", solution, history, native, page, demos)) + assert tried[str(harness)] == "done" # the checkout it runs from + + +@needs_bwrap +def test_the_validator_runs_on_aspires_own_venv_inside_its_sandbox(tmp_path, monkeypatch): + """ASPIRE's venv interpreter is a link to its base installation: confined, it keeps the venv (whose cap_harness + the validator imports), for the service's `bin/sim validate` and the judge's Engine alike.""" + root = fake_aspire(tmp_path) + venv = root / ".venv-robosuite" + shutil.move(str(venv / "bin" / "cap-harness"), str(tmp_path / "cap-harness")) + shutil.rmtree(venv) + subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(venv)], check=True) + shutil.move(str(tmp_path / "cap-harness"), str(venv / "bin" / "cap-harness")) + site = next(venv.glob("lib/python3*/site-packages")) + (site / "cap_harness").mkdir() + (site / "cap_harness" / "__init__.py").write_text("") + (site / "cap_harness" / "runtime.py").write_text( + "import ast\nclass _ProgramValidator(ast.NodeVisitor):\n def visit_Import(self, node):\n" + " if any(a.name != 'math' for a in node.names):\n raise ValueError('only math')\n") + assert (venv / "bin" / "python").is_symlink() + S = _load("aspire_sim_service_venv", EXAMPLE / "agent" / "sim_service.py") + service = S.Service(root, {ROW["id"]: ROW}, [tmp_path / "sessions"], S.Ledger(tmp_path / "ledger.json"), {}, 2) + work = tmp_path / "sessions" / "rollout" / f"{ROW['id']}-1a2b" / "workspaces" / "w" / "work" + work.mkdir(parents=True) + assert service.register({"workspace": str(work), "task": ROW["id"]})["ok"] + (work / "good.py").write_text("import math\nresult = {}\n") + (work / "bad.py").write_text("import os\nresult = {}\n") + good = service.run({"workspace": str(work), "cmd": "validate", "program": "good.py"}) + bad = service.run({"workspace": str(work), "cmd": "validate", "program": "bad.py"}) + assert good["code"] == 0 and "program accepted" in good["stdout"], good + assert bad["code"] != 0 and "only math" in bad["stderr"], bad + sim = _sim_module() + engine = sim.Engine(root) + assert engine.validate(work / "good.py") is None and "only math" in engine.validate(work / "bad.py") + + +@needs_bwrap +def test_the_engine_reads_no_link_and_no_run_dir_outside_its_run(tmp_path): + """The judge's and the direct actor's Engine read what a confined program could write: once it has ended, links + in its output are removed and a run_dir outside the run is not read, so neither a host file nor another run's + success reaches the records.""" + sim = _sim_module() + root = fake_aspire(tmp_path) + elsewhere = tmp_path / "another-run" / "seed_0001" + elsewhere.mkdir(parents=True) + (elsewhere / "outcome.json").write_text('{"task_success": true, "program_ok": true}') + secret = tmp_path / "secret.txt" + secret.write_text("host secret") + out = tmp_path / "stage2" / "runs" + program = tmp_path / "stage2" / "program.py" + program.parent.mkdir() + program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# RUNDIR {elsewhere}\n") + records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1, 2], out=out) + assert [r["task_success"] for r in records] == [False, False] and "not in this run's output" in records[0]["error"] + assert secret.read_text() == "host secret" and not (out / "records.json").is_symlink() + program.write_text(f"result = {{}}\n# FAIL\n# LINK run-1/outcome.json {elsewhere / 'outcome.json'}\n") + records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "again") + assert records[0]["task_success"] is False and not (tmp_path / "again" / "run-1" / "outcome.json").exists() + # a directory made unlistable keeps no link from being removed: it is made listable, or the run fails + program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# CHMOD . 0o111\n") + sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "locked") + assert secret.read_text() == "host secret" and not (tmp_path / "locked" / "records.json").is_symlink() + + +@needs_bwrap +def test_the_simulators_sandbox_shows_the_renderer_and_reaches_remote_services_by_proxy(tmp_path, monkeypatch): + """An EGL vendor library extracted by hand (its ICD and its libraries, file by file, its library found as GLVND + finds it) and ASPIRE_SIM_READ are shown; without a GPU an NVIDIA-only ICD selection is dropped; a service on + another host is reached through an allowlist proxy for that host only.""" + sim = _sim_module() + home = tmp_path / "home" # the ICD and the libraries extracted where other files are too + (home / "lib").mkdir(parents=True) + (home / "credentials").write_text("key") + icd = home / "10_nvidia.json" + icd.write_text(json.dumps({"ICD": {"library_path": str(home / "lib" / "libEGL_nvidia.so.0")}})) + for name in ("libEGL_nvidia.so.0", "libnvidia-eglcore.so.580", "notes.txt"): + (home / "lib" / name).write_text("") + extra = tmp_path / "extra" + extra.mkdir() + env = {"PATH": os.environ["PATH"], "__EGL_VENDOR_LIBRARY_FILENAMES": str(icd), + "ASPIRE_SIM_READ": str(extra), "CAP_HARNESS_SAM3_URL": "http://provider.internal:9000"} + held, args, inside = sim.confine(["true"], root=fake_aspire(tmp_path), env=env, + gpu=_fake_gpu(tmp_path, monkeypatch)) + try: + ro = set(args[i + 1] for i, a in enumerate(args) if a == "--ro-bind") + assert {str(icd), str(home / "lib" / "libEGL_nvidia.so.0"), str(home / "lib" / "libnvidia-eglcore.so.580"), + str(extra)} <= ro # the driver's files, file by file, and the explicit grant + assert not ro & {str(home), str(home / "lib"), str(home / "credentials"), str(home / "lib" / "notes.txt")} + assert inside["__EGL_VENDOR_LIBRARY_FILENAMES"] == str(icd) # with a GPU (below: gpu=[8]) it stays + assert inside["HTTPS_PROXY"].startswith("http://127.0.0.1:") and held.held[0].allow == ["provider.internal"] + finally: + held.close() + # an ICD naming its library bare: GLVND finds it on LD_LIBRARY_PATH, and so is it shown + icd.write_text(json.dumps({"ICD": {"library_path": "libEGL_nvidia.so.0"}})) + held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", gpu=_fake_gpu(tmp_path, monkeypatch), + env={**env, "LD_LIBRARY_PATH": str(home / "lib")}) + held.close() + assert str(home / "lib" / "libEGL_nvidia.so.0") in [args[i + 1] for i, a in enumerate(args) if a == "--ro-bind"] + # without a GPU an NVIDIA-only ICD selection is dropped, so EGL finds Mesa's software renderer + held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", env=env) + held.close() + assert "__EGL_VENDOR_LIBRARY_FILENAMES" not in inside + + +@needs_bwrap +def test_the_service_lets_a_run_write_only_its_output_and_delivers_no_replaced_output(tmp_path, service, monkeypatch): + """The run's own directory (its program) is read-only to it and only its output writable; an output directory + the program replaced (by a link to a host directory) is not delivered.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + (work / "p.py").write_text("result = {}\n") + sim = type(service.service).run.__globals__["sim"] + grants, real = [], sim.confine + monkeypatch.setattr(sim, "confine", lambda cmd, **k: grants.append(k) or real(cmd, **k)) + assert _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "o").returncode == 0 + own = Path(grants[-1]["readable"][0]) + assert grants[-1]["writable"] == [own / "out"] + host = tmp_path / "host-dir" + host.mkdir() + (host / "id_rsa").write_text("key") + + def replaced(cmd, env, timeout, own, out=None, gpu=()): + shutil.rmtree(out) + out.symlink_to(host) + return 0, "", "" + monkeypatch.setattr(service.service, "_execute", replaced) + ran = _sim(work, "run", "--seed", "102", "--program", "p.py", "--output-root", "o2") + assert ran.returncode == 2 and "replaced while it ran" in ran.stderr + assert (host / "id_rsa").read_text() == "key" and not (work / "o2").exists() + + +@needs_bwrap +def test_a_program_named_pyvenv_cfg_grants_nothing(tmp_path, service): + """A session names its program: one called pyvenv.cfg that says `home = ` is the run's own file, not a venv + of the sandbox's, so stays out of the simulator's sandbox.""" + T = _task_module() + work = _workspace(service, ROW) + T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) + operator = tmp_path / "operator" + operator.mkdir() + (operator / "credentials").write_text("key") + (work / "pyvenv.cfg").write_text(f'marker = """\nhome = {operator}\n"""\nresult = {{}}\n' + f"# READ {operator / 'credentials'}\n") + ran = _sim(work, "run", "--seed", "101", "--program", "pyvenv.cfg", "--output-root", "out") + assert ran.returncode == 0, ran.stderr + assert json.loads((work / "out" / "run.json").read_text())["tried"][str(operator / "credentials")] != "done" + + +def test_the_service_refuses_to_start_on_a_gpu_the_machine_lacks(tmp_path): + """A wrong ASPIRE_SIM_GPU stops the service at its start, not each session's run after its budget is spent.""" + (tmp_path / "tasks.jsonl").write_text(json.dumps(ROW) + "\n") + done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "sim_service.py"), "--aspire-root", + str(fake_aspire(tmp_path)), "--tasks", str(tmp_path / "tasks.jsonl"), "--workspaces", + str(tmp_path / "w"), "--state", str(tmp_path / "state"), "--gpu", "999", "--port", "0"], + capture_output=True, text=True, timeout=60, env={**os.environ, "PYTHONPATH": str(ROOT)}) + assert done.returncode != 0 and ("/dev/nvidia999" in done.stderr or "bubblewrap" in done.stderr) + + +@needs_bwrap +def test_the_simulators_sandbox_reaches_only_the_services_ports(tmp_path): + """The judge runs sim.Engine in hermit's process, whose environment names its own loopback endpoints: only the + services' ports (ASPIRE's, and a CAP_HARNESS_*_URL's) are forwarded into cap-harness's sandbox.""" + sim = _sim_module() + root = fake_aspire(tmp_path) + box, args, _ = sim.confine(["true"], root=root, env={ + "PATH": os.environ["PATH"], "HA_TOOL_URL": "http://127.0.0.1:4999/mcp", + "ASPIRE_SIM_URL": "http://127.0.0.1:8120", "CAP_HARNESS_SAM3_URL": "http://127.0.0.1:9114", + "CAP_HARNESS_CUROBO_URL": "http://localhost"}) # no port: its scheme's + box.close() + forwarded = sorted(int(args[i + 1].split("=")[0]) for i, a in enumerate(args) if a == "--forward") + assert forwarded == [80, 8114, 8115, 8116, 8118, 9114] + for where in ({"hidden": [root / "runs"]}, {"writable": [root / "scratch" / "run"]}): # runs stay outside it + with pytest.raises(ValueError, match="inside ASPIRE's checkout"): + sim.confine(["true"], root=root, env={"PATH": os.environ["PATH"]}, **where) + + +@needs_bwrap +def test_a_run_takes_the_registrations_max_steps_and_stops_at_the_timeout(tmp_path, monkeypatch): + monkeypatch.setenv("TEST_PROVIDER_API_KEY", "k") # the host's secrets: not for a generated program + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "8") # the host's numbering: the sandbox's GPUs count from 0 + svc = _Service(tmp_path, fake_aspire(tmp_path), run_timeout=2) + try: + work = _workspace(svc, ROW) + assert _register(svc.url, {"workspace": str(work), "task": ROW["id"], "max_steps": 300})["ok"] + (work / "bin").mkdir() + shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", work / "bin" / "sim_cli.py") + (work / "bin" / "sim.json").write_text(json.dumps({"workspace": str(work), "url": svc.url})) + sim = lambda *args: subprocess.run([sys.executable, str(work / "bin" / "sim_cli.py"), *args], + capture_output=True, text=True, cwd=work) + (work / "p.py").write_text("result = {}\n") + ran = sim("run", "--seed", "101", "--program", "p.py", "--output-root", "o", "--max-steps", "5000") + record = json.loads((work / "o" / "run.json").read_text()) + assert ran.returncode == 0 and record["max_steps"] == "300" + assert "TEST_PROVIDER_API_KEY" not in record["env"] and "PATH" in record["env"] + assert "CUDA_VISIBLE_DEVICES" not in record["env"] + assert not Path(record["home"]).is_relative_to(work) and not Path(record["home"]).exists() # its own, gone + assert list((tmp_path / "sandbox").glob("cap-harness-*.json")) # the run's sandbox record + (work / "slow.py").write_text("SLEEP\n") + started = time.time() + slow = sim("run", "--seed", "102", "--program", "slow.py", "--output-root", "s") + assert slow.returncode == 124 and "stopped after 2 s" in slow.stderr and time.time() - started < 30 + finally: + svc.close() + + +def test_the_judge_runs_the_program_on_the_rows_held_out_seeds_within_its_budget(tmp_path, monkeypatch): + J = _load("cap_judge", EXAMPLE / "benchmark" / "judge.py") + runs, engines = [], [] + + class Engine: + def __init__(self, *a, **k): + engines.append(k) + + def validate(self, program): + assert program.is_absolute() and program.is_file() # cap-harness runs in ASPIRE's checkout + return "ValueError: only math" if "import os" in program.read_text() else None + + def run(self, program, *, suite, seeds, out, task_id=0): + runs.append(seeds) + return [{"task_success": s % 2 == 1} for s in seeds] + monkeypatch.setattr(J.sim, "Engine", Engine) + monkeypatch.setenv("ASPIRE_ROOT", str(tmp_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ASPIRE_OUT", "out") # relative, as a user may give it + out = lambda facts: SimpleNamespace(extra={"task": facts}, output=None) + actor = {"fix_code": "result = {}\n", "unseen": {"passed": 1, "total": 1}} + assert J.score(ROW, out(actor))["score"] == 2 / 3 and runs[-1] == [1, 2, 3] # the row's held-out seeds + assert J.score(ROW, out({**actor, "fix_code": "import os\n"}))["score"] == 0.0 + assert J.score(ROW, out({**actor, "max_steps": 300}))["score"] == 2 / 3 and engines[-1]["max_steps"] == 300 + assert engines[-1]["benchmark"] == "robosuite" + assert J.score(LIBERO_ROW, out(actor))["score"] == 2 / 3 and engines[-1]["benchmark"] == "libero-pro" + assert J.score(ROW, out(actor))["feedback"] == "2/3 held-out seeds; unseen-check 1/1" + # A row with an attempt budget counts only a program its budget ran (the service's ledger), under its own label + work = str(tmp_path / "w") + budget = {**BUDGET, "eval_seeds": "101-105", "score_label": "verifier"} + ran = {"fix_code": "result = {}\n", "workspace": work} + ledger = tmp_path / "out" / "sim-service" / "ledger.json" + ledger.parent.mkdir(parents=True) + ledger.write_text(json.dumps({work: {"programs": [hashlib.sha256(b"result = {}\n").hexdigest()]}})) + scored = J.score(budget, out(ran)) + assert scored["score"] == 3 / 5 and runs[-1] == [101, 102, 103, 104, 105] + assert scored["feedback"] == "3/5 verifier seeds" + third = J.score(budget, out({**ran, "fix_code": "result = {'v': 3}\n"})) # beyond its two programs + assert third["score"] == 0.0 and "attempt budget" in third["feedback"] + direct = {k: v for k, v in ran.items() if k != "workspace"} # a session that runs its own programs + assert J.score(budget, out({**direct, "fix_code": "result = {'v': 3}\n"}))["score"] == 3 / 5 + assert J.score(ROW, out({}))["score"] == 0.0 # no program: not scored + withheld = J.score(ROW, out({**actor, "not_scored": "verifier verdict void"})) # a method's rule + assert withheld == {"score": 0.0, "feedback": "verifier verdict void", "seeds": 3} + + +def test_the_robosuite_tasks_are_task_directories_read_back_as_the_rows_they_were(tmp_path, monkeypatch): + """prepare.py writes each task and its rerun as a task directory (Harbor's format, no network, judged on the + host); the engine's fields.py, every config's data.view, reads each back as exactly the row it wrote.""" + from hermit.task import Benchmark + P = _load("robosuite_prepare", ROBOSUITE / "utils" / "benchmark" / "prepare.py") + Fd = _load("cap_fields", EXAMPLE / "agent" / "fields.py") + monkeypatch.setattr(P, "HERE", tmp_path) + monkeypatch.setattr(sys, "argv", ["prepare.py", "--preset", "smoke"]) + P.main() + found = Fd.task_rows(tmp_path / "tasks") + seeds = {"debug_seeds": "101-103", "unseen_seeds": "126", "eval_seeds": "1-3"} + task = {"id": "robosuite-cube_stack", "suite": "cube_stack", "task_id": 0} + assert json.dumps(found["robosuite-cube_stack"]) == json.dumps({**task, **seeds}) # its keys in order, types + assert json.dumps(found["rerun-robosuite-cube_stack"]) == json.dumps( + {**task, "id": "rerun-robosuite-cube_stack", "rerun_of": "robosuite-cube_stack", **seeds}) + split = lambda name: (tmp_path / "splits" / name).read_text().split() + assert len(found) == 14 and split("smoke.txt") == ["robosuite-cube_lifting", "robosuite-cube_stack"] + assert split("smoke_rerun.txt") == ["rerun-robosuite-cube_lifting", "rerun-robosuite-cube_stack"] + assert split("rerun.txt") == [f"rerun-{i}" for i in split("all.txt")] and len(split("all.txt")) == 7 + loaded = Benchmark.load(tmp_path / "tasks", view=Fd.task_row).tasks["robosuite-cube_stack"] + assert loaded.private["tests"] is None and loaded.private["needs"]["network"]["agent"] == "no-network" + + +def test_prepare_refuses_held_out_seeds_a_session_could_run(): + for args in (["--eval", "1-200"], ["--unseen", "120-130"], ["--dev", "1-5"]): + done = subprocess.run([sys.executable, str(ROBOSUITE / "utils" / "benchmark" / "prepare.py"), *args], + capture_output=True, text=True) + assert done.returncode == 2 and "overlap" in done.stderr, args + + +def test_prune_drops_the_runs_images_and_keeps_their_records(tmp_path): + T = _task_module() + run = tmp_path / "run" / "work" / "debug" / "0101" / "run-x" + (run / "media" / "keyframes").mkdir(parents=True) + (run / "media" / "keyframes" / "a.png").write_bytes(b"png") + (run / "run.json").write_text('{"seed": 101}') + (tmp_path / "run" / "work" / "notes" / "media").mkdir(parents=True) # not a run's: kept + T.prune(tmp_path / "run") + assert not (run / "media").exists() and (run / "run.json").is_file() + assert (tmp_path / "run" / "work" / "notes" / "media").is_dir() diff --git a/examples/libero/.gitignore b/examples/libero/.gitignore index a1e03960..62859799 100644 --- a/examples/libero/.gitignore +++ b/examples/libero/.gitignore @@ -1 +1,4 @@ runs/ +# ASPIRE's tasks and splits, written by aspire/tasks.sh at launch +aspire/tasks/ +aspire/splits/ diff --git a/examples/libero/README.md b/examples/libero/README.md index 6b64c097..64f8f7dd 100644 --- a/examples/libero/README.md +++ b/examples/libero/README.md @@ -11,26 +11,28 @@ on held-out seeds by the task's own success check. | Folder | What it is | |---|---| -| `utils/benchmark/prepare.py` | The 210 tasks (a suite and a task id each) with their seeds, as task directories in Harbor's format (`tasks/`, read as their rows by the engine's `aspire_fields.py`), and splits; the verifier's tasks and the reruns | -| `utils/agent/` | A method-neutral acting prompt (`prompt.md`, written here: LIBERO has none) and the workspace without ASPIRE's method pages, for a method that brings no prompt of its own | +| `utils/benchmark/prepare.py` | The 210 tasks (a suite and a task id each) with their seeds, as task directories in Harbor's format (`tasks/`, each with its rerun, read as their rows by the engine's `fields.py`), and splits | +| `utils/agent/` | A method-neutral acting prompt (`prompt.md`, written here: LIBERO has none), for a method that brings no prompt of its own; the workspace is the engine's | | `utils/benchmark/task-info/` | What a task and its record are, for the learner | | `utils/launch.sh` | Starts a method folder's run: the simulator service and hermit | -| [`aspire/`](aspire/) | The ASPIRE method: its fix-loop prompts, verifier and coordinator. `run.yaml` (Object, Spatial, Goal, Long: 40 tasks), `run_90.yaml` (LIBERO-90), `run_pro.yaml` (LIBERO-Pro: 80), `run_pro_long.yaml` (LIBERO-Long-Pro), `run_luna.yaml` (`run.yaml` on Luna), `run_sample_luna.yaml` (six of the forty on Luna, for a run of a day or two), `smoke*.yaml`, `launch.sh` | +| [`aspire/`](aspire/) | The ASPIRE method (its session's task functions, verifier and coordinator are ASPIRE's own; `tasks.sh` adds its verifiers to the tasks). `run.yaml` (Object, Spatial, Goal, Long: 40 tasks), `run_90.yaml` (LIBERO-90), `run_pro.yaml` (LIBERO-Pro: 80), `run_pro_long.yaml` (LIBERO-Long-Pro), `run_luna.yaml` (`run.yaml` on Luna), `run_sample_luna.yaml` (six of the forty on Luna, for a run of a day or two), `smoke*.yaml`, `launch.sh` | | [`prompt_only/`](prompt_only/) | The prompt-only template: one `ADAPTER.md` the built-in Codex Evolver revises after each task, with the method-neutral prompt. `run.yaml`, `run_pro.yaml`, `smoke.yaml`, `smoke_luna.yaml`, `launch.sh` | The `*_luna.yaml` configs run GPT-5.6-Luna on the machine's ChatGPT login; the others GLM 5.3 Flash on Fireworks. ## The engine and the protocol -The engine is the [Robosuite example](../aspire_robosuite/)'s: its simulator service (`bin/sim` in the session's -workspace), its task functions (the workspace, the record), its judge and its sandboxes, which that README describes. +The engine is the [cap-harness engine](../cap_harness/), which Robosuite shares: its simulator service (`bin/sim` in +the session's workspace), its task functions (the workspace, the record), its judge and its sandboxes, which its +README describes. It takes the benchmark from the task row (`"benchmark": "libero-pro"`) and with it ASPIRE's settings for LIBERO (its fix-loop template's Benchmark Profiles): - `.venv-libero/bin/cap-harness`, with `LIBERO_CONFIG_PATH=$ASPIRE_ROOT/.libero` and `TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1`; - `--init-mode seeded` on every run, the session's and the judge's, so that each seed is a scene of its own; -- ASPIRE's LIBERO seeds: development 51-65, unseen-check 66-80, held-out 1-50, the verifier's 51-55; +- ASPIRE's LIBERO seeds: development 51-65, unseen-check 66-80, held-out 1-50 (and ASPIRE's verifier's, 51-55, + which its `tasks.sh` gives); - `docs/libero-pro.md` in the session's workspace without its "Task examples" section, which names ASPIRE's worked programs per task. The task sandbox is shown the staged copy only, not the checkout's page; a page without that section is refused rather than passed whole. @@ -44,17 +46,14 @@ distinct for every task: where a task's placement leaves little room, a held-out layout. A method's number is measured the same way whatever its prompt: the workspace, `bin/sim`, the seeds, the scored -program (`fix_code.py`) and the judge are the engine's. ASPIRE's configs give the session ASPIRE's own prompts (the -fix-loop subagent and the evolutionary search); `prompt_only/` gives it a method-neutral one (`utils/agent/prompt.md`: -the task, the workspace, what is scored) and `evosearch: false`, with which the engine neither stages ASPIRE's -evolutionary-search page nor scores a search's program: the scored program is `fix_code.py`. Its workspace leaves out -ASPIRE's assignment (`utils/agent/task.py:prepare`). Its task sandbox can still read the engine's folder -(`../aspire_robosuite/utils/agent`), which holds the engine's adaptations of ASPIRE's prompts, beside the code the -session runs. +program (`fix_code.py`) and the judge are the engine's, and nothing of a method is in them. ASPIRE's configs give the +session ASPIRE's own task functions (`hermit.agent.evolver.methods.aspire.actor`: the fix-loop subagent and the +evolutionary search, whose best program it scores); `prompt_only/` gives it a method-neutral prompt +(`utils/agent/prompt.md`: the task, the workspace, what is scored) and the engine's workspace and record as they are. ## Setup -As [Robosuite's](../aspire_robosuite/README.md#setup) (the providers, the render device, Codex and the sandboxes), +As [the engine's](../cap_harness/README.md#setup) (the providers, the render device, Codex and the sandboxes), with LIBERO's venv instead of Robosuite's in the ASPIRE checkout: ```bash @@ -67,7 +66,7 @@ Serve Contact-GraspNet on 8115 beside the other three (`scripts/supervise_servic contact_graspnet`); the launcher refuses to start without it, since `generate_grasps` would have no backend to call. Segmentation is ASPIRE's SAM3 service (`--providers sam3`): LIBERO's scenes hold several similar objects, which -the Robosuite example's stand-in does not tell apart. +the engine's stand-in does not tell apart. ## Run @@ -83,8 +82,8 @@ ASPIRE_SIM_GPU=0 ASPIRE_CONFIG=run_pro.yaml ./launch.sh # LIBERO-Pro ./launch.sh eval --adapter root # the same with the empty one ``` -`utils/launch.sh` starts the Robosuite example's launcher on this folder and the method folder; its variables +`utils/launch.sh` starts the engine's launcher on this folder and the method folder; its variables (`ASPIRE_SIM_ENV`, `ASPIRE_SIM_GPU`, `ASPIRE_OUT`, `ASPIRE_EVAL_SEEDS`, ...) are described there. A method on LIBERO -is a folder of configs beside these: `utils/agent/task.py` as its prompt and workspace, or its own. Run one method -folder at a time on a checkout and an `ASPIRE_OUT`: they share the task rows, the staged docs and the service's -ledger, which each launch writes. +is a folder of configs beside these: `utils/agent/task.py` as its prompt and the engine's workspace, or its own (and a +`tasks.sh` if it adds rows of its own, as ASPIRE's verifiers). Run one method folder at a time on a checkout and an +`ASPIRE_OUT`: they share the task rows, the staged docs and the service's ledger, which each launch writes. diff --git a/examples/libero/aspire/run.yaml b/examples/libero/aspire/run.yaml index 0f8450db..9db2f09b 100644 --- a/examples/libero/aspire/run.yaml +++ b/examples/libero/aspire/run.yaml @@ -3,11 +3,11 @@ # (development 51-65, unseen-check 66-80, held-out 1-50, written into the task rows by launch.sh; every run # --init-mode seeded); Codex acting and coordinating, GLM 5.3 Flash. Started by launch.sh. data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -15,10 +15,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -30,7 +31,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -40,7 +41,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_90.yaml b/examples/libero/aspire/run_90.yaml index dacaf106..1b3639a3 100644 --- a/examples/libero/aspire/run_90.yaml +++ b/examples/libero/aspire/run_90.yaml @@ -1,11 +1,11 @@ # ASPIRE on LIBERO-90: the ninety tasks of LIBERO's libero_90 suite, as run.yaml runs the other four (the same # seeds, roles and model). Started by launch.sh: ASPIRE_CONFIG=run_90.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_90.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_90_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_90.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_90_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -13,10 +13,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -28,7 +29,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -38,7 +39,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_luna.yaml b/examples/libero/aspire/run_luna.yaml index e297052a..b2edec2a 100644 --- a/examples/libero/aspire/run_luna.yaml +++ b/examples/libero/aspire/run_luna.yaml @@ -1,11 +1,11 @@ # run.yaml with the models changed and nothing else: GPT-5.6-Luna at medium effort acts and coordinates on the # machine's own ChatGPT login (`codex login`; no API key). Started by launch.sh: ASPIRE_CONFIG=run_luna.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -13,10 +13,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -28,7 +29,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -38,7 +39,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_pro.yaml b/examples/libero/aspire/run_pro.yaml index 62537701..166e6fc2 100644 --- a/examples/libero/aspire/run_pro.yaml +++ b/examples/libero/aspire/run_pro.yaml @@ -4,11 +4,11 @@ # rows by launch.sh; every run --init-mode seeded); Codex acting and coordinating, GLM 5.3 Flash. Started by # launch.sh. data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_pro.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_pro_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_pro.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_pro_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -16,10 +16,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -31,7 +32,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -41,7 +42,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_pro_long.yaml b/examples/libero/aspire/run_pro_long.yaml index 8964ba6b..f2043d26 100644 --- a/examples/libero/aspire/run_pro_long.yaml +++ b/examples/libero/aspire/run_pro_long.yaml @@ -1,11 +1,11 @@ # ASPIRE on LIBERO-Long-Pro: the twenty tasks of libero_10_swap and libero_10_task, as run_pro.yaml runs all eighty (the # same seeds, roles and model). Started by launch.sh: ASPIRE_CONFIG=run_pro_long.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_pro_long.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_pro_long_rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_pro_long.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_pro_long_rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -13,10 +13,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -28,7 +29,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -38,7 +39,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/run_sample_luna.yaml b/examples/libero/aspire/run_sample_luna.yaml index 27164d2b..c30ae837 100644 --- a/examples/libero/aspire/run_sample_luna.yaml +++ b/examples/libero/aspire/run_sample_luna.yaml @@ -3,11 +3,11 @@ # ./launch.sh. Its baseline, the same tasks with ASPIRE's empty library: # ASPIRE_CONFIG=run_sample_luna.yaml ./launch.sh eval --adapter root data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/libero_sample.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/libero_sample_rerun.txt # every task again with an adapter given + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/libero_sample.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/libero_sample_rerun.txt # every task again with an adapter given step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -15,10 +15,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -30,7 +31,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -40,7 +41,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke.yaml b/examples/libero/aspire/smoke.yaml index 5648f30e..354a3fed 100644 --- a/examples/libero/aspire/smoke.yaml +++ b/examples/libero/aspire/smoke.yaml @@ -2,11 +2,11 @@ # (launch.sh writes them into the task rows: ASPIRE_SEEDS=smoke); Codex acting (ASPIRE's fix-loop subagent and # verifier prompts) and coordinating, GLM 5.3 Flash. Started by launch.sh: ASPIRE_CONFIG=smoke.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke_luna.yaml b/examples/libero/aspire/smoke_luna.yaml index 2f746c60..ef39b42b 100644 --- a/examples/libero/aspire/smoke_luna.yaml +++ b/examples/libero/aspire/smoke_luna.yaml @@ -2,11 +2,11 @@ # machine's own ChatGPT login (`codex login`; no API key). Started by launch.sh: ASPIRE_CONFIG=smoke_luna.yaml # ./launch.sh (its seeds are the smoke's). data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke_pro.yaml b/examples/libero/aspire/smoke_pro.yaml index 638e5f0b..ae1b77fb 100644 --- a/examples/libero/aspire/smoke_pro.yaml +++ b/examples/libero/aspire/smoke_pro.yaml @@ -2,11 +2,11 @@ # (launch.sh writes them into the task rows: ASPIRE_SEEDS=smoke); Codex acting (ASPIRE's fix-loop subagent and # verifier prompts) and coordinating, GLM 5.3 Flash. Started by launch.sh: ASPIRE_CONFIG=smoke_pro.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke_pro.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_pro_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke_pro.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_pro_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/smoke_pro_luna.yaml b/examples/libero/aspire/smoke_pro_luna.yaml index 6e1e381f..40335396 100644 --- a/examples/libero/aspire/smoke_pro_luna.yaml +++ b/examples/libero/aspire/smoke_pro_luna.yaml @@ -2,11 +2,11 @@ # machine's own ChatGPT login (`codex login`; no API key). Started by launch.sh: ASPIRE_CONFIG=smoke_pro_luna.yaml # ./launch.sh (its seeds are the smoke's). data: - tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke_pro.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_pro_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke_pro.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_pro_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../../aspire_robosuite/utils/agent/task.py:prompt - prepare: ../../aspire_robosuite/utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../../aspire_robosuite/utils/agent # the engine's task functions, bin/sim's client, the prompt pages + - ../../cap_harness/utils/agent # the engine's task functions and bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/libero/aspire/tasks.sh b/examples/libero/aspire/tasks.sh new file mode 100644 index 00000000..eb3be258 --- /dev/null +++ b/examples/libero/aspire/tasks.sh @@ -0,0 +1,6 @@ +# Sourced by the engine's launcher (../../cap_harness/launch.sh) in this folder, after the benchmark's prepare.py: +# ASPIRE's verifiers beside LIBERO's tasks (hermit/agent/evolver/methods/aspire/rows.py, on ASPIRE's first five +# development seeds), in tasks/ and splits/ here, which this folder's configs read. +"$HERMIT" -m hermit.agent.evolver.methods.aspire.rows --benchmark ../utils/benchmark --out . --verify-seeds 51-55 \ + --engine ../../cap_harness/utils/benchmark/tasks.py +TASKS="$PWD/tasks" diff --git a/examples/libero/prompt_only/run.yaml b/examples/libero/prompt_only/run.yaml index f8668473..ebf54ea9 100644 --- a/examples/libero/prompt_only/run.yaml +++ b/examples/libero/prompt_only/run.yaml @@ -4,7 +4,7 @@ # Flash. Started by launch.sh. data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/libero.txt test: ../utils/benchmark/splits/libero_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -15,10 +15,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -31,13 +30,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/prompt_only/run_pro.yaml b/examples/libero/prompt_only/run_pro.yaml index f62382b6..b7f52fe5 100644 --- a/examples/libero/prompt_only/run_pro.yaml +++ b/examples/libero/prompt_only/run_pro.yaml @@ -3,7 +3,7 @@ # ./launch.sh data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/libero_pro.txt test: ../utils/benchmark/splits/libero_pro_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -14,10 +14,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -30,13 +29,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/prompt_only/smoke.yaml b/examples/libero/prompt_only/smoke.yaml index 83c28c19..e9863631 100644 --- a/examples/libero/prompt_only/smoke.yaml +++ b/examples/libero/prompt_only/smoke.yaml @@ -4,7 +4,7 @@ # ASPIRE_CONFIG=smoke.yaml ./launch.sh data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/smoke.txt test: ../utils/benchmark/splits/smoke_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -15,10 +15,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -31,13 +30,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/prompt_only/smoke_luna.yaml b/examples/libero/prompt_only/smoke_luna.yaml index 527e6d2c..7def1039 100644 --- a/examples/libero/prompt_only/smoke_luna.yaml +++ b/examples/libero/prompt_only/smoke_luna.yaml @@ -3,7 +3,7 @@ # ./launch.sh (its seeds are the smoke's). data: tasks: ../utils/benchmark/tasks - view: ../../aspire_robosuite/utils/agent/aspire_fields.py:aspire_task # each task as the row it always was + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was train: ../utils/benchmark/splits/smoke.txt test: ../utils/benchmark/splits/smoke_rerun.txt # `launch.sh eval`: the tasks again, with an adapter given step_size: 1 @@ -14,10 +14,9 @@ agent: concurrency: 1 params: prompt: ../utils/agent/task.py:prompt # a method-neutral acting prompt: the task, the workspace, what is scored - prepare: ../utils/agent/task.py:prepare # the engine's workspace, without ASPIRE's method pages - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # no search: the engine scores fix_code.py and stages no search page + prepare: ../../cap_harness/utils/agent/task.py:prepare # the engine's workspace + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {name: gpt-5.6-luna, login: chatgpt, effort: medium} @@ -30,13 +29,13 @@ agent: environment: # the task sandbox: the engine's code, the staged docs, the scene-snapshot program read: - ../utils/agent - - ../../aspire_robosuite/utils/agent + - ../../cap_harness/utils/agent - $ASPIRE_DOCS - $ASPIRE_ROOT/.claude/skills/iterative-debugging/scripts/scene_snapshot.py - $ASPIRE_ROOT/.claude/skills/iterative-debugging/skills/task-exploration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../../aspire_robosuite/utils/benchmark/judge.py # on the host: the program once on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: the program once on the held-out seeds template: prompt-only@1.0 evolver: name: codex # the built-in Codex AgentEvolver: it revises ADAPTER.md after each task diff --git a/examples/libero/utils/agent/task.py b/examples/libero/utils/agent/task.py index fbdc758b..437d4034 100644 --- a/examples/libero/utils/agent/task.py +++ b/examples/libero/utils/agent/task.py @@ -1,17 +1,15 @@ -"""A method-neutral acting prompt for LIBERO, for a method that brings no acting prompt of its own (prompt-only, and -the others that learn only what the session is given as context): the task, the workspace, the simulator, the rules -and what is scored, and nothing of a method. LIBERO has no acting prompt; this one is written here. The workspace, -the simulator service, the record and the judge are the engine's (`examples/aspire_robosuite/utils/agent/task.py`), -as for ASPIRE's own prompts, so a method's number on LIBERO is measured the same way whatever its prompt; `prepare` -leaves out of the workspace what is ASPIRE's method rather than the task's. +"""LIBERO's acting prompt, for a method that brings no acting prompt of its own (prompt-only, and the others that learn +only what the session is given as context): the task, the workspace, the simulator, the rules and what is scored, +and nothing of a method. LIBERO has no acting prompt; this one is written here. The workspace, the simulator +service, the record and the judge are the cap-harness engine's (`examples/cap_harness/utils/agent/task.py`), so a +method's number on LIBERO is measured the same way whatever its prompt. agent: params: prompt: ../utils/agent/task.py:prompt - prepare: ../utils/agent/task.py:prepare - answer: ../../aspire_robosuite/utils/agent/task.py:answer - collect: ../../aspire_robosuite/utils/agent/task.py:collect - evosearch: false # the engine neither stages ASPIRE's search page nor scores a search's program + prepare: ../../cap_harness/utils/agent/task.py:prepare + answer: ../../cap_harness/utils/agent/task.py:answer + collect: ../../cap_harness/utils/agent/task.py:collect """ from __future__ import annotations @@ -20,48 +18,22 @@ import sys HERE = Path(__file__).resolve().parent -ENGINE = HERE.parents[2] / "aspire_robosuite" / "utils" / "agent" -# What the engine's prepare writes that is ASPIRE's method, not the task's: ASPIRE's assignment block (its -# coordinator's terms). Its evolutionary search is left out by `evosearch: false`, which `_settings` requires, so that -# the engine's collect scores fix_code.py, the file this prompt names. -ASPIRE_ONLY = ("task.md",) +ENGINE = HERE.parents[2] / "cap_harness" / "utils" / "agent" def _engine(): """The engine's task functions, loaded from their own folder (they import sim.py beside them).""" if str(ENGINE) not in sys.path: sys.path.insert(0, str(ENGINE)) - spec = importlib.util.spec_from_file_location("aspire_robosuite_task", ENGINE / "task.py") + spec = importlib.util.spec_from_file_location("cap_harness_task", ENGINE / "task.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module -def _settings(engine, task: dict, params: dict) -> dict: - s = engine.settings(task, params) - if params.get("evosearch", True): - raise ValueError("agent.params.evosearch must be false: this prompt runs no evolutionary search, and the " - "engine's collect scores the search's program where one is left") - if s["role"] == "verifier": - raise ValueError(f"{task['id']}: a verifier row belongs to a method that verifies (ASPIRE's); " - "this prompt acts on a task") - return s - - -def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: - """The engine's workspace (bin/sim registered with the service, the docs, the scene-snapshot program), without the - search's page (`evosearch: false`) and ASPIRE_ONLY.""" - engine = _engine() - _settings(engine, task, params) - mounts = engine.prepare(task, workspace, params, base) - for name in ASPIRE_ONLY: - (Path(workspace) / name).unlink(missing_ok=True) - return mounts - - def prompt(task: dict, params: dict, base: Path) -> str: engine = _engine() - s = _settings(engine, task, params) + s = engine.settings(task, params) steps = (f" An episode runs at most {int(s['max_steps'])} simulator steps." if s["max_steps"] else "") return (HERE / "prompt.md").read_text().format( suite=s["suite"], task_id=s["task_id"], task_dir=engine.task_dir(s), dev=s["dev"], diff --git a/examples/libero/utils/benchmark/prepare.py b/examples/libero/utils/benchmark/prepare.py index 35136984..202be729 100644 --- a/examples/libero/utils/benchmark/prepare.py +++ b/examples/libero/utils/benchmark/prepare.py @@ -5,9 +5,9 @@ python prepare.py # ASPIRE's seeds python prepare.py --preset smoke # the smoke's: three development seeds, one unseen, three held-out - # tasks// (task directories), + # tasks// (task directories, and each one's rerun-/), # splits/{all,libero,libero_90,libero_pro,libero_pro_long,,smoke,smoke_pro,libero_sample}.txt, - # their *_rerun.txt (rerun.txt for all), verify.txt + # their *_rerun.txt (rerun.txt for all) LIBERO's own five suites (Liu et al., 2023): `libero_object`, `libero_spatial`, `libero_goal` and `libero_10` (LIBERO-Long), ten tasks each, which splits/libero.txt holds in that order, and `libero_90` (splits/libero_90.txt). @@ -18,8 +18,8 @@ splits/libero_pro_long.txt for LIBERO-Long-Pro (`libero_10_swap`, `libero_10_task`). A suite and its perturbations share task ids, so a row's id carries the suite. -The tasks, the verifier's and the reruns are written by the Robosuite example's prepare.py (`write`), whose engine -this benchmark runs on: one task directory each, read back as its row by its utils/agent/aspire_fields.py. +The tasks are written by the cap-harness engine's helpers (../../../cap_harness/utils/benchmark/tasks.py), the engine +this benchmark runs on: one task directory each, read back as its row by the engine's utils/agent/fields.py. """ from __future__ import annotations @@ -27,8 +27,8 @@ import sys HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE.parents[2] / "aspire_robosuite" / "utils" / "benchmark")) -from prepare import seeds_from, write # noqa: E402 +sys.path.insert(0, str(HERE.parents[2] / "cap_harness" / "utils" / "benchmark")) +from tasks import seeds_from, write # noqa: E402 BENCHMARK = "libero-pro" # cap-harness's name for its LIBERO adapter SUITES = {"libero_object": 10, "libero_spatial": 10, "libero_goal": 10, "libero_10": 10, "libero_90": 90} @@ -37,12 +37,12 @@ # A sample of the forty that a run on one GPU finishes in a day or two: task ids 0 and 5 of Object, Spatial and Goal # (Long's two-stage tasks are left out for their length) SAMPLE = [f"{suite}-t{n:02d}" for suite in ("libero_object", "libero_spatial", "libero_goal") for n in (0, 5)] -PRESETS = {"aspire": {"dev": "51-65", "unseen": "66-80", "eval": "1-50", "verify": "51-55"}, - "smoke": {"dev": "51-53", "unseen": "66", "eval": "1-3", "verify": "51-55"}} +PRESETS = {"aspire": {"dev": "51-65", "unseen": "66-80", "eval": "1-50"}, + "smoke": {"dev": "51-53", "unseen": "66", "eval": "1-3"}} def main() -> None: - seeds, verify = seeds_from(PRESETS) + seeds = seeds_from(PRESETS) # `t`: a task id is not read as the hex suffix of a rollout directory's name (the simulator service matches it) tasks = [{"id": f"{suite}-t{n:02d}", "benchmark": BENCHMARK, "suite": suite, "task_id": n} for suite, count in {**SUITES, **PRO_SUITES}.items() for n in range(count)] @@ -50,7 +50,7 @@ def main() -> None: ids = [t["id"] for t in tasks] original = of(*[s for s in SUITES if s != "libero_90"]) pro = of(*PRO_SUITES) - write(HERE, tasks, seeds, verify, + write(HERE, tasks, seeds, {"all": ids, "libero": original, "libero_90": of("libero_90"), "libero_pro": pro, "libero_pro_long": of("libero_10_swap", "libero_10_task"), **{suite: of(suite) for suite in (*SUITES, *PRO_SUITES) if suite != "libero_90"}, diff --git a/examples/libero/utils/launch.sh b/examples/libero/utils/launch.sh index 1bcd66fd..da429df6 100755 --- a/examples/libero/utils/launch.sh +++ b/examples/libero/utils/launch.sh @@ -9,8 +9,8 @@ # ./launch.sh eval --adapter final # the tasks again with the final adapter (test split) # ./launch.sh eval --adapter root # the same with the empty one # -# The engine is the Robosuite example's (../../aspire_robosuite): its launcher runs the method folder's configs, with -# the seeds of utils/benchmark/prepare.py, and takes the same variables (ASPIRE_SIM_ENV, ASPIRE_SIM_GPU, ASPIRE_OUT, +# The engine is the cap-harness engine (../../cap_harness): its launcher runs the method folder's configs, with the +# seeds of utils/benchmark/prepare.py, and takes its variables (ASPIRE_SIM_ENV, ASPIRE_SIM_GPU, ASPIRE_OUT, # ASPIRE_EVAL_SEEDS, ASPIRE_SIM_PORT, HERMIT_PYTHON; see its header). ASPIRE_ROOT needs .venv-libero. set -euo pipefail HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" @@ -19,7 +19,7 @@ METHOD="${1:?the method folder}"; shift [[ -x "$ASPIRE_ROOT/.venv-libero/bin/cap-harness" ]] \ || { echo "$ASPIRE_ROOT/.venv-libero/bin/cap-harness: run ASPIRE's scripts/bootstrap_libero.sh" >&2; exit 1; } # Contact-GraspNet, LIBERO's grasp backend (ASPIRE's SERVICE_PORTS for libero-pro), on its fixed port; the -# Robosuite launcher checks the other services +# engine's launcher checks the other services curl -sf -o /dev/null --max-time 3 "http://127.0.0.1:8115/openapi.json" \ || { echo "ASPIRE's Contact-GraspNet service at http://127.0.0.1:8115 is not up (see ../README.md)" >&2; exit 1; } -ASPIRE_EXAMPLE="$HERE/.." ASPIRE_METHOD="$METHOD" exec "$HERE/../../aspire_robosuite/aspire/launch.sh" "$@" +ASPIRE_EXAMPLE="$HERE/.." ASPIRE_METHOD="$METHOD" exec "$HERE/../../cap_harness/launch.sh" "$@" diff --git a/examples/robodojo_sim/README.md b/examples/robodojo_sim/README.md index 22a571bc..22ac6381 100644 --- a/examples/robodojo_sim/README.md +++ b/examples/robodojo_sim/README.md @@ -25,7 +25,7 @@ what is left is listed under [Not verified](#not-verified). | `utils/policy/install.py` | Copies the policy into the RoboDojo checkout (`XPolicyLab/policy/hermit_program`) | | `utils/agent/dojo.py` | The engine: one evaluation of a program (`scripts/robodojo.sh eval`), its result per layout | | `utils/agent/split_eval.sh` | One evaluation with the simulator in RoboDojo's container (`ROBODOJO_IMAGE`) | -| `utils/agent/dojo_service.py`, `sim_cli.py` | The simulator service on the host and `bin/sim`, the actor's client of it (the ASPIRE example's service with this engine) | +| `utils/agent/dojo_service.py`, `sim_cli.py` | The simulator service on the host and `bin/sim`, the actor's client of it (the cap-harness engine's service, `../cap_harness/utils/agent/sim_service.py`, with this engine) | | `utils/agent/task.py`, `prompts/`, `docs/interface.md` | The Codex actor's task functions, its prompt and the interface page it is given | | `utils/benchmark/prepare.py` | The forty-two tasks with their development and held-out layouts, as task directories in Harbor's format (`tasks/`, each read as its row through `utils/agent/robodojo_fields.py`, the configs' `data.view`), and `splits/` | | `utils/benchmark/judge.py` | The program on the held-out layout sets, on the host | diff --git a/examples/robodojo_sim/utils/agent/dojo_service.py b/examples/robodojo_sim/utils/agent/dojo_service.py index 70313889..c5df9749 100644 --- a/examples/robodojo_sim/utils/agent/dojo_service.py +++ b/examples/robodojo_sim/utils/agent/dojo_service.py @@ -1,6 +1,6 @@ """RoboDojo's simulator as a host service for the actors in the task sandbox: `bin/sim` (sim_cli.py) asks it to -evaluate a program on the session's development layouts. It is the ASPIRE example's simulator service -(examples/aspire_robosuite/utils/agent/sim_service.py: the registration of a session by its rollout directory and +evaluate a program on the session's development layouts. It is the cap-harness engine's simulator service +(examples/cap_harness/utils/agent/sim_service.py: the registration of a session by its rollout directory and task id, the program read and the output written without following a link the session left, the ledger) with RoboDojo's engine (dojo.Engine) in place of cap-harness: @@ -29,8 +29,8 @@ import dojo # noqa: E402 from robodojo_fields import task_rows # noqa: E402 -sys.path.append(str(HERE.parents[2] / "aspire_robosuite" / "utils" / "agent")) -import sim_service as base # noqa: E402 (the ASPIRE example's) +sys.path.append(str(HERE.parents[2] / "cap_harness" / "utils" / "agent")) +import sim_service as base # noqa: E402 (the cap-harness engine's) class Service(base.Service): @@ -81,7 +81,7 @@ def _launch(self, body, session, program: Path, own_out: Path, seeds: list[int]) if len(records) < len(seeds): lines.append(f"{len(seeds) - len(records)} layout(s) have no result (the evaluation ended early)") lines.append(f"{passed}/{len(records)} layouts succeeded") - # As the ASPIRE service reports a run it stopped: 124, and why + # As the cap-harness service reports a run it stopped: 124, and why summary = own_out / "records.json" # dojo.Engine.run's account of the run run = json.loads(summary.read_text()) if summary.is_file() else {} if run.get("timed_out"): diff --git a/examples/robodojo_sim/utils/tests/test_robodojo.py b/examples/robodojo_sim/utils/tests/test_robodojo.py index 93f45260..1649c1f2 100644 --- a/examples/robodojo_sim/utils/tests/test_robodojo.py +++ b/examples/robodojo_sim/utils/tests/test_robodojo.py @@ -522,7 +522,7 @@ def run(self, program, *, task, layout_seed, layouts, out, private=None): assert done["ok"] and "1/2 layouts succeeded" in done["stdout"] and ran == [("stack_bowls", 2, 2)] assert "not assigned to you" in request([0, 1, 2, 3])["error"] # more than its development layouts assert "from the first" in request([1, 2])["stderr"] and len(ran) == 1 - # A run past its timeout is said, as the ASPIRE service says it (124); after stop() no simulator starts + # A run past its timeout is said, as the cap-harness service says it (124); after stop() no simulator starts timed_out = {"value": True} def summarized(program, *, task, layout_seed, layouts, out, private=None): diff --git a/examples/robosuite/.gitignore b/examples/robosuite/.gitignore new file mode 100644 index 00000000..62859799 --- /dev/null +++ b/examples/robosuite/.gitignore @@ -0,0 +1,4 @@ +runs/ +# ASPIRE's tasks and splits, written by aspire/tasks.sh at launch +aspire/tasks/ +aspire/splits/ diff --git a/examples/robosuite/README.md b/examples/robosuite/README.md new file mode 100644 index 00000000..d5bc3c51 --- /dev/null +++ b/examples/robosuite/README.md @@ -0,0 +1,60 @@ +# Robosuite + +The Robosuite tasks of [ASPIRE](https://research.nvidia.com/labs/gear/aspire/)'s Code-as-Policy harness (Agentic +Skills Discovery for Robotics, [arXiv:2607.00272](https://arxiv.org/abs/2607.00272), +[code](https://github.com/NVlabs/ASPIRE)) as a benchmark: for each task a session writes a program (Python against +the harness's perception, grasping and motion API), develops it on development seeds, checks it once on unseen seeds, +and is scored once on held-out seeds. The tasks run on the [cap-harness engine](../cap_harness/) (its simulator +service, `bin/sim`, its judge), which LIBERO shares; its README has the setup. + +| Folder | What it is | +|---|---| +| `utils/benchmark/prepare.py` | The seven Robosuite tasks with their seeds, as task directories in Harbor's format (`tasks/`, each with its rerun, each read as its row through the engine's `utils/agent/fields.py`, the configs' `data.view`); `splits/` | +| `utils/benchmark/task-info/` | What a task and its record are, for the learner | +| `aspire/` | ASPIRE on Robosuite: `run.yaml` (ASPIRE's seeds), `smoke.yaml`, the `*_direct.yaml` comparison, `launch.sh`, and `tasks.sh`, which adds ASPIRE's verifiers to the tasks | + +## ASPIRE on Robosuite + +The curriculum run as ASPIRE runs it ([`aspire`](../../docs/methods/aspire.md)): ASPIRE's fix-loop subagent (the +actor), its skill verifier and its coordinator, each a Codex session given +ASPIRE's own prompt. For each task the actor debugs its program from the per-primitive traces and RGB keyframes; the +harness then scores it once on held-out seeds. After each task the coordinator updates the shared skill library, +admitting the actor's proposals only if a fresh verifier re-solves the task from the library and the proposals alone, +and chooses the next task. The actor's task functions are ASPIRE's (`hermit/agent/evolver/methods/aspire/actor.py`, +built on the engine's), and its verifier rows are ASPIRE's too (`aspire/tasks.sh` derives them from the tasks +`prepare.py` wrote). + +On the host that serves the providers, with the engine set up: + +```bash +export FIREWORKS_API_KEY=... +cd examples/robosuite/aspire +ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh +ASPIRE_ROOT=/path/to/ASPIRE ASPIRE_SIM_ENV=/path/to/sim_env.yaml ASPIRE_CONFIG=smoke.yaml ./launch.sh eval --adapter final +``` + +`./launch.sh eval --adapter final` reruns every task with the final library (the test split's `rerun-*` rows): +ASPIRE's backward transfer, compared per task with the task's first pass during training (each training step's +score in the run's `steps.jsonl`; the rerun's in the eval's results). `--adapter root` is the empty library and +`--adapter none` no library at all: the baselines on the same seeds. + +The protocol is ASPIRE's: each task learns on its development seeds and is scored once on held-out seeds; the +reruns score the same held-out seeds again, so the comparison reuses that partition (as ASPIRE's Phase 3 does). +The first evaluation ran `run.yaml` on held-out seeds 1-50 (half of ASPIRE's 1-100), and its reruns on the same +seeds. The held-out seeds are the task rows' (`prepare.py --eval`), so they are set when the launcher writes the +rows: `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh`, and `ASPIRE_EVAL_SEEDS=1-50 ./launch.sh eval --adapter final` for the +reruns (an `agent.params.eval_seeds` override has no effect). + +### Protocol + +| | `run.yaml` | `smoke.yaml` | +|---|---|---| +| Tasks | the 7 Robosuite tasks, in the order the coordinator chooses (it starts from `splits/all.txt`'s first) | `cube_lifting`, `cube_stack` | +| Development seeds | 101-125 (ASPIRE's) | 101-103 | +| Unseen-check seeds | 126-130 (ASPIRE's), run once by the actor | 126 | +| Held-out seeds | 1-100 (ASPIRE's; `ASPIRE_EVAL_SEEDS` for others), run once by the judge | 1-3 | +| Fix loop | ASPIRE's: 3 versions per failure mode, 8 in total | the same | +| Evolutionary search | ASPIRE's, for a task the fix loop leaves below 90%: K = 8, top 3, stop at 80% or 5 rounds | the same | +| Verifier | ASPIRE's: 5 seeds (101-105), 2 attempts, verified at ≥ 4/5 | the same | +| Actor, verifier, coordinator | Codex, GLM 5.3 Flash (Fireworks), reasoning effort medium | the same | + diff --git a/examples/robosuite/aspire/launch.sh b/examples/robosuite/aspire/launch.sh new file mode 100755 index 00000000..d3105286 --- /dev/null +++ b/examples/robosuite/aspire/launch.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Train, or evaluate, ASPIRE on Robosuite: the cap-harness engine's launcher (../../cap_harness/launch.sh, whose header +# gives the commands and variables) with this folder's configs, e.g. ASPIRE_ROOT=/path/to/ASPIRE ./launch.sh +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ASPIRE_EXAMPLE="$HERE/.." ASPIRE_METHOD=aspire exec "$HERE/../../cap_harness/launch.sh" "$@" diff --git a/examples/aspire_robosuite/aspire/run.yaml b/examples/robosuite/aspire/run.yaml similarity index 70% rename from examples/aspire_robosuite/aspire/run.yaml rename to examples/robosuite/aspire/run.yaml index 037fee61..ab4e94b8 100644 --- a/examples/aspire_robosuite/aspire/run.yaml +++ b/examples/robosuite/aspire/run.yaml @@ -2,11 +2,11 @@ # coordinator chooses (its curriculum); ASPIRE's seeds (development 101-125, unseen-check 126-130, held-out 1-100, # written into the task rows by launch.sh); Codex acting and coordinating, GLM 5.3 Flash. Started by launch.sh. data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/all.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/all.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../utils/agent/task.py:prompt - prepare: ../utils/agent/task.py:prepare - answer: ../utils/agent/task.py:answer - collect: ../utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../utils/agent + - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/aspire_robosuite/aspire/run_direct.yaml b/examples/robosuite/aspire/run_direct.yaml similarity index 71% rename from examples/aspire_robosuite/aspire/run_direct.yaml rename to examples/robosuite/aspire/run_direct.yaml index 140c3df7..70fd19cd 100644 --- a/examples/aspire_robosuite/aspire/run_direct.yaml +++ b/examples/robosuite/aspire/run_direct.yaml @@ -1,21 +1,23 @@ -# The structured direct-chat actor (utils/agent/agent.py) in place of Codex: a comparison with ASPIRE's coding agent. +# ASPIRE's structured direct-chat actor (hermit/agent/evolver/methods/aspire/direct.py) in place of Codex: a +# comparison with ASPIRE's coding agent. # ASPIRE on Robosuite: the seven Robosuite tasks of ASPIRE's harness as one curriculum, one task per step; each # task's actor debugs its program on ASPIRE's development seeds (101-125) and is scored once on held-out seeds; after # every task the coordinator repairs and grows the skill library through the verifier gate. Started by launch.sh, # inside the host that runs ASPIRE's perception, IK and planning services (see ../README.md). data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/all.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/all.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/rerun.txt # every task again with the final library: ASPIRE's backward transfer step_size: 1 # one task per step: the coordinator acts on each task as it finishes shuffle: false # the curriculum in the split's order task_info: ../utils/benchmark/task-info agent: - name: ../utils/agent/agent.py:AspireActor + name: hermit.agent.evolver.methods.aspire.direct:AspireActor concurrency: 1 # one simulator sweep at a time on the engine's GPU params: + engine: ../../cap_harness/utils/agent/sim.py # the engine this actor runs its programs on aspire_root: $ASPIRE_ROOT # an ASPIRE checkout with .venv-robosuite (launch.sh) sim_env: {} # the engine's environment; launch.sh fills it from ASPIRE_SIM_ENV model: fireworks/accounts/fireworks/models/glm-5p3-flash @@ -25,7 +27,7 @@ agent: evo: {k: 8, rounds: 5, threshold: 0.8, qualify_below: 0.9, parents: 3} workers: 4 judge: - path: ../utils/benchmark/judge.py + path: ../../cap_harness/utils/benchmark/judge.py template: aspire evolver: # the coordinator, as in run.yaml name: aspire diff --git a/examples/aspire_robosuite/aspire/smoke.yaml b/examples/robosuite/aspire/smoke.yaml similarity index 70% rename from examples/aspire_robosuite/aspire/smoke.yaml rename to examples/robosuite/aspire/smoke.yaml index 35cff3a4..10773d33 100644 --- a/examples/aspire_robosuite/aspire/smoke.yaml +++ b/examples/robosuite/aspire/smoke.yaml @@ -2,11 +2,11 @@ # the task rows: ASPIRE_SEEDS=smoke); Codex acting (ASPIRE's fix-loop subagent and verifier prompts) and coordinating, # GLM 5.3 Flash. Started by launch.sh: ASPIRE_CONFIG=smoke.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt # the verifier's rows: run only by the coordinator, through run_agent + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info @@ -14,10 +14,11 @@ agent: base: codex concurrency: 1 params: - prompt: ../utils/agent/task.py:prompt - prepare: ../utils/agent/task.py:prepare - answer: ../utils/agent/task.py:answer - collect: ../utils/agent/task.py:collect + prompt: hermit.agent.evolver.methods.aspire.actor:prompt + prepare: hermit.agent.evolver.methods.aspire.actor:prepare + answer: hermit.agent.evolver.methods.aspire.actor:answer + collect: hermit.agent.evolver.methods.aspire.actor:collect + engine: ../../cap_harness/utils/agent/task.py # the engine's task functions ASPIRE's build on aspire_root: $ASPIRE_ROOT aspire_docs: $ASPIRE_DOCS model: {ref: fireworks/accounts/fireworks/models/glm-5p3-flash, effort: medium} @@ -29,7 +30,7 @@ agent: project_doc_max_bytes: 131072 # AGENTS.md carries the whole library (the template caps it at 100,000 bytes) environment: # the task sandbox: the actor's code, ASPIRE's prompts and docs, nothing else of it read: - - ../utils/agent + - ../../cap_harness/utils/agent # the engine's task functions, bin/sim's client - $ASPIRE_DOCS # ASPIRE's docs as staged by launch.sh: nothing that names its worked programs - $ASPIRE_ROOT/.claude/skills/iterative-debugging/subagent-prompt.md - $ASPIRE_ROOT/.claude/skills/iterative-debugging/verifier-prompt.md @@ -39,7 +40,7 @@ environment: # the task sandbox: the actor's code, ASPIRE - $ASPIRE_ROOT/.claude/skills/evosearch/skills/evosearch-iteration.md env: [ASPIRE_SIM_URL] # the host's simulator service (launch.sh): its 127.0.0.1 port is forwarded judge: - path: ../utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds + path: ../../cap_harness/utils/benchmark/judge.py # on the host: Stage 2 on the held-out seeds template: aspire evolver: name: aspire diff --git a/examples/aspire_robosuite/aspire/smoke_direct.yaml b/examples/robosuite/aspire/smoke_direct.yaml similarity index 62% rename from examples/aspire_robosuite/aspire/smoke_direct.yaml rename to examples/robosuite/aspire/smoke_direct.yaml index 8d6b7b61..549a445a 100644 --- a/examples/aspire_robosuite/aspire/smoke_direct.yaml +++ b/examples/robosuite/aspire/smoke_direct.yaml @@ -1,19 +1,21 @@ -# The structured direct-chat actor (utils/agent/agent.py) in place of Codex: a comparison with ASPIRE's coding agent. +# ASPIRE's structured direct-chat actor (hermit/agent/evolver/methods/aspire/direct.py) in place of Codex: a +# comparison with ASPIRE's coding agent. # Smoke: two tasks, three development seeds, three held-out seeds, two versions and no search; the verifier's own # rows (5 seeds, 2 attempts). Started by launch.sh: ASPIRE_CONFIG=smoke_direct.yaml ./launch.sh data: - tasks: ../utils/benchmark/tasks - view: ../utils/agent/aspire_fields.py:aspire_task # each task as the row it always was - train: ../utils/benchmark/splits/smoke.txt - val: ../utils/benchmark/splits/verify.txt - test: ../utils/benchmark/splits/smoke_rerun.txt + tasks: tasks # the benchmark's tasks and ASPIRE's verifiers (tasks.sh) + view: ../../cap_harness/utils/agent/fields.py:task_row # each task as the row it always was + train: splits/smoke.txt + val: splits/verify.txt + test: splits/smoke_rerun.txt step_size: 1 shuffle: false task_info: ../utils/benchmark/task-info agent: - name: ../utils/agent/agent.py:AspireActor + name: hermit.agent.evolver.methods.aspire.direct:AspireActor concurrency: 1 params: + engine: ../../cap_harness/utils/agent/sim.py # the engine this actor runs its programs on aspire_root: $ASPIRE_ROOT sim_env: {} model: fireworks/accounts/fireworks/models/glm-5p3-flash @@ -22,7 +24,7 @@ agent: search: debug workers: 3 judge: - path: ../utils/benchmark/judge.py + path: ../../cap_harness/utils/benchmark/judge.py template: aspire evolver: # the coordinator, as in run.yaml name: aspire diff --git a/examples/robosuite/aspire/tasks.sh b/examples/robosuite/aspire/tasks.sh new file mode 100644 index 00000000..44a345c9 --- /dev/null +++ b/examples/robosuite/aspire/tasks.sh @@ -0,0 +1,6 @@ +# Sourced by the engine's launcher (../../cap_harness/launch.sh) in this folder, after the benchmark's prepare.py: +# ASPIRE's verifiers beside Robosuite's tasks (hermit/agent/evolver/methods/aspire/rows.py, on ASPIRE's first five +# development seeds), in tasks/ and splits/ here, which this folder's configs read. +"$HERMIT" -m hermit.agent.evolver.methods.aspire.rows --benchmark ../utils/benchmark --out . --verify-seeds 101-105 \ + --engine ../../cap_harness/utils/benchmark/tasks.py +TASKS="$PWD/tasks" diff --git a/examples/aspire_robosuite/utils/.gitignore b/examples/robosuite/utils/.gitignore similarity index 100% rename from examples/aspire_robosuite/utils/.gitignore rename to examples/robosuite/utils/.gitignore diff --git a/examples/aspire_robosuite/utils/benchmark/.gitignore b/examples/robosuite/utils/benchmark/.gitignore similarity index 100% rename from examples/aspire_robosuite/utils/benchmark/.gitignore rename to examples/robosuite/utils/benchmark/.gitignore diff --git a/examples/robosuite/utils/benchmark/prepare.py b/examples/robosuite/utils/benchmark/prepare.py new file mode 100644 index 00000000..cafd27f1 --- /dev/null +++ b/examples/robosuite/utils/benchmark/prepare.py @@ -0,0 +1,47 @@ +"""Write the Robosuite tasks of ASPIRE's harness (cap_harness/robosuite/registry.py) as hermit tasks: one task +directory (Harbor's format) per task with its seeds, which the simulator service and the judge take from the task's +row (ASPIRE's Robosuite defaults: development 101-125, unseen-check 126-130, held-out 1-100), through the cap-harness +engine's helpers (../../../cap_harness/utils/benchmark/tasks.py). + + python prepare.py # ASPIRE's seeds + python prepare.py --preset smoke # the smoke's: three development seeds, one unseen, three held-out + python prepare.py --dev 101-103 --unseen 126 --eval 1-3 # any others + # tasks// (and rerun-/), splits/{all,smoke,single_arm}.txt and their *_rerun.txt (rerun.txt for all) + +The order of splits/all.txt is a curriculum from simple to hard (single-arm pick before stacking and assembly, the +fixed-tool wipe, then the bimanual tasks). +""" +from __future__ import annotations + +from pathlib import Path +import sys + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parents[2] / "cap_harness" / "utils" / "benchmark")) +from tasks import seeds_from, write # noqa: E402 + +TASKS = [ # suite, what it is (task-info only; the session reads the language from get_task_context()) + ("cube_lifting", "Lift"), + ("cube_stack", "Stack"), + ("cube_restack", "Stack, restacked"), + ("nut_assembly", "NutAssemblySquare"), + ("spill_wipe", "Wipe, fixed tool"), + ("two_arm_lift", "TwoArmLift, bimanual"), + ("two_arm_handover", "TwoArmHandover, bimanual"), +] + + +# The seeds of a run: ASPIRE's Robosuite defaults, and the smoke's +PRESETS = {"aspire": {"dev": "101-125", "unseen": "126-130", "eval": "1-100"}, + "smoke": {"dev": "101-103", "unseen": "126", "eval": "1-3"}} + + +def main() -> None: + seeds = seeds_from(PRESETS) + tasks = [{"id": f"robosuite-{suite}", "suite": suite, "task_id": 0} for suite, _ in TASKS] + ids = [t["id"] for t in tasks] + write(HERE, tasks, seeds, {"all": ids, "single_arm": ids[:5], "smoke": ids[:2]}) + + +if __name__ == "__main__": + main() diff --git a/examples/aspire_robosuite/utils/benchmark/task-info/README.md b/examples/robosuite/utils/benchmark/task-info/README.md similarity index 100% rename from examples/aspire_robosuite/utils/benchmark/task-info/README.md rename to examples/robosuite/utils/benchmark/task-info/README.md diff --git a/hermit/agent/acting/workspace.py b/hermit/agent/acting/workspace.py index 106558d7..26a55d30 100644 --- a/hermit/agent/acting/workspace.py +++ b/hermit/agent/acting/workspace.py @@ -17,7 +17,8 @@ # give a reference or a login) effort: medium # Codex's model_reasoning_effort; absent: the CLI's default prompt: environment/prompt.md # the task prompt; `{field}` placeholders take the task's fields; or - # `file.py:function` (task) -> the prompt + # `file.py:function` (task) -> the prompt; every function here may + # also be `package.module:function` (a method's own) files: [environment/fixtures] # optional: copied into every workspace before the run mounts: # optional: read-only data in the workspace - {source: data/corpus, at: docs} @@ -104,9 +105,15 @@ def __missing__(self, key): return template.format_map(Fields({k: v for k, v in task.items() if not isinstance(v, (dict, list))})) +# A task function by module, as a method that brings its own task functions names them (`package.module:function`) +_MODULE_FUNCTION = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+:[A-Za-z_]\w*") + + def _task_function(value, base: Path): - """A `file.py:function` value as the function, else None.""" - return load_callable(value, Path(base)) if isinstance(value, str) and ".py:" in value else None + """A `file.py:function` or `package.module:function` value as the function, else None.""" + if not isinstance(value, str) or not (".py:" in value or _MODULE_FUNCTION.fullmatch(value)): + return None + return load_callable(value, Path(base)) def _call(fn, *args, params: dict, base: Path): diff --git a/hermit/agent/evolver/methods/aspire/actor.py b/hermit/agent/evolver/methods/aspire/actor.py new file mode 100644 index 00000000..667a9ba5 --- /dev/null +++ b/hermit/agent/evolver/methods/aspire/actor.py @@ -0,0 +1,199 @@ +"""ASPIRE's acting sessions (agent.base: codex): its fix-loop subagent, its evolutionary search and its skill +verifier, as ASPIRE's own prompts run them, on any benchmark of the cap-harness engine. These are the task functions +an ASPIRE config names (agent.params.prompt, prepare, answer, collect); they build on the engine's own +(agent.params.engine: its utils/agent/task.py, e.g. examples/cap_harness/utils/agent/task.py), which give every +method's session its workspace, `bin/sim` and the facts the judge reads. They run in the task sandbox with the +session. + + prepare the engine's workspace, then ASPIRE's: task.md (the assignment), the search's procedure page, and for a + verifier the proposals under test as proposed_skills.md + prompt the adaptation page (prompts/codex_header.md), the assignment, then ASPIRE's template verbatim; for an + actor, Part 2: the evolutionary search (prompts/evosearch_header.md and its template) + answer the agent's last message (the record's output is the facts collect returns) + collect the engine's facts, and ASPIRE's: findings, skill report, the verifier's report, the search's; the + selected program (the search's best where it ran, else fix_code.py) is the one the judge scores, and a + verifier's program only with its own verdict `verified` + +A task's role is its row's: `verifier` for the verifier's rows, which ASPIRE derives from a benchmark's tasks +(rows.py); an actor otherwise (a rerun is an actor with the final library). ASPIRE's checkout comes from +agent.params.aspire_root (its prompt templates must be declared readable: environment.read). +""" +from __future__ import annotations + +from functools import lru_cache +import importlib.util +import json +import os +from pathlib import Path +import shutil +import sys + +HERE = Path(__file__).resolve().parent +PROMPTS = HERE / "prompts" +TEMPLATES = {"actor": "subagent-prompt.md", "verifier": "verifier-prompt.md"} +SKILLS = Path(".claude/skills/iterative-debugging") +EVOSEARCH = Path(".claude/skills/evosearch") +# The files of ASPIRE's checkout its sessions read beyond the engine's (environment.read): its prompt templates and +# the search's procedure +ASPIRE_FILES = [str(SKILLS / "subagent-prompt.md"), str(SKILLS / "verifier-prompt.md"), + str(EVOSEARCH / "subagent-prompt.md"), str(EVOSEARCH / "skills" / "evosearch-iteration.md")] +# ASPIRE's files the log keeps beyond the engine's +KEEP = ("findings_report.md", "skill_report.json", "verification_report.json", "verification.md") + + +@lru_cache(maxsize=None) +def _load(path: str): + module_path = Path(path) + if str(module_path.parent) not in sys.path: + sys.path.insert(0, str(module_path.parent)) + spec = importlib.util.spec_from_file_location(f"_cap_engine_{abs(hash(path))}", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def engine(params: dict, base: Path): + """The engine's task functions: agent.params.engine, a path relative to the config's folder.""" + path = params.get("engine") + if not path: + raise ValueError("agent.params.engine: the engine's task functions (e.g. " + "../../cap_harness/utils/agent/task.py)") + return _load(str((Path(base) / os.path.expandvars(str(path))).resolve())) + + +def settings(task: dict, params: dict, base: Path) -> dict: + """The engine's settings of the task, with ASPIRE's role.""" + role = "verifier" if task.get("role") == "verifier" else "actor" # a rerun is an actor with the final library + return {**engine(params, base).settings(task, params), "role": role} + + +def task_dir(s: dict, eng) -> str: + """The actor's $TASK_DIR as ASPIRE's template defines it, with $RUN_ROOT the workspace; the verifier's is `.`.""" + return eng.task_dir(s) if s["role"] == "actor" else "." + + +def evo_dir(s: dict) -> str: + """The evolutionary search's $TASK_DIR, as its template defines it, with EVOSEARCH_DIR `evosearch`.""" + return f"evosearch/{s['benchmark']}/{s['suite']}/task_{s['task_id']}" + + +def assignment(s: dict, eng) -> str: + if s["role"] == "verifier": + return (f"## Your assignment\n\n- BENCHMARK: `{s['benchmark']}`\n- SUITE: `{s['suite']}`\n" + f"- TASK_ID: `{s['task_id']}`\n- GPU: handled by `bin/sim`\n- VERIFY_DIR: `.` (this workspace)\n" + f"- Development seeds: **{s['dev']}** — wherever the template says 51, 52, 53, 54, 55 (or 51-55, or " + f"seed 51), use these.\n- The proposals under test: `proposed_skills.md` in this workspace.\n") + return (f"## Task Assignment\n\nBENCHMARK: {s['benchmark']}\nSUITE: {s['suite']}\nTASK_ID: {s['task_id']}\n" + f"GPU: handled by bin/sim\nRUN_ROOT: . (this workspace)\n\n" + f"MAX_STEPS: {s['max_steps'] or 1000}\nDEV_SEEDS: {s['dev']}\nUNSEEN_SEEDS: {s['unseen'] or 'none'}\n" + f"HELDOUT_COUNT: {len(eng.sim_seeds(s['heldout']))} (coordinator-owned; you never run them)\n") + + +def prompt(task: dict, params: dict, base: Path) -> str: + eng = engine(params, base) + s = settings(task, params, base) + template = (s["root"] / SKILLS / TEMPLATES[s["role"]]).read_text() + body = template.split("", 1)[1] + body = body.split("", 1)[0].strip() + if s["role"] == "actor": + body = body.split("\n---\n", 1)[1] if body.startswith("## Task Assignment") else body # ours replaces it + header = (PROMPTS / "codex_header.md").read_text().format( + role_name="fix-loop subagent" if s["role"] == "actor" else "skill verifier", + template=TEMPLATES[s["role"]], benchmark=s["benchmark"], benchmark_doc=s["doc"], + your_dir=(f"`$RUN_ROOT` is this workspace (`.`), so `$TASK_DIR` is `{task_dir(s, eng)}`, as the template " + f"defines it." if s["role"] == "actor" else "`$VERIFY_DIR` is this workspace (`.`)."), + proposed_line=("The proposals under test are `proposed_skills.md` in this workspace (the template's " + "`$VERIFY_DIR/proposed_skills.md`); they are not in the library." if s["role"] == "verifier" + else ""), + finish_line=("only after both parts of this session: Part 1, the template below, and Part 2 after it, the " + "evolutionary search, which runs only when Part 1 leaves `fix_code.py` below 90% of the " + "development seeds. Reply with the return message of each part you ran." + if s["role"] == "actor" and params.get("evosearch", True) else + "by replying with the template's return message."), + heldout_line=("is not yours: the harness scores your selected program on the held-out seeds after you " + "finish (the template's Stage 2)." if s["role"] == "actor" else + "does not apply: your verdict is scored on the program you select, on your seeds; write it " + "as `fix_code.py` in this workspace as well.")) + text = f"{header}\n\n{assignment(s, eng)}\n---\n\n{body}\n" + if s["role"] == "actor" and params.get("evosearch", True): + text += "\n---\n\n" + evosearch_part(s, eng) + return text + + +def evosearch_part(s: dict, eng) -> str: + """Part 2 of the actor's session: ASPIRE's evolutionary-search subagent (§2.3), for a task the fix loop left below + 90% of its development seeds. Its template is the fenced block of evosearch/subagent-prompt.md, verbatim after + its assignment block, which the page before it replaces.""" + template = (s["root"] / EVOSEARCH / "subagent-prompt.md").read_text() + body = template[template.index("## Task Assignment"):template.rindex("\n```")] + body = body.split("\n---\n", 1)[1].strip() + header = (PROMPTS / "evosearch_header.md").read_text().format( + benchmark=s["benchmark"], suite=s["suite"], task_id=s["task_id"], dev=s["dev"], task_dir=task_dir(s, eng), + evo_dir=evo_dir(s)) + return f"{header}\n---\n\n{body}\n" + + +def prepare(task: dict, workspace: Path, params: dict, base: Path) -> list[dict]: + eng = engine(params, base) + s = settings(task, params, base) + mounts = eng.prepare(task, workspace, params, base) + work = Path(workspace) + if params.get("evosearch", True): # the search's procedure, for a prompt that runs it + shutil.copy2(s["root"] / EVOSEARCH / "skills" / "evosearch-iteration.md", + work / "docs" / "evosearch-iteration.md") + if s["role"] == "verifier": + # The coordinator stages the proposals under test as the adapter's skills/proposed.md: the verifier's + # proposed_skills.md, kept out of the library it reads (interface.py leaves the file out). + staged = sorted(work.glob("hermit/*/skills/proposed.md")) + (work / "proposed_skills.md").write_text(staged[0].read_text() if staged else "(no proposals were staged)\n") + (work / "task.md").write_text(assignment(s, eng)) + return mounts + + +def answer(last_message: str, task: dict) -> dict: + return {"answer": last_message.strip()[-4000:]} + + +def collect(task: dict, workspace: Path, log: Path, params: dict, base: Path) -> dict: + """The engine's files and facts, and ASPIRE's: what the judge and the coordinator read.""" + eng = engine(params, base) + s = settings(task, params, base) + work = Path(workspace) / task_dir(s, eng) + facts = eng.collect_files(task, work, log, (*eng.KEEP, *KEEP)) + files = Path(log) / "files" + read = lambda name: (work / name).read_text(errors="replace") if (work / name).is_file() else None + facts = {"role": s["role"], "benchmark": s["benchmark"], "suite": s["suite"], + "findings": facts["findings"] or read("findings_report.md"), + "skill_report": _json(read("skill_report.json")), "verification": _json(read("verification_report.json")), + "unseen": facts["unseen"], "workspace": str(Path(workspace).resolve()), "max_steps": s["max_steps"]} + evo = Path(workspace) / evo_dir(s) + search = s["role"] == "actor" and params.get("evosearch", True) # a prompt that runs the search + if search and evo.is_dir(): + for name in ("evosearch_best_code.py", "findings.md", "task_analysis.md", "BLOCKED"): + if (evo / name).is_file(): + (files / "evosearch").mkdir(parents=True, exist_ok=True) + shutil.copy2(evo / name, files / "evosearch" / name) + facts["evosearch"] = {"findings": (evo / "findings.md").read_text(errors="replace") + if (evo / "findings.md").is_file() else None, + "best_code": (evo / "evosearch_best_code.py").is_file()} + # The selected program: the search's best when the search ran (it falls back to fix_code.py itself), else the fix + # loop's; with `evosearch: false`, fix_code.py alone. The judge scores it. + best = evo / "evosearch_best_code.py" + program = best if search and best.is_file() else work / "fix_code.py" + facts["scored_program"] = str(program.relative_to(workspace)) if program.is_file() else None + facts["fix_code"] = program.read_text(errors="replace") if program.is_file() else None + # A verifier's program counts only with its own verdict `verified` (ASPIRE: `refuted` and `void` are never + # promoted): without it, nothing is scored. + if s["role"] == "verifier" and (facts["verification"] or {}).get("verdict") != "verified": + facts["not_scored"] = f"verifier verdict {(facts['verification'] or {}).get('verdict') or 'missing'}" + facts["scored_program"] = facts["fix_code"] = None + if params.get("prune_media", True): + eng.prune(Path(workspace).parent) + return facts + + +def _json(text): + try: + return json.loads(text) if text else None + except ValueError: + return {"unparsed": text[-4000:]} diff --git a/examples/aspire_robosuite/utils/agent/agent.py b/hermit/agent/evolver/methods/aspire/direct.py similarity index 87% rename from examples/aspire_robosuite/utils/agent/agent.py rename to hermit/agent/evolver/methods/aspire/direct.py index b942cd73..d64afccf 100644 --- a/examples/aspire_robosuite/utils/agent/agent.py +++ b/hermit/agent/evolver/methods/aspire/direct.py @@ -1,17 +1,19 @@ -"""ASPIRE's actor on one Robosuite task: write a Code-as-Policy program, run it in the robot execution engine on the -task's development seeds, diagnose from the per-primitive traces and keyframes, repair, and keep the best version +"""ASPIRE's actor on one Robosuite task of the cap-harness engine, in direct chat (its prompts and its runs are +Robosuite's): write a Code-as-Policy program, run it in the robot execution engine on the task's development seeds, +diagnose from the per-primitive traces and keyframes, repair, and keep the best version (ASPIRE §2.1, the iterative-debugging loop of NVlabs/ASPIRE .claude/skills/iterative-debugging/subagent-prompt.md); optionally continue with the evolutionary search over the program (ASPIRE §2.3, Algorithm 1; .claude/skills/evosearch/); then write the findings the coordinator distills skills from. The judge runs the -selected program on the held-out seeds (utils/benchmark/judge.py). +selected program on the held-out seeds (the engine's utils/benchmark/judge.py). The adapter is the shared skill library: its context (ADAPTER.md, and the skills it exports) is the "Shared skill library" section of the actor's instructions, the only thing that carries from one task to the next. adapter=None runs without one. Every model call is one Direct call (Layer 1, docs/framework/agent-layers.md); the loop is this file's. -The robot execution engine is ASPIRE's harness in its own venv (sim.py), started in this process: this actor needs -the GPU, so its configs run it on the host (environment.runner: host), a comparison, not the method. +The robot execution engine is ASPIRE's harness in its own venv (the engine's sim.py, agent.params.engine: e.g. +../../cap_harness/utils/agent/sim.py), started in this process: this actor needs the GPU, so its configs run it on +the host (environment.runner: host), a comparison, not the method. Output: the development rate of the selected program. Log: a compact directory (every program version, each sweep's records and per-seed digests, the keyframes the model was shown, findings.md, skill_report.json, the transcript); ASPIRE's full @@ -20,6 +22,7 @@ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor +import importlib.util import json import os from pathlib import Path @@ -30,12 +33,9 @@ import time from hermit.agent.acting import Agent -from hermit.task import RunOutput +from hermit.common import RunOutput HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE)) -import sim # noqa: E402 -from sim import seeds_of # noqa: E402,F401 DEFAULT_MODEL = "fireworks/accounts/fireworks/models/glm-5p3-flash" PROMPTS = HERE / "prompts" @@ -75,10 +75,25 @@ def json_of(reply: str): return None +def engine_module(path: Path): + """The engine's sim.py (agent.params.engine), loaded from its own folder once.""" + path = Path(path).resolve() + name = f"_cap_sim_{abs(hash(str(path)))}" + if name not in sys.modules: + if str(path.parent) not in sys.path: + sys.path.insert(0, str(path.parent)) + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return sys.modules[name] + + class AspireActor(Agent): - """`agent.name: ../utils/agent/agent.py:AspireActor` + """`agent.name: hermit.agent.evolver.methods.aspire.direct:AspireActor` - params: aspire_root (an ASPIRE checkout with .venv-robosuite), sim_env (the engine's environment), model, + params: engine (the engine's sim.py, relative to the config), aspire_root (an ASPIRE checkout with + .venv-robosuite), sim_env (the engine's environment), model, effort, max_tokens, debug_seeds (a task row's own seeds win; its held-out seeds are the row's, for the judge), max_versions, search (debug | evo), evo {k, rounds, threshold, qualify_below, parents}, workers, max_steps, failed_shown, keyframes, evidence_chars, @@ -133,18 +148,23 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput: root = p.get("aspire_root") or os.environ.get("ASPIRE_ROOT") if not root: raise ValueError("agent.params.aspire_root (or ASPIRE_ROOT): an ASPIRE checkout") + if not p.get("engine"): + raise ValueError("agent.params.engine: the engine's sim.py (e.g. ../../cap_harness/utils/agent/sim.py)") + sim = engine_module(self.path(os.path.expandvars(str(p["engine"])))) engine = sim.Engine(self.path(os.path.expandvars(root)), env=p.get("sim_env"), workers=int(p.get("workers", 4)), max_steps=p.get("max_steps"), timeout_s=float(p.get("sweep_timeout_s", 3600))) # A task row's own settings win over agent.params: the verifier's rows carry its seeds and budget. p = {**p, **{k: task[k] for k in ROW_KEYS if k in task}} - debug = seeds_of(p.get("debug_seeds") or "101-125") + if task.get("rerun_of"): # a rerun with the final library (ASPIRE §2.4): no findings to distill + p["findings"] = False + debug = sim.seeds_of(p.get("debug_seeds") or "101-125") if not debug: raise ValueError(f"{task['id']}: no development seeds") stamp = time.strftime("%Y%m%dT%H%M%S") + f"-{time.time_ns() % 10**6:06d}" log = self.log_dir() / f"{task['id']}-{stamp}" scratch = self.path(p.get("scratch") or (self.log_dir() / "scratch")) / f"{task['id']}-{stamp}" - run = _Run(self, engine, task, log, scratch, debug, self._skills(adapter, task), p) + run = _Run(self, sim, engine, task, log, scratch, debug, self._skills(adapter, task), p) try: run.explore() @@ -164,12 +184,14 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput: rate = best.passed / len(debug) # the development rate; the judge runs the held-out seeds (Stage 2) role = "verifier" if p.get("role") == "verifier" else "actor" - # the facts in the Codex actor's schema (task.py:collect), which the judge and the coordinator read; a - # verifier's verdict is ASPIRE's rule on its own seeds (>= 4 of 5) + # the facts in the Codex actor's schema (actor.py:collect), which the judge and the coordinator read; a + # verifier's verdict is ASPIRE's rule on its own seeds (>= 4 of 5), and its program counts only when verified facts = {"role": role, "suite": task["suite"], "findings": findings, "skill_report": report, "fix_code": best.source, "unseen": None, "max_steps": p.get("max_steps"), **({"verification": {"verdict": "verified" if best.passed >= 0.8 * len(debug) else "refuted"}} if role == "verifier" else {})} + if role == "verifier" and facts["verification"]["verdict"] != "verified": + facts.update(not_scored=f"verifier verdict {facts['verification']['verdict']}", fix_code=None) extra = {"task": facts, "selected": best.name, "debug_passed": best.passed, "debug_total": len(debug), "versions": [v.summary() for v in run.versions], "usage": run.totals} return RunOutput(output=rate, log=log, wall_s=time.time() - started, **run.spent(), @@ -213,9 +235,9 @@ def summary(self) -> dict: class _Run: """The state of one task's debugging: the versions, the transcript, the log directory.""" - def __init__(self, actor: AspireActor, engine: sim.Engine, task: dict, log: Path, scratch: Path, + def __init__(self, actor: AspireActor, sim, engine, task: dict, log: Path, scratch: Path, debug: list[int], skills: str, params: dict): - self.actor, self.engine, self.task, self.log, self.scratch = actor, engine, task, log, scratch + self.actor, self.sim, self.engine, self.task, self.log, self.scratch = actor, sim, engine, task, log, scratch self.debug, self.skills = debug, skills self.suite, self.task_id = task["suite"], int(task.get("task_id", 0)) self.versions: list[_Version] = [] @@ -279,7 +301,7 @@ def sweep(self, name: str, source: str, seeds: list[int], *, parent=None, note=" out = self.log / "sweeps" / name out.mkdir(parents=True, exist_ok=True) (out / "records.json").write_text(json.dumps(records, indent=1)) - (out / "digest.txt").write_text("\n\n".join(sim.digest(r) for r in records)) + (out / "digest.txt").write_text("\n\n".join(self.sim.digest(r) for r in records)) return _Version(name, source, records, parent, note) def evidence(self, version: _Version) -> list[dict]: @@ -292,15 +314,15 @@ def evidence(self, version: _Version) -> list[dict]: parts: list[dict] = [] budget = int(self.params.get("evidence_chars", 40000)) # per version shown: the digests share it for record in shown: - text.append(_clip_middle(sim.digest(record), max(2000, budget // max(1, len(shown))))) + text.append(_clip_middle(self.sim.digest(record), max(2000, budget // max(1, len(shown))))) n_frames = int(self.params.get("keyframes", 2)) for record in shown[:2]: - for label, png in sim.keyframes(record.get("run_dir"), limit=n_frames): + for label, png in self.sim.keyframes(record.get("run_dir"), limit=n_frames): kept = self.log / "keyframes" / version.name / f"seed{record['seed']}-{label}.png" kept.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(png, kept) parts.append({"type": "text", "text": f"keyframe {version.name} seed {record['seed']} {label}:"}) - parts.append(sim.image_part(kept)) + parts.append(self.sim.image_part(kept)) return [{"type": "text", "text": "\n\n".join(text)}, *parts] # --- Stage 0: explore once ---------------------------------------------------------------------------------- @@ -308,14 +330,14 @@ def explore(self): probe = (Path(self.engine.root) / ".claude/skills/iterative-debugging/scripts/scene_snapshot.py").read_text() version = self.sweep("probe", probe, self.debug[:1]) record = version.records[0] - self.scene = sim.digest(record, call_chars=1500) + self.scene = self.sim.digest(record, call_chars=1500) for camera in ("agentview", "robot0_eye_in_hand"): - for label, png in sim.keyframes(record.get("run_dir"), camera=camera, limit=1): + for label, png in self.sim.keyframes(record.get("run_dir"), camera=camera, limit=1): kept = self.log / "keyframes" / "probe" / f"{camera}-{label}.png" kept.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(png, kept) self.scene_images += [{"type": "text", "text": f"initial scene, camera {camera}:"}, - sim.image_part(kept)] + self.sim.image_part(kept)] def _task_text(self) -> str: return (f"Task: robosuite suite `{self.suite}`, task id {self.task_id}. Development seeds: {self.debug}.\n\n" @@ -401,7 +423,7 @@ def evolve(self, start: _Version, k: int = 8, rounds: int = 5, threshold: float def findings(self, best: _Version): template = (PROMPTS / "findings.md").read_text().split("-->", 1)[1].strip() history = self.changelog() + "\n\n" + "\n\n".join( - f"{v.name} failures:\n" + "\n".join(sim.digest(r, call_chars=200) for r in v.failed()[:2]) + f"{v.name} failures:\n" + "\n".join(self.sim.digest(r, call_chars=200) for r in v.failed()[:2]) for v in self.versions if v.failed()) prompt = template.format(selected=best.name, debug_passed=best.passed, debug_total=len(best.records), suite=self.suite, history=history[-60000:], program=best.source) diff --git a/examples/aspire_robosuite/utils/agent/prompts/actor_system.md b/hermit/agent/evolver/methods/aspire/prompts/actor_system.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/actor_system.md rename to hermit/agent/evolver/methods/aspire/prompts/actor_system.md diff --git a/examples/aspire_robosuite/utils/agent/prompts/codex_header.md b/hermit/agent/evolver/methods/aspire/prompts/codex_header.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/codex_header.md rename to hermit/agent/evolver/methods/aspire/prompts/codex_header.md diff --git a/examples/aspire_robosuite/utils/agent/prompts/evosearch_header.md b/hermit/agent/evolver/methods/aspire/prompts/evosearch_header.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/evosearch_header.md rename to hermit/agent/evolver/methods/aspire/prompts/evosearch_header.md diff --git a/examples/aspire_robosuite/utils/agent/prompts/findings.md b/hermit/agent/evolver/methods/aspire/prompts/findings.md similarity index 100% rename from examples/aspire_robosuite/utils/agent/prompts/findings.md rename to hermit/agent/evolver/methods/aspire/prompts/findings.md diff --git a/hermit/agent/evolver/methods/aspire/rows.py b/hermit/agent/evolver/methods/aspire/rows.py new file mode 100644 index 00000000..9626f915 --- /dev/null +++ b/hermit/agent/evolver/methods/aspire/rows.py @@ -0,0 +1,97 @@ +"""ASPIRE's own task rows beside a benchmark's: for every task, its verifier. A benchmark of the cap-harness engine +writes only its tasks and their reruns (its utils/benchmark/prepare.py); an ASPIRE run reads them and these, which +this writes into a folder of its own (the ASPIRE configs' `tasks/` and `splits/`), from the benchmark's prepared +tasks and splits, with the engine's task writer (`--engine`: its utils/benchmark/tasks.py, which reads the tasks back +as rows and writes a row as a task directory; agents never import the task layer). The ASPIRE folder's tasks.sh runs +it after the benchmark's prepare.py (the engine's launch.sh): + + python -m hermit.agent.evolver.methods.aspire.rows --benchmark ../utils/benchmark --out . --verify-seeds 101-105 \ + --engine ../../cap_harness/utils/benchmark/tasks.py + +- `verify-`: the verifier of each task (ASPIRE verifier-prompt.md): a fresh session that re-solves the task from + the library and the proposals alone, on its own development seeds (`--verify-seeds`, ASPIRE's first five) in at + most two programs (the simulator service's attempt budget, `max_programs`), with no findings; the judge scores its + program on those seeds. Only the coordinator runs them (the configs' `data.val`, through run_agent). + +The benchmark's tasks and splits are copied as they are (its reruns are ASPIRE's Phase 3 rerun with the final +library, §2.4), and splits/verify.txt holds every verifier. +""" +from __future__ import annotations + +import argparse +from functools import lru_cache +import importlib.util +from pathlib import Path +import re +import shutil +import sys + +SEEDS = ("debug_seeds", "unseen_seeds", "eval_seeds") + + +def seeds_of(value) -> set[int]: + """The seeds a set names: `101-125`, `1,3,5`, or none.""" + out: set[int] = set() + for part in re.split(r"[,\s]+", str(value or "").strip()): + if not part or part == "none": + continue + low, _, high = part.partition("-") + out.update(range(int(low), int(high or low) + 1)) + return out + + +@lru_cache(maxsize=None) +def engine(path: str): + """The engine's task writer (its utils/benchmark/tasks.py), loaded from its own folder once.""" + module_path = Path(path).resolve() + if str(module_path.parent) not in sys.path: + sys.path.insert(0, str(module_path.parent)) + spec = importlib.util.spec_from_file_location(f"_cap_tasks_{abs(hash(str(module_path)))}", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def what(row: dict) -> str: + return f"{row.get('benchmark') or 'robosuite'} suite `{row['suite']}`, task id {row['task_id']}" + + +def verifier(row: dict, seeds: str) -> dict: + base = {k: v for k, v in row.items() if k not in SEEDS} + return {**base, "id": f"verify-{row['id']}", "role": "verifier", "debug_seeds": seeds, "eval_seeds": seeds, + "score_label": "verifier", "max_programs": 2, "max_versions": 2, "search": "debug", "findings": False} + + +def write(benchmark: Path, out: Path, verify_seeds: str, engine_tasks: Path) -> None: + eng = engine(str(engine_tasks)) + rows = eng.rows(benchmark / "tasks") + order = (benchmark / "splits" / "all.txt").read_text().split() # the benchmark's own order of its tasks + tasks = {tid: rows[tid] for tid in order} + for row in tasks.values(): + clash = seeds_of(verify_seeds) & seeds_of(row.get("eval_seeds")) + if clash: + raise ValueError(f"{row['id']}: the verifier's seeds overlap the held-out seeds: {sorted(clash)[:10]}") + for name in ("tasks", "splits"): # this script's own output: rewritten whole + if (out / name).exists(): + shutil.rmtree(out / name) + shutil.copytree(benchmark / "tasks", out / "tasks") + for row in tasks.values(): + text = f"Verify the proposals for {what(row)}: solve it again from the library and the proposals alone." + eng.write_task(out / "tasks", verifier(row, verify_seeds), text) + shutil.copytree(benchmark / "splits", out / "splits") + (out / "splits" / "verify.txt").write_text("".join(f"verify-{i}\n" for i in tasks)) + print(f"{len(tasks)} verifiers -> {out / 'tasks'}") + + +def main(argv=None) -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--benchmark", type=Path, required=True, help="the benchmark's utils/benchmark (tasks/, splits/)") + ap.add_argument("--out", type=Path, required=True, help="where tasks/ and splits/ go") + ap.add_argument("--verify-seeds", required=True, help="the verifier's development seeds (ASPIRE's first five)") + ap.add_argument("--engine", type=Path, required=True, help="the engine's task writer (utils/benchmark/tasks.py)") + a = ap.parse_args(argv) + write(a.benchmark, a.out, a.verify_seeds, a.engine) + + +if __name__ == "__main__": + main() diff --git a/tests/README.md b/tests/README.md index c4fc2885..4f45e3ba 100644 --- a/tests/README.md +++ b/tests/README.md @@ -33,6 +33,10 @@ selected with the benchmark’s `evosafe` example (plus `core` and the existing installed instead; model responses come from a local HTTP server. The Docker hook check also skips without an engine and the `python:3.12-slim` image. +The cap-harness engine that Robosuite and LIBERO share has its tests in `examples/cap_harness/utils/tests/` (its +stand-ins, `cap_fakes.py`, are ASPIRE's tests' too); a change to the engine runs the groups of the methods with an +example on one of its benchmarks (`ENGINES` in `selection.py`), and `core`. + CI (`.github/workflows/tests.yml`) runs `--changed` on a pull request and the whole suite on main, daily and on demand. diff --git a/tests/core/test_cli_agent.py b/tests/core/test_cli_agent.py index 27c7e6ad..b12321b9 100644 --- a/tests/core/test_cli_agent.py +++ b/tests/core/test_cli_agent.py @@ -158,6 +158,15 @@ def test_a_run_with_the_tasks_hooks_records_the_hosts_log(fake): assert [s.kind for s in steps] == ["reasoning", "command", "command", "message"] +def test_a_task_function_may_be_named_by_its_module_and_a_prompt_stays_text(tmp_path): + """A method that brings its own task functions names them by module (`package.module:function`, ASPIRE's + hermit.agent.evolver.methods.aspire.actor); a prompt written as text is never read as one.""" + assert cli._task_function("os.path:basename", tmp_path)("a/b.txt") == "b.txt" + for text in ("Answer: yes", "Solve it:now", "see notes.md:3", "json:loads"): + assert cli._task_function(text, tmp_path) is None, text + assert cli._prompt({"prompt": "Solve it:now"}, tmp_path, {}) == "Solve it:now" + + def test_the_answer_pattern_and_why_there_is_none(fake): tmp_path = fake params = {"web_search": False, "prompt": "go", "answer": "(.*?)", "workspaces": "ws"} diff --git a/tests/core/test_environment.py b/tests/core/test_environment.py index 89064276..3bc6aa90 100644 --- a/tests/core/test_environment.py +++ b/tests/core/test_environment.py @@ -350,7 +350,12 @@ def test_no_shipped_config_runs_a_method_on_an_agent_that_lacks_what_it_needs(): supports = supports_of(name, path.parent, agent.get("params"), may_import=False) if supports is None and ":" in name: file, _, symbol = name.partition(":") - supports = _declared_supports((path.parent / file).resolve(), symbol) + if file.endswith(".py"): + source = (path.parent / file).resolve() + else: # `package.module:Class`: its source, not imported + import importlib.util + source = Path(importlib.util.find_spec(file).origin) + supports = _declared_supports(source, symbol) assert supports is not None and needs <= set(supports), f"{path.relative_to(root)}: {name} lacks {needs}" checked += 1 assert checked >= 10 # MWM, ASPIRE, ReHarness and AHE's examples diff --git a/tests/core/test_selection.py b/tests/core/test_selection.py index 941e695b..c48640c1 100644 --- a/tests/core/test_selection.py +++ b/tests/core/test_selection.py @@ -61,6 +61,10 @@ def test_a_benchmarks_shared_files_run_the_methods_with_an_example_there(): "ace", "ahe", "continual_harness", "harness2", "meta_harness_adapter", "opro", "optimizers", "reharness", "rrsi_adapter", "core"} assert groups_for("examples/servelearnbench/utils/x.py") == {"mwm", "core"} + # an engine several benchmarks run on: the methods with an example under any of them + assert groups_for("examples/cap_harness/utils/agent/sim.py") == {"aspire", "core"} + assert groups_for("examples/libero/utils/agent/task.py") == {"aspire", "core"} + assert groups_for("examples/robosuite/aspire/run.yaml") == {"aspire", "core"} groups, why = select(["hermit/cli.py", "examples/officeqa/rrsi_adapter/README.md", "docs/index.md"]) assert groups == {"core", "rrsi_adapter"} and why == {"core": "hermit/cli.py", "rrsi_adapter": "examples/officeqa/rrsi_adapter/README.md"} diff --git a/tests/methods/aspire/test_aspire.py b/tests/methods/aspire/test_aspire.py index 516e9cf6..20515f4b 100644 --- a/tests/methods/aspire/test_aspire.py +++ b/tests/methods/aspire/test_aspire.py @@ -1,29 +1,58 @@ """ASPIRE: the aspire template (the skill library, its reader and its check), the coordinator's hook into the -Trainer (next_tasks), and the example's pieces: the Codex task functions (prompt assembly, the workspace and its -registration), the simulator service's rules as bin/sim meets them (a real service on a loopback port, over a fake -checkout), the judge's Stage 2 (a stand-in engine), and pure helpers; no simulator, no model, no network.""" -import hashlib +Trainer (next_tasks), ASPIRE's acting sessions on the cap-harness engine (actor.py: the prompt, the workspace beyond the +engine's, the facts the coordinator and the judge read), its verifier rows (rows.py), its direct-chat actor +(direct.py) and its configs on Robosuite and LIBERO. The engine is the stand-in of its own tests +(examples/cap_harness/utils/tests/cap_fakes.py): no simulator, no model, no network.""" import importlib.util import json import os from pathlib import Path import shutil -import subprocess import sys -import tempfile -import time from types import SimpleNamespace import pytest from hermit.agent.evolver.base import StepResult +from hermit.agent.evolver.methods.aspire import actor as A +from hermit.agent.evolver.methods.aspire import direct as D from hermit.agent.evolver.methods.aspire import evolver as E +from hermit.agent.evolver.methods.aspire import rows as Rw from hermit.spec import check_spec ROOT = Path(__file__).resolve().parents[3] TEMPLATE = ROOT / "hermit" / "adapter" / "templates" / "aspire" -EXAMPLE = ROOT / "examples" / "aspire_robosuite" / "utils" -LIBERO = ROOT / "examples" / "libero" # LIBERO on the same engine: its tasks and its methods' configs +ENGINE = ROOT / "examples" / "cap_harness" / "utils" +ROBOSUITE = ROOT / "examples" / "robosuite" +LIBERO = ROOT / "examples" / "libero" + + +def _fakes(): + spec_ = importlib.util.spec_from_file_location("cap_fakes", ENGINE / "tests" / "cap_fakes.py") + module = importlib.util.module_from_spec(spec_) + spec_.loader.exec_module(module) + return module + + +F = _fakes() +tmp_path = F.tmp_path # a fixture: a directory outside /tmp, for the template check's sandbox +_load, fake_aspire, _Service, _workspace, _sim = F._load, F.fake_aspire, F._Service, F._workspace, F._sim +ROW, LIBERO_ROW = F.ROW, F.LIBERO_ROW +# ASPIRE's verifier of ROW, as rows.py derives it +VERIFY = Rw.verifier(ROW, "101-105") +# The engine as ASPIRE's configs name it (agent.params.engine), relative to the config's folder +PARAMS = {"engine": "../../cap_harness/utils/agent/task.py"} +BASE = ROBOSUITE / "aspire" + + +@pytest.fixture +def service(tmp_path, monkeypatch): + """The engine's simulator service, serving ASPIRE's verifier row besides the benchmarks' rows.""" + svc = _Service(tmp_path, fake_aspire(tmp_path), rows=(VERIFY,)) + monkeypatch.setenv("ASPIRE_SIM_URL", svc.url) + yield svc + svc.close() + ENTRY = """## Re-localize after the approach **Failure signature.** The gripper closes on air: the cube moved when the arm passed over it. @@ -40,17 +69,6 @@ """ -@pytest.fixture -def tmp_path(): - """A directory outside /tmp: the template check runs under bubblewrap with an empty tmpfs at /tmp.""" - base = Path.home() / ".cache" / "hermit-tests" - base.mkdir(parents=True, exist_ok=True) - path = Path(tempfile.mkdtemp(dir=base)) - yield path - if not os.environ.get("KEEP_TEST_DIRS"): - shutil.rmtree(path, ignore_errors=True) - - def seed(tmp: Path) -> Path: adapter = tmp / "adapter" shutil.copytree(TEMPLATE, adapter, ignore=shutil.ignore_patterns("__pycache__")) @@ -62,8 +80,6 @@ def spec(path) -> dict: return {"passed": r.passed, "issues": [{"path": i.path, "problem": i.problem} for i in r.issues]} -# ---------- the template ---------- - def test_template_root_passes_its_own_check(): assert check_spec(TEMPLATE, TEMPLATE).passed @@ -140,8 +156,6 @@ def test_reader_leaves_out_the_staged_proposals(tmp_path): assert "Re-localize" not in module.library()["text"] -# ---------- the coordinator ---------- - def test_next_tasks_reads_only_a_regular_file(tmp_path): """output/next_tasks.json is the learner's, read on the host: a link or a pipe in its place names no task.""" path = tmp_path / "next_tasks.json" @@ -155,6 +169,7 @@ def test_next_tasks_reads_only_a_regular_file(tmp_path): os.mkfifo(path) assert E.next_tasks(path) is None # returns at once, not blocked on the pipe + def test_the_coordinator_chooses_tasks_and_returns_its_named_next_task(tmp_path, monkeypatch): from hermit.agent.evolver.agent import AgentEvolver out = tmp_path / "output" @@ -179,33 +194,21 @@ def test_the_coordinator_prompt_carries_the_guides_sections(): assert placeholder in text -# ---------- the example's actor helpers ---------- - -def _actor_module(): - sys.path.insert(0, str(EXAMPLE / "agent")) - try: - spec_ = importlib.util.spec_from_file_location("aspire_actor", EXAMPLE / "agent" / "agent.py") - module = importlib.util.module_from_spec(spec_) - spec_.loader.exec_module(module) - return module - finally: - sys.path.remove(str(EXAMPLE / "agent")) - - def test_actor_reads_seeds_and_the_outermost_program_fence(): - A = _actor_module() - assert A.seeds_of("101-103 7") == [101, 102, 103, 7] - assert A.seeds_of([1, "3-4"]) == [1, 3, 4] + sim = D.engine_module(ENGINE / "agent" / "sim.py") # the engine the direct actor runs its programs on + assert sim.seeds_of("101-103 7") == [101, 102, 103, 7] + assert sim.seeds_of([1, "3-4"]) == [1, 3, 4] + assert D.engine_module(ENGINE / "agent" / "sim.py") is sim # loaded once reply = "diagnosis\n```python\ns = '''\n```python\nx\n```\n'''\nresult = {}\n```\ntrailing" - assert A.program_of(reply).endswith("result = {}\n") - assert A.program_of("no program") is None - assert A.json_of('text ```json\n{"a": 1}\n```') == {"a": 1} + assert D.program_of(reply).endswith("result = {}\n") + assert D.program_of("no program") is None + assert D.json_of('text ```json\n{"a": 1}\n```') == {"a": 1} def test_the_direct_actor_keeps_its_models_settings_and_an_unknown_cost_unknown(monkeypatch): import threading from hermit.agent.model import llm - A = _actor_module() + A = D specs = [] monkeypatch.setattr(llm.ChatClient, "from_spec", classmethod(lambda cls, spec, **k: specs.append(spec))) monkeypatch.setattr("hermit.agent.base.interface.make", lambda *a, **k: object()) @@ -223,7 +226,8 @@ def test_the_direct_actor_keeps_its_models_settings_and_an_unknown_cost_unknown( def test_a_failed_direct_task_keeps_what_it_spent(tmp_path, monkeypatch): - A = _actor_module() + A = D + sim = D.engine_module(ENGINE / "agent" / "sim.py") class Engine: def __init__(self, root, **k): @@ -235,9 +239,12 @@ def __init__(self, root, **k): def explore(run): # one paid call, then an outage past the retries run.account([SimpleNamespace(input_tokens=10, output_tokens=5, cost_usd=0.5)], "explore", "r", "", "stop") raise RuntimeError("model unavailable: 503") - monkeypatch.setattr(A.sim, "Engine", Engine) + monkeypatch.setattr(sim, "Engine", Engine) monkeypatch.setattr(A._Run, "explore", explore) - actor = A.AspireActor({"aspire_root": str(tmp_path), "scratch": str(tmp_path / "scratch")}, base=tmp_path) + actor = A.AspireActor({"aspire_root": str(tmp_path), "scratch": str(tmp_path / "scratch"), + "engine": str(ENGINE / "agent" / "sim.py")}, base=tmp_path) + with pytest.raises(ValueError, match="agent.params.engine"): # it names the engine it runs on + A.AspireActor({"aspire_root": str(tmp_path)}, base=tmp_path).run(dict(ROW), None) monkeypatch.setattr(actor, "log_dir", lambda: tmp_path / "logs") monkeypatch.setattr(actor, "session", lambda adapter: None) out = actor.run(dict(ROW), None) @@ -249,887 +256,185 @@ def explore(run): # one paid call, the run_one(dict(ROW), None, lambda task, adapter: actor.run(task, adapter), lambda task, out: {"score": 0}) -def test_sim_compact_makes_traces_readable(): - sim = _sim_module() - value = {"$type": "Pose", "position": {"$array_inline": [0.123456, 1.0, 2.0], "dtype": "float64", "shape": [3]}, - "obs": {"$type": "Observation", "cameras": {}}, "big": {"$array_inline": list(range(40)), "shape": [40]}} - out = sim.compact(value) - assert out == {"type": "Pose", "position": [0.1235, 1.0, 2.0], "obs": "", - "big": ""} - - -def test_the_failure_keyframe_is_the_failing_calls_own(tmp_path): - sim = _sim_module() - run = tmp_path / "run" - (run / "source").mkdir(parents=True) - trace = [{"tool": "open_gripper", "ok": True}] + [{"tool": "move_to_pose", "ok": i != 2} for i in range(4)] - (run / "source" / "program-result.json").write_text(json.dumps({"calls": trace})) - names = ["reset", "before-open-gripper", "after-open-gripper"] - for i in range(3): # the third move raised: no after-frame of its own - names += ["before-move-to-pose"] + (["after-move-to-pose"] if i < 2 else []) - for n, name in enumerate(names, 1): - d = run / "media" / "keyframes" / f"{n:06d}-{name}" - d.mkdir(parents=True) - (d / "agentview.png").write_bytes(b"png") - picked = [label for label, _ in sim.keyframes(run, limit=1)] - assert picked == [f"{names.index('before-move-to-pose', 7) + 1:06d}-before-move-to-pose"] - - -# ---------- the Codex actor's task functions ---------- - -def _load(name: str, path: Path): - sys.path.insert(0, str(path.parent)) - try: - spec_ = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec_) - spec_.loader.exec_module(module) - return module - finally: - sys.path.remove(str(path.parent)) - - -def _task_module(): - return _load("aspire_task", EXAMPLE / "agent" / "task.py") - - -def fake_aspire(tmp: Path) -> Path: - """The files of an ASPIRE checkout the task functions read: the two templates, the docs, the snapshot program.""" - root = tmp / "ASPIRE" - skills = root / ".claude" / "skills" / "iterative-debugging" - (skills / "scripts").mkdir(parents=True) - (root / "docs").mkdir() - marks = ("", - "") - (skills / "subagent-prompt.md").write_text(f"intro\n{marks[0]}\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" - f"## What You Are\nfix-loop text\n{marks[1]}\n") - (skills / "verifier-prompt.md").write_text(f"intro\n{marks[0]}\nYou are verifying. Seeds 51-55.\n{marks[1]}\n") - (skills / "scripts" / "scene_snapshot.py").write_text("result = {}\n") - for name in ("api-reference.md", "run-artifacts.md", "robosuite.md"): - (root / "docs" / name).write_text(f"# {name}\nrun examples/robosuite/cube_lifting_seed1.py\nkept\n") - (skills / "skills").mkdir() - (skills / "skills" / "task-exploration.md").write_text("# exploration\n") - evosearch = root / ".claude" / "skills" / "evosearch" - (evosearch / "skills").mkdir(parents=True) - (evosearch / "subagent-prompt.md").write_text("intro\n\n```\n## Task Assignment\n\nBENCHMARK: \n\n---\n\n" - "## EVAL SET LOCKOUT\nsearch text\n```\n") - (evosearch / "skills" / "evosearch-iteration.md").write_text("# iteration\n") - (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\nkept\n\n## Task examples\n\n| `close_microwave.py` | " - "`libero_90:33` | Close the open microwave. |\n") - for venv in (".venv-robosuite", ".venv-libero"): - harness = root / venv / "bin" / "cap-harness" - harness.parent.mkdir(parents=True) - harness.write_text(FAKE_HARNESS.format(python=sys.executable)) - harness.chmod(0o755) - return root - - -# cap-harness without a simulator: the run's files where ASPIRE writes them (run.json, results.jsonl naming the run's -# directory, outcome.json), the program and settings it was given; a program saying SLEEP runs until stopped, and its -# `# WRITE ` / `# READ ` lines are tried as a program that escaped ASPIRE's validator would (in run.json), -# and after the run `# LINK ` leaves a link in its output, `# RUNDIR ` names another run_dir and -# `# CHMOD ` changes a mode (last). -FAKE_HARNESS = """#!{python} -import argparse, json, os, pathlib, sys, time -ap = argparse.ArgumentParser() -for flag in ("--benchmark", "--suite", "--task-id", "--program", "--output-root", "--max-steps", "--seed", "--workers", - "--results-jsonl", "--log-dir", "--init-mode"): - ap.add_argument(flag) -ap.add_argument("--seeds", nargs="+") -ap.add_argument("--flat-run-dir", action="store_true") -ap.add_argument("--no-videos", action="store_true") -ap.add_argument("cmd") -a = ap.parse_args() -code = pathlib.Path(a.program).read_text() -if "SLEEP" in code: - time.sleep(60) -tried = {{}} -for line in code.splitlines(): - what, _, path = line.partition(" ")[2].partition(" ") - try: - if what == "WRITE": - pathlib.Path(path).write_text("escaped") - elif what == "READ": - pathlib.Path(path).read_text() - else: - continue - tried[path] = "done" - except OSError as exc: - tried[path] = type(exc).__name__ -out = pathlib.Path(a.output_root) -run = out / "run-1" -run.mkdir(parents=True, exist_ok=True) -(out / "run.json").write_text(json.dumps({{"program": a.program, "max_steps": a.max_steps, "code": code, - "env": sorted(os.environ), "home": os.environ.get("HOME"), - "tried": tried}})) -(run / "outcome.json").write_text(json.dumps({{"task_success": True}})) -rows = [{{"seed": int(s), "run_dir": str(run)}} for s in (a.seeds or [a.seed])] -for line in code.splitlines(): - what, _, rest = line.partition(" ")[2].partition(" ") - if what == "LINK": - name, target = rest.split() - (out / name).unlink(missing_ok=True) - (out / name).symlink_to(target) - elif what == "RUNDIR": - rows = [dict(r, run_dir=rest) for r in rows] -pathlib.Path(a.results_jsonl or out / "results.jsonl").write_text("".join(json.dumps(r) + chr(10) for r in rows)) -for line in code.splitlines(): - what, _, rest = line.partition(" ")[2].partition(" ") - if what == "CHMOD": - name, mode = rest.split() - os.chmod(out / name, int(mode, 8)) -print("ran", a.program, "into", out, "max_steps", a.max_steps) -""" - - -def _bwrap_works() -> bool: - """A bubblewrap that can make its namespaces here: every simulator run is confined (sim.confine).""" - if not shutil.which("bwrap"): - return False - return subprocess.run(["bwrap", "--unshare-user", "--unshare-net", "--ro-bind", "/", "/", "true"], - capture_output=True).returncode == 0 - - -needs_bwrap = pytest.mark.skipif(not _bwrap_works(), reason="needs a working bubblewrap: simulator runs are confined") - - -LIBERO_ROW = {"id": "libero_90-t33", "benchmark": "libero-pro", "suite": "libero_90", "task_id": 33, - "debug_seeds": "51-53", "unseen_seeds": "66", "eval_seeds": "1-3"} -ROW = {"id": "robosuite-cube_lifting", "suite": "cube_lifting", "task_id": 0, "debug_seeds": "101-103", - "unseen_seeds": "126", "eval_seeds": "1-3"} -VERIFY = {"id": "verify-robosuite-cube_lifting", "suite": "cube_lifting", "task_id": 0, "role": "verifier", - "debug_seeds": "101-105", "eval_seeds": "none"} - - def test_the_actor_prompt_is_the_header_the_assignment_and_aspires_template(tmp_path): - T = _task_module() - params = {"aspire_root": str(fake_aspire(tmp_path))} - text = T.prompt(ROW, params, tmp_path) + T = A + params = {**PARAMS, "aspire_root": str(fake_aspire(tmp_path))} + text = T.prompt(ROW, params, BASE) assert "fix-loop subagent" in text and "`$TASK_DIR` is `robosuite/cube_lifting/task_0`" in text assert "DEV_SEEDS: 101-103" in text and "UNSEEN_SEEDS: 126" in text and "HELDOUT_COUNT: 3" in text assert "## What You Are\nfix-loop text" in text and "BENCHMARK: " not in text # ours replaces theirs # Part 2, the evolutionary search, follows the fix loop with its own page and its template after its assignment assert text.index("fix-loop text") < text.index("# Part 2: the evolutionary search") < text.index("search text") assert "`TASK_DIR` is `evosearch/robosuite/cube_lifting/task_0`" in text - assert "# Part 2" not in T.prompt(ROW, {**params, "evosearch": False}, tmp_path) - verifier = T.prompt(VERIFY, params, tmp_path) + assert "# Part 2" not in T.prompt(ROW, {**params, "evosearch": False}, BASE) + verifier = T.prompt(VERIFY, params, BASE) assert "skill verifier" in verifier and "Development seeds: **101-105**" in verifier assert "You are verifying. Seeds 51-55." in verifier and "# Part 2" not in verifier + libero = A.prompt(LIBERO_ROW, params, BASE) # LIBERO's tasks on the same engine + assert "BENCHMARK: libero-pro" in libero and "`$TASK_DIR` is `libero-pro/libero_90/task_33`" in libero + assert "the benchmark (libero-pro)" in libero and "`docs/libero-pro.md`" in libero and "robosuite" not in libero + assert "`TASK_DIR` is `evosearch/libero-pro/libero_90/task_33`" in libero and "DEV_SEEDS: 51-53" in libero + with pytest.raises(ValueError, match="agent.params.engine"): + A.prompt(ROW, {"aspire_root": params["aspire_root"]}, BASE) -class _Service: - """The simulator service on a free loopback port, over a fake checkout whose cap-harness writes a run's files - without a simulator (FAKE_HARNESS).""" - - def __init__(self, tmp: Path, root: Path, **options): - import threading - from http.server import ThreadingHTTPServer - S = _load("aspire_sim_service", EXAMPLE / "agent" / "sim_service.py") - self.sessions = tmp / "sessions" - self.sessions.mkdir() - self.service = service = S.Service(root, {r["id"]: r for r in (ROW, VERIFY, LIBERO_ROW)}, [self.sessions], - S.Ledger(tmp / "ledger.json"), {}, 2, **options) - self.server = ThreadingHTTPServer(("127.0.0.1", 0), S.handler(service)) - threading.Thread(target=self.server.serve_forever, daemon=True).start() - self.url = f"http://127.0.0.1:{self.server.server_address[1]}" - - def close(self): - self.server.shutdown() - - -@pytest.fixture -def service(tmp_path, monkeypatch): - svc = _Service(tmp_path, fake_aspire(tmp_path)) - monkeypatch.setenv("ASPIRE_SIM_URL", svc.url) - yield svc - svc.close() - - -def _workspace(service, task: dict, hexa: str = "56d9fea2") -> Path: - """A session's workspace where the framework puts it: //-/workspaces/<...>/work.""" - work = service.sessions / "rollout" / f"{task['id']}-{hexa}" / "workspaces" / f"{task['id']}-da10428a" / "work" - (work / "hermit" / "aspire" / "skills").mkdir(parents=True) - return work - - -def _register(url: str, body: dict) -> dict: - import urllib.request - request = urllib.request.Request(url + "/register", data=json.dumps(body).encode()) - with urllib.request.urlopen(request) as reply: - return json.loads(reply.read()) - - -def _sim(work: Path, *args): - return subprocess.run([str(work / "bin" / "sim"), *args], capture_output=True, text=True, cwd=work) - - -def test_prepare_registers_the_session_and_the_service_holds_its_rules(tmp_path, service): - T = _task_module() +def test_aspires_workspace_is_the_engines_with_its_assignment_its_search_page_and_the_proposals(tmp_path, service): + params = {**PARAMS, "aspire_root": str(tmp_path / "ASPIRE")} work = _workspace(service, ROW) - params = {"aspire_root": str(tmp_path / "ASPIRE")} - T.prepare(ROW, work, params, tmp_path) - for name in ("task.md", "docs/api-reference.md", "docs/evosearch-iteration.md", "docs/task-exploration.md", + A.prepare(ROW, work, params, BASE) + for name in ("task.md", "docs/evosearch-iteration.md", "docs/api-reference.md", "docs/task-exploration.md", "scripts/scene_snapshot.py", "bin/sim", "bin/sim.json"): - assert (work / name).is_file(), name - assert "examples/" not in (work / "docs" / "robosuite.md").read_text() - assert "kept" in (work / "docs" / "robosuite.md").read_text() - (work / "p.py").write_text("result = {}\n") - held_out = _sim(work, "run-batch", "--seeds", "1-3", "--program", "p.py", "--output-root", "out") - assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr - outside = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", str(tmp_path / "elsewhere")) - assert outside.returncode == 2 and "inside the workspace" in outside.stderr - _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u", "--init-mode", "saved") - again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") - assert again.returncode == 2 and "run once" in again.stderr - T.prepare(ROW, work, params, tmp_path) # a retry's prepare: the registration as it was - again = _sim(work, "run", "--seed", "126", "--program", "p.py", "--output-root", "u") - assert again.returncode == 2 and "run once" in again.stderr # what the session spent stays spent - - -def test_a_libero_task_runs_on_its_own_venv_seeded_with_its_own_docs(tmp_path, service, monkeypatch): - T = _task_module() - params = {"aspire_root": str(tmp_path / "ASPIRE")} - text = T.prompt(LIBERO_ROW, params, tmp_path) - assert "BENCHMARK: libero-pro" in text and "`$TASK_DIR` is `libero-pro/libero_90/task_33`" in text - assert "the benchmark (libero-pro)" in text and "`docs/libero-pro.md`" in text and "robosuite" not in text - assert "`TASK_DIR` is `evosearch/libero-pro/libero_90/task_33`" in text and "DEV_SEEDS: 51-53" in text - work = _workspace(service, LIBERO_ROW) - T.prepare(LIBERO_ROW, work, params, tmp_path) - page = (work / "docs" / "libero-pro.md").read_text() - assert "kept" in page and "Task examples" not in page and "microwave" not in page # no worked solution named - assert not (work / "docs" / "robosuite.md").exists() - ran = [] - monkeypatch.setattr(service.service, "_execute", lambda cmd, env, *a, **k: ran.append((cmd, env)) or (0, "", "")) - (work / "p.py").write_text("result = {}\n") - _sim(work, "run-batch", "--seeds", "51-52", "--program", "p.py", "--output-root", "out", "--init-mode", "saved") - cmd, env = ran[-1] - assert cmd[0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "cap-harness") - option = lambda flag: cmd[cmd.index(flag) + 1] - assert (option("--benchmark"), option("--suite"), option("--task-id")) == ("libero-pro", "libero_90", "33") - assert option("--init-mode") == "seeded" # the engine's, whatever the session passed - assert env["LIBERO_CONFIG_PATH"] == str(tmp_path / "ASPIRE" / ".libero") - assert env["TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD"] == "1" - _sim(work, "validate", "--program", "p.py") - assert ran[-1][0][0] == str(tmp_path / "ASPIRE" / ".venv-libero" / "bin" / "python") - # Robosuite's runs are as they were: its venv, no init mode - robosuite = _workspace(service, ROW) - T.prepare(ROW, robosuite, params, tmp_path) - (robosuite / "p.py").write_text("result = {}\n") - _sim(robosuite, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - cmd, env = ran[-1] - assert cmd[0].endswith(".venv-robosuite/bin/cap-harness") and "--init-mode" not in cmd - assert "LIBERO_CONFIG_PATH" not in env - - -def test_the_task_sandbox_is_shown_the_staged_docs_and_never_a_page_that_lost_its_cut(tmp_path, service): - """ASPIRE's benchmark pages name its worked programs per task: an actor that could read one would be handed - a strategy for its task. The configs grant the staged copies only, and a page without its cut is refused.""" - T = _task_module() - root = tmp_path / "ASPIRE" - (tmp_path / "rows.jsonl").write_text(json.dumps(ROW) + "\n" + json.dumps(LIBERO_ROW) + "\n") - staged = tmp_path / "staged" - done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", str(root), - "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], - capture_output=True, text=True) - assert done.returncode == 0, done.stderr - assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", - "run-artifacts.md"] - assert all("examples/" not in p.read_text() and "microwave" not in p.read_text() for p in staged.iterdir()) - stage = lambda: subprocess.run([sys.executable, str(EXAMPLE / "agent" / "stage_docs.py"), "--aspire-root", - str(root), "--tasks", str(tmp_path / "rows.jsonl"), "--out", str(staged)], - capture_output=True, text=True) - before = {p.name: p.stat().st_ino for p in staged.iterdir()} - assert stage().returncode == 0 and {p.name: p.stat().st_ino for p in staged.iterdir()} == before # in place - (staged / "extra").mkdir() # anything else there: staged anew, without it - (staged / "notes.bin").write_bytes(b"\xff\xfe") - assert stage().returncode == 0 - assert sorted(p.name for p in staged.iterdir()) == ["api-reference.md", "libero-pro.md", "robosuite.md", - "run-artifacts.md"] - (staged / "libero-pro.md").write_text("# the staged page\n") - work = _workspace(service, LIBERO_ROW) - T.prepare(LIBERO_ROW, work, {"aspire_root": str(root), "aspire_docs": str(staged)}, tmp_path) - assert (work / "docs" / "libero-pro.md").read_text() == "# the staged page\n" # the staged copy, as it is - (root / "docs" / "libero-pro.md").write_text("# LIBERO-Pro\n\n## Worked programs\n| `close_microwave.py` |\n") - with pytest.raises(ValueError, match="Task examples"): - _sim_module().workspace_docs(root, "libero-pro") - # No Codex config of either example grants a page of the checkout's own docs, and each loads - from hermit.config import Config - configs = [p for folder in (EXAMPLE.parent / "aspire", LIBERO / "aspire", LIBERO / "prompt_only") - for p in sorted(folder.glob("*.yaml"))] - assert len(configs) == 18 - for path in configs: - data = Config.load(path).data - read = [str(p) for p in (data.get("environment") or {}).get("read") or []] - assert not any("/docs/" in p for p in read), path - if "_direct" not in path.name: - assert "$ASPIRE_DOCS" in read and data["agent"]["params"]["aspire_docs"] == "$ASPIRE_DOCS", path - # Codex reads AGENTS.md (the adapter's context) only up to this: past it every session is refused - assert data["agent"]["params"]["project_doc_max_bytes"] >= 65536, path - if path.parent.name == "prompt_only": # no search: none staged, granted or scored - assert data["agent"]["params"]["evosearch"] is False and not any("evosearch" in p for p in read), path - if path.is_relative_to(EXAMPLE.parent): - for key in ("train", "val", "test"): # the splits prepare.py writes - name = Path(data["data"][key]).name - assert name in {"all.txt", "rerun.txt", "verify.txt", "smoke.txt", "smoke_rerun.txt"}, (path, name) - - -def test_the_engine_takes_the_benchmarks_venv_and_refuses_one_it_does_not_run(tmp_path, monkeypatch): - sim = _sim_module() - root = fake_aspire(tmp_path) - engine = sim.Engine(root, benchmark="libero-pro") - assert engine.cli == root.resolve() / ".venv-libero" / "bin" / "cap-harness" - assert engine._env()["LIBERO_CONFIG_PATH"] == str(root.resolve() / ".libero") - assert sim.init_args("libero-pro") == ["--init-mode", "seeded"] and sim.init_args(None) == [] - assert sim.Engine(root).cli.parent.parent.name == ".venv-robosuite" - commands = [] # the command line of a sweep, the judge's Stage 2 included - - def recorded(cmd, **_): - commands.append(cmd) - raise RuntimeError("recorded") - - monkeypatch.setattr(sim, "confine", recorded) - for each in (engine, sim.Engine(root)): - with pytest.raises(RuntimeError, match="recorded"): - each.run(tmp_path / "p.py", suite="s", seeds=[1, 2], out=tmp_path / "o") - at = commands[0].index("--init-mode") - assert commands[0][at:at + 2] == ["--init-mode", "seeded"] and "--init-mode" not in commands[1] - shutil.rmtree(root / ".venv-libero") - with pytest.raises(FileNotFoundError, match="bootstrap_libero.sh"): - sim.Engine(root, benchmark="libero-pro") - with pytest.raises(ValueError, match="behavior"): - sim.Engine(root, benchmark="behavior") - - -def test_the_libero_tasks_are_liberos_five_suites_and_liberos_pros_eighty_with_aspires_seeds(tmp_path, monkeypatch): - """The tasks and splits prepare.py writes, read as their rows (aspire_fields.py): what the simulator service, the - judge and every method's configs read.""" - from hermit.config import Config - from hermit.task import Benchmark - P = _load("libero_prepare", LIBERO / "utils" / "benchmark" / "prepare.py") - F = _load("aspire_fields", EXAMPLE / "agent" / "aspire_fields.py") - assert sum(P.SUITES.values()) == 130 and P.SUITES["libero_90"] == 90 - pro = [f"libero_{family}_{kind}" for family in ("object", "spatial", "goal", "10") for kind in ("swap", "task")] - assert list(P.PRO_SUITES) == pro and set(P.PRO_SUITES.values()) == {10} - monkeypatch.setattr(P, "HERE", tmp_path) - monkeypatch.setattr(sys, "argv", ["prepare.py"]) - P.main() - found = F.task_rows(tmp_path / "tasks") # each task directory as its row - rows = list(found.values()) - tasks = [found[i] for i in (tmp_path / "splits" / "all.txt").read_text().split()] - assert len(tasks) == 210 and len(rows) == 630 and len({r["id"] for r in rows}) == 630 # a suite and its - assert all(r["benchmark"] == "libero-pro" for r in rows) # perturbations share their task ids - assert list(found["libero_goal-t03"]) == ["id", "benchmark", "suite", "task_id", "debug_seeds", "unseen_seeds", - "eval_seeds"] # the row's keys, in its order - assert found["verify-libero_goal-t03"]["findings"] is False and found["rerun-libero_goal-t03"]["role"] == "rerun" - assert (tmp_path / "tasks" / "libero_goal-t03" / "instruction.md").is_file() - assert all((r["debug_seeds"], r["unseen_seeds"], r["eval_seeds"]) == ("51-65", "66-80", "1-50") for r in tasks) - split = lambda name: (tmp_path / "splits" / name).read_text().split() - by = lambda *suites: [r["id"] for r in tasks if r["suite"] in suites] - assert split("all.txt") == [r["id"] for r in tasks] - assert split("libero.txt") == by("libero_object", "libero_spatial", "libero_goal", "libero_10") - assert split("libero_90.txt") == by("libero_90") and split("libero_pro.txt") == by(*pro) - assert split("libero_pro_long.txt") == by("libero_10_swap", "libero_10_task") - assert split("smoke.txt") == split("libero.txt")[:2] and split("smoke_pro.txt") == split("libero_pro.txt")[:2] - assert split("libero_sample.txt") == [f"{s}-t{n:02d}" for s in ("libero_object", "libero_spatial", "libero_goal") - for n in (0, 5)] - for path in sorted((LIBERO / "aspire").glob("*.yaml")) + sorted((LIBERO / "prompt_only").glob("*.yaml")): - config = Config.load(path) - data = config.data["data"] - assert config.path(data["tasks"]) == (LIBERO / "utils" / "benchmark" / "tasks").resolve(), path - assert config.path(data["view"].split(":")[0]) == (EXAMPLE / "agent" / "aspire_fields.py").resolve(), path - monkeypatch.setitem(data, "tasks", str(tmp_path / "tasks")) # the tasks prepared here - for key in ("train", "val", "test"): - if data.get(key): - monkeypatch.setitem(data, key, str(tmp_path / "splits" / Path(data[key]).name)) - assert {t.id: t.given() for t in Benchmark.from_config(config).tasks.values()} == found, path - train, test = split(Path(data["train"]).name), split(Path(data["test"]).name) - assert train and set(train) <= {r["id"] for r in tasks}, path - assert test == [f"rerun-{i}" for i in train], path # the reruns are of the tasks it trained on - if data.get("val"): # the coordinator's verifiers (ASPIRE) - assert set(split(Path(data["val"]).name)) == {f"verify-{r['id']}" for r in tasks}, path - seeds, verify = P.seeds_from(P.PRESETS, []) - assert seeds == {"debug_seeds": "51-65", "unseen_seeds": "66-80", "eval_seeds": "1-50"} and verify == "51-55" - assert P.seeds_from(P.PRESETS, ["--preset", "smoke", "--eval", "1-5"])[0]["eval_seeds"] == "1-5" - with pytest.raises(SystemExit): - P.seeds_from(P.PRESETS, ["--eval", "1-60"]) # held-out seeds a session could run - - -def test_the_neutral_libero_prompt_names_the_task_the_seeds_and_the_scored_program_and_nothing_of_a_method( - tmp_path, service): - """prompt_only/'s acting prompt and workspace: the engine's, without ASPIRE's fix-loop, library or search.""" - P = _load("libero_task", LIBERO / "utils" / "agent" / "task.py") - T = _task_module() - root = tmp_path / "ASPIRE" # the service fixture's ASPIRE tree - params = {"aspire_root": str(root), "evosearch": False} # as prompt_only's configs set them - row = {**LIBERO_ROW, "suite": "libero_object_task", "task_id": 4, "id": "libero_object_task-t04"} - text = P.prompt(row, params, tmp_path) - assert "Suite `libero_object_task`, task id `4`" in text and "get_task_context()" in text - assert "**51-53**" in text and "**66**" in text and "3 held-out seeds" in text - scored = f"{T.task_dir(T.settings(row, params))}/fix_code.py" - assert f"Your program is `{scored}`" in text # the file the engine's collect scores - unseen = f"--output-root {scored.rsplit('/', 1)[0]}/debug/holdout" # where the engine's collect reads it - assert unseen in text and "docs/libero-pro.md" in text and "simulator steps" not in text - assert "at most 600 simulator steps" in P.prompt(row, {**params, "max_steps": 600}, tmp_path) - assert not any(word in text for word in ("skill", "library", "coordinator", "fix-loop", "evolutionary")) - with pytest.raises(ValueError, match="verifier"): - P.prompt({**row, "role": "verifier", "id": "verify-x"}, params, tmp_path) - for call in (lambda: P.prompt(row, {"aspire_root": str(root)}, tmp_path), # without evosearch: false the - lambda: P.prepare(LIBERO_ROW, tmp_path / "w", {"aspire_root": str(root)}, tmp_path)): # engine - with pytest.raises(ValueError, match="evosearch must be false"): # would score a search's - call() # program - work = _workspace(service, LIBERO_ROW) - P.prepare(LIBERO_ROW, work, params, tmp_path) - assert (work / "bin" / "sim").is_file() and (work / "docs" / "libero-pro.md").is_file() - assert (work / "docs" / "task-exploration.md").is_file() # the scene procedure stays - assert not (work / "docs" / "evosearch-iteration.md").exists() and not (work / "task.md").exists() - # The scored program is fix_code.py even where a search's best is left; ASPIRE's own configs score the search's - s = T.settings(LIBERO_ROW, params) - (work / T.task_dir(s)).mkdir(parents=True, exist_ok=True) - (work / T.task_dir(s) / "fix_code.py").write_text("# fix\n") - (work / T.evo_dir(s)).mkdir(parents=True) - (work / T.evo_dir(s) / "evosearch_best_code.py").write_text("# best\n") - neutral = T.collect(LIBERO_ROW, work, tmp_path / "log-a", {**params, "prune_media": False}) - assert neutral["scored_program"].endswith("/fix_code.py") and neutral["fix_code"] == "# fix\n" - assert "evosearch" not in neutral - aspire = T.collect(LIBERO_ROW, work, tmp_path / "log-b", {"aspire_root": str(root), "prune_media": False}) - assert aspire["scored_program"].endswith("/evosearch_best_code.py") and aspire["fix_code"] == "# best\n" - - -def test_a_session_cannot_forge_a_registration_for_more_seeds(tmp_path, service): - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - forged = work / "x" # a fresh directory the session can write - forged.mkdir() - (forged / "p.py").write_text("result = {}\n") - answer = _register(service.url, {"workspace": str(forged), "task": ROW["id"], "dev_seeds": [1, 2, 3]}) - assert not answer["ok"] and "registered its session already" in answer["error"] - other = _workspace(service, ROW, "0badc0de") # another rollout (the framework's): the row's seeds, - assert _register(service.url, {"workspace": str(other), "task": ROW["id"], "dev_seeds": [1, 2, 3], - "max_programs": None})["ok"] # whatever the request says - (other / "bin").mkdir() - shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", other / "bin" / "sim_cli.py") - (other / "bin" / "sim.json").write_text(json.dumps({"workspace": str(other), "url": service.url})) - (other / "p.py").write_text("result = {}\n") - held_out = subprocess.run([sys.executable, str(other / "bin" / "sim_cli.py"), "run-batch", "--seeds", "1-3", - "--program", "p.py", "--output-root", "out"], capture_output=True, text=True, cwd=other) - assert held_out.returncode == 2 and "not assigned to you" in held_out.stderr - mismatched = _workspace(service, VERIFY, "12345678") # a rollout registers only the task its name carries - answer = _register(service.url, {"workspace": str(mismatched), "task": ROW["id"]}) - assert not answer["ok"] and "is not one of task" in answer["error"] - - -def test_a_verifier_may_run_two_programs_and_sees_the_staged_proposals(tmp_path, service): - T = _task_module() - work = _workspace(service, VERIFY) - (work / "hermit" / "aspire" / "skills" / "proposed.md").write_text(ENTRY) - T.prepare(VERIFY, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - assert (work / "proposed_skills.md").read_text() == ENTRY + assert (work / name).is_file(), name # ASPIRE's pages beside the engine's workspace + assert (work / "task.md").read_text() == A.assignment(A.settings(ROW, params, BASE), A.engine(params, BASE)) + plain = _workspace(service, ROW, "0badc0de") + A.prepare(ROW, plain, {**params, "evosearch": False}, BASE) + assert not (plain / "docs" / "evosearch-iteration.md").exists() + # The verifier sees the proposals the coordinator staged in its copy of the library, and runs two programs + verifier = _workspace(service, VERIFY) + (verifier / "hermit" / "aspire" / "skills").mkdir(parents=True) + (verifier / "hermit" / "aspire" / "skills" / "proposed.md").write_text(ENTRY) + A.prepare(VERIFY, verifier, params, BASE) + assert (verifier / "proposed_skills.md").read_text() == ENTRY codes = [] for name in ("a.py", "a.py", "c.py"): # the same path, rewritten: a new program - (work / name).write_text(f"result = {{'v': {len(codes)}}}\n") - codes.append(_sim(work, "run-batch", "--seeds", "101-105", "--program", name, "--output-root", "out")) + (verifier / name).write_text(f"result = {{'v': {len(codes)}}}\n") + codes.append(_sim(verifier, "run-batch", "--seeds", "101-105", "--program", name, "--output-root", "out")) assert "attempt budget" not in codes[0].stderr and "attempt budget" not in codes[1].stderr assert codes[2].returncode == 2 and "attempt budget is 2 programs" in codes[2].stderr -@needs_bwrap -def test_the_service_follows_no_link_the_session_puts_in_its_workspace(tmp_path, service): - """A run is written by the service into its own directory and moved into the workspace: a link the session put - where the run's files go is replaced, not followed, and a program that is a link is not read.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - victim = tmp_path / "victim.txt" - victim.write_text("the host's own file\n") - (work / "out").mkdir() - (work / "out" / "run.json").symlink_to(victim) # where cap-harness writes - (work / "out" / "run-1").symlink_to(tmp_path) # a directory of the run's - (work / "p.py").write_text("result = {}\n") - ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - assert victim.read_text() == "the host's own file\n" and not (tmp_path / "outcome.json").exists() - assert ran.returncode == 2 and "a link in its path" in ran.stderr - (work / "out" / "run-1").unlink() - ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - assert ran.returncode == 0 and victim.read_text() == "the host's own file\n" - record = json.loads((work / "out" / "run.json").read_text()) - assert not (work / "out" / "run.json").is_symlink() and record["program"] == str(work / "p.py") - rows = (work / "out" / "results.jsonl").read_text() - assert json.loads(rows)["run_dir"] == str(work / "out" / "run-1") # the workspace's paths, not the service's - assert str(work / "p.py") in ran.stdout and str(service.service.scratch) not in ran.stdout - secret = tmp_path / "secret.py" - secret.write_text("KEY = 'host'\n") - (work / "q.py").symlink_to(secret) - refused = _sim(work, "validate", "--program", "q.py") - assert refused.returncode == 2 and "no program at q.py" in refused.stderr - assert list(service.service.scratch.iterdir()) == [] # the service's copies are gone - import urllib.request - service.service.nonce = "n-1" - with urllib.request.urlopen(service.url + "/health") as reply: # launch.sh knows its own service by it - assert json.loads(reply.read())["nonce"] == "n-1" - service.service.stop() - late = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - assert late.returncode == 1 and "stopping" in late.stderr # no run starts once the service stops - - -def test_bin_sim_quotes_the_interpreter(tmp_path, service, monkeypatch): - T = _task_module() - work = _workspace(service, ROW) - monkeypatch.setattr(T.sys, "executable", "/opt/hermit env/bin/python") - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - assert "exec '/opt/hermit env/bin/python' " in (work / "bin" / "sim").read_text() - - -@needs_bwrap -def test_a_program_that_escaped_the_validator_still_cannot_reach_the_host(tmp_path, service): - """cap-harness runs in a sandbox of its own (sim.confine): a program in it writes only its run's own output and - reads ASPIRE's checkout, not the host's files, another session's workspace or the judge's runs.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - secret, stage2 = tmp_path / "secret.txt", tmp_path / "out" / "stage2" / "robosuite-x" / "stage2.json" - secret.write_text("host secret\n") - stage2.parent.mkdir(parents=True) - stage2.write_text("{}") - other = _workspace(service, ROW, "0badc0de") - (other / "fix_code.py").write_text("result = {'theirs': 1}\n") - solution = tmp_path / "ASPIRE" / "examples" / "robosuite" / "cube_lifting_seed1.py" # a worked solution - solution.parent.mkdir(parents=True) - solution.write_text("result = {'worked': 1}\n") - history = tmp_path / "ASPIRE" / ".git" / "HEAD" # the git history holds it too - history.parent.mkdir() - history.write_text("ref: refs/heads/main\n") - native = tmp_path / "ASPIRE" / "outputs" / "run-1" / "source" / "program.py" # a run of ASPIRE's own CLI - native.parent.mkdir(parents=True) - native.write_text("result = {'native': 1}\n") - page = tmp_path / "ASPIRE" / "docs" / "libero-pro.md" # names the worked programs per task - demos = tmp_path / "ASPIRE" / "third_party" / "LIBERO-PRO" / "libero" / "datasets" / "demo.hdf5" - demos.parent.mkdir(parents=True) - demos.write_text("demonstration\n") - harness = tmp_path / "ASPIRE" / ".venv-robosuite" / "bin" / "cap-harness" - targets = {"WRITE": [tmp_path / "planted.txt", tmp_path / "ASPIRE" / "planted.py", work / "planted.txt"], - "READ": [secret, stage2, other / "fix_code.py", solution, history, native, page, demos, harness]} - (work / "p.py").write_text("result = {}\n" + "".join(f"# {w} {p}\n" for w, ps in targets.items() for p in ps)) - ran = _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "out") - tried = json.loads((work / "out" / "run.json").read_text())["tried"] - assert ran.returncode == 0, ran.stderr - assert not (tmp_path / "planted.txt").exists() and not (tmp_path / "ASPIRE" / "planted.py").exists() - assert tried[str(work / "planted.txt")] != "done" and not (work / "planted.txt").exists() # its workspace too - assert all(tried[str(p)] != "done" - for p in (secret, stage2, other / "fix_code.py", solution, history, native, page, demos)) - assert tried[str(harness)] == "done" # the checkout it runs from - - -@needs_bwrap -def test_the_validator_runs_on_aspires_own_venv_inside_its_sandbox(tmp_path, monkeypatch): - """ASPIRE's venv interpreter is a link to its base installation: confined, it keeps the venv (whose cap_harness - the validator imports), for the service's `bin/sim validate` and the judge's Engine alike.""" - root = fake_aspire(tmp_path) - venv = root / ".venv-robosuite" - shutil.move(str(venv / "bin" / "cap-harness"), str(tmp_path / "cap-harness")) - shutil.rmtree(venv) - subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(venv)], check=True) - shutil.move(str(tmp_path / "cap-harness"), str(venv / "bin" / "cap-harness")) - site = next(venv.glob("lib/python3*/site-packages")) - (site / "cap_harness").mkdir() - (site / "cap_harness" / "__init__.py").write_text("") - (site / "cap_harness" / "runtime.py").write_text( - "import ast\nclass _ProgramValidator(ast.NodeVisitor):\n def visit_Import(self, node):\n" - " if any(a.name != 'math' for a in node.names):\n raise ValueError('only math')\n") - assert (venv / "bin" / "python").is_symlink() - S = _load("aspire_sim_service_venv", EXAMPLE / "agent" / "sim_service.py") - service = S.Service(root, {ROW["id"]: ROW}, [tmp_path / "sessions"], S.Ledger(tmp_path / "ledger.json"), {}, 2) - work = tmp_path / "sessions" / "rollout" / f"{ROW['id']}-1a2b" / "workspaces" / "w" / "work" - work.mkdir(parents=True) - assert service.register({"workspace": str(work), "task": ROW["id"]})["ok"] - (work / "good.py").write_text("import math\nresult = {}\n") - (work / "bad.py").write_text("import os\nresult = {}\n") - good = service.run({"workspace": str(work), "cmd": "validate", "program": "good.py"}) - bad = service.run({"workspace": str(work), "cmd": "validate", "program": "bad.py"}) - assert good["code"] == 0 and "program accepted" in good["stdout"], good - assert bad["code"] != 0 and "only math" in bad["stderr"], bad - sim = _sim_module() - engine = sim.Engine(root) - assert engine.validate(work / "good.py") is None and "only math" in engine.validate(work / "bad.py") - - -def _sim_module(): - sys.path.insert(0, str(EXAMPLE / "agent")) - try: - import sim - return sim - finally: - sys.path.remove(str(EXAMPLE / "agent")) - - -@needs_bwrap -def test_the_engine_reads_no_link_and_no_run_dir_outside_its_run(tmp_path): - """The judge's and the direct actor's Engine read what a confined program could write: once it has ended, links - in its output are removed and a run_dir outside the run is not read, so neither a host file nor another run's - success reaches the records.""" - sim = _sim_module() - root = fake_aspire(tmp_path) - elsewhere = tmp_path / "another-run" / "seed_0001" - elsewhere.mkdir(parents=True) - (elsewhere / "outcome.json").write_text('{"task_success": true, "program_ok": true}') - secret = tmp_path / "secret.txt" - secret.write_text("host secret") - out = tmp_path / "stage2" / "runs" - program = tmp_path / "stage2" / "program.py" - program.parent.mkdir() - program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# RUNDIR {elsewhere}\n") - records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1, 2], out=out) - assert [r["task_success"] for r in records] == [False, False] and "not in this run's output" in records[0]["error"] - assert secret.read_text() == "host secret" and not (out / "records.json").is_symlink() - program.write_text(f"result = {{}}\n# FAIL\n# LINK run-1/outcome.json {elsewhere / 'outcome.json'}\n") - records = sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "again") - assert records[0]["task_success"] is False and not (tmp_path / "again" / "run-1" / "outcome.json").exists() - # a directory made unlistable keeps no link from being removed: it is made listable, or the run fails - program.write_text(f"result = {{}}\n# FAIL\n# LINK records.json {secret}\n# CHMOD . 0o111\n") - sim.Engine(root, gpu=[]).run(program, suite="cube_lifting", seeds=[1], out=tmp_path / "locked") - assert secret.read_text() == "host secret" and not (tmp_path / "locked" / "records.json").is_symlink() - - -def _fake_gpu(tmp_path, monkeypatch) -> list[int]: - """GPU 8 as hermit's sandbox sees devices (sandbox.DEV): a machine without one can still build the command.""" - import hermit.sandbox as sandbox_module - dev = tmp_path / "dev" - dev.mkdir(exist_ok=True) - for name in ("nvidiactl", "nvidia8"): - (dev / name).touch() - monkeypatch.setattr(sandbox_module, "DEV", str(dev)) - return [8] - - -@needs_bwrap -def test_the_simulators_sandbox_shows_the_renderer_and_reaches_remote_services_by_proxy(tmp_path, monkeypatch): - """An EGL vendor library extracted by hand (its ICD and its libraries, file by file, its library found as GLVND - finds it) and ASPIRE_SIM_READ are shown; without a GPU an NVIDIA-only ICD selection is dropped; a service on - another host is reached through an allowlist proxy for that host only.""" - sim = _sim_module() - home = tmp_path / "home" # the ICD and the libraries extracted where other files are too - (home / "lib").mkdir(parents=True) - (home / "credentials").write_text("key") - icd = home / "10_nvidia.json" - icd.write_text(json.dumps({"ICD": {"library_path": str(home / "lib" / "libEGL_nvidia.so.0")}})) - for name in ("libEGL_nvidia.so.0", "libnvidia-eglcore.so.580", "notes.txt"): - (home / "lib" / name).write_text("") - extra = tmp_path / "extra" - extra.mkdir() - env = {"PATH": os.environ["PATH"], "__EGL_VENDOR_LIBRARY_FILENAMES": str(icd), - "ASPIRE_SIM_READ": str(extra), "CAP_HARNESS_SAM3_URL": "http://provider.internal:9000"} - held, args, inside = sim.confine(["true"], root=fake_aspire(tmp_path), env=env, - gpu=_fake_gpu(tmp_path, monkeypatch)) - try: - ro = set(args[i + 1] for i, a in enumerate(args) if a == "--ro-bind") - assert {str(icd), str(home / "lib" / "libEGL_nvidia.so.0"), str(home / "lib" / "libnvidia-eglcore.so.580"), - str(extra)} <= ro # the driver's files, file by file, and the explicit grant - assert not ro & {str(home), str(home / "lib"), str(home / "credentials"), str(home / "lib" / "notes.txt")} - assert inside["__EGL_VENDOR_LIBRARY_FILENAMES"] == str(icd) # with a GPU (below: gpu=[8]) it stays - assert inside["HTTPS_PROXY"].startswith("http://127.0.0.1:") and held.held[0].allow == ["provider.internal"] - finally: - held.close() - # an ICD naming its library bare: GLVND finds it on LD_LIBRARY_PATH, and so is it shown - icd.write_text(json.dumps({"ICD": {"library_path": "libEGL_nvidia.so.0"}})) - held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", gpu=_fake_gpu(tmp_path, monkeypatch), - env={**env, "LD_LIBRARY_PATH": str(home / "lib")}) - held.close() - assert str(home / "lib" / "libEGL_nvidia.so.0") in [args[i + 1] for i, a in enumerate(args) if a == "--ro-bind"] - # without a GPU an NVIDIA-only ICD selection is dropped, so EGL finds Mesa's software renderer - held, args, inside = sim.confine(["true"], root=tmp_path / "ASPIRE", env=env) - held.close() - assert "__EGL_VENDOR_LIBRARY_FILENAMES" not in inside - - -@needs_bwrap -def test_the_service_lets_a_run_write_only_its_output_and_delivers_no_replaced_output(tmp_path, service, monkeypatch): - """The run's own directory (its program) is read-only to it and only its output writable; an output directory - the program replaced (by a link to a host directory) is not delivered.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - (work / "p.py").write_text("result = {}\n") - sim = type(service.service).run.__globals__["sim"] - grants, real = [], sim.confine - monkeypatch.setattr(sim, "confine", lambda cmd, **k: grants.append(k) or real(cmd, **k)) - assert _sim(work, "run", "--seed", "101", "--program", "p.py", "--output-root", "o").returncode == 0 - own = Path(grants[-1]["readable"][0]) - assert grants[-1]["writable"] == [own / "out"] - host = tmp_path / "host-dir" - host.mkdir() - (host / "id_rsa").write_text("key") - - def replaced(cmd, env, timeout, own, out=None, gpu=()): - shutil.rmtree(out) - out.symlink_to(host) - return 0, "", "" - monkeypatch.setattr(service.service, "_execute", replaced) - ran = _sim(work, "run", "--seed", "102", "--program", "p.py", "--output-root", "o2") - assert ran.returncode == 2 and "replaced while it ran" in ran.stderr - assert (host / "id_rsa").read_text() == "key" and not (work / "o2").exists() - - -@needs_bwrap -def test_a_program_named_pyvenv_cfg_grants_nothing(tmp_path, service): - """A session names its program: one called pyvenv.cfg that says `home = ` is the run's own file, not a venv - of the sandbox's, so stays out of the simulator's sandbox.""" - T = _task_module() - work = _workspace(service, ROW) - T.prepare(ROW, work, {"aspire_root": str(tmp_path / "ASPIRE")}, tmp_path) - operator = tmp_path / "operator" - operator.mkdir() - (operator / "credentials").write_text("key") - (work / "pyvenv.cfg").write_text(f'marker = """\nhome = {operator}\n"""\nresult = {{}}\n' - f"# READ {operator / 'credentials'}\n") - ran = _sim(work, "run", "--seed", "101", "--program", "pyvenv.cfg", "--output-root", "out") - assert ran.returncode == 0, ran.stderr - assert json.loads((work / "out" / "run.json").read_text())["tried"][str(operator / "credentials")] != "done" - - -def test_the_service_refuses_to_start_on_a_gpu_the_machine_lacks(tmp_path): - """A wrong ASPIRE_SIM_GPU stops the service at its start, not each session's run after its budget is spent.""" - (tmp_path / "tasks.jsonl").write_text(json.dumps(ROW) + "\n") - done = subprocess.run([sys.executable, str(EXAMPLE / "agent" / "sim_service.py"), "--aspire-root", - str(fake_aspire(tmp_path)), "--tasks", str(tmp_path / "tasks.jsonl"), "--workspaces", - str(tmp_path / "w"), "--state", str(tmp_path / "state"), "--gpu", "999", "--port", "0"], - capture_output=True, text=True, timeout=60, env={**os.environ, "PYTHONPATH": str(ROOT)}) - assert done.returncode != 0 and ("/dev/nvidia999" in done.stderr or "bubblewrap" in done.stderr) - - -@needs_bwrap -def test_the_simulators_sandbox_reaches_only_the_services_ports(tmp_path): - """The judge runs sim.Engine in hermit's process, whose environment names its own loopback endpoints: only the - services' ports (ASPIRE's, and a CAP_HARNESS_*_URL's) are forwarded into cap-harness's sandbox.""" - sim = _sim_module() +def test_aspires_facts_score_the_searchs_best_and_a_verifiers_program_only_when_verified(tmp_path): root = fake_aspire(tmp_path) - box, args, _ = sim.confine(["true"], root=root, env={ - "PATH": os.environ["PATH"], "HA_TOOL_URL": "http://127.0.0.1:4999/mcp", - "ASPIRE_SIM_URL": "http://127.0.0.1:8120", "CAP_HARNESS_SAM3_URL": "http://127.0.0.1:9114", - "CAP_HARNESS_CUROBO_URL": "http://localhost"}) # no port: its scheme's - box.close() - forwarded = sorted(int(args[i + 1].split("=")[0]) for i, a in enumerate(args) if a == "--forward") - assert forwarded == [80, 8114, 8115, 8116, 8118, 9114] - for where in ({"hidden": [root / "runs"]}, {"writable": [root / "scratch" / "run"]}): # runs stay outside it - with pytest.raises(ValueError, match="inside ASPIRE's checkout"): - sim.confine(["true"], root=root, env={"PATH": os.environ["PATH"]}, **where) - - -@needs_bwrap -def test_a_run_takes_the_registrations_max_steps_and_stops_at_the_timeout(tmp_path, monkeypatch): - monkeypatch.setenv("TEST_PROVIDER_API_KEY", "k") # the host's secrets: not for a generated program - monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "8") # the host's numbering: the sandbox's GPUs count from 0 - svc = _Service(tmp_path, fake_aspire(tmp_path), run_timeout=2) - try: - work = _workspace(svc, ROW) - assert _register(svc.url, {"workspace": str(work), "task": ROW["id"], "max_steps": 300})["ok"] - (work / "bin").mkdir() - shutil.copy2(EXAMPLE / "agent" / "sim_cli.py", work / "bin" / "sim_cli.py") - (work / "bin" / "sim.json").write_text(json.dumps({"workspace": str(work), "url": svc.url})) - sim = lambda *args: subprocess.run([sys.executable, str(work / "bin" / "sim_cli.py"), *args], - capture_output=True, text=True, cwd=work) - (work / "p.py").write_text("result = {}\n") - ran = sim("run", "--seed", "101", "--program", "p.py", "--output-root", "o", "--max-steps", "5000") - record = json.loads((work / "o" / "run.json").read_text()) - assert ran.returncode == 0 and record["max_steps"] == "300" - assert "TEST_PROVIDER_API_KEY" not in record["env"] and "PATH" in record["env"] - assert "CUDA_VISIBLE_DEVICES" not in record["env"] - assert not Path(record["home"]).is_relative_to(work) and not Path(record["home"]).exists() # its own, gone - assert list((tmp_path / "sandbox").glob("cap-harness-*.json")) # the run's sandbox record - (work / "slow.py").write_text("SLEEP\n") - started = time.time() - slow = sim("run", "--seed", "102", "--program", "slow.py", "--output-root", "s") - assert slow.returncode == 124 and "stopped after 2 s" in slow.stderr and time.time() - started < 30 - finally: - svc.close() - - -def test_the_judge_runs_stage_two_on_the_rows_seeds_and_a_verifier_only_when_verified(tmp_path, monkeypatch): - J = _load("aspire_judge", EXAMPLE / "benchmark" / "judge.py") - runs, engines = [], [] + params = {**PARAMS, "aspire_root": str(root), "prune_media": False} + eng = A.engine(params, BASE) + work = tmp_path / "work" + s = A.settings(ROW, params, BASE) + (work / A.task_dir(s, eng)).mkdir(parents=True) + (work / A.task_dir(s, eng) / "fix_code.py").write_text("# fix\n") + (work / A.task_dir(s, eng) / "skill_report.json").write_text('{"graded": []}') + (work / A.evo_dir(s)).mkdir(parents=True) + (work / A.evo_dir(s) / "evosearch_best_code.py").write_text("# best\n") + facts = A.collect(ROW, work, tmp_path / "log-a", params, BASE) + assert facts["scored_program"].endswith("/evosearch_best_code.py") and facts["fix_code"] == "# best\n" + assert facts["role"] == "actor" and facts["skill_report"] == {"graded": []} and facts["evosearch"]["best_code"] + assert (tmp_path / "log-a" / "files" / "skill_report.json").is_file() + plain = A.collect(ROW, work, tmp_path / "log-b", {**params, "evosearch": False}, BASE) + assert plain["fix_code"] == "# fix\n" and "evosearch" not in plain + # A verifier's program counts only with its own verdict `verified`; without it nothing is scored + vwork = tmp_path / "verify" + vwork.mkdir() + (vwork / "fix_code.py").write_text("# v\n") + (vwork / "verification_report.json").write_text('{"verdict": "refuted"}') + refuted = A.collect(VERIFY, vwork, tmp_path / "log-c", params, BASE) + assert refuted["fix_code"] is None and refuted["not_scored"] == "verifier verdict refuted" + (vwork / "verification_report.json").write_text('{"verdict": "verified"}') + verified = A.collect(VERIFY, vwork, tmp_path / "log-d", params, BASE) + assert verified["fix_code"] == "# v\n" and "not_scored" not in verified and verified["role"] == "verifier" + + +def test_the_direct_verifiers_program_counts_only_when_verified(tmp_path, monkeypatch): + sim = D.engine_module(ENGINE / "agent" / "sim.py") class Engine: - def __init__(self, *a, **k): - engines.append(k) - - def validate(self, program): - assert program.is_absolute() and program.is_file() # cap-harness runs in ASPIRE's checkout - return "ValueError: only math" if "import os" in program.read_text() else None - - def run(self, program, *, suite, seeds, out, task_id=0): - runs.append(seeds) - return [{"task_success": s % 2 == 1} for s in seeds] - monkeypatch.setattr(J.sim, "Engine", Engine) - monkeypatch.setenv("ASPIRE_ROOT", str(tmp_path)) - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ASPIRE_OUT", "out") # relative, as a user may give it - out = lambda facts: SimpleNamespace(extra={"task": facts}, output=None) - actor = {"role": "actor", "fix_code": "result = {}\n", "unseen": {"passed": 1, "total": 1}} - assert J.score(ROW, out(actor))["score"] == 2 / 3 and runs[-1] == [1, 2, 3] # the row's held-out seeds - assert J.score(ROW, out({**actor, "fix_code": "import os\n"}))["score"] == 0.0 - assert J.score(ROW, out({**actor, "max_steps": 300}))["score"] == 2 / 3 and engines[-1]["max_steps"] == 300 - assert engines[-1]["benchmark"] == "robosuite" - assert J.score(LIBERO_ROW, out(actor))["score"] == 2 / 3 and engines[-1]["benchmark"] == "libero-pro" - work = str(tmp_path / "w") - verified = {"role": "verifier", "fix_code": "result = {}\n", "verification": {"verdict": "verified"}, - "workspace": work} - ledger = tmp_path / "out" / "sim-service" / "ledger.json" - ledger.parent.mkdir(parents=True) - ledger.write_text(json.dumps({work: {"programs": [hashlib.sha256(b"result = {}\n").hexdigest()]}})) - assert J.score(VERIFY, out(verified))["score"] == 3 / 5 and runs[-1] == [101, 102, 103, 104, 105] - assert J.score(VERIFY, out({**verified, "verification": {"verdict": "void"}}))["score"] == 0.0 - third = J.score(VERIFY, out({**verified, "fix_code": "result = {'v': 3}\n"})) # beyond its two programs - assert third["score"] == 0.0 and "attempt budget" in third["feedback"] - direct = {k: v for k, v in verified.items() if k != "workspace"} # the direct actor's: it runs its own versions - assert J.score(VERIFY, out({**direct, "fix_code": "result = {'v': 3}\n"}))["score"] == 3 / 5 - assert J.score(ROW, out({"role": "actor"}))["score"] == 0.0 # no program: not scored - - -def test_the_robosuite_tasks_are_task_directories_read_back_as_the_rows_they_were(tmp_path, monkeypatch): - """prepare.py writes each task, verifier and rerun as a task directory (Harbor's format, no network, judged on the - host); aspire_fields.py, every config's data.view, reads each back as exactly the row tasks.jsonl held.""" - from hermit.config import Config - from hermit.task import Benchmark - P = _load("robosuite_prepare", EXAMPLE / "benchmark" / "prepare.py") - F = _load("aspire_fields", EXAMPLE / "agent" / "aspire_fields.py") - monkeypatch.setattr(P, "HERE", tmp_path) + def __init__(self, root, **k): + self.root = Path(root) + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "api-reference.md").write_text("# api\n") + passes = {"n": 0} + + def debug_loop(run, max_versions): + records = [{"task_success": i < passes["n"], "program_ok": True} for i in range(len(run.debug))] + version = D._Version("v00", "result = {}\n", records) + run.versions.append(version) + return version + monkeypatch.setattr(sim, "Engine", Engine) + monkeypatch.setattr(D._Run, "explore", lambda run: None) + monkeypatch.setattr(D._Run, "debug_loop", debug_loop) + actor = D.AspireActor({"aspire_root": str(tmp_path), "scratch": str(tmp_path / "scratch"), + "engine": str(ENGINE / "agent" / "sim.py")}, base=tmp_path) + monkeypatch.setattr(actor, "log_dir", lambda: tmp_path / "logs") + monkeypatch.setattr(actor, "session", lambda adapter: None) + out = actor.run(dict(VERIFY), None) # 0 of 5: refuted, its program withheld + assert out.extra["task"]["not_scored"] == "verifier verdict refuted" and out.extra["task"]["fix_code"] is None + passes["n"] = 4 + out = actor.run(dict(VERIFY), None) # 4 of 5: verified + assert out.extra["task"]["verification"]["verdict"] == "verified" and out.extra["task"]["fix_code"] + assert "not_scored" not in out.extra["task"] + + +def test_aspires_verifiers_are_derived_from_a_benchmarks_tasks(tmp_path, monkeypatch): + """rows.py: every task of the benchmark (its splits/all.txt, in that order) gets its verifier beside it; the + benchmark's tasks, reruns and splits are copied as they are.""" + P = _load("robosuite_prepare", ROBOSUITE / "utils" / "benchmark" / "prepare.py") + monkeypatch.setattr(P, "HERE", tmp_path / "bench") monkeypatch.setattr(sys, "argv", ["prepare.py", "--preset", "smoke"]) + (tmp_path / "bench").mkdir() P.main() - found = F.task_rows(tmp_path / "tasks") - seeds = {"debug_seeds": "101-103", "unseen_seeds": "126", "eval_seeds": "1-3"} + out = tmp_path / "aspire" + writer = ENGINE / "benchmark" / "tasks.py" # the engine's task writer + Rw.main(["--benchmark", str(tmp_path / "bench"), "--out", str(out), "--verify-seeds", "101-105", + "--engine", str(writer)]) + Fd = _load("cap_fields", ENGINE / "agent" / "fields.py") + found = Fd.task_rows(out / "tasks") task = {"id": "robosuite-cube_stack", "suite": "cube_stack", "task_id": 0} - assert json.dumps(found["robosuite-cube_stack"]) == json.dumps({**task, **seeds}) # its keys in order, types assert json.dumps(found["verify-robosuite-cube_stack"]) == json.dumps( {**task, "id": "verify-robosuite-cube_stack", "role": "verifier", "debug_seeds": "101-105", - "eval_seeds": "none", "max_versions": 2, "search": "debug", "findings": False}) - assert json.dumps(found["rerun-robosuite-cube_stack"]) == json.dumps( - {**task, "id": "rerun-robosuite-cube_stack", "role": "rerun", "findings": False, **seeds}) - assert len(found) == 21 and (tmp_path / "splits" / "smoke.txt").read_text().split() == \ - ["robosuite-cube_lifting", "robosuite-cube_stack"] - loaded = Benchmark.load(tmp_path / "tasks", view=F.aspire_task).tasks["robosuite-cube_stack"] - assert loaded.private["tests"] is None and loaded.private["needs"]["network"]["agent"] == "no-network" - for path in sorted((EXAMPLE.parent / "aspire").glob("*.yaml")): + "eval_seeds": "101-105", "score_label": "verifier", "max_programs": 2, "max_versions": 2, + "search": "debug", "findings": False}) # its keys in order: the row the service reads + bench = Fd.task_rows(tmp_path / "bench" / "tasks") + assert {k: v for k, v in found.items() if not k.startswith("verify-")} == bench + assert len(found) == 21 and (out / "tasks" / "verify-robosuite-cube_stack" / "instruction.md").is_file() + order = (tmp_path / "bench" / "splits" / "all.txt").read_text().split() + assert (out / "splits" / "verify.txt").read_text().split() == [f"verify-{i}" for i in order] + for split in (tmp_path / "bench" / "splits").iterdir(): + assert (out / "splits" / split.name).read_text() == split.read_text() + Rw.write(tmp_path / "bench", out, "101-105", writer) # again: rewritten whole, the same + assert Fd.task_rows(out / "tasks") == found + with pytest.raises(ValueError, match="overlap the held-out seeds"): + Rw.write(tmp_path / "bench", out, "1-5", writer) + + +def test_the_aspire_configs_read_its_verifiers_and_name_its_session_and_the_engine(tmp_path, monkeypatch): + """Every ASPIRE config on Robosuite and LIBERO: its tasks are the benchmark's with ASPIRE's verifiers (tasks.sh), + the coordinator's are the verifier rows, the test split the reruns of what it trained on, and the session's task + functions are ASPIRE's over the engine's.""" + from hermit.config import Config + from hermit.task import Benchmark + from hermit.agent.acting import workspace as cli + configs = sorted((ROBOSUITE / "aspire").glob("*.yaml")) + sorted((LIBERO / "aspire").glob("*.yaml")) + assert len(configs) == 14 + prepared = {} + for bench, seeds in ((ROBOSUITE, "101-105"), (LIBERO, "51-55")): + P = _load(f"prepare_{bench.name}", bench / "utils" / "benchmark" / "prepare.py") + monkeypatch.setattr(P, "HERE", tmp_path / bench.name) + monkeypatch.setattr(sys, "argv", ["prepare.py", "--preset", "smoke"]) + (tmp_path / bench.name).mkdir() + P.main() + hook = (bench / "aspire" / "tasks.sh").read_text() + assert f"--verify-seeds {seeds}" in hook and "--engine ../../cap_harness/utils/benchmark/tasks.py" in hook + Rw.write(tmp_path / bench.name, tmp_path / bench.name / "aspire", seeds, ENGINE / "benchmark" / "tasks.py") + prepared[bench] = tmp_path / bench.name / "aspire" + for path in configs: config = Config.load(path) - data = config.data["data"] - assert config.path(data["tasks"]) == (EXAMPLE / "benchmark" / "tasks").resolve(), path - monkeypatch.setitem(data, "tasks", str(tmp_path / "tasks")) + data, params = config.data["data"], config.data["agent"]["params"] + here = prepared[path.parents[1]] + assert config.path(data["tasks"]) == (path.parent / "tasks").resolve(), path for key in ("train", "val", "test"): - monkeypatch.setitem(data, key, str(tmp_path / "splits" / Path(data[key]).name)) - assert {t.id: t.given() for t in Benchmark.from_config(config).tasks.values()} == found, path - - -def test_prepare_refuses_held_out_seeds_a_session_could_run(): - for args in (["--eval", "1-200"], ["--unseen", "120-130"], ["--eval", "1-100", "--verify", "50-54"]): - done = subprocess.run([sys.executable, str(EXAMPLE / "benchmark" / "prepare.py"), *args], capture_output=True, - text=True) - assert done.returncode == 2 and "overlap" in done.stderr, args - - -def test_prune_drops_the_runs_images_and_keeps_their_records(tmp_path): - T = _task_module() - run = tmp_path / "run" / "work" / "debug" / "0101" / "run-x" - (run / "media" / "keyframes").mkdir(parents=True) - (run / "media" / "keyframes" / "a.png").write_bytes(b"png") - (run / "run.json").write_text('{"seed": 101}') - (tmp_path / "run" / "work" / "notes" / "media").mkdir(parents=True) # not a run's: kept - T.prune(tmp_path / "run") - assert not (run / "media").exists() and (run / "run.json").is_file() - assert (tmp_path / "run" / "work" / "notes" / "media").is_dir() + assert config.path(data[key]).parent == (path.parent / "splits").resolve(), (path, key) + monkeypatch.setitem(data, key, str(here / "splits" / Path(data[key]).name)) + monkeypatch.setitem(data, "tasks", str(here / "tasks")) + tasks = Benchmark.from_config(config) + split = lambda key: (here / "splits" / Path(data[key]).name).read_text().split() + assert split("val") == [i for i in (here / "splits" / "verify.txt").read_text().split()], path + assert split("test") == [f"rerun-{i}" for i in split("train")], path + assert all(tasks.tasks[i].given()["role"] == "verifier" for i in split("val")), path + if "_direct" in path.name: + assert config.data["agent"]["name"] == "hermit.agent.evolver.methods.aspire.direct:AspireActor", path + assert config.path(params["engine"]) == (ENGINE / "agent" / "sim.py").resolve(), path + continue + assert config.path(params["engine"]) == (ENGINE / "agent" / "task.py").resolve(), path + for name in ("prompt", "prepare", "answer", "collect"): + assert cli._task_function(params[name], config.base) is getattr(A, name), (path, name) + read = [str(p) for p in (config.data.get("environment") or {}).get("read") or []] + for name in A.ASPIRE_FILES: # its templates and the search's page, declared + assert f"$ASPIRE_ROOT/{name}" in read, (path, name) diff --git a/tests/selection.py b/tests/selection.py index 27adaccc..37f93bde 100644 --- a/tests/selection.py +++ b/tests/selection.py @@ -26,8 +26,7 @@ def fnmatch(path: str, pattern: str) -> bool: # group -> the paths (patterns, as `fnmatch` above reads them) that are the method's own METHODS = { - "aspire": ["hermit/agent/evolver/methods/aspire/*", "hermit/adapter/templates/aspire/*", - "examples/aspire_robosuite/*", "examples/libero/*"], + "aspire": ["hermit/agent/evolver/methods/aspire/*", "hermit/adapter/templates/aspire/*", "examples/*/aspire/*"], "she": ["hermit/agent/evolver/methods/she/*", "hermit/adapter/templates/she/*", "examples/*/she/*"], "ace": ["hermit/agent/evolver/methods/ace/*", "hermit/adapter/templates/ace/*", "examples/*/ace/*"], "ahe": ["hermit/agent/evolver/methods/ahe_adapter/*", "hermit/adapter/templates/ahe/*", @@ -56,6 +55,9 @@ def fnmatch(path: str, pattern: str) -> bool: "she": {"evosafe"}, # AgentDyn runner tests freeze SHE configs "evosafe": {"she"}, # SHE reuses the CLI roles and verdict contract "harness2": {"continual_harness"}} # Continual Harness deploys with the harness2_codex template +# an engine several benchmarks run on (examples//) -> those benchmarks: a change to it runs the groups of every +# method with an example under one of them +ENGINES = {"cap_harness": ("robosuite", "libero", "robodojo_sim")} # what every method is built on: a change here runs every group (tests/core/test_selection.py checks that every # framework module these import, or a method's tests import, is here too) SHARED = ["hermit/agent/__init__.py", "hermit/agent/base/*", "hermit/environment/*", @@ -93,9 +95,10 @@ def groups_for(path: str) -> set[str] | None: if path.startswith("examples/") and path.count("/") <= 2 and path.endswith("README.md"): return set() # the examples' and a benchmark's description if path.startswith("examples/"): - # a benchmark's shared files (utils/, data): every method with an example under that benchmark - bench = path.split("/")[1] - users = {name for name, patterns in METHODS.items() for p in patterns + # a benchmark's shared files (utils/, data): every method with an example under that benchmark (an engine's: + # under any benchmark on it) + benches = ENGINES.get(path.split("/")[1], (path.split("/")[1],)) + users = {name for name, patterns in METHODS.items() for p in patterns for bench in benches if p.startswith("examples/*/") and _has_example(bench, p.split("/")[2])} return users | {"core"} if any(fnmatch(path, p) for p in CORE):