Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/_examples/first_model/judge.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Private labels are kept in the judge, outside learner-visible task rows."""
import json

from hermit.task import JudgeResult, RunOutput
from hermit.task import JudgeResult
from hermit.common import RunOutput

LABELS = {
"train-001": "positive",
Expand Down
2 changes: 1 addition & 1 deletion docs/_examples/first_model/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from hermit import Session
from hermit.agent.model.llm import ChatClient
from hermit.task import RunOutput
from hermit.common import RunOutput


def run(task: dict, adapter: Path | None) -> RunOutput:
Expand Down
3 changes: 2 additions & 1 deletion docs/_examples/first_training/judge.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Compute the target privately rather than placing answers in task rows."""
from hermit.task import JudgeResult, RunOutput
from hermit.task import JudgeResult
from hermit.common import RunOutput


def score(task: dict, output: RunOutput) -> JudgeResult:
Expand Down
2 changes: 1 addition & 1 deletion docs/_examples/first_training/pipeline.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""A deterministic teaching agent that interprets one adapter rule."""
from pathlib import Path
from hermit import Session
from hermit.task import RunOutput
from hermit.common import RunOutput


def run(task: dict, adapter: Path | None) -> RunOutput:
Expand Down
3 changes: 2 additions & 1 deletion docs/_examples/interface_task/judge.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Scores the offline echo task."""
from hermit.task import JudgeResult, RunOutput
from hermit.task import JudgeResult
from hermit.common import RunOutput


def score(task: dict, output: RunOutput) -> JudgeResult:
Expand Down
2 changes: 1 addition & 1 deletion docs/_examples/interface_task/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from pathlib import Path

from hermit import Session
from hermit.task import RunOutput
from hermit.common import RunOutput


def run(task: dict, adapter: Path | None) -> RunOutput:
Expand Down
3 changes: 2 additions & 1 deletion docs/_examples/mini_swe_agent/judge.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Private answers are kept in the judge, outside learner-visible task rows."""
import json

from hermit.task import JudgeResult, RunOutput
from hermit.task import JudgeResult
from hermit.common import RunOutput

ANSWERS = {
"train-001": 195,
Expand Down
4 changes: 3 additions & 1 deletion docs/framework/agent-layers.md
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,8 @@ Configurations that name an agent, an Evolver or a template (`agent.name: codex`
| `make("direct", settings, model)`, `hermit.agent.base.direct` | `Direct(settings, model)`, `hermit.agent.model.direct` (Layer 1); `make("direct")` is refused; the old module path forwards for one release |
| `hermit.agent.base.container` | `hermit.environment.docker` (the old path forwards for one release) |
| `agent.name: harbor` (Hermit's agents injected into `harbor run`) | Codex, Claude Code, `answer`, `react` or a custom agent that takes the `environment` argument, in the task's Harbor environment (`environment.harbor`; Pi, OpenCode, dsh, mini-swe-agent and Hermes refuse it), verified by Harbor's verifier (`judge: {path: hermit.task.verify, name: environment}`) |
| `Mount`, `Limits` from `hermit.agent.base.interface` | `hermit.environment` (still importable from the interface) |
| `Environment`, `Mount`, `Limits`, `Unsupported`, `as_environment` from `hermit.agent.base.interface` | `hermit.environment` (still importable from the interface) |
| `as_messages` from `hermit.agent.base.interface` | `hermit.agent.model.direct` (still importable from the interface) |
| `RunOutput`, `load_callable` from `hermit.task` | `hermit.common` (still importable from `hermit.task`) |
| a benchmark's own copy of the tool loop | a `ReactAgent` subclass with `@tool` methods and hooks |
| `@tool(description, parameters)` | also `@tool` alone: the description from the docstring, the schema from the signature |
2 changes: 1 addition & 1 deletion examples/agentdojo/utils/tests/test_replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import pytest

from hermit.agent.acting import load_agent
from hermit.task import load_callable
from hermit.common import load_callable

ROOT = Path(__file__).resolve().parents[4]
UTILS = ROOT / 'examples/agentdojo/utils'
Expand Down
3 changes: 2 additions & 1 deletion examples/agentdyn/utils/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import time
import uuid

from hermit.task import JudgeResult, RunOutput
from hermit.task import JudgeResult
from hermit.common import RunOutput
from hermit.agent.evolver.methods.evosafe.hooks import ToolCallContext, ToolCallResult

# The sibling modules (victim, auditor) are imported by file name, however this file was loaded.
Expand Down
2 changes: 1 addition & 1 deletion examples/aspire_robosuite/utils/agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import time

from hermit.agent.acting import Agent
from hermit.task import RunOutput
from hermit.common import RunOutput

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
Expand Down
3 changes: 2 additions & 1 deletion examples/claude_subscription/utils/benchmark/judge.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Private labels are kept in the judge, outside learner-visible task rows."""
import json

from hermit.task import JudgeResult, RunOutput
from hermit.task import JudgeResult
from hermit.common import RunOutput

LABELS = {
"train-001": "positive",
Expand Down
2 changes: 1 addition & 1 deletion examples/clbench_database/utils/agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import sys

from hermit.agent.acting import Agent
from hermit.task import RunOutput
from hermit.common import RunOutput

sys.path.insert(0, str(Path(__file__).resolve().parent))
from clbench_host import run_host # noqa: E402
Expand Down
2 changes: 1 addition & 1 deletion examples/clbench_poker/utils/agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import sys

from hermit.agent.acting import Agent
from hermit.task import RunOutput
from hermit.common import RunOutput

sys.path.insert(0, str(Path(__file__).resolve().parent))
from clbench_host import run_host # noqa: E402
Expand Down
2 changes: 1 addition & 1 deletion examples/officeqa/ahe_adapter/trials.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from pathlib import Path
import shutil

from hermit.task import load_callable
from hermit.common import load_callable

UTILS = Path(__file__).resolve().parent.parent / "utils"
HOOK_LOGS = ".hermit-hooks" # the hook runner's log base under the session's workspace
Expand Down
2 changes: 1 addition & 1 deletion examples/officeqa/utils/agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import types

from hermit.agent.acting import Agent
from hermit.task import RunOutput
from hermit.common import RunOutput
from officeqa_fields import officeqa_task # beside this file (load_callable puts its folder on the path)

HERE = Path(__file__).resolve().parent
Expand Down
2 changes: 1 addition & 1 deletion examples/servelearnbench/utils/agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

from hermit import Session
from hermit.agent.acting import Agent
from hermit.task import RunOutput
from hermit.common import RunOutput

HERE = Path(__file__).resolve().parent

Expand Down
4 changes: 2 additions & 2 deletions hermit/agent/acting/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@
import uuid

from hermit.agent.acting.agent import ALL, Agent
from hermit.agent.base.interface import BaseAgent, LoginFailed, Unsupported, make
from hermit.environment import Container, Environment, Limits, Mount, Local, docker as container
from hermit.agent.base.interface import BaseAgent, LoginFailed, make
from hermit.environment import Container, Environment, Limits, Mount, Local, Unsupported, docker as container
from hermit.agent.base.login import LOGINS, NO_LOGIN
from hermit.agent.model.usage import Ledger
from hermit.common import RunOutput, load_callable
Expand Down
4 changes: 2 additions & 2 deletions hermit/agent/base/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,9 @@

from hermit.agent.base import login as codex_login
from hermit.agent.base.credentials import held
from hermit.environment import docker as container
from hermit.environment import Limits, Mount, Unsupported, docker as container
from hermit.environment.base import Environment, ExecResult, _descendants, as_environment
from hermit.agent.base.interface import BaseAgent, Limits, Mount, Result, Unsupported
from hermit.agent.base.interface import BaseAgent, Result
from hermit.agent.base.records import (parse_claude, parse_codex, parse_dsh, parse_hermes, parse_mini_swe_agent,
parse_opencode, parse_pi, price_usage)
from hermit.agent.base.tool import Tool
Expand Down
2 changes: 1 addition & 1 deletion hermit/agent/base/endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ def answer(self, request: dict, authorization: str | None, abort: threading.Even
upstream gave no answer."""
from hermit.agent.base.cli import CliAgent
if isinstance(self.agent, CliAgent):
from hermit.agent.base.interface import Unsupported
from hermit.environment import Unsupported
try:
return cli_answer(self.agent, request, abort, place=self.place)
except (ValueError, Unsupported) as exc: # the session's settings: the client reads why
Expand Down
5 changes: 3 additions & 2 deletions hermit/agent/base/react.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@
import time
from typing import Any, Sequence

from hermit.agent.base.interface import BaseAgent, Result, Unsupported, as_environment, as_messages
from hermit.agent.base.interface import BaseAgent, Result
from hermit.agent.base.tool import Tool
from hermit.agent.base.trajectory import Step
from hermit.agent.model.direct import client_of
from hermit.agent.model.direct import as_messages, client_of
from hermit.environment import Unsupported, as_environment


def message_steps(message: dict, calls: bool = True) -> list[Step]:
Expand Down
4 changes: 2 additions & 2 deletions hermit/agent/evolver/methods/ahe_adapter/debugger.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@
import threading
import time

from hermit.agent.base.interface import LoginFailed, Unsupported, make
from hermit.environment import placed
from hermit.agent.base.interface import LoginFailed, make
from hermit.environment import Unsupported, placed
from hermit.agent.base.trajectory import CODEX_STREAM, STREAMS, log_steps
from hermit.agent.evolver.base import Usage
from hermit.agent.evolver.methods.ahe_adapter import trajectory as _trajectory
Expand Down
3 changes: 2 additions & 1 deletion hermit/isolate.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
program_paths, sandbox_fault, sanitized_copy, set_hidden)
from hermit.environment.bwrap import task_sandbox
from hermit.environment.spec import EnvironmentSpec
from hermit.task import RunOutput, load_callable, output_record
from hermit.common import RunOutput, load_callable
from hermit.task import output_record
from hermit.task.verify import LOGS, SANDBOX_ENV, TESTS

SCRIPT_VERIFIER = ("hermit.task.verify", "script") # judge: {path, name} of a task directory's own verifier
Expand Down
3 changes: 2 additions & 1 deletion hermit/settings_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
from hermit.agent.acting.agent import load_agent
from hermit.isolate import agent_pipeline, config_paths, host_pipeline, judge_for, task_environments, \
validate as validate_sandboxes
from hermit.task import Benchmark, load_callable
from hermit.task import Benchmark
from hermit.common import load_callable
from hermit.trainer.rollout import run_one
from hermit.trainer.trainer import needs_errors, warnings_for

Expand Down
6 changes: 4 additions & 2 deletions hermit/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ def _run(job: dict) -> dict:
if job["kind"] == "agent":
from hermit.adapter.session import ENV_PYTHON, ENV_SANDBOX
from hermit.agent.acting.agent import load_agent
from hermit.task import RunOutput, output_record
from hermit.common import RunOutput
from hermit.task import output_record
# The adapter's tools never run in this process, nor in any it starts (inherited through the environment).
os.environ[ENV_SANDBOX] = "1"
os.environ[ENV_PYTHON] = sys.executable
Expand All @@ -38,7 +39,8 @@ def _run(job: dict) -> dict:
return {"ok": True, "output": output_record(out)}
if job["kind"] == "judge":
from dataclasses import asdict
from hermit.task import RunOutput, load_callable, normalise_judge
from hermit.common import RunOutput, load_callable
from hermit.task import normalise_judge
rec = dict(job["output"])
out = RunOutput(**{**rec, "log": Path(rec["log"]) if rec.get("log") else None})
verdict = normalise_judge(load_callable(job["spec"], Path(job["base"]))(dict(job["task"]), out))
Expand Down
2 changes: 1 addition & 1 deletion tests/core/test_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def test_load_agent_builtin_class_function_and_alias(tmp_path):
claude = load_agent("claude-code", {"model": "m"}, tmp_path)
assert isinstance(claude, CliAgent) and claude.params["backend"] == "claude" and claude.base == tmp_path.resolve()
(tmp_path / "agent.py").write_text(
"from hermit.agent.acting import Agent\nfrom hermit.task import RunOutput\n"
"from hermit.agent.acting import Agent\nfrom hermit.common import RunOutput\n"
"class Mine(Agent):\n def run(self, task, adapter):\n return RunOutput(output=(self.params['k'], str(self.base.name), adapter))\n"
"def plain(task, adapter):\n return RunOutput(output='plain')\n")
mine = load_agent("agent.py:Mine", {"k": 1}, tmp_path)
Expand Down
4 changes: 2 additions & 2 deletions tests/core/test_base_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
import pytest

from fakes import FunctionModel
from hermit.agent.base.interface import Limits, Mount, Unsupported, make
from hermit.agent.base.interface import make
from hermit.agent.base.tool import Tool
from hermit.agent.base.tool_server import ToolServer
from hermit.environment import Container, Local, Sandboxed
from hermit.environment import Container, Limits, Local, Mount, Sandboxed, Unsupported
from hermit.agent.model.direct import Direct


Expand Down
4 changes: 2 additions & 2 deletions tests/core/test_claude_subscription.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@
from hermit.agent.acting import workspace as acting
from hermit.agent.base import cli as base_cli, login as subscription
from hermit.agent.base.endpoint import cli_endpoint, cli_model, cli_problems, endpoint_spec
from hermit.agent.base.interface import Limits, LoginFailed, Result, Unsupported, make
from hermit.agent.base.interface import LoginFailed, Result, make
from hermit.agent.base.records import parse_log
from hermit.agent.model.backends import normalize_config
from hermit.agent.model.llm import ChatAborted, ChatClient, ChatHTTPError
from hermit.config import Config, layered
from hermit.sandbox import Sandbox

from claude_stand import INDEX, ROOT, STREAMS, TOKEN, bwrap_works, stand # noqa: F401 (stand: the fixture)
from hermit.environment import Container, Local, Sandboxed
from hermit.environment import Container, Limits, Local, Sandboxed, Unsupported

EXAMPLE = ROOT / "examples" / "claude_subscription"

Expand Down
3 changes: 1 addition & 2 deletions tests/core/test_cli_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@
from hermit.agent.base import trajectory
from hermit.agent.acting import workspace as cli
from hermit.agent.base import cli as base_cli
from hermit.agent.base.interface import Limits as LIMITS
from hermit.agent.base import login
from hermit.environment import docker as container
from hermit.environment import Sandboxed
from hermit.environment import Limits as LIMITS, Sandboxed
from hermit.agent.base.interface import make

ROOT = Path(__file__).resolve().parents[2]
Expand Down
8 changes: 4 additions & 4 deletions tests/core/test_dsh_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@

from hermit.agent.base import cli as base_cli
from hermit.agent.base import trajectory
from hermit.agent.base.interface import Unsupported, make
from hermit.agent.base.interface import make
from hermit.agent.base.tool import Tool
from hermit.environment import Container, Local, Sandboxed, placed
from hermit.environment import Container, Local, Sandboxed, Unsupported, placed
from hermit.agent.model.backends import bind_agent_params, registry

ROOT = Path(__file__).resolve().parents[2]
Expand Down Expand Up @@ -559,7 +559,7 @@ def test_what_dsh_cannot_use_is_refused_rather_than_dropped(fake, monkeypatch, t
def test_a_tasks_data_is_bound_in_the_workspace_where_bubblewrap_can_bind_it(fake, monkeypatch, tmp_path):
"""dsh's glob and grep run ripgrep without --follow: without isolation a task's data inside the workspace is
bound there, a real read-only directory that a search walks into (as for Claude Code), not linked."""
from hermit.agent.base.interface import Limits, Mount
from hermit.environment import Limits, Mount
if base_cli._bwrap_binds() is None:
pytest.skip("needs a working bubblewrap")
data = tmp_path / "data"
Expand Down Expand Up @@ -1099,7 +1099,7 @@ def test_with_the_dsh_cli_a_tasks_data_is_found_by_dshs_own_search(tmp_path, mon
"""dsh's glob and grep run ripgrep without --follow, so a task's data linked into the workspace was never listed
or matched. Without isolation it is bound there, a real directory: dsh's own search finds it, and dsh's own
sandbox, nested in that mount namespace, runs a command that reads it."""
from hermit.agent.base.interface import Limits, Mount
from hermit.environment import Limits, Mount
if base_cli._bwrap_binds() is None:
pytest.skip("needs a working bubblewrap")
data, work = tmp_path / "data", tmp_path / "work"
Expand Down
4 changes: 2 additions & 2 deletions tests/core/test_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

import pytest

from hermit.agent.base.interface import Unsupported, make
from hermit.environment import Container, Limits, Local, Sandboxed, as_environment, placed
from hermit.agent.base.interface import make
from hermit.environment import Container, Limits, Local, Sandboxed, Unsupported, as_environment, placed
from hermit.agent.model.usage import Usage


Expand Down
4 changes: 2 additions & 2 deletions tests/core/test_hermes_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

import pytest

from hermit.environment import Container, placed
from hermit.environment import Container, Unsupported, placed
import yaml

from hermit.agent.base import cli as base_cli
from hermit.agent.base import trajectory
from hermit.agent.base.interface import Unsupported, make
from hermit.agent.base.interface import make
from hermit.agent.base.records import HERMES_AUX, parse_hermes, parse_log
from hermit.agent.base.tool import Tool
from hermit.agent.model.backends import bind_agent_params, registry
Expand Down
Loading
Loading