diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..d7811b1 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,6 @@ +{ + "name": "drums", + "description": "Isolated desktops for computer-use agents: start a computer from a template, act with GUI, shell and files, verify against a task checker, fork and reset, and read run results, all through the Drums MCP server.", + "version": "0.3.0", + "author": { "name": "Drums Labs" } +} diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..67c1927 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "drums": { + "command": "drums", + "args": ["mcp"], + "env": { "DRUMS_URL": "${DRUMS_URL}", "DRUMS_TOKEN": "${DRUMS_TOKEN}" } + } + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index d89df46..1f595f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,23 @@ All notable changes to the `drums` Python SDK. The SDK talks to drumsd over its HTTP API; the drumsd version each SDK version was built against is noted where it matters. +## 0.3.0 - unreleased (after 0.2.5 is on PyPI) + +The MCP server and the `drums` command. The hosted MCP endpoint on the gateway (`/mcp`) is live already; +this release adds the local server and the client methods for the new gateway routes. + +- `drums mcp` (the same as `python -m drums.mcp`): a stdio MCP server with the 34 workspace and desktop tools of + `drums/mcp_tools.json`, the list the gateway serves. `--env ` binds the desktop tools to one computer; + `--print-config claude|claude-local|cursor|codex` prints the matching client snippet with a placeholder key. + Destructive tools (`destroy_computer`, `delete_snapshot`, `revoke_api_key`) are refused unless `confirm` is true, + before any request is sent. +- `drums` console script (`drums mcp`, `drums version`); `mcp_tools.json` ships in the wheel. +- `Client.runs`, `create_run`, `run`, `api_keys`, `create_api_key`, `revoke_api_key`, `audit` for the gateway's + `/v1/runs`, `/v1/api-keys` and `/v1/audit` routes; `call()` takes extra headers. +- The repository is a Claude Code plugin (`.claude-plugin/plugin.json`, `.mcp.json`, the `drums-workflow` skill). +- `drums.hybrid` `write_file` sends chunks of 8000 base64 characters (the gateway caps a step argument at 8192); + `gui` code is at most 16384 characters. + ## 0.2.5 - first release on PyPI The first version published on PyPI, as `drums-sdk` (`pip install drums-sdk`; the import name stays `drums`). Versions 0.2.0 to 0.2.4 shipped as wheels sent with invites. diff --git a/README.md b/README.md index aa4644e..375955d 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ for an invalid episode instead of returning a reward. ([guide](https://console.drums.run/docs/guides/group-rollouts-with-fork)) - Run a policy over many tasks: [Evaluate on OSWorld](https://console.drums.run/docs/guides/evaluate-on-osworld) - `drums.aio` is the asyncio variant of the same calls; `drums.hybrid` adds shell, file and app-API tools over one - desktop; `python -m drums.mcp` serves them to an agent over MCP (stdio). + desktop; `drums mcp` serves the whole workflow to an agent over MCP (stdio), see [MCP server](#mcp-server). - Every method: [Python SDK reference](https://console.drums.run/docs/reference/python-sdk). Every endpoint: [HTTP API](https://console.drums.run/docs/reference/http-api). If something fails: [Errors](https://console.drums.run/docs/reference/errors). @@ -99,6 +99,100 @@ for an invalid episode instead of returning a reward. - **Errors.** Every failure is a typed exception under `drums.DrumsError` carrying the server's status, message and body; errors about a desktop carry its record and gate report. +## MCP server + +`drums mcp` (the same as `python -m drums.mcp`) is an MCP server on stdio that gives an agent the whole workflow: +start a computer, act in it, verify against a task checker, fork and reset it, and read run results. It talks to +the Drums gateway (or a plain drumsd for the desktop tools) at `DRUMS_URL` with `DRUMS_TOKEN`. + +``` +drums mcp # all tools; desktop tools take computer_id +drums mcp --env # desktop tools default to this computer +drums mcp --create [--task FILE] # create one computer, bind it, close it on exit +drums mcp --print-config claude|claude-local|cursor|codex +``` + +The same tool list is served remotely by the gateway at `/mcp` (MCP Streamable HTTP, authenticated +with a workspace API key; Drums MCP uses API keys, not OAuth). Replace `` with your workspace's gateway +URL and `drums_sk_...` with a key from the console. + +Claude Code, remote: + +``` +claude mcp add --transport http drums /mcp --header "Authorization: Bearer drums_sk_..." +``` + +Claude Code, local (this package): + +``` +claude mcp add drums -e DRUMS_URL= -e DRUMS_TOKEN=drums_sk_... -- drums mcp +``` + +Cursor (`~/.cursor/mcp.json` or `.cursor/mcp.json`): + +```json +{"mcpServers": {"drums": {"url": "/mcp", "headers": {"Authorization": "Bearer drums_sk_..."}}}} +``` + +Codex (`~/.codex/config.toml`): + +```toml +[mcp_servers.drums] +command = "drums" +args = ["mcp"] +env = { DRUMS_URL = "", DRUMS_TOKEN = "drums_sk_..." } +``` + +This repository is also a Claude Code plugin (`.claude-plugin/plugin.json`, `.mcp.json`, and the +`drums-workflow` skill under `skills/`), which installs the server and a skill that teaches the loop below. + +### Tools + +Workspace tools (the gateway's HTTP API; `*` needs the Drums gateway and answers "not available on this server" +against a plain drumsd): + +| tool | what it does | +|---|---| +| `list_templates` | base desktop images a computer can start from | +| `list_computers`, `get_computer` | the workspace's computers: state, validity, task | +| `create_computer` | a new computer from a template or snapshot, optionally with a task | +| `destroy_computer` | close a computer for good (destructive) | +| `list_snapshots`, `create_snapshot`, `delete_snapshot` | saved desktop states (delete is destructive) | +| `list_tasks`, `get_task` | the verified computer-use tasks on the host | +| `list_runs`, `create_run`, `get_run` * | runs: submit a RunSpec, read status, counts and events | +| `list_evaluations`, `create_evaluation`, `get_evaluation` * | runs with kind `evaluation` | +| `list_api_keys`, `create_api_key`, `revoke_api_key` * | API keys (metadata only; revoke is destructive) | +| `read_audit_log` * | who did what, newest first; MCP calls appear as `mcp.` | + +Desktop tools (each takes `computer_id`): + +| tool | what it does | +|---|---| +| `gui` | pyautogui code in the desktop (1920x1080) | +| `screenshot` | a PNG of the desktop (image content) | +| `shell` | a bash command as the desktop user (no root, no internet) | +| `read_file`, `write_file`, `list_dir` | the desktop's files (`~` is `/home/user`) | +| `office_convert`, `spreadsheet_read` | LibreOffice conversions and sheet reads | +| `browser_tabs`, `browser_open` | Chrome tabs through its DevTools port (1337) | +| `load_task` | load a task into a running computer (setup runs, a new episode starts) | +| `verify` | score the desktop with the task's checker: score, valid, trusted, audit flags | +| `fork` | n children that continue from this exact state; the parent keeps running | +| `reset` | back to the clean template state; the task's setup runs again | + +The canonical list with every input schema is `drums/mcp_tools.json`; `tools/list` returns exactly it, in order. + +### Safety + +- `destroy_computer`, `delete_snapshot` and `revoke_api_key` are refused unless `confirm` is `true`, before any + request is sent. Agents should tell the user before passing it. +- Arguments are checked against the tool's schema before anything is sent; a violation is a tool error naming the + field. A tool failure is always a tool result with `isError`, never a protocol error. +- `--print-config` prints placeholders (`` when no URL is known, always `drums_sk_...`), never a key. +- The token comes from `DRUMS_TOKEN`, else the file `DRUMS_TOKEN_FILE`, else `/etc/drums/token`; it is never + printed or written by the server. +- Keys are workspace-scoped on the gateway: a key never sees another workspace's computers. +- `create_api_key` returns the plaintext key once; nothing stores it. + ## Development ```bash diff --git a/drums/__init__.py b/drums/__init__.py index e76adc3..eb5fa36 100644 --- a/drums/__init__.py +++ b/drums/__init__.py @@ -47,7 +47,7 @@ EnvGone, ForkError, GateFailed, GuestError, InvalidEpisode, NotFound, NotQuiescent, Overloaded, RestoreFailed, TaskSetupFailed, TaskUnsupported) -__version__ = "0.2.5" +__version__ = "0.3.0" DEFAULT_URL = os.environ.get("DRUMS_URL") or os.environ.get("DRUMSD_URL") or "http://127.0.0.1:7070" DEFAULT_IMAGE = "osworld-ubuntu" @@ -168,11 +168,12 @@ def __exit__(self, *exc): self.close() # transport -------------------------------------------------------------------------------------------------------- - def call(self, method: str, path: str, body=None, raw: bool = False, timeout: float | None = None): + def call(self, method: str, path: str, body=None, raw: bool = False, timeout: float | None = None, headers: dict | None = None): """One request. GETs are retried while drumsd is unreachable (up to retry_s: a restarting drumsd comes back and - reconciles); other methods are never retried, since their outcome is unknown after a dropped connection.""" + reconciles); other methods are never retried, since their outcome is unknown after a dropped connection. + `headers` are added to the request (e.g. Idempotency-Key).""" data = None if body is None else json.dumps(body).encode() - headers = {"Content-Type": "application/json"} + headers = {"Content-Type": "application/json", **(headers or {})} if self.token: headers["Authorization"] = f"Bearer {self.token}" deadline = time.monotonic() + (self.retry_s if method == "GET" else 0.0) @@ -266,6 +267,46 @@ def env_records(self, include_destroyed: bool = False, page_size: int = 500): if not after: return + # gateway: runs, API keys, audit log --------------------------------------------------------------------------------- + # These routes exist on the Drums gateway only; a plain drumsd answers 404 (NotFound). + def runs(self, kind: str | None = None, status: str | None = None, limit: int = 50) -> dict: + """The workspace's runs, newest first (GET /v1/runs): {"items": [run rows], "next_cursor"}. `kind` filters + (e.g. "evaluation"), `status` too (queued, running, completed, failed, cancelled).""" + q = {k: v for k, v in (("kind", kind), ("status", status), ("limit", limit)) if v is not None} + return self.call("GET", "/v1/runs?" + urllib.parse.urlencode(q)) + + def create_run(self, spec: dict, idempotency_key: str | None = None) -> dict: + """Submit a RunSpec (POST /v1/runs): {"run_id", "operation_id", "status", "created"}. With `idempotency_key` + (the Idempotency-Key header) the same key with the same spec returns the same run.""" + headers = {"Idempotency-Key": idempotency_key} if idempotency_key else None + return self.call("POST", "/v1/runs", spec, headers=headers) + + def run(self, run_id: str) -> dict: + """One run (GET /v1/runs/{id}): its row, the attempts summary and the last events.""" + return self.call("GET", f"/v1/runs/{urllib.parse.quote(str(run_id))}") + + def api_keys(self) -> dict: + """The workspace's API keys, metadata only (GET /v1/api-keys): {"items": [{id, name, prefix, scopes, ...}]}.""" + return self.call("GET", "/v1/api-keys") + + def create_api_key(self, name: str, scopes: list, expires_in_days: int | None = None) -> dict: + """A new API key (POST /v1/api-keys): {"id", "prefix", "key", "expires_at"}. `key` is the plaintext, shown + once; the scopes cannot exceed this client's own.""" + body = {"name": name, "scopes": list(scopes)} + if expires_in_days is not None: + body["expires_in_days"] = expires_in_days + return self.call("POST", "/v1/api-keys", body) + + def revoke_api_key(self, key_id: str) -> dict: + """Revoke an API key (DELETE /v1/api-keys/{id}): {"id", "revoked_at"}. A key cannot revoke itself.""" + return self.call("DELETE", f"/v1/api-keys/{urllib.parse.quote(str(key_id))}") + + def audit(self, action: str | None = None, limit: int = 50, cursor: str | None = None) -> dict: + """The workspace's audit log, newest first (GET /v1/audit): {"items": [events], "next_cursor"}. `action` + filters by prefix (e.g. "mcp", "computer.create").""" + q = {k: v for k, v in (("action", action), ("limit", limit), ("cursor", cursor)) if v is not None} + return self.call("GET", "/v1/audit?" + urllib.parse.urlencode(q)) + def _task(self, task): if task is None or isinstance(task, dict): return task diff --git a/drums/cli.py b/drums/cli.py new file mode 100644 index 0000000..1d18bc0 --- /dev/null +++ b/drums/cli.py @@ -0,0 +1,66 @@ +"""The `drums` command. + + drums mcp [--env ID] [--url URL] [--create] [--task FILE] # the MCP server on stdio (== python -m drums.mcp) + drums mcp --print-config claude|claude-local|cursor|codex # a client config snippet (placeholders, never a key) + drums version +""" +from __future__ import annotations + +import argparse +import os +import sys + +from . import __version__ + +PLACEHOLDER_URL = "" +PLACEHOLDER_KEY = "drums_sk_..." +CLIENTS = ("claude", "claude-local", "cursor", "codex") + + +def gateway_url(url: str | None = None) -> str: + """--url, else $DRUMS_URL, else the placeholder . Never a token.""" + u = (url or os.environ.get("DRUMS_URL") or "").strip().rstrip("/") + return u or PLACEHOLDER_URL + + +def print_config(client: str, url: str | None = None) -> str: + """The config snippet for one MCP client. The key is always the placeholder `drums_sk_...`: paste your own.""" + g = gateway_url(url) + if client == "claude": + return (f'claude mcp add --transport http drums {g}/mcp --header "Authorization: Bearer {PLACEHOLDER_KEY}"\n') + if client == "claude-local": + return f"claude mcp add drums -e DRUMS_URL={g} -e DRUMS_TOKEN={PLACEHOLDER_KEY} -- drums mcp\n" + if client == "cursor": + return ('{"mcpServers": {"drums": {"url": "' + g + '/mcp", "headers": {"Authorization": "Bearer ' + PLACEHOLDER_KEY + '"}}}}\n') + if client == "codex": + return ("[mcp_servers.drums]\n" + 'command = "drums"\n' + 'args = ["mcp"]\n' + 'env = { DRUMS_URL = "' + g + '", DRUMS_TOKEN = "' + PLACEHOLDER_KEY + '" }\n') + raise ValueError(f"unknown client {client!r}; one of {', '.join(CLIENTS)}") + + +def build_parser() -> argparse.ArgumentParser: + from .mcp import add_arguments + ap = argparse.ArgumentParser(prog="drums", description="Drums: isolated desktops for computer-use agents.") + sub = ap.add_subparsers(dest="command", metavar="command") + add_arguments(sub.add_parser("mcp", help="run the MCP server on stdio (or print a client config with --print-config)")) + sub.add_parser("version", help="print the SDK version") + return ap + + +def main(argv=None) -> int: + ap = build_parser() + a = ap.parse_args(sys.argv[1:] if argv is None else argv) + if a.command == "version": + sys.stdout.write(f"drums {__version__}\n") + return 0 + if a.command == "mcp": + from .mcp import run + return run(a) + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/drums/hybrid.py b/drums/hybrid.py index 455f1d1..3009fd7 100644 --- a/drums/hybrid.py +++ b/drums/hybrid.py @@ -32,7 +32,7 @@ HOME = "/home/user" MAX_READ = 2 * 1024 * 1024 -CHUNK = 60_000 +CHUNK = 8_000 # one argv chunk: the gateway caps each step command argument at 8192 chars (contracts drumsActionSchema) SOFFICE = ["soffice", "-env:UserInstallation=file:///tmp/drums-hybrid-lo", "--headless", "--norestore"] diff --git a/drums/mcp.py b/drums/mcp.py index 97e55ae..b552c44 100644 --- a/drums/mcp.py +++ b/drums/mcp.py @@ -1,54 +1,394 @@ -"""An MCP server (stdio, JSON-RPC 2.0) that gives an agent one Drums desktop with hybrid tools. +"""The Drums MCP server (stdio, JSON-RPC 2.0): the workspace and desktop tools of drums/mcp_tools.json. - python -m drums.mcp --env s123 # bind an existing env - python -m drums.mcp --create [--task task.json] # create one (closed when the server exits) + drums mcp # == python -m drums.mcp; talks to $DRUMS_URL with $DRUMS_TOKEN + drums mcp --env # desktop tools default to this computer when computer_id is omitted + drums mcp --create [--task FILE] # create one computer, bind it, close it when the server exits -Tools: everything in drums.hybrid.TOOLS (gui, screenshot, shell, read_file, write_file, list_dir, office_convert, -spreadsheet_read, browser_tabs, browser_open) plus `verify` (the task's reward with its audit). Screenshots come back as -MCP image content. Token: $DRUMS_TOKEN, else the file $DRUMS_TOKEN_FILE, else /etc/drums/token; URL: $DRUMS_URL. Standard library only. +Two layers, one tool list (drums/mcp_tools.json, the same list the gateway's /gw/mcp serves): +- workspace tools (list_templates, create_computer, list_runs, create_api_key, read_audit_log, ...) call the HTTP API + through `Client.call`; runs, evaluations, API keys and the audit log exist only on the Drums gateway, so against a + plain drumsd those tools answer "not available on this server (needs the Drums gateway)"; +- desktop tools (gui, screenshot, shell, read_file, ..., verify, fork, reset, load_task) act on one computer through + drums.hybrid.HybridTools and drums.Env. + +Framing: one JSON-RPC message per line on stdin, one reply per line on stdout (what MCP stdio clients send); a JSON +array on a line is a batch. Notifications get no reply, unknown methods -32601, unreadable lines -32700. A tool +failure is a tool result with isError true (never a JSON-RPC error). Screenshots are image content (PNG), everything +else one text item holding JSON. Destructive tools (destroy_computer, delete_snapshot, revoke_api_key) are refused +before any request unless `confirm` is true. Token: $DRUMS_TOKEN, else the file $DRUMS_TOKEN_FILE, else +/etc/drums/token. Standard library only. """ from __future__ import annotations import argparse import base64 import json +import os +import re import sys +import urllib.parse -from . import connect -from .hybrid import TOOLS, HybridTools +from . import Client, Env, __version__, connect +from .errors import DrumsError, NotFound +from .hybrid import HybridTools PROTOCOL = "2025-06-18" -VERIFY = {"name": "verify", "description": "Score the desktop's current state with the task's checker; returns score, valid and the reward audit.", - "parameters": {"type": "object", "properties": {}}} +TOOLS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp_tools.json") +NOT_AVAILABLE = "not available on this server (needs the Drums gateway)" +READ_ONLY = {"screenshot", "browser_tabs", "verify"} +_TOOLS = None + + +class ToolError(Exception): + """A tool failure: becomes a tool result with isError true and this one-line text.""" + + +# ---- the tool list ------------------------------------------------------------------------------------------------- +def load_tools() -> list: + """The canonical tool list (drums/mcp_tools.json), as loaded: name, layer, scope, destructive, description, inputSchema.""" + global _TOOLS + if _TOOLS is None: + with open(TOOLS_PATH, encoding="utf-8") as f: + _TOOLS = json.load(f)["tools"] + return _TOOLS + + +def tool_by_name() -> dict: + return {t["name"]: t for t in load_tools()} + + +def annotations(tool: dict) -> dict: + name = tool["name"] + listing = name.startswith(("list_", "get_", "read_")) + return {"readOnlyHint": listing or name in READ_ONLY, "destructiveHint": bool(tool.get("destructive")), + "idempotentHint": listing, "openWorldHint": False} + + +def tool_list() -> list: + """What tools/list answers: the tools of mcp_tools.json in order, with name, description, inputSchema and annotations.""" + return [{"name": t["name"], "description": t["description"], "inputSchema": t["inputSchema"], "annotations": annotations(t)} + for t in load_tools()] + + +# ---- argument validation (the JSON schema subset mcp_tools.json uses) ---------------------------------------------- +def _type_ok(v, t: str) -> bool: + if t == "object": + return isinstance(v, dict) + if t == "array": + return isinstance(v, list) + if t == "string": + return isinstance(v, str) + if t == "boolean": + return isinstance(v, bool) + if t == "integer": + return (isinstance(v, int) and not isinstance(v, bool)) or (isinstance(v, float) and v.is_integer()) + if t == "number": + return isinstance(v, (int, float)) and not isinstance(v, bool) + return True + + +def _check(v, schema: dict, label: str): + """The first problem with `v` against `schema` as a sentence naming the field, or None.""" + t = schema.get("type") + if t is not None and not _type_ok(v, t): + return f"field '{label}' must be {'an' if t[0] in 'aeiou' else 'a'} {t}" + if "const" in schema and v != schema["const"]: + return f"field '{label}' must be {json.dumps(schema['const'])}" + if "enum" in schema and v not in schema["enum"]: + return f"field '{label}' must be one of {', '.join(map(str, schema['enum']))}" + if isinstance(v, (int, float)) and not isinstance(v, bool): + if "minimum" in schema and v < schema["minimum"]: + return f"field '{label}' must be >= {schema['minimum']}" + if "maximum" in schema and v > schema["maximum"]: + return f"field '{label}' must be <= {schema['maximum']}" + if isinstance(v, str): + if "minLength" in schema and len(v) < schema["minLength"]: + return f"field '{label}' must be at least {schema['minLength']} characters" + if "maxLength" in schema and len(v) > schema["maxLength"]: + return f"field '{label}' must be at most {schema['maxLength']} characters" + if "pattern" in schema and not re.search(schema["pattern"], v): + return f"field '{label}' does not match {schema['pattern']}" + if isinstance(v, list): + if "minItems" in schema and len(v) < schema["minItems"]: + return f"field '{label}' needs at least {schema['minItems']} item(s)" + if "maxItems" in schema and len(v) > schema["maxItems"]: + return f"field '{label}' takes at most {schema['maxItems']} item(s)" + if schema.get("uniqueItems") and len({json.dumps(x, sort_keys=True) for x in v}) != len(v): + return f"field '{label}' must not repeat an item" + if "items" in schema: + for i, x in enumerate(v): + e = _check(x, schema["items"], f"{label}[{i}]") + if e: + return e + if isinstance(v, dict): + props = schema.get("properties") or {} + for k in schema.get("required") or []: + if k not in v: + return f"field '{k}' is required" if label == "arguments" else f"field '{label}.{k}' is required" + if schema.get("additionalProperties") is False: + for k in v: + if k not in props: + return f"unknown field '{k}'" if label == "arguments" else f"unknown field '{label}.{k}'" + for k, x in v.items(): + if k in props: + e = _check(x, props[k], k if label == "arguments" else f"{label}.{k}") + if e: + return e + return None + + +def validate(schema: dict, args) -> str | None: + """The first schema violation of a tool's arguments, naming the field; None when they are fine.""" + if not isinstance(args, dict): + return "arguments must be an object" + return _check(args, schema, "arguments") + +# ---- the server ----------------------------------------------------------------------------------------------------- +def _is_template(rec: dict) -> bool: + """A checkpoint record is a template (base image) unless it was captured from a computer (a snapshot).""" + kind = str(rec.get("kind") or "").lower() + if kind in ("template", "base", "image", "base_image"): + return True + if kind in ("snapshot", "state", "saved_state", "fork"): + return False + if rec.get("installed"): + return True + if rec.get("captured_from") or str(rec.get("id") or "").startswith("snp-"): + return False + return True -def tool_list(): - return [{"name": t["name"], "description": t["description"], "inputSchema": t["parameters"]} for t in TOOLS + [VERIFY]] + +def _rows(reply, *keys): + """A list reply, or the list under the first of `keys` (gateway replies are {items, next_cursor}).""" + if isinstance(reply, list): + return reply + if isinstance(reply, dict): + for k in keys + ("items",): + if isinstance(reply.get(k), list): + return reply[k] + return [] + + +def _slim(rec): + """A computer record without its full gate report (the validity verdict and failed gates stay).""" + if isinstance(rec, dict): + return {k: v for k, v in rec.items() if k != "gates"} + return rec + + +def _q(**params) -> str: + p = {k: ("true" if v is True else "false" if v is False else str(v)) for k, v in params.items() if v is not None} + return "?" + urllib.parse.urlencode(p) if p else "" class Server: - def __init__(self, env): - self.env = env - self.tools = HybridTools(env) - - def call(self, name, args): - if name == "verify": - r = self.env.verify(strict=False) - return [{"type": "text", "text": json.dumps({"score": r.score, "valid": r.valid, "trusted": r.trusted, - "audit_flags": r.audit_flags, "checker_error": r.checker_error})}] - out = self.tools.call(name, args) + """The MCP server over one Client. `default_computer_id` (from --env) fills computer_id for desktop tools when the + caller leaves it out. `env_factory(computer_id)` returns the object desktop tools act on (default: an Env over the + client; tests pass a stand-in).""" + + def __init__(self, client: Client | None = None, default_computer_id: str | None = None, env_factory=None, version: str = __version__): + self.client = client + self.default_computer_id = default_computer_id + self.env_factory = env_factory + self.version = version + self.tools = tool_by_name() + + # helpers ---------------------------------------------------------------------------------------------------------- + def _env(self, computer_id: str): + if self.env_factory is not None: + return self.env_factory(computer_id) + return Env(self._client(), {"id": computer_id}) + + def _client(self) -> Client: + if self.client is None: + raise ToolError("no Drums server configured (set DRUMS_URL and DRUMS_TOKEN)") + return self.client + + def _api(self, method: str, path: str, body=None, headers=None): + return self._client().call(method, path, body, headers=headers) + + def _gateway(self, method: str, path: str, body=None, headers=None): + """A call to a route only the gateway has: a plain drumsd's 404 (no body, or 'no such route') is reported as such.""" + try: + return self._api(method, path, body, headers) + except NotFound as e: + msg = str(getattr(e, "message", "") or "") + if e.body is None or not isinstance(e.body, dict) or "no such route" in msg or "method not allowed" in msg: + raise ToolError(NOT_AVAILABLE) from None + raise + + def _task_doc(self, args: dict, required: bool): + if isinstance(args.get("task"), dict): + return args["task"] + if args.get("task_id"): + return self._api("GET", f"/v1/tasks/{urllib.parse.quote(str(args['task_id']))}") + if required: + raise ToolError("one of task_id or task is required") + return None + + def _checkpoints(self) -> list: + return _rows(self._api("GET", "/v1/checkpoints"), "checkpoints") + + # workspace tools ---------------------------------------------------------------------------------------------------- + def list_templates(self, a): + rows = [dict(r, checkpoint=r.get("id")) for r in self._checkpoints() if isinstance(r, dict) and _is_template(r)] + return {"templates": rows} + + def list_snapshots(self, a): + return {"snapshots": [r for r in self._checkpoints() if isinstance(r, dict) and not _is_template(r)]} + + def list_computers(self, a): + q = _q(include_destroyed=True if a.get("include_destroyed") else None, limit=a.get("limit")) + r = self._api("GET", "/v1/slots" + q) + rows = [_slim(x) for x in _rows(r, "computers", "slots")] + if not a.get("include_destroyed"): + rows = [x for x in rows if not (isinstance(x, dict) and x.get("state") == "destroyed")] + out = {"computers": rows} + if isinstance(r, dict) and (r.get("next_after") or r.get("next_cursor")): + out["next"] = r.get("next_after") or r.get("next_cursor") + return out + + def create_computer(self, a): + body = {"checkpoint": a["template"], "task": self._task_doc(a, required=False)} + if a.get("ttl_s") is not None: + body["ttl_s"] = a["ttl_s"] + if a.get("setup") is not None: + body["setup"] = a["setup"] + return _slim(self._api("POST", "/v1/slots", body)) + + def get_computer(self, a): + return self._api("GET", f"/v1/slots/{urllib.parse.quote(a['computer_id'])}") + + def destroy_computer(self, a): + return _slim(self._api("DELETE", f"/v1/slots/{urllib.parse.quote(a['computer_id'])}")) + + def create_snapshot(self, a): + body = {"id": a["name"]} if a.get("name") else {} + return self._api("POST", f"/v1/slots/{urllib.parse.quote(a['computer_id'])}/capture", body) + + def delete_snapshot(self, a): + return self._api("DELETE", f"/v1/checkpoints/{urllib.parse.quote(a['snapshot_id'])}") + + def list_tasks(self, a): + rows = _rows(self._api("GET", "/v1/tasks" + _q(domain=a.get("domain"))), "tasks") + limit = int(a.get("limit") or 200) + return {"tasks": rows[:limit], "total": len(rows)} + + def get_task(self, a): + return self._api("GET", f"/v1/tasks/{urllib.parse.quote(a['task_id'])}") + + def _list_runs(self, a, kind=None): + return self._gateway("GET", "/v1/runs" + _q(kind=kind, status=a.get("status"), limit=a.get("limit") or 50)) + + def _create_run(self, spec, key): + return self._gateway("POST", "/v1/runs", spec, headers={"Idempotency-Key": key} if key else None) + + def list_runs(self, a): + return self._list_runs(a) + + def create_run(self, a): + return self._create_run(a["spec"], a.get("idempotency_key")) + + def get_run(self, a): + return self._gateway("GET", f"/v1/runs/{urllib.parse.quote(a['run_id'])}") + + def list_evaluations(self, a): + return self._list_runs(a, kind="evaluation") + + def create_evaluation(self, a): + return self._create_run(dict(a["spec"], kind="evaluation"), a.get("idempotency_key")) + + get_evaluation = get_run + + def list_api_keys(self, a): + return self._gateway("GET", "/v1/api-keys") + + def create_api_key(self, a): + body = {"name": a["name"], "scopes": a["scopes"]} + if a.get("expires_in_days") is not None: + body["expires_in_days"] = a["expires_in_days"] + return self._gateway("POST", "/v1/api-keys", body) + + def revoke_api_key(self, a): + return self._gateway("DELETE", f"/v1/api-keys/{urllib.parse.quote(a['key_id'])}") + + def read_audit_log(self, a): + return self._gateway("GET", "/v1/audit" + _q(action=a.get("action"), limit=a.get("limit") or 50, cursor=a.get("cursor"))) + + # desktop tools ------------------------------------------------------------------------------------------------------ + def verify(self, a): + r = self._env(a["computer_id"]).verify(strict=False) + return {"score": r.score, "valid": r.valid, "trusted": r.trusted, "audit_flags": r.audit_flags, "checker_error": r.checker_error} + + def fork(self, a): + fr = self._env(a["computer_id"]).fork(int(a.get("n") or 1), strict=False) + out = {"children": [getattr(k, "id", k) for k in fr], "seconds": getattr(fr, "seconds", None)} + if getattr(fr, "state", None): + out["state"] = str(fr.state) + failed = [str(x) for x in (getattr(fr, "failed", None) or [])] + if failed: + out["failed"] = failed + return out + + def reset(self, a): + env = self._env(a["computer_id"]) + r = env.reset() + return _slim(getattr(r, "record", r)) + + def load_task(self, a): + doc = self._task_doc(a, required=True) + env = self._env(a["computer_id"]) + r = env.load_task(doc) + return _slim(getattr(r, "record", r)) + + def _hybrid(self, name, a): + args = {k: v for k, v in a.items() if k != "computer_id"} + return HybridTools(self._env(a["computer_id"])).call(name, args) + + # dispatch ----------------------------------------------------------------------------------------------------------- + def call(self, name: str, args) -> list: + """One tool call: returns the content list, or raises ToolError (isError) with a one-line reason.""" + tool = self.tools.get(name) + if tool is None: + raise ToolError(f"unknown tool {name!r}") + args = dict(args or {}) if isinstance(args, dict) else args + if isinstance(args, dict) and "computer_id" in (tool["inputSchema"].get("properties") or {}) \ + and args.get("computer_id") in (None, "") and self.default_computer_id: + args["computer_id"] = self.default_computer_id + if tool.get("destructive") and not (isinstance(args, dict) and args.get("confirm") is True): + raise ToolError(f"refused: pass confirm: true to {name}") + problem = validate(tool["inputSchema"], args) + if problem: + raise ToolError(problem) + try: + if tool["layer"] == "desktop" and name not in ("verify", "fork", "reset", "load_task"): + out = self._hybrid(name, args) + else: + out = getattr(self, name)(args) + except ToolError: + raise + except DrumsError as e: + raise ToolError(str(e)[:1000]) from None + except Exception as e: # noqa: BLE001 - a tool failure is a tool result, not a protocol error + raise ToolError(f"{type(e).__name__}: {e}"[:1000]) from None if isinstance(out, (bytes, bytearray)): - return [{"type": "image", "data": base64.b64encode(out).decode(), "mimeType": "image/png"}] - return [{"type": "text", "text": json.dumps(out)}] + return [{"type": "image", "data": base64.b64encode(bytes(out)).decode(), "mimeType": "image/png"}] + return [{"type": "text", "text": json.dumps(out, default=str)}] def handle(self, msg): + """One JSON-RPC message -> its reply dict, or None for a notification (or anything that is not a request).""" + if not isinstance(msg, dict): + return {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "invalid request"}} mid, method, params = msg.get("id"), msg.get("method"), msg.get("params") or {} - if mid is None: # a notification (e.g. notifications/initialized): no reply + if mid is None or method is None: # a notification (notifications/initialized, ...) or a client's response: no reply return None + if not isinstance(params, dict): + params = {} try: if method == "initialize": res = {"protocolVersion": params.get("protocolVersion") or PROTOCOL, "capabilities": {"tools": {}}, - "serverInfo": {"name": "drums", "version": "0.2"}} + "serverInfo": {"name": "drums", "version": self.version}} elif method == "ping": res = {} elif method == "tools/list": @@ -56,15 +396,18 @@ def handle(self, msg): elif method == "tools/call": try: res = {"content": self.call(params.get("name"), params.get("arguments") or {}), "isError": False} - except Exception as e: # noqa: BLE001 - a tool failure is a tool result, not a protocol error - res = {"content": [{"type": "text", "text": f"{type(e).__name__}: {e}"[:1000]}], "isError": True} + except ToolError as e: + res = {"content": [{"type": "text", "text": str(e)}], "isError": True} else: return {"jsonrpc": "2.0", "id": mid, "error": {"code": -32601, "message": f"method not found: {method}"}} return {"jsonrpc": "2.0", "id": mid, "result": res} except Exception as e: # noqa: BLE001 return {"jsonrpc": "2.0", "id": mid, "error": {"code": -32603, "message": str(e)[:500]}} - def serve(self, inp=sys.stdin, out=sys.stdout): + def serve(self, inp=None, out=None): + """Read newline-delimited JSON-RPC from `inp` (stdin) and write replies to `out` (stdout) until EOF.""" + inp = sys.stdin if inp is None else inp + out = sys.stdout if out is None else out for line in inp: line = line.strip() if not line: @@ -72,40 +415,73 @@ def serve(self, inp=sys.stdin, out=sys.stdout): try: msg = json.loads(line) except ValueError: - out.write(json.dumps({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error"}}) + "\n") - out.flush() - continue - r = self.handle(msg) - if r is not None: - out.write(json.dumps(r) + "\n") + reply = {"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "parse error"}} + else: + if isinstance(msg, list): # a batch: one reply array, notifications left out + replies = [r for r in (self.handle(m) for m in msg) if r is not None] + reply = replies or None + else: + reply = self.handle(msg) + if reply is not None: + out.write(json.dumps(reply) + "\n") out.flush() -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - g = ap.add_mutually_exclusive_group(required=True) - g.add_argument("--env") - g.add_argument("--create", action="store_true") - ap.add_argument("--task") - a = ap.parse_args(argv) - import os +# ---- entry point ---------------------------------------------------------------------------------------------------- +def read_token() -> str | None: + """$DRUMS_TOKEN, else the contents of $DRUMS_TOKEN_FILE, else /etc/drums/token when readable.""" tok = os.environ.get("DRUMS_TOKEN") + if tok: + return tok tf = os.environ.get("DRUMS_TOKEN_FILE") or ("/etc/drums/token" if os.access("/etc/drums/token", os.R_OK) else None) - if tok is None and tf: - tok = open(tf).read().strip() - c = connect(os.environ.get("DRUMS_URL"), tok, close_on_exit=bool(a.create)) - if a.env: - from . import Env - env = Env(c, c.call("GET", f"/v1/slots/{a.env}")) - else: - env = c.create(task=json.load(open(a.task)) if a.task else None) + if tf and os.path.exists(tf): + with open(tf) as f: + return f.read().strip() or None + return None + + +def add_arguments(ap: argparse.ArgumentParser) -> argparse.ArgumentParser: + ap.add_argument("--env", metavar="ID", help="default computer_id for the desktop tools") + ap.add_argument("--url", metavar="URL", help="the gateway or drumsd URL (default $DRUMS_URL)") + ap.add_argument("--create", action="store_true", help="create one computer, bind it, and close it when the server exits") + ap.add_argument("--task", metavar="FILE", help="with --create: a task document (JSON file) to load") + ap.add_argument("--print-config", choices=["claude", "claude-local", "cursor", "codex"], metavar="CLIENT", + help="print the client config snippet (claude, claude-local, cursor or codex) and exit") + return ap + + +def run(a) -> int: + """Run the server from parsed arguments (drums.cli builds them too).""" + if a.print_config: + from .cli import print_config + sys.stdout.write(print_config(a.print_config, a.url)) + return 0 + if a.env and a.create: + sys.stderr.write("drums mcp: --env and --create exclude each other\n") + return 2 + # retry_s: an unreachable server fails a tool call within seconds (a trainer's 60 s is too long for a chat turn) + c = connect(a.url or os.environ.get("DRUMS_URL"), read_token(), check=False, close_on_exit=bool(a.create), retry_s=5.0) + env, bound = None, a.env + if a.create: + task = None + if a.task: + with open(a.task) as f: + task = json.load(f) + env = c.create(task=task) env.wait_valid() + bound = env.id try: - Server(env).serve() + Server(c, default_computer_id=bound).serve() finally: - if a.create: + if env is not None: env.close() + return 0 + + +def main(argv=None) -> int: + ap = add_arguments(argparse.ArgumentParser(prog="python -m drums.mcp", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)) + return run(ap.parse_args(argv)) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/drums/mcp_tools.json b/drums/mcp_tools.json new file mode 100644 index 0000000..50bcff9 --- /dev/null +++ b/drums/mcp_tools.json @@ -0,0 +1,481 @@ +{ + "$comment": "Canonical Drums MCP tool list. Both servers (gateway /gw/mcp in TypeScript, `drums mcp` stdio in Python) load this file verbatim and must list exactly these tools with exactly these inputSchemas. `scope` is the API-key scope the gateway requires; `destructive` tools require `confirm: true`; `layer` is workspace or desktop. Desktop tools take `computer_id` (the Python server fills it from --env when bound).", + "version": "0.3", + "tools": [ + { + "name": "list_templates", + "layer": "workspace", + "scope": "computers:read", + "description": "List the templates (base desktop images) this workspace can start a computer from. Use a template's `checkpoint` as the `template` argument of create_computer.", + "inputSchema": { "type": "object", "properties": {}, "additionalProperties": false } + }, + { + "name": "list_computers", + "layer": "workspace", + "scope": "computers:read", + "description": "List the live computers (desktops) of this workspace: id, name, state, validity, task, created_at. Pass include_destroyed to also see closed ones.", + "inputSchema": { + "type": "object", + "properties": { + "include_destroyed": { "type": "boolean", "description": "Also list destroyed computers (default false).", "default": false }, + "limit": { "type": "integer", "minimum": 1, "maximum": 500, "default": 100 } + }, + "additionalProperties": false + } + }, + { + "name": "create_computer", + "layer": "workspace", + "scope": "computers:write", + "description": "Start a new isolated desktop (computer) from a template or a snapshot, optionally with a task loaded. Returns the computer record; its `id` is the computer_id every desktop tool takes. The desktop is ready when state is `ready` and validity is `valid`.", + "inputSchema": { + "type": "object", + "properties": { + "template": { "type": "string", "description": "A template checkpoint from list_templates (for example osworld-ubuntu) or a snapshot id from list_snapshots.", "minLength": 1, "maxLength": 200 }, + "task_id": { "type": "string", "description": "A task id from list_tasks to load into the desktop at start (its setup runs before the episode).", "maxLength": 200 }, + "task": { "type": "object", "description": "A full task document (OSWorld format) instead of task_id.", "additionalProperties": true }, + "ttl_s": { "type": "integer", "description": "Lifetime in seconds; the computer is closed when it expires (0 or absent: the workspace default).", "minimum": 0, "maximum": 2592000 }, + "setup": { "type": "boolean", "description": "Run the task's setup (default true when a task is given)." } + }, + "required": ["template"], + "additionalProperties": false + } + }, + { + "name": "get_computer", + "layer": "workspace", + "scope": "computers:read", + "description": "Read one computer's record: state, validity verdict and reasons, task, lease, epoch.", + "inputSchema": { + "type": "object", + "properties": { "computer_id": { "type": "string", "description": "The computer id.", "minLength": 1, "maxLength": 64 } }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "destroy_computer", + "layer": "workspace", + "scope": "computers:write", + "destructive": true, + "description": "Close a computer for good (its desktop state is lost). Destructive: the call is refused unless confirm is true.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "confirm": { "type": "boolean", "description": "Must be true. Say so to the user before destroying.", "const": true } + }, + "required": ["computer_id", "confirm"], + "additionalProperties": false + } + }, + { + "name": "list_snapshots", + "layer": "workspace", + "scope": "computers:read", + "description": "List the saved desktop states (snapshots) of this workspace. A snapshot id can be the `template` of create_computer.", + "inputSchema": { "type": "object", "properties": {}, "additionalProperties": false } + }, + { + "name": "create_snapshot", + "layer": "workspace", + "scope": "computers:write", + "description": "Save a computer's current desktop state as a snapshot (the computer keeps running). Returns the snapshot record.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "name": { "type": "string", "description": "A name for the snapshot (optional).", "maxLength": 80 } + }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "delete_snapshot", + "layer": "workspace", + "scope": "computers:write", + "destructive": true, + "description": "Delete a snapshot. Destructive: refused unless confirm is true.", + "inputSchema": { + "type": "object", + "properties": { + "snapshot_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "confirm": { "type": "boolean", "const": true } + }, + "required": ["snapshot_id", "confirm"], + "additionalProperties": false + } + }, + { + "name": "list_tasks", + "layer": "workspace", + "scope": "computers:read", + "description": "List the verified computer-use tasks available on this host (id, domain, instruction). Load one with create_computer(task_id) or load_task.", + "inputSchema": { + "type": "object", + "properties": { + "domain": { "type": "string", "description": "Filter by domain (for example chrome, os, libreoffice_calc).", "maxLength": 80 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 2000, "default": 200 } + }, + "additionalProperties": false + } + }, + { + "name": "get_task", + "layer": "workspace", + "scope": "computers:read", + "description": "Read one task document (instruction, setup, checker).", + "inputSchema": { + "type": "object", + "properties": { "task_id": { "type": "string", "minLength": 1, "maxLength": 200 } }, + "required": ["task_id"], + "additionalProperties": false + } + }, + { + "name": "load_task", + "layer": "desktop", + "scope": "computers:write", + "description": "Load a task into a running computer (runs its setup and starts a new episode). After this, `verify` scores the desktop against this task's checker.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "task_id": { "type": "string", "maxLength": 200 }, + "task": { "type": "object", "additionalProperties": true } + }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "list_runs", + "layer": "workspace", + "scope": "runs:read", + "description": "List the runs (training, scripted tests) of this workspace, newest first: id, name, kind, status, counts.", + "inputSchema": { + "type": "object", + "properties": { + "status": { "type": "string", "description": "Filter by status (queued, running, completed, failed, cancelled).", "maxLength": 40 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 200, "default": 50 } + }, + "additionalProperties": false + } + }, + { + "name": "create_run", + "layer": "workspace", + "scope": "runs:write", + "description": "Submit a run (RunSpec: name, kind, task_set_id, task_ids, template_version_id, agent, execution, budget). It is queued and the host's runner picks it up. Returns run_id and operation_id.", + "inputSchema": { + "type": "object", + "properties": { + "spec": { "type": "object", "description": "The RunSpec, the same document the console's New run form submits.", "additionalProperties": true }, + "idempotency_key": { "type": "string", "description": "Resubmitting with the same key returns the same run.", "maxLength": 128 } + }, + "required": ["spec"], + "additionalProperties": false + } + }, + { + "name": "get_run", + "layer": "workspace", + "scope": "runs:read", + "description": "Read a run's status and results: status, stop reason, attempt counts (done, succeeded, verified valid), metrics and the last events.", + "inputSchema": { + "type": "object", + "properties": { "run_id": { "type": "string", "minLength": 1, "maxLength": 64 } }, + "required": ["run_id"], + "additionalProperties": false + } + }, + { + "name": "list_evaluations", + "layer": "workspace", + "scope": "runs:read", + "description": "List evaluation runs of this workspace, newest first.", + "inputSchema": { + "type": "object", + "properties": { + "status": { "type": "string", "maxLength": 40 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 200, "default": 50 } + }, + "additionalProperties": false + } + }, + { + "name": "create_evaluation", + "layer": "workspace", + "scope": "runs:write", + "description": "Submit an evaluation (a run with kind evaluation): the same RunSpec as create_run; `kind` is set to evaluation.", + "inputSchema": { + "type": "object", + "properties": { + "spec": { "type": "object", "additionalProperties": true }, + "idempotency_key": { "type": "string", "maxLength": 128 } + }, + "required": ["spec"], + "additionalProperties": false + } + }, + { + "name": "get_evaluation", + "layer": "workspace", + "scope": "runs:read", + "description": "Read an evaluation's status and results (per-task success and verified validity).", + "inputSchema": { + "type": "object", + "properties": { "run_id": { "type": "string", "minLength": 1, "maxLength": 64 } }, + "required": ["run_id"], + "additionalProperties": false + } + }, + { + "name": "list_api_keys", + "layer": "workspace", + "scope": null, + "description": "List this workspace's API keys (metadata only: id, name, prefix, scopes, created, last used, expiry; never a secret).", + "inputSchema": { "type": "object", "properties": {}, "additionalProperties": false } + }, + { + "name": "create_api_key", + "layer": "workspace", + "scope": null, + "description": "Create an API key in this workspace. The plaintext key is in this reply once and nowhere else. Scopes cannot exceed the calling key's own scopes.", + "inputSchema": { + "type": "object", + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 80 }, + "scopes": { + "type": "array", + "items": { "type": "string", "enum": ["computers:read", "computers:write", "runs:read", "runs:write", "templates:read", "templates:write", "usage:read", "audit:read"] }, + "minItems": 1, + "uniqueItems": true + }, + "expires_in_days": { "type": "integer", "minimum": 1, "maximum": 365 } + }, + "required": ["name", "scopes"], + "additionalProperties": false + } + }, + { + "name": "revoke_api_key", + "layer": "workspace", + "scope": null, + "destructive": true, + "description": "Revoke an API key; it is refused on its next call. Destructive: refused unless confirm is true.", + "inputSchema": { + "type": "object", + "properties": { + "key_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "confirm": { "type": "boolean", "const": true } + }, + "required": ["key_id", "confirm"], + "additionalProperties": false + } + }, + { + "name": "read_audit_log", + "layer": "workspace", + "scope": "audit:read", + "description": "Read this workspace's audit log, newest first: who did what, when, with what outcome. MCP calls appear with actor mcp and action mcp..", + "inputSchema": { + "type": "object", + "properties": { + "action": { "type": "string", "description": "Filter by action prefix, for example mcp or computer.create.", "pattern": "^[a-z_]+(\\.[a-z_.]+)?$", "maxLength": 80 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 200, "default": 50 }, + "cursor": { "type": "string", "description": "next_cursor from the previous page.", "maxLength": 200 } + }, + "additionalProperties": false + } + }, + { + "name": "gui", + "layer": "desktop", + "scope": "computers:write", + "description": "Run a GUI action in the desktop: Python using pyautogui (e.g. pyautogui.click(100, 200), pyautogui.write('text'), pyautogui.hotkey('ctrl', 's')). Screen is 1920x1080 pixels. Take a screenshot afterwards to see the result.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "code": { "type": "string", "minLength": 1, "maxLength": 16384 } + }, + "required": ["computer_id", "code"], + "additionalProperties": false + } + }, + { + "name": "screenshot", + "layer": "desktop", + "scope": "computers:read", + "description": "Take a screenshot of the desktop (PNG image).", + "inputSchema": { + "type": "object", + "properties": { "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 } }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "shell", + "layer": "desktop", + "scope": "computers:write", + "description": "Run a bash command in the desktop as the desktop user (no root, no internet). Returns returncode, stdout, stderr.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "command": { "type": "string", "minLength": 1, "maxLength": 20000 }, + "timeout_s": { "type": "number", "minimum": 1, "maximum": 120, "default": 60 } + }, + "required": ["computer_id", "command"], + "additionalProperties": false + } + }, + { + "name": "read_file", + "layer": "desktop", + "scope": "computers:write", + "description": "Read a file in the desktop (~ is /home/user). Text comes back as text, binary as base64.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "path": { "type": "string", "minLength": 1, "maxLength": 4096 } + }, + "required": ["computer_id", "path"], + "additionalProperties": false + } + }, + { + "name": "write_file", + "layer": "desktop", + "scope": "computers:write", + "description": "Write a text file in the desktop (creates folders; overwrites).", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "path": { "type": "string", "minLength": 1, "maxLength": 4096 }, + "content": { "type": "string", "maxLength": 2000000 } + }, + "required": ["computer_id", "path", "content"], + "additionalProperties": false + } + }, + { + "name": "list_dir", + "layer": "desktop", + "scope": "computers:write", + "description": "List a folder in the desktop: name, dir, size.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "path": { "type": "string", "minLength": 1, "maxLength": 4096 } + }, + "required": ["computer_id", "path"], + "additionalProperties": false + } + }, + { + "name": "office_convert", + "layer": "desktop", + "scope": "computers:write", + "description": "Convert a document with LibreOffice (e.g. docx->pdf, xlsx->csv, csv->xlsx). Writes next to the source unless outdir is given.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "path": { "type": "string", "minLength": 1, "maxLength": 4096 }, + "format": { "type": "string", "minLength": 1, "maxLength": 40 }, + "outdir": { "type": "string", "maxLength": 4096 } + }, + "required": ["computer_id", "path", "format"], + "additionalProperties": false + } + }, + { + "name": "spreadsheet_read", + "layer": "desktop", + "scope": "computers:write", + "description": "Read the first sheet of a spreadsheet (xlsx, ods, csv) as rows of text, through LibreOffice.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "path": { "type": "string", "minLength": 1, "maxLength": 4096 } + }, + "required": ["computer_id", "path"], + "additionalProperties": false + } + }, + { + "name": "browser_tabs", + "layer": "desktop", + "scope": "computers:write", + "description": "List the open Chrome tabs (url, title). Needs Chrome started with --remote-debugging-port=1337.", + "inputSchema": { + "type": "object", + "properties": { "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 } }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "browser_open", + "layer": "desktop", + "scope": "computers:write", + "description": "Open a URL in a new Chrome tab (Chrome must be running with --remote-debugging-port=1337).", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "url": { "type": "string", "minLength": 1, "maxLength": 4096 } + }, + "required": ["computer_id", "url"], + "additionalProperties": false + } + }, + { + "name": "verify", + "layer": "desktop", + "scope": "computers:write", + "description": "Score the desktop's current state with the loaded task's checker: score (0..1), valid (every gate of the episode passed), trusted, audit flags.", + "inputSchema": { + "type": "object", + "properties": { "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 } }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "fork", + "layer": "desktop", + "scope": "computers:write", + "description": "Fork a running desktop into n children that each continue from this exact state (apps open, files as they are). Returns the children's computer ids. The parent keeps running.", + "inputSchema": { + "type": "object", + "properties": { + "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 }, + "n": { "type": "integer", "minimum": 1, "maximum": 64, "default": 1 } + }, + "required": ["computer_id"], + "additionalProperties": false + } + }, + { + "name": "reset", + "layer": "desktop", + "scope": "computers:write", + "description": "Reset a desktop to its clean template state and start a new episode (the loaded task's setup runs again).", + "inputSchema": { + "type": "object", + "properties": { "computer_id": { "type": "string", "minLength": 1, "maxLength": 64 } }, + "required": ["computer_id"], + "additionalProperties": false + } + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index d617a55..afdf6d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "drums-sdk" -version = "0.2.5" +version = "0.3.0" # Distribution name drums-sdk (PyPI rejects "drums" as too similar to an existing project); import name: drums. description = "Drums SDK: isolated desktop environments for computer-use RL. Create a desktop, act, screenshot, fork, verify." readme = "README.md" @@ -41,5 +41,11 @@ Repository = "https://github.com/Drums-Labs/drums-python-sdk" Changelog = "https://github.com/Drums-Labs/drums-python-sdk/blob/main/CHANGELOG.md" Issues = "https://github.com/Drums-Labs/drums-python-sdk/issues" +[project.scripts] +drums = "drums.cli:main" + [tool.setuptools] packages = ["drums"] + +[tool.setuptools.package-data] +drums = ["mcp_tools.json"] diff --git a/skills/drums-workflow/SKILL.md b/skills/drums-workflow/SKILL.md new file mode 100644 index 0000000..7ed2f8d --- /dev/null +++ b/skills/drums-workflow/SKILL.md @@ -0,0 +1,68 @@ +--- +name: drums-workflow +description: Use when the user wants to run, test or train a computer-use agent on an isolated Drums desktop (start a computer, act in it, verify against a task, fork or reset it, read run results) through the Drums MCP tools. +--- + +# Drums workflow + +Drums gives you isolated Linux desktops ("computers") with a task checker. The MCP server exposes two layers of tools: + +- Workspace tools: `list_templates`, `list_computers`, `create_computer`, `get_computer`, `destroy_computer`, + `list_snapshots`, `create_snapshot`, `delete_snapshot`, `list_tasks`, `get_task`, `list_runs`, `create_run`, + `get_run`, `list_evaluations`, `create_evaluation`, `get_evaluation`, `list_api_keys`, `create_api_key`, + `revoke_api_key`, `read_audit_log`. +- Desktop tools (each takes `computer_id`): `gui`, `screenshot`, `shell`, `read_file`, `write_file`, `list_dir`, + `office_convert`, `spreadsheet_read`, `browser_tabs`, `browser_open`, `verify`, `fork`, `reset`, `load_task`. + +Runs, evaluations, API keys and the audit log need the Drums gateway. Against a plain drumsd those tools answer +"not available on this server (needs the Drums gateway)"; the computer and desktop tools work on both. + +## The loop + +1. Pick a template: `list_templates`. Use a template's `checkpoint` (for example `osworld-ubuntu`) as `template`. +2. Pick a task (optional): `list_tasks` (filter with `domain`, e.g. `chrome`, `os`, `libreoffice_calc`), then + `get_task` to read the instruction, setup and checker. +3. Start a computer: `create_computer` with `template` and `task_id` (or a full `task` document). Keep the returned + `id`: it is the `computer_id` for every desktop tool. Set `ttl_s` when the work has a known length. +4. Wait until it is ready: call `get_computer` until `state` is `ready` and the validity verdict is `valid`. Do not + act on a computer whose verdict is `invalid` or whose state is `quarantined`; tell the user and start another. +5. Act in the desktop: + - `gui` runs pyautogui code (screen 1920x1080, origin top left). Take a `screenshot` after every GUI action and + look at it before the next one. + - `shell` runs a bash command as the desktop user (no root, no internet). `read_file`, `write_file`, `list_dir` + work on the same filesystem; `~` is `/home/user`. + - `office_convert` and `spreadsheet_read` use LibreOffice; `browser_tabs` and `browser_open` need Chrome running + with `--remote-debugging-port=1337`. +6. Score it: `verify` runs the loaded task's checker and returns `score` (0..1), `valid` (every gate of the episode + passed), `trusted` and `audit_flags`. Use the score only when `valid` is true. +7. Branch at a decision point: `fork` with `n` makes n children that continue from this exact state (apps open, files + as they are). The parent keeps running. Each child's id is a new `computer_id`; record which child tried what. +8. Try again from clean: `reset` returns the computer to its template state and runs the task's setup again. + `load_task` puts a different task into a running computer. +9. Save a state worth keeping: `create_snapshot`; it appears in `list_snapshots` and can be the `template` of a + later `create_computer`. +10. Clean up: `destroy_computer` with `confirm: true` for every computer you started (children too) once the user is + done with them. Destroyed computers cannot be recovered. +11. Read results: `get_run` / `get_evaluation` for a run's status, attempt counts and last events; `read_audit_log` + (filter `action: "mcp"`) to see what the MCP session did, newest first. + +## Runs and evaluations + +`create_run` submits a RunSpec (name, kind, task_set_id, task_ids, template_version_id, agent, execution, budget); +the host's runner picks it up. `create_evaluation` is the same with `kind` set to `evaluation`. Both return `run_id`; +poll `get_run` / `get_evaluation`. `list_runs` and `list_evaluations` filter by `status`. + +## Rules + +- Never call `destroy_computer`, `delete_snapshot` or `revoke_api_key` without telling the user first. They are + refused unless `confirm` is `true`; passing it is your statement that the user agreed. +- Prefer `shell`, `read_file` and `list_dir` over GUI actions when you only need to check something; use the GUI + for what the task actually asks a user to do. +- Keep every computer id you create (and every fork child id) and report them; a lost id is a running desktop + nobody can clean up until its lease expires. +- Take a screenshot after each GUI action; do not chain blind clicks. +- `create_api_key` returns the plaintext key once. Show it to the user and never write it into files or logs. +- A computer's validity verdict decides whether a `verify` score counts. Report `valid: false` as "no usable + result", never as a failure of the task. +- Every desktop tool call needs `computer_id` unless the server was started with `--env`; pass it explicitly when + more than one computer is in play. diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..79184c0 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,87 @@ +"""The `drums` command: version, and `drums mcp --print-config` for every client (placeholders only, never a token).""" +import contextlib +import io +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import drums # noqa: E402 +from drums import cli # noqa: E402 + +SECRET = "drums_sk_abcdefghijkl_" + "Z" * 43 + + +def run(argv, env=None): + out, err = io.StringIO(), io.StringIO() + saved = {k: os.environ.get(k) for k in ("DRUMS_TOKEN", "DRUMS_URL", "DRUMS_TOKEN_FILE")} + for k in saved: + os.environ.pop(k, None) + os.environ.update(env or {}) + try: + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + rc = cli.main(argv) + except SystemExit as e: + rc = e.code + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + return rc, out.getvalue(), err.getvalue() + + +class Cli(unittest.TestCase): + def test_version(self): + rc, out, _ = run(["version"]) + self.assertEqual(rc, 0) + self.assertEqual(out, f"drums {drums.__version__}\n") + + def test_print_config_never_prints_a_token(self): + env = {"DRUMS_TOKEN": SECRET, "DRUMS_URL": "https://beta.drums.run/gw"} + for client in cli.CLIENTS: + rc, out, _ = run(["mcp", "--print-config", client], env) + self.assertEqual(rc, 0, client) + self.assertIn("drums_sk_...", out, client) + self.assertNotIn(SECRET, out, client) + self.assertNotIn("Z" * 10, out, client) + self.assertIn("https://beta.drums.run/gw", out, client) + self.assertNotIn("", out, client) + + def test_print_config_exact_snippets(self): + env = {"DRUMS_TOKEN": SECRET} + self.assertEqual(run(["mcp", "--print-config", "claude"], env)[1], + 'claude mcp add --transport http drums /mcp --header "Authorization: Bearer drums_sk_..."\n') + self.assertEqual(run(["mcp", "--print-config", "claude-local"], env)[1], + "claude mcp add drums -e DRUMS_URL= -e DRUMS_TOKEN=drums_sk_... -- drums mcp\n") + self.assertEqual(run(["mcp", "--print-config", "cursor"], env)[1], + '{"mcpServers": {"drums": {"url": "/mcp", "headers": {"Authorization": "Bearer drums_sk_..."}}}}\n') + self.assertEqual(run(["mcp", "--print-config", "codex"], env)[1], + '[mcp_servers.drums]\ncommand = "drums"\nargs = ["mcp"]\nenv = { DRUMS_URL = "", DRUMS_TOKEN = "drums_sk_..." }\n') + # --url wins over $DRUMS_URL; a trailing slash is dropped + rc, out, _ = run(["mcp", "--url", "https://gw.example/gw/", "--print-config", "cursor"], {"DRUMS_URL": "https://other.example"}) + self.assertIn('"url": "https://gw.example/gw/mcp"', out) + + def test_bad_client_and_bad_subcommand(self): + rc, out, err = run(["mcp", "--print-config", "emacs"]) + self.assertNotEqual(rc, 0) + rc, out, err = run([]) + self.assertEqual(rc, 2) + self.assertIn("mcp", out) + rc, out, err = run(["mcp", "--env", "c1", "--create"]) + self.assertEqual(rc, 2) + self.assertIn("exclude", err) + + def test_module_entry_point_matches(self): + from drums import mcp + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = mcp.main(["--print-config", "codex"]) + self.assertEqual(rc, 0) + self.assertIn('command = "drums"', out.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_hybrid_mcp.py b/tests/test_hybrid_mcp.py index e1a84dd..439117b 100644 --- a/tests/test_hybrid_mcp.py +++ b/tests/test_hybrid_mcp.py @@ -30,7 +30,10 @@ def step(self, action, check=True): cmd = [a.replace("/home/user", self.root) for a in cmd] if cmd[0] == "timeout": # not on every test machine; the desktop has it cmd = cmd[2:] - p = subprocess.run(cmd, capture_output=True, text=True) + try: + p = subprocess.run(cmd, capture_output=True, text=True) + except FileNotFoundError: # a missing binary is exit 127 on the desktop, not an exception + return R({"returncode": 127, "output": "", "error": f"{cmd[0]}: command not found"}) return R({"returncode": p.returncode, "output": p.stdout, "error": p.stderr}) def screenshot(self): @@ -74,7 +77,9 @@ class Mcp(unittest.TestCase): def test_json_rpc_session(self): import tempfile with tempfile.TemporaryDirectory() as d: - srv = mcp.Server(FakeEnv(d)) + fake = FakeEnv(d) + # bound to one desktop (--env s1): desktop tools need no computer_id; every env lookup is the stand-in + srv = mcp.Server(client=None, default_computer_id="s1", env_factory=lambda cid: fake) msgs = [{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-06-18"}}, {"jsonrpc": "2.0", "method": "notifications/initialized"}, {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..cefc450 --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,523 @@ +"""drums.mcp, the two-layer server: tools/list == mcp_tools.json, every tool once against a fake gateway or a +stand-in desktop, confirm refusals before any request, --env binding, 404 -> "not available", schema violations.""" +import base64 +import io +import json +import os +import sys +import threading +import unittest +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import drums # noqa: E402 +from drums import mcp # noqa: E402 + +TOKEN = "k" * 43 +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOLS_JSON = os.path.join(HERE, "..", "drums", "mcp_tools.json") + +PNG = b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + (1920).to_bytes(4, "big") + (1080).to_bytes(4, "big") + b"\x08\x02\x00\x00\x00" +CHECKPOINTS = [{"id": "osworld-ubuntu", "installed": True, "captured_from": None, "created_epoch": 1.0}, + {"id": "snp-1111", "installed": False, "captured_from": "c1", "created_epoch": 2.0}, + {"id": "fork-22", "captured_from": "c2", "created_epoch": 3.0}] + + +def computer(i, state="ready", **kw): + return {"id": f"c{i}", "state": state, "episode": 1, "validity": {"verdict": "valid", "gates": []}, "gates": {"big": "report"}, **kw} + + +class FakeGateway(BaseHTTPRequestHandler): + """Emulates the /v1 routes the tools use and records every request as (method, path, body, headers).""" + calls = [] + no_runs = False # a plain drumsd: /v1/runs, /v1/api-keys, /v1/audit are not routes (404, empty body) + + def log_message(self, *a): + pass + + def _send(self, code, obj=None, raw=None): + b = raw if raw is not None else (b"" if obj is None else json.dumps(obj).encode()) + self.send_response(code) + self.send_header("Content-Length", str(len(b))) + self.end_headers() + self.wfile.write(b) + + def _record(self, method): + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(n) or b"null") + FakeGateway.calls.append((method, self.path, body, dict(self.headers))) + if self.headers.get("Authorization") != f"Bearer {TOKEN}": + self._send(401, {"error": "missing or wrong bearer token", "code": "unauthorized"}) + return None + return body + + def _route(self, method, body): + u = urllib.parse.urlsplit(self.path) + path, q = u.path, dict(urllib.parse.parse_qsl(u.query)) + parts = path.strip("/").split("/") + gateway_only = len(parts) >= 2 and parts[1] in ("runs", "api-keys", "audit") + if gateway_only and FakeGateway.no_runs: + return self._send(404) # axum's default: no body at all + # ---- slots + if path == "/v1/slots" and method == "GET": + rows = [computer(1), computer(2, state="destroyed")] + if q.get("include_destroyed") != "true": + rows = [r for r in rows if r["state"] != "destroyed"] + return self._send(200, {"items": rows[: int(q.get("limit") or 100)], "next_after": None}) + if path == "/v1/slots" and method == "POST": + return self._send(201, computer(7, checkpoint=body.get("checkpoint"), task=body.get("task"))) + if path.startswith("/v1/slots/"): + cid = parts[2] + if cid == "missing": + return self._send(404, {"error": f"no env {cid} on this drumsd", "code": "not_found"}) + if len(parts) == 3 and method == "GET": + return self._send(200, computer(1)) + if len(parts) == 3 and method == "DELETE": + return self._send(200, computer(1, state="destroyed")) + op = parts[3] + if op == "step": + return self._send(200, {"result": {"returncode": 0, "output": "", "error": ""}, "seconds": 0.1}) + if op == "screenshot": + return self._send(200, raw=PNG) + if op == "reset": + return self._send(200, computer(1, episode=2)) + if op == "task": + return self._send(200, computer(1, task=body.get("task"))) + if op == "verify": + return self._send(200, {"slot": cid, "score": 1.0, "valid": True, "validity": {"verdict": "valid", "gates": []}, + "audit": {"trusted": True, "flags": []}, "checker_error": None}) + if op == "capture": + return self._send(200, {"id": "snp-new", "captured_from": cid, "name": body.get("id")}) + if op == "branch": + n = int(body.get("n") or 1) + return self._send(200, {"children": [computer(10 + i) for i in range(n)], "checkpoint": {"id": "fork-99"}, "seconds": 0.42}) + # ---- checkpoints + if path == "/v1/checkpoints" and method == "GET": + return self._send(200, CHECKPOINTS) + if path.startswith("/v1/checkpoints/") and method == "DELETE": + return self._send(200, {"id": parts[2], "deleted": True}) + # ---- tasks + if path == "/v1/tasks" and method == "GET": + rows = [{"id": "t1", "domain": "chrome", "instruction": "open a tab"}, {"id": "t2", "domain": "os", "instruction": "make a folder"}] + if q.get("domain"): + rows = [r for r in rows if r["domain"] == q["domain"]] + return self._send(200, rows) + if path.startswith("/v1/tasks/"): + return self._send(200, {"id": parts[2], "domain": "chrome", "instruction": "open a tab", "config": [], "evaluator": {"func": "x"}}) + # ---- runs + if path == "/v1/runs" and method == "GET": + return self._send(200, {"items": [{"id": "r1", "kind": q.get("kind") or "scripted_test", "status": q.get("status") or "queued"}], "next_cursor": None}) + if path == "/v1/runs" and method == "POST": + return self._send(202, {"run_id": "r2", "operation_id": "op2", "status": "queued", "created": True}) + if path.startswith("/v1/runs/"): + if parts[2] == "nope": + return self._send(404, {"error": "no such run", "code": "not_found"}) + return self._send(200, {"id": parts[2], "status": "completed", "attempts": {"done": 3}, "events": []}) + # ---- api keys + if path == "/v1/api-keys" and method == "GET": + return self._send(200, {"items": [{"id": "k1", "name": "ci", "prefix": "drums_sk_abc", "scopes": ["runs:read"]}]}) + if path == "/v1/api-keys" and method == "POST": + return self._send(201, {"id": "k2", "prefix": "drums_sk_new", "key": "drums_sk_new_plaintext", "expires_at": None}) + if path.startswith("/v1/api-keys/") and method == "DELETE": + return self._send(200, {"id": parts[2], "revoked_at": "2026-11-11T00:00:00Z"}) + # ---- audit + if path == "/v1/audit" and method == "GET": + return self._send(200, {"items": [{"action": "mcp.list_computers", "actor": {"kind": "mcp"}, "outcome": "success"}], "next_cursor": "n1"}) + self._send(404, {"error": "no such route", "code": "not_found"}) + + def do_GET(self): + b = self._record("GET") + if b is not None or self.headers.get("Authorization") == f"Bearer {TOKEN}": + self._route("GET", b) + + def do_POST(self): + b = self._record("POST") + if self.headers.get("Authorization") == f"Bearer {TOKEN}": + self._route("POST", b) + + def do_DELETE(self): + b = self._record("DELETE") + if self.headers.get("Authorization") == f"Bearer {TOKEN}": + self._route("DELETE", b) + + +class GatewayTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.httpd = ThreadingHTTPServer(("127.0.0.1", 0), FakeGateway) + threading.Thread(target=cls.httpd.serve_forever, daemon=True).start() + cls.url = f"http://127.0.0.1:{cls.httpd.server_address[1]}" + + @classmethod + def tearDownClass(cls): + cls.httpd.shutdown() + cls.httpd.server_close() + + def setUp(self): + FakeGateway.calls = [] + FakeGateway.no_runs = False + self.client = drums.Client(self.url, TOKEN, retry_s=0.5) + self.srv = mcp.Server(self.client) + + # helpers ------------------------------------------------------------------------------------------------------------ + def rpc(self, srv, method, params=None, mid=1): + return srv.handle({"jsonrpc": "2.0", "id": mid, "method": method, "params": params or {}}) + + def tool(self, name, args=None, srv=None): + """tools/call -> (result dict, parsed text or image bytes).""" + r = self.rpc(srv or self.srv, "tools/call", {"name": name, "arguments": args or {}}) + res = r["result"] + c = res["content"][0] + if c["type"] == "image": + return res, base64.b64decode(c["data"]) + if res["isError"]: + return res, c["text"] + return res, json.loads(c["text"]) + + def last(self): + return FakeGateway.calls[-1] + + # protocol ----------------------------------------------------------------------------------------------------------- + def test_tools_list_equals_mcp_tools_json_in_order_with_annotations(self): + with open(TOOLS_JSON) as f: + want = json.load(f)["tools"] + got = self.rpc(self.srv, "tools/list")["result"]["tools"] + self.assertEqual(len(got), 34) + self.assertEqual([t["name"] for t in got], [t["name"] for t in want]) + for g, w in zip(got, want): + self.assertEqual(g["description"], w["description"]) + self.assertEqual(g["inputSchema"], w["inputSchema"]) + self.assertEqual(g["inputSchema"]["type"], "object") + ann = g["annotations"] + self.assertEqual(set(ann), {"readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"}) + self.assertEqual(ann["destructiveHint"], bool(w.get("destructive"))) + self.assertFalse(ann["openWorldHint"]) + byname = {t["name"]: t["annotations"] for t in got} + for n in ("list_templates", "get_run", "read_audit_log", "screenshot", "browser_tabs", "verify"): + self.assertTrue(byname[n]["readOnlyHint"], n) + for n in ("create_computer", "gui", "shell", "fork", "destroy_computer"): + self.assertFalse(byname[n]["readOnlyHint"], n) + self.assertTrue(byname["list_runs"]["idempotentHint"]) + self.assertFalse(byname["screenshot"]["idempotentHint"]) + for n in ("destroy_computer", "delete_snapshot", "revoke_api_key"): + self.assertTrue(byname[n]["destructiveHint"], n) + + def test_initialize_ping_unknown_method_notifications_and_parse_errors(self): + r = self.rpc(self.srv, "initialize", {"protocolVersion": "2024-11-05", "capabilities": {}})["result"] + self.assertEqual(r["protocolVersion"], "2024-11-05") + self.assertEqual(r["capabilities"], {"tools": {}}) + self.assertEqual(r["serverInfo"], {"name": "drums", "version": drums.__version__}) + self.assertEqual(self.rpc(self.srv, "initialize")["result"]["protocolVersion"], mcp.PROTOCOL) + self.assertEqual(self.rpc(self.srv, "ping")["result"], {}) + self.assertEqual(self.rpc(self.srv, "resources/list")["error"]["code"], -32601) + self.assertIsNone(self.srv.handle({"jsonrpc": "2.0", "method": "notifications/initialized"})) + self.assertIsNone(self.srv.handle({"jsonrpc": "2.0", "id": 9, "result": {}})) # a client's response: ignored + out = io.StringIO() + self.srv.serve(io.StringIO('{"jsonrpc": "2.0", "id": 1, "method": "ping"}\n{not json\n' + '[{"jsonrpc": "2.0", "id": 2, "method": "ping"}, {"jsonrpc": "2.0", "method": "notifications/x"}]\n'), out) + rep = [json.loads(x) for x in out.getvalue().splitlines()] + self.assertEqual(rep[0]["id"], 1) + self.assertEqual(rep[1]["error"]["code"], -32700) + self.assertEqual([r["id"] for r in rep[2]], [2]) # batch: one array, no notification reply + self.assertEqual(self.rpc(self.srv, "tools/call", {"name": "no_such_tool"})["result"]["isError"], True) + + # workspace tools -------------------------------------------------------------------------------------------------- + def test_templates_and_snapshots_split_the_checkpoint_list(self): + res, out = self.tool("list_templates") + self.assertEqual(self.last()[:2], ("GET", "/v1/checkpoints")) + self.assertEqual([t["checkpoint"] for t in out["templates"]], ["osworld-ubuntu"]) + res, out = self.tool("list_snapshots") + self.assertEqual([s["id"] for s in out["snapshots"]], ["snp-1111", "fork-22"]) + + def test_list_get_create_destroy_computer(self): + res, out = self.tool("list_computers") + self.assertEqual(self.last()[:2], ("GET", "/v1/slots")) + self.assertEqual([c["id"] for c in out["computers"]], ["c1"]) + self.assertNotIn("gates", out["computers"][0]) + res, out = self.tool("list_computers", {"include_destroyed": True, "limit": 5}) + self.assertEqual(self.last()[1], "/v1/slots?include_destroyed=true&limit=5") + self.assertEqual([c["id"] for c in out["computers"]], ["c1", "c2"]) + + res, out = self.tool("create_computer", {"template": "osworld-ubuntu", "task_id": "t1", "ttl_s": 600, "setup": True}) + self.assertEqual(FakeGateway.calls[-2][:2], ("GET", "/v1/tasks/t1")) # task_id resolved first + m, p, body, _ = self.last() + self.assertEqual((m, p), ("POST", "/v1/slots")) + self.assertEqual(body, {"checkpoint": "osworld-ubuntu", "task": {"id": "t1", "domain": "chrome", "instruction": "open a tab", "config": [], "evaluator": {"func": "x"}}, + "ttl_s": 600, "setup": True}) + self.assertEqual(out["id"], "c7") + res, out = self.tool("create_computer", {"template": "snp-1111", "task": {"id": "inline", "config": []}}) + self.assertEqual(self.last()[2], {"checkpoint": "snp-1111", "task": {"id": "inline", "config": []}}) + res, out = self.tool("create_computer", {"template": "osworld-ubuntu"}) + self.assertEqual(self.last()[2], {"checkpoint": "osworld-ubuntu", "task": None}) + + res, out = self.tool("get_computer", {"computer_id": "c1"}) + self.assertEqual(self.last()[:2], ("GET", "/v1/slots/c1")) + self.assertEqual(out["state"], "ready") + res, out = self.tool("get_computer", {"computer_id": "missing"}) + self.assertTrue(res["isError"]) + self.assertIn("no env missing", out) + + res, out = self.tool("destroy_computer", {"computer_id": "c1", "confirm": True}) + self.assertEqual(self.last()[:2], ("DELETE", "/v1/slots/c1")) + self.assertEqual(out["state"], "destroyed") + + def test_snapshots_create_and_delete(self): + res, out = self.tool("create_snapshot", {"computer_id": "c1", "name": "after-login"}) + self.assertEqual(self.last()[:3], ("POST", "/v1/slots/c1/capture", {"id": "after-login"})) + self.assertEqual(out["id"], "snp-new") + res, out = self.tool("create_snapshot", {"computer_id": "c1"}) + self.assertEqual(self.last()[2], {}) + res, out = self.tool("delete_snapshot", {"snapshot_id": "snp-1111", "confirm": True}) + self.assertEqual(self.last()[:2], ("DELETE", "/v1/checkpoints/snp-1111")) + self.assertTrue(out["deleted"]) + + def test_tasks(self): + res, out = self.tool("list_tasks") + self.assertEqual(self.last()[:2], ("GET", "/v1/tasks")) + self.assertEqual([t["id"] for t in out["tasks"]], ["t1", "t2"]) + res, out = self.tool("list_tasks", {"domain": "os", "limit": 1}) + self.assertEqual(self.last()[1], "/v1/tasks?domain=os") + self.assertEqual([t["id"] for t in out["tasks"]], ["t2"]) + res, out = self.tool("get_task", {"task_id": "t2"}) + self.assertEqual(self.last()[:2], ("GET", "/v1/tasks/t2")) + self.assertEqual(out["evaluator"], {"func": "x"}) + + def test_runs_and_evaluations(self): + res, out = self.tool("list_runs") + self.assertEqual(self.last()[1], "/v1/runs?limit=50") + self.assertEqual(out["items"][0]["id"], "r1") + res, out = self.tool("list_runs", {"status": "running", "limit": 5}) + self.assertEqual(self.last()[1], "/v1/runs?status=running&limit=5") + res, out = self.tool("list_evaluations", {"limit": 3}) + self.assertEqual(self.last()[1], "/v1/runs?kind=evaluation&limit=3") + self.assertEqual(out["items"][0]["kind"], "evaluation") + + spec = {"name": "n", "kind": "scripted_test", "task_ids": ["t1"]} + res, out = self.tool("create_run", {"spec": spec, "idempotency_key": "abcdefgh-1"}) + m, p, body, h = self.last() + self.assertEqual((m, p, body), ("POST", "/v1/runs", spec)) + self.assertEqual(h.get("Idempotency-Key"), "abcdefgh-1") + self.assertEqual(out["run_id"], "r2") + res, out = self.tool("create_run", {"spec": spec}) + self.assertIsNone(self.last()[3].get("Idempotency-Key")) + res, out = self.tool("create_evaluation", {"spec": spec}) + self.assertEqual(self.last()[2]["kind"], "evaluation") + self.assertEqual(spec["kind"], "scripted_test") # the caller's spec is not changed + + res, out = self.tool("get_run", {"run_id": "r1"}) + self.assertEqual(self.last()[:2], ("GET", "/v1/runs/r1")) + self.assertEqual(out["attempts"], {"done": 3}) + res, out = self.tool("get_evaluation", {"run_id": "r1"}) + self.assertEqual(self.last()[:2], ("GET", "/v1/runs/r1")) + res, out = self.tool("get_run", {"run_id": "nope"}) # a gateway's own 404 is passed on + self.assertTrue(res["isError"]) + self.assertIn("no such run", out) + self.assertNotIn("not available", out) + + def test_api_keys_and_audit(self): + res, out = self.tool("list_api_keys") + self.assertEqual(self.last()[:2], ("GET", "/v1/api-keys")) + self.assertEqual(out["items"][0]["prefix"], "drums_sk_abc") + res, out = self.tool("create_api_key", {"name": "ci", "scopes": ["runs:read", "computers:read"], "expires_in_days": 30}) + self.assertEqual(self.last()[:3], ("POST", "/v1/api-keys", {"name": "ci", "scopes": ["runs:read", "computers:read"], "expires_in_days": 30})) + self.assertEqual(out["key"], "drums_sk_new_plaintext") + res, out = self.tool("create_api_key", {"name": "ci", "scopes": ["runs:read"]}) + self.assertEqual(self.last()[2], {"name": "ci", "scopes": ["runs:read"]}) + res, out = self.tool("revoke_api_key", {"key_id": "k1", "confirm": True}) + self.assertEqual(self.last()[:2], ("DELETE", "/v1/api-keys/k1")) + self.assertEqual(out["id"], "k1") + res, out = self.tool("read_audit_log") + self.assertEqual(self.last()[1], "/v1/audit?limit=50") + self.assertEqual(out["items"][0]["action"], "mcp.list_computers") + res, out = self.tool("read_audit_log", {"action": "mcp", "limit": 10, "cursor": "n1"}) + self.assertEqual(self.last()[1], "/v1/audit?action=mcp&limit=10&cursor=n1") + + def test_gateway_only_tools_against_a_plain_drumsd(self): + FakeGateway.no_runs = True + for name, args in (("list_runs", {}), ("create_run", {"spec": {}}), ("get_run", {"run_id": "r1"}), ("list_evaluations", {}), + ("create_evaluation", {"spec": {}}), ("get_evaluation", {"run_id": "r1"}), ("list_api_keys", {}), + ("create_api_key", {"name": "a", "scopes": ["runs:read"]}), ("revoke_api_key", {"key_id": "k", "confirm": True}), + ("read_audit_log", {})): + res, out = self.tool(name, args) + self.assertTrue(res["isError"], name) + self.assertEqual(out, "not available on this server (needs the Drums gateway)", name) + + # desktop tools through the real Env over the fake gateway ------------------------------------------------------- + def test_desktop_tools_over_the_api(self): + res, out = self.tool("screenshot", {"computer_id": "c1"}) + self.assertEqual(self.last()[:2], ("GET", "/v1/slots/c1/screenshot")) + self.assertEqual(res["content"][0]["type"], "image") + self.assertEqual(res["content"][0]["mimeType"], "image/png") + self.assertEqual(out, PNG) + + res, out = self.tool("gui", {"computer_id": "c1", "code": "pyautogui.click(1, 2)"}) + self.assertEqual(self.last()[:3], ("POST", "/v1/slots/c1/step", {"action": {"type": "pyautogui", "code": "pyautogui.click(1, 2)"}, "screenshot": False})) + self.assertEqual(out["returncode"], 0) + res, out = self.tool("shell", {"computer_id": "c1", "command": "ls ~", "timeout_s": 5}) + self.assertEqual(self.last()[2]["action"], {"type": "execute", "command": ["timeout", "5", "bash", "-c", "ls ~"], "shell": False}) + self.assertEqual(out["returncode"], 0) + + res, out = self.tool("verify", {"computer_id": "c1"}) + self.assertEqual(self.last()[:3], ("POST", "/v1/slots/c1/verify", {})) + self.assertEqual(out, {"score": 1.0, "valid": True, "trusted": True, "audit_flags": [], "checker_error": None}) + + res, out = self.tool("fork", {"computer_id": "c1", "n": 3}) + self.assertEqual(self.last()[:3], ("POST", "/v1/slots/c1/branch", {"n": 3})) + self.assertEqual(out["children"], ["c10", "c11", "c12"]) + self.assertEqual(out["seconds"], 0.42) + self.assertEqual(out["state"], "fork-99") + res, out = self.tool("fork", {"computer_id": "c1"}) + self.assertEqual(self.last()[2], {"n": 1}) + + res, out = self.tool("reset", {"computer_id": "c1"}) + self.assertEqual(self.last()[:2], ("POST", "/v1/slots/c1/reset")) + self.assertEqual(out["episode"], 2) + self.assertNotIn("gates", out) + + res, out = self.tool("load_task", {"computer_id": "c1", "task_id": "t1"}) + self.assertEqual(FakeGateway.calls[-2][:2], ("GET", "/v1/tasks/t1")) + self.assertEqual(self.last()[:2], ("POST", "/v1/slots/c1/task")) + self.assertEqual(self.last()[2]["task"]["id"], "t1") + res, out = self.tool("load_task", {"computer_id": "c1", "task": {"id": "inline"}}) + self.assertEqual(self.last()[2], {"task": {"id": "inline"}}) + res, out = self.tool("load_task", {"computer_id": "c1"}) + self.assertTrue(res["isError"]) + self.assertIn("task_id or task", out) + + # safety ----------------------------------------------------------------------------------------------------------- + def test_destructive_tools_refuse_without_confirm_before_any_request(self): + for name, args in (("destroy_computer", {"computer_id": "c1"}), ("destroy_computer", {"computer_id": "c1", "confirm": False}), + ("destroy_computer", {"computer_id": "c1", "confirm": "yes"}), ("delete_snapshot", {"snapshot_id": "snp-1111"}), + ("revoke_api_key", {"key_id": "k1"}), ("revoke_api_key", {"key_id": "k1", "confirm": 1})): + res, out = self.tool(name, args) + self.assertTrue(res["isError"], name) + self.assertEqual(out, f"refused: pass confirm: true to {name}") + self.assertEqual(FakeGateway.calls, []) + + def test_schema_violations_name_the_field_and_send_nothing(self): + cases = [("get_computer", {}, "computer_id"), ("fork", {"computer_id": "c1", "n": 0}, "'n'"), ("fork", {"computer_id": "c1", "n": 65}, "'n'"), + ("fork", {"computer_id": "c1", "n": "2"}, "'n'"), ("shell", {"computer_id": "c1", "command": ""}, "'command'"), + ("shell", {"computer_id": "c1", "command": "ls", "timeout_s": 500}, "'timeout_s'"), ("gui", {"computer_id": "c1", "code": 5}, "'code'"), + ("list_computers", {"limit": 501}, "'limit'"), ("list_computers", {"bogus": 1}, "'bogus'"), + ("create_computer", {"template": ""}, "'template'"), ("create_computer", {"template": "x", "task": "t1"}, "'task'"), + ("create_api_key", {"name": "a", "scopes": []}, "'scopes'"), ("create_api_key", {"name": "a", "scopes": ["root"]}, "'scopes[0]'"), + ("create_api_key", {"name": "a", "scopes": ["runs:read", "runs:read"]}, "'scopes'"), + ("create_api_key", {"name": "a", "scopes": ["runs:read"], "expires_in_days": 400}, "'expires_in_days'"), + ("read_audit_log", {"action": "Not Valid"}, "'action'"), ("create_run", {}, "'spec'"), ("create_run", {"spec": []}, "'spec'"), + ("get_computer", {"computer_id": "x" * 65}, "'computer_id'"), ("list_tasks", {"domain": 3}, "'domain'")] + for name, args, field in cases: + res, out = self.tool(name, args) + self.assertTrue(res["isError"], (name, args)) + self.assertIn(field, out, (name, args, out)) + self.assertEqual(FakeGateway.calls, []) + + def test_env_binding_fills_computer_id_for_desktop_tools_only(self): + bound = mcp.Server(self.client, default_computer_id="c1") + res, out = self.tool("screenshot", {}, srv=bound) + self.assertEqual(self.last()[1], "/v1/slots/c1/screenshot") + res, out = self.tool("verify", {}, srv=bound) + self.assertEqual(self.last()[1], "/v1/slots/c1/verify") + res, out = self.tool("get_computer", {}, srv=bound) # a workspace tool that takes computer_id is filled too + self.assertEqual(self.last()[1], "/v1/slots/c1") + res, out = self.tool("screenshot", {"computer_id": "c2"}, srv=bound) # an explicit id wins + self.assertEqual(self.last()[1], "/v1/slots/c2/screenshot") + res, out = self.tool("list_tasks", {}, srv=bound) # no computer_id in the schema: nothing added + self.assertEqual(self.last()[1], "/v1/tasks") + res, out = self.tool("screenshot", {}) # unbound: the schema's required field is missing + self.assertTrue(res["isError"]) + self.assertIn("'computer_id'", out) + + def test_wrong_token_is_a_tool_error_not_a_protocol_error(self): + srv = mcp.Server(drums.Client(self.url, "wrong", retry_s=0.5)) + r = self.rpc(srv, "tools/call", {"name": "list_computers", "arguments": {}}) + self.assertIn("result", r) + self.assertTrue(r["result"]["isError"]) + self.assertIn("401", r["result"]["content"][0]["text"]) + + def test_no_client_configured(self): + res, out = self.tool("list_computers", {}, srv=mcp.Server(None)) + self.assertTrue(res["isError"]) + self.assertIn("DRUMS_URL", out) + + +class FakeDesktopTests(unittest.TestCase): + """Every desktop tool once through the FakeEnv of test_hybrid_mcp (the stdlib snippets run on this machine).""" + + def setUp(self): + import tempfile + from test_hybrid_mcp import FakeEnv + self.tmp = tempfile.TemporaryDirectory() + + class Env(FakeEnv): + def __init__(s, root): + super().__init__(root) + s.record = {"id": "c1", "state": "ready", "episode": 1, "gates": {"x": 1}} + + def fork(s, n, strict=True): + return drums.ForkResult([drums.Env(None, {"id": f"c{10 + i}"}) for i in range(n)], "fork-1", {"id": "fork-1"}, [], 0.2) + + def reset(s): + s.record = dict(s.record, episode=s.record["episode"] + 1) + return s + + def load_task(s, task): + s.record = dict(s.record, task=task) + return s + self.env = Env(self.tmp.name) + self.ids = [] + + def factory(cid): + self.ids.append(cid) + return self.env + self.srv = mcp.Server(client=None, default_computer_id="c1", env_factory=factory) + + def tearDown(self): + self.tmp.cleanup() + + def tool(self, name, args=None): + res = self.srv.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": name, "arguments": args or {}}})["result"] + c = res["content"][0] + return res, (base64.b64decode(c["data"]) if c["type"] == "image" else c["text"] if res["isError"] else json.loads(c["text"])) + + def test_every_desktop_tool_once(self): + res, out = self.tool("write_file", {"path": "~/Desktop/notes.txt", "content": "hello\n"}) + self.assertTrue(out["ok"]) + self.assertEqual(self.ids, ["c1"]) # --env binding + res, out = self.tool("read_file", {"computer_id": "c1", "path": "~/Desktop/notes.txt"}) + self.assertEqual(out["text"], "hello\n") + res, out = self.tool("list_dir", {"path": "~/Desktop"}) + self.assertEqual([e["name"] for e in out["entries"]], ["notes.txt"]) + res, out = self.tool("shell", {"command": "echo hi"}) + self.assertEqual(out["stdout"].strip(), "hi") + res, out = self.tool("gui", {"code": "pyautogui.click(1, 2)"}) + self.assertEqual(out["returncode"], 0) + self.assertEqual(self.env.steps[-1], {"type": "pyautogui", "code": "pyautogui.click(1, 2)"}) + res, out = self.tool("screenshot", {}) + self.assertEqual(res["content"][0]["type"], "image") + self.assertEqual(out, b"\x89PNG fake") + res, out = self.tool("browser_tabs", {}) # no Chrome here: an error record, not a crash + self.assertIn("error", out) + res, out = self.tool("browser_open", {"url": "http://127.0.0.1:8000/"}) + self.assertIn("error", out) + res, out = self.tool("office_convert", {"path": "~/Desktop/notes.txt", "format": "pdf"}) + self.assertFalse(res["isError"]) # soffice is not here: ok False, with the log + self.assertIn("ok", out) + res, out = self.tool("spreadsheet_read", {"path": "~/Desktop/notes.txt"}) + self.assertFalse(res["isError"]) + res, out = self.tool("verify", {}) + self.assertEqual(out["score"], 1.0) + res, out = self.tool("fork", {"n": 2}) + self.assertEqual(out, {"children": ["c10", "c11"], "seconds": 0.2, "state": "fork-1"}) + res, out = self.tool("reset", {}) + self.assertEqual(out["episode"], 2) + self.assertNotIn("gates", out) + res, out = self.tool("load_task", {"task": {"id": "inline", "config": []}}) + self.assertEqual(out["task"]["id"], "inline") + res, out = self.tool("load_task", {"task_id": "t1"}) # needs the API to resolve the id: no client here + self.assertTrue(res["isError"]) + self.assertIn("DRUMS_URL", out) + + +if __name__ == "__main__": + unittest.main()