diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 8b5f101..c8ec477 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "plugins": [ { "name": "compartment", - "description": "Encrypted, fully offline memory for Claude Code. One vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client; 14 memory tools plus the /compartmentalize skill. Requires `pip install compartment && compartment init` first.", + "description": "Encrypted, fully offline memory for Claude Code. One vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client; 14 memory tools plus the compartmentalize skill. Requires `pip install compartment && compartment init` first.", "author": { "name": "Max Freedom Pollard" }, "category": "productivity", "source": "./", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index aad7aa1..e64c93b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,9 @@ { + "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "compartment", "displayName": "Compartment", - "description": "Encrypted, fully offline memory for Claude Code: one vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client. Adds 14 memory tools and the /compartmentalize skill. Requires `pip install compartment && compartment init` first (the vault passphrase is yours to choose).", + "version": "4.13.0", + "description": "Encrypted, fully offline memory for Claude Code: one vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client. Adds 14 memory tools and the compartmentalize skill (/compartment:compartmentalize). Requires `pip install compartment && compartment init` first (the vault passphrase is yours to choose).", "author": { "name": "Max Freedom Pollard", "url": "https://github.com/MaxFreedomPollard" diff --git a/.gitattributes b/.gitattributes index e26e145..f6534f4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,11 @@ src/compartment/models/** -text *.onnx -text *.mpack -text *.jsonl -text + +# Scripts a POSIX shell or a container reads. A Windows checkout with +# core.autocrlf rewrites them to CRLF, and bash then fails on `$'\r'` - the +# usual way to meet it is running install-linux.sh from that checkout inside +# WSL, or `docker build` from Windows copying CRLF files into the image. +*.sh text eol=lf +Dockerfile text eol=lf +.github/workflows/*.yml text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 33d4e98..81ddf00 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -23,6 +23,7 @@ body: - pip / pipx / uv (PyPI) - Compartment.pkg (macOS one-click) - Compartment.dmg + - A plugin or extension store (Claude Code, Gemini CLI, Hermes, Claude Desktop bundle...) - Docker - From source validations: @@ -30,8 +31,9 @@ body: - type: input id: platform attributes: - label: OS and Python - placeholder: "macOS 26.2 / Python 3.12.4" + label: OS, CPU and Python + description: Include the architecture (Apple silicon / Intel / ARM64 / x86_64), and say so if it runs under WSL, in a container, or from an agent's sandboxed shell. + placeholder: "macOS 26.2, Apple silicon / Python 3.12.4" validations: required: true - type: dropdown @@ -47,6 +49,12 @@ body: - Cline / Roo Code - Codex CLI - Gemini CLI + - VS Code / GitHub Copilot + - Windsurf + - Zed + - OpenCode + - Goose + - LM Studio - Other MCP client - CLI only - The menu bar / tray app diff --git a/.github/release-notes/v4.13.0.md b/.github/release-notes/v4.13.0.md new file mode 100644 index 0000000..f57b4b0 --- /dev/null +++ b/.github/release-notes/v4.13.0.md @@ -0,0 +1,111 @@ +Compartment 4.13.0 is a compatibility release: it is about Compartment installing, starting and staying connected on every system and in every agent, including the places it used to fail quietly - agents' sandboxed shells, Windows on ARM, Intel Macs, containers, and GUI apps that cannot see your PATH. 4.12.0 was merged but never published, so its 79 stability fixes ship here too. + +## Downloads + +| File | For | +|---|---| +| `Compartment-4.13.0.pkg` | macOS on Apple silicon, one click, with the menu bar app | +| `Compartment-4.13.0.dmg` | macOS on Apple silicon, drag to Applications | +| `Compartment-4.13.0-intel.pkg` | **new:** macOS on Intel, one click | +| `Compartment-4.13.0-intel.dmg` | **new:** macOS on Intel, drag to Applications | +| `compartment-4.13.0.mcpb` | **new:** Claude Desktop extension (attached a few minutes after publishing) | +| `compartment-4.13.0-py3-none-any.whl` | pip and offline installs, any OS | +| `compartment-4.13.0.tar.gz` | source | +| `SHA256SUMS.txt` | checksums of all of the above | + +Everything here was built from this tag on GitHub's machines, and PyPI carries the very same wheel and sdist. + +```bash +pipx install compartment && compartment init # or: uv tool install compartment +``` + +The Mac app needs macOS 13.4 or newer. The README's new **Where it runs** table gives every system, CPU and Python version CI tests. + +## Highlights + +- **Works from an agent's own shell.** Claude Code and Codex run commands in a sandbox that blocks the macOS boot clock, Unix sockets and some file modes. Every `compartment` command an agent ran there used to fail with "Cannot identify this boot session". It now uses the vault you unlocked, embeds in-process, and never hangs. +- **Clients can always start the server.** `compartment integrate` writes the full path of the installed command. GUI apps started from the Dock or Start menu do not inherit your PATH, and a bare `compartment` failed there with "spawn compartment ENOENT". +- **`pip install compartment` works on Windows on ARM and Intel Macs again.** Two indirect dependencies (cryptography, argon2-cffi-bindings) stopped shipping wheels there, and pip tried to compile them. A new CI check fails if any supported platform would need a compiler again. +- **Intel Mac installer.** The Mac app is built for Apple silicon and Intel, and no longer bundles libraries that need a newer macOS than it says it supports. +- **A plugin in every store.** + - New: a Claude Desktop extension (`.mcpb`). + - Updated: the Claude Code plugin now carries a version, plus better Codex and MCP Registry listings, and corrected Gemini CLI and LM Studio listings. + - Ready to submit: Docker MCP Catalog and Continue entries. + - The full list is in the new `docs/PLUGINS.md`. + +## Agents and clients + +- 33 clients, adding Kiro, Amazon Q Developer CLI, Junie, Factory and Crush. Continue gets its own block file, and Goose gets a real YAML extension entry. +- **New config locations:** Claude Desktop's Microsoft Store build, Cline 4, Kilo Code 7, Cline/Roo/Kilo inside Cursor, Windsurf, VSCodium and Flathub VS Code, and Zed on macOS. +- **Moved config homes followed:** `CODEX_HOME`, `GEMINI_CLI_HOME`, `QWEN_HOME`, `COPILOT_HOME`, `OPENCODE_CONFIG_DIR`, `GOOSE_PATH_ROOT`, `CLAUDE_CONFIG_DIR`, and OpenClaw's state and config variables. +- **OpenClaw:** the server is written under `mcp.servers`, where current OpenClaw reads it. An old entry is moved there. +- **Config edits keep what is already there:** + - Comments in Zed and VS Code files, and the user's own keys in a Codex entry. + - Each file's own line endings. + - Files saved by Notepad or PowerShell (byte order mark, UTF-16) are read correctly. + - `integrate codex` can no longer corrupt config.toml. +- **Claude Code capture hook:** quoted correctly for Git Bash on Windows and non-ASCII user names. It also follows `CLAUDE_CONFIG_DIR` and `autoMemoryDirectory`. +- **MCP:** + - Every tool says it never reaches the internet. + - Tool errors reach the model with their reason. + - A missing vault says "run `compartment init`" instead of "locked". + - `compartment serve --vault PATH` works. + - Closing a client's terminal keeps the session's audit rows. + - mcp 1.x no longer logs a stderr line per request. + - The real floor is mcp 1.21.2: nothing older runs beside today's pydantic. + +## Operating systems + +- **Windows:** + - A machine up for more than 24.9 days no longer relocks the vault and deletes the unlock credential. + - A process started without USERNAME no longer deletes everyone's credential. + - No console windows behind the tray or the panel's buttons, and start at sign-in uses pythonw. + - With fast user switching, another user's tray is not taken for yours. + - Saves outlast OneDrive and antivirus holding the vault. + - The lock always takes byte zero. + - The Visual C++ message names the ARM64 installer on ARM. +- **macOS:** + - An acknowledged store survives a power cut (F_FULLFSYNC). + - The boot time is read in-process instead of forking `sysctl` on every vault open. +- **Linux:** + - Start at login keeps working on Xfce, MATE, Cinnamon and LXQt. + - It is refused, with a reason, inside Flatpak or a strict Snap. + - A missing Tk names the exact package for your distribution. + - `compartment dash` works on WSL (it opens the Windows browser) and on headless machines (it prints how to reach it). +- **Containers:** + - A container run as an arbitrary user with no home folder works, with `COMPARTMENT_PASSPHRASE` or `COMPARTMENT_SESSION_DIR`. + - Inference threads respect CPU quotas. + - The Docker image reads a host user's vault, runs with `--network none` and keeps the offline guard on. +- **BSD, Alpine, Termux:** + - FreeBSD, NetBSD, DragonFly and OpenBSD get the boot clock. + - OpenBSD's on-disk shared memory is refused for the unlock secret. + - The installers point Alpine, FreeBSD and Termux at their own onnxruntime package. +- **Installers:** + - They handle "externally-managed-environment" Pythons (Debian 12+, Ubuntu 23.04+, Fedora, Arch, Homebrew). + - They pick a Python that works on the machine. + - They survive being re-run or piped into a shell. +- **Offline guard:** it now refuses every socket family that can reach another machine, not just IP: Bluetooth, VSOCK/Hyper-V (WSL 2's host) and packet sockets. + +## Storage + +- A disk that fills up or a sync that fails part way through a store no longer leaves a vault that will not open. +- Text containing half an emoji (a lone surrogate) is stored and searched instead of failing. +- Imports with odd dates, NaN, or Unicode line separators no longer break search. +- A save by another process is noticed even on filesystems with coarse timestamps. +- A store whose journal write failed is never treated as written. +- A vault on a filesystem that cannot lock says so at once. +- Above 20,000 records the exact index stays when usearch has no SIMD (as inside a sandbox). Rebuilding HNSW there took 50 s at 25,000 records on every open. +- A Python whose SQLite lacks FTS5 gets a clear message. +- `COMPARTMENT_PASSPHRASE` is still tried when the boot-session lookup fails. + +## For maintainers + +- **Release workflow:** a tag now builds every download, Intel included, and drafts the release. Publishing it uploads those same files to PyPI, then builds and attaches the Claude Desktop extension. +- **CI coverage:** + - Every declared Python (3.11-3.14) on Windows, macOS and Linux, plus ARM64 Linux, Windows on ARM and an Intel Mac. + - The macOS app on both architectures. + - The built wheel installed with pipx and driven over MCP on each system. + - The suite at the lowest dependency versions the bounds allow. + - A weekly wheel-coverage check. +- **Failure reports:** a red job states its failing tests on the run page. +- **One-time store submissions** that need an account: listed in `docs/PLUGINS.md`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0adc973..0024e0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,20 @@ name: CI on: push: pull_request: + # Weekly, because a dependency's new release can break an install on some + # platform without any commit here (see wheel-coverage). + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: + +# One run per branch at a time. A push to a pull request's branch fires both +# `push` and `pull_request`, and every new commit used to leave the previous +# runs going too, so the macOS jobs - the slowest runners to come free - +# queued behind runs nobody would read. Keyed on the branch for both events, +# the newest run cancels the rest; the weekly run has its own group. +concurrency: + group: ci-${{ github.event_name == 'schedule' && 'weekly' || (github.head_ref || github.ref_name) }} + cancel-in-progress: true jobs: test-offline: @@ -14,11 +28,34 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python: ["3.11", "3.12"] + # Every Python the package declares. 3.13 and 3.14 were classifiers + # nothing ran, and 3.14 is what a fresh Homebrew, Fedora or python.org + # install hands people now. + python: ["3.11", "3.12", "3.13", "3.14"] + include: + # ARM64 Linux (Graviton, Ampere, Raspberry Pi 5, Apple silicon VMs) + # and Windows on ARM (Snapdragon laptops): onnxruntime ships wheels + # for both, and "it installed but the native library will not load" + # is the failure only a real machine of that architecture shows. + - os: ubuntu-24.04-arm + python: "3.12" + - os: windows-11-arm + python: "3.12" + # Intel Macs. onnxruntime's last macOS x86_64 wheels are 1.23.2, + # built for Python up to 3.13, so this is the newest pairing that + # installs there, and it proves the code still runs on that older + # onnxruntime as well as on the newest one above. + - os: macos-15-intel + python: "3.13" runs-on: ${{ matrix.os }} + # The suite takes minutes. A test that hangs (a pipe never closed, a + # child never reaped) used to hold a runner for six hours - and on macOS, + # where runners are scarce, every other macOS job behind it. pytest's + # --timeout below fails the one test with its stack; this caps the rest. + timeout-minutes: 45 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python }} - name: Install (network allowed for dependency download only) @@ -26,17 +63,17 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: pytest tests/ -q --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/ -q --timeout=300 -o faulthandler_timeout=120 --cov=compartment --cov-report=xml --cov-append - name: Dashboard tests (loopback socket; offline guard intentionally off) # `compartment dash` binds 127.0.0.1 by design and is never run under the # offline guard, so its tests run here with the guard off. - run: pytest tests/test_dash.py -q --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/test_dash.py -q --timeout=300 -o faulthandler_timeout=120 --cov=compartment --cov-report=xml --cov-append - name: Coverage to Codecov # Never fail the build on a reporting service: the offline guard and the # test suite are the gate here, not the upload. - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v7 continue-on-error: true with: files: ./coverage.xml @@ -44,6 +81,157 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false + real-sockets: + # The shared embedding daemon and the stdio server need real Unix sockets + # and pipes, which agent sandboxes refuse, so these are the tests that + # only ever run on a CI runner. They run here on their own, verbose, with + # every thread's stack dumped for a test that stalls, so a hang names + # itself within minutes instead of holding a runner until its timeout. + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: pip install -e .[dev] + - name: Daemon and stdio tests, verbose + env: + CI_REPORT_STALL_S: "75" + run: python tools/ci_report.py tests/test_embed_daemon.py tests/test_server_stdio.py -v --timeout=120 -o faulthandler_timeout=60 + + wheel-smoke: + # The suite runs against an editable checkout, which hides packaging + # mistakes: a model file or the starter pack missing from the wheel, an + # entry point that only resolves from source, a server that starts in a + # test but not as a console script. This builds the wheel, installs it + # the way the README tells people to (pipx), and uses it like a person + # and like an MCP client would, offline guard on. + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 25 + env: + COMPARTMENT_ASSERT_OFFLINE: "1" + COMPARTMENT_PASSPHRASE: ci-only-not-a-secret + PYTHONIOENCODING: utf-8 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Build the wheel + shell: bash + run: | + python -m pip install --upgrade build + python -m build --wheel + ls -l dist + - name: Install it with pipx, as the README says + shell: bash + env: + COMPARTMENT_ASSERT_OFFLINE: "" + run: | + pipx install --python python dist/*.whl + pipx ensurepath + - name: Create, write, read, verify + shell: bash + run: | + compartment --version + compartment init --passphrase ci-only-not-a-secret --no-app + compartment store --source "from chat" "Deploys go out Tuesday and Thursday mornings." + compartment search "when do deploys go out" | tee search.txt + grep -q "Tuesday and Thursday" search.txt + compartment selftest + compartment status + - name: An MCP client's handshake against the installed server + shell: bash + run: python tools/mcp_smoke.py --command compartment + + macos-app: + # The same build the Release workflow runs at tag time, so a broken + # installer shows up on the pull request that broke it rather than on + # release day. The .pkg and .dmg are kept for a week for anyone who + # wants to try a change before it ships. + if: github.event_name != 'schedule' + strategy: + fail-fast: false + matrix: + include: + - runner: macos-latest + arch: apple-silicon + - runner: macos-15-intel + arch: intel + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.0 # exact: no floating major tags are published + - name: A standalone, non-framework Python to embed + shell: bash + run: | + uv python install 3.12 + echo "EMBED_PY=$(uv python find --managed-python 3.12)" >> "$GITHUB_ENV" + - name: Build Compartment.app, the .pkg and the .dmg + shell: bash + run: | + "$EMBED_PY" tools/build_macos_app.py --python "$EMBED_PY" --pkg --dmg + ls -l build + - uses: actions/upload-artifact@v7 + with: + name: compartment-macos-${{ matrix.arch }} + path: | + build/Compartment-*.pkg + build/Compartment-*.dmg + retention-days: 7 + if-no-files-found: error + + wheel-coverage: + # pip installs the newest release of every dependency and compiles it when + # that release has no wheel for the machine. That is how a plain + # `pip install compartment` broke on Windows on ARM and Intel Macs: an + # indirect dependency stopped shipping wheels there. This resolves the + # dependencies for every supported platform and Python, as pip would and + # wheels-only, and fails when they differ. A dependency release can break + # this without a commit here, so it also runs weekly. + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: python -m pip install uv + - run: python tools/wheel_coverage.py + + lowest-dependencies: + # pyproject.toml's lower bounds are a promise to everyone whose distro or + # lock file holds an older library. It used to promise mcp>=1.0, and no + # release before 1.21.2 runs the server beside today's pydantic. This + # installs the lowest version of every direct dependency the bounds + # allow and runs the suite, so a bound is a tested fact, not a guess. + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + - name: Install at the lowest versions the bounds allow + run: | + python -m pip install uv + uv pip install --system --resolution lowest-direct -e ".[dev]" + python -m pip list + - name: Test suite with offline guard active + env: + COMPARTMENT_ASSERT_OFFLINE: "1" + run: python tools/ci_report.py tests/ -q --timeout=300 -o faulthandler_timeout=120 --ignore=tests/test_dash.py + linux-panel: # Linux has no notification area worth relying on, so it draws the same # panel as an ordinary window. "It opens and stays open" is the failure @@ -52,8 +240,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: The Tk package, which pip cannot install @@ -129,8 +317,8 @@ jobs: # locked, and the next command sat asking for input. timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Install with the tray extra @@ -178,16 +366,25 @@ jobs: # `compartment init` above already started the tray app, which is the # whole point of it. So a launch now must hand off and exit 0, and # the copy that was already there must still be running afterwards. + # Counted by command line, not by process name: init starts the tray + # with pythonw.exe (no console window), so there is no process called + # "compartment" to count. shell: pwsh run: | - $before = @(Get-Process compartment -ErrorAction SilentlyContinue).Count + function Get-Tray { + @(Get-CimInstance Win32_Process | Where-Object { + $_.CommandLine -and $_.CommandLine -match 'compartment' -and + $_.CommandLine -match '\btray\b' }) + } + $before = (Get-Tray).Count + Get-Tray | ForEach-Object { Write-Host " running: $($_.Name): $($_.CommandLine)" } if ($before -lt 1) { Write-Error "init did not start the tray app"; exit 1 } compartment tray --show if ($LASTEXITCODE -ne 0) { Write-Error "the second copy failed instead of standing down"; exit 1 } - $after = @(Get-Process compartment -ErrorAction SilentlyContinue).Count + $after = (Get-Tray).Count if ($after -ne $before) { Write-Error "second copy changed the count: $before -> $after"; exit 1 } - Write-Host "one copy before, one copy after" - Get-Process compartment -ErrorAction SilentlyContinue | Stop-Process -Force + Write-Host "the same $before process(es) before and after" + Get-Tray | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } Start-Sleep -Seconds 2 - name: Start the tray app and prove it is still alive @@ -285,7 +482,7 @@ jobs: [System.Drawing.Imaging.ImageFormat]::Png) Write-Host "panel: ${w}x${ht}" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: windows-tray-screenshot path: | diff --git a/.github/workflows/publish-registry.yml b/.github/workflows/publish-registry.yml index 601c4d1..e096dff 100644 --- a/.github/workflows/publish-registry.yml +++ b/.github/workflows/publish-registry.yml @@ -25,7 +25,7 @@ jobs: publish-registry: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install mcp-publisher run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0e8075b..6fb5a30 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,17 +23,44 @@ jobs: environment: pypi permissions: id-token: write # required for Trusted Publishing (OIDC) + contents: read # to download the release's own files steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" - - name: Build wheel + sdist + # PyPI gets the very wheel and sdist attached to the GitHub release + # (release.yml builds them from the tag and lists their SHA-256 sums), + # so "pip install compartment" and the release page hand out identical + # bytes. A manual run, or a release drafted by hand without them, + # falls back to building from this checkout as before. + - name: Take the release's wheel + sdist, or build them + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + shell: bash run: | - python -m pip install --upgrade build - python -m build + mkdir -p dist + if [ -n "$TAG" ]; then + gh release download "$TAG" -R "$GITHUB_REPOSITORY" -D dist \ + -p "compartment-*.whl" -p "compartment-*.tar.gz" || true + fi + if ls dist/*.whl >/dev/null 2>&1 && ls dist/*.tar.gz >/dev/null 2>&1; then + echo "publishing the release's own files:" + else + rm -rf dist + python -m pip install --upgrade build + python -m build + echo "built from the checkout:" + fi + ls -l dist + if [ -n "$TAG" ]; then + v="${TAG#v}" + ls dist | grep -q "compartment-$v" || { + echo "::error::dist/ holds no compartment-$v files for tag $TAG"; exit 1; } + fi - name: Check metadata run: | @@ -68,9 +95,9 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -86,3 +113,46 @@ jobs: with: packages-dir: packaging/engram-memory-vault/dist skip-existing: true # only the first release of a given version uploads + + # The Claude Desktop extension. Its uv.lock pins this release from PyPI + # with hashes, so it can only be resolved once PyPI serves the version: + # this waits for it, builds dist/compartment-.mcpb and attaches it to + # the release the rest of the downloads are already on. + claude-desktop-bundle: + runs-on: ubuntu-latest + needs: build-and-publish + if: github.event_name == 'release' + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.release.tag_name }} + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Wait for the version to reach PyPI + shell: bash + env: + TAG: ${{ github.event.release.tag_name }} + run: | + v="${TAG#v}" + for attempt in $(seq 1 60); do + code=$(curl -s -o /dev/null -w '%{http_code}' \ + "https://pypi.org/pypi/compartment/$v/json" || echo 000) + [ "$code" = "200" ] && { echo "on PyPI after $attempt attempt(s)"; exit 0; } + sleep 10 + done + echo "::error::compartment $v never appeared on PyPI"; exit 1 + + - name: Build the bundle and attach it to the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + shell: bash + run: | + python -m pip install uv + python tools/build_mcpb.py + gh release upload "$TAG" dist/compartment-*.mcpb --clobber diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..32a614b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,183 @@ +name: Release + +# Every release ships every download (RELEASING.md). They used to be built by +# hand on one laptop, which tied each release to that laptop and could only +# ever produce installers for the laptop's own architecture. This builds all +# of them on GitHub's machines from the tagged commit - the wheel, the sdist, +# and the macOS .pkg and .dmg for Apple silicon AND for Intel - and attaches +# them, with their SHA-256 sums, to a DRAFT release. +# +# Publishing the draft stays a human decision. Publishing is what triggers +# publish.yml (PyPI, which uploads these very wheel and sdist files) and +# publish-registry.yml (the MCP registry). +# +# git tag v4.13.0 && git push origin v4.13.0 # then review the draft + +on: + push: + tags: ["v*.*.*"] + workflow_dispatch: + inputs: + tag: + description: "An existing tag to build or rebuild, e.g. v4.13.0" + required: true + +permissions: + contents: read + +concurrency: + group: release-${{ inputs.tag || github.ref_name }} + cancel-in-progress: false + +jobs: + check: + name: The tag and every version field agree + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.v.outputs.tag }} + version: ${{ steps.v.outputs.version }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag || github.ref }} + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - id: v + name: The tag names the package version + shell: bash + env: + TAG: ${{ inputs.tag || github.ref_name }} + run: | + version=$(python -c "import re, pathlib; print(re.search(r'__version__\s*=\s*\"([^\"]+)\"', pathlib.Path('src/compartment/__init__.py').read_text(encoding='utf-8')).group(1))") + if [ "$TAG" != "v$version" ]; then + echo "::error::tag $TAG does not match the package version $version" + exit 1 + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Every manifest carries that version + # The lockstep test reads files only, so it runs without the package's + # dependencies and without the suite's conftest. + run: | + python -m pip install pytest + python -m pytest -q -p no:cacheprovider --noconftest tests/test_version_lockstep.py + + python-dist: + name: Wheel and sdist + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.check.outputs.tag }} + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Build and check the metadata + run: | + python -m pip install --upgrade build twine + python -m build + python -m twine check --strict dist/* + ls -l dist + - name: The wheel installs and starts + env: + COMPARTMENT_ASSERT_OFFLINE: "1" + run: | + python -m pip install dist/*.whl + compartment --version | grep -F "${{ needs.check.outputs.version }}" + - uses: actions/upload-artifact@v7 + with: + name: python-dist + path: dist/* + if-no-files-found: error + + macos-app: + name: macOS app (${{ matrix.arch }}) + needs: check + strategy: + fail-fast: false + matrix: + include: + - runner: macos-latest + arch: apple-silicon + # onnxruntime's last macOS x86_64 wheels are 1.23.2 for Python up + # to 3.13; the build selects it on its own. + - runner: macos-15-intel + arch: intel + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.check.outputs.tag }} + - uses: astral-sh/setup-uv@v10.3.0 # exact: no floating major tags are published + - name: A standalone, non-framework Python to embed + # tools/build_macos_app.py explains why it must not be a framework + # build: the app would then report itself as org.python.python. + shell: bash + run: | + uv python install 3.12 + echo "EMBED_PY=$(uv python find --managed-python 3.12)" >> "$GITHUB_ENV" + - name: Build Compartment.app, the .pkg and the .dmg + # On an Intel runner the script names them Compartment--intel.* + # itself, and the installer refuses to run on the other architecture. + shell: bash + run: | + "$EMBED_PY" tools/build_macos_app.py --python "$EMBED_PY" --pkg --dmg + ls -l build + - name: Exactly one .pkg and one .dmg of this version + shell: bash + env: + V: ${{ needs.check.outputs.version }} + run: | + mkdir -p out + cp build/Compartment-"$V"*.pkg build/Compartment-"$V"*.dmg out/ + ls -l out + test "$(ls out | wc -l)" -eq 2 + - uses: actions/upload-artifact@v7 + with: + name: macos-${{ matrix.arch }} + path: out/* + if-no-files-found: error + + draft: + name: Draft the release with every download + needs: [check, python-dist, macos-app] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.check.outputs.tag }} + - uses: actions/download-artifact@v8 + with: + path: assets + merge-multiple: true + - name: Checksums + working-directory: assets + run: | + sha256sum * > SHA256SUMS.txt + cat SHA256SUMS.txt + - name: Create the draft, or refresh its downloads on a rebuild + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.check.outputs.tag }} + VERSION: ${{ needs.check.outputs.version }} + shell: bash + run: | + notes=".github/release-notes/$TAG.md" + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" assets/* --clobber + echo "refreshed the downloads on the existing release $TAG" + else + if [ -f "$notes" ]; then + gh release create "$TAG" --draft --verify-tag \ + --title "Compartment $VERSION" --notes-file "$notes" assets/* + else + gh release create "$TAG" --draft --verify-tag \ + --title "Compartment $VERSION" --generate-notes assets/* + fi + echo "drafted $TAG; review it and press Publish" + fi diff --git a/Dockerfile b/Dockerfile index b2dd465..1599326 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,13 @@ # # The default embedding model ships inside the wheel, so the network is needed # only while this image is being built. At runtime the container makes no -# outbound connection. +# outbound connection, and it is held to that twice: the offline guard is on +# (COMPARTMENT_ASSERT_OFFLINE=1, below), and the run line gives it no network. # # Build: docker build -t compartment . -# Run: docker run -i --rm \ +# docker buildx build --platform linux/amd64,linux/arm64 -t compartment . +# Run: docker run -i --rm --network none \ +# --user "$(id -u):$(id -g)" \ # -v "$HOME/.compartment:/data" \ # -e COMPARTMENT_PASSPHRASE \ # compartment @@ -18,26 +21,47 @@ # /data. Create it once on the host with `compartment init` before the first # container run, because init prompts for a passphrase and that passphrase is # the only key to the vault. +# +# --user: the vault is mode 0600 and owned by you, so on a Linux host the +# image's own unprivileged user (uid 10001) cannot read a bind-mounted one; +# running as your own uid can. (Docker Desktop on macOS and Windows maps +# ownership itself and needs no --user.) A uid the image has never heard of +# gets HOME=/, which is fine: with the passphrase in the environment nothing +# is written outside /data. +# +# linux/amd64 and linux/arm64 only: onnxruntime, which Compartment cannot run +# without, publishes Linux wheels for x86_64 and aarch64 and nothing else. -FROM python:3.11-slim AS build +FROM python:3.13-slim AS build WORKDIR /src -COPY . /src +# Only what the wheel is built from. `COPY .` also brought in whatever the +# checkout had lying about - a local build/ directory, whose stale build/lib +# setuptools folds into the wheel, a virtualenv, .git - into the build. +COPY pyproject.toml README.md LICENSE /src/ +COPY src /src/src RUN pip install --no-cache-dir build \ && python -m build --wheel --outdir /dist -FROM python:3.11-slim +FROM python:3.13-slim # Unbuffered so the stdio transport is not held up by Python's block buffering. +# The offline guard is sealed on for every process in the container. The +# shared embedding daemon is for a machine full of agents sharing one model; +# a container runs one server, and the daemon would be one more process +# wanting a writable session directory. ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 + PYTHONDONTWRITEBYTECODE=1 \ + COMPARTMENT_ASSERT_OFFLINE=1 \ + COMPARTMENT_EMBED_DAEMON=0 COPY --from=build /dist/*.whl /tmp/ RUN pip install --no-cache-dir /tmp/*.whl \ && rm -rf /tmp/*.whl # Runs unprivileged. /data is the mount point for the host's vault, so the uid -# here has to be able to read and write whatever is mounted there. +# here has to be able to read and write whatever is mounted there (see --user +# above). RUN useradd --create-home --uid 10001 compartment \ && mkdir -p /data \ && chown compartment:compartment /data @@ -45,5 +69,9 @@ USER compartment VOLUME ["/data"] +# A stdio server has no listener for a probe to reach, and its health is the +# client's pipe: the MCP client sees at once when the server stops answering. +HEALTHCHECK NONE + # No EXPOSE and no port: stdio only, zero listeners. ENTRYPOINT ["compartment", "--vault", "/data/memory.vault", "--caller", "docker", "serve"] diff --git a/FORMAT.md b/FORMAT.md index f03e678..7ec8813 100644 --- a/FORMAT.md +++ b/FORMAT.md @@ -99,7 +99,10 @@ u32 section_count, then per section: Each entry: `AEAD_seal(master_key, canonical_json(entry), aad="compartment-journal:"+vault_id+":"+u64_be(seq))` where `seq` starts at 0 after each compaction and increments per entry. Entries are fsync'd on -append (acknowledged). Ops: +append (acknowledged; F_FULLFSYNC on Apple platforms, where fsync stops at +the drive's volatile cache). An append the disk refuses part way (ENOSPC, EDQUOT, +a failed sync) is truncated back to where it began, so an unacknowledged +write leaves no bytes behind. Ops: ```json {"op":"store","audit":AuditRow,"record":{"id","ns","text","vec":base64_f32, @@ -146,7 +149,8 @@ requires the chain to extend the anchor. The anchor is mandatory: its absence is reported as tampering, not as an older vault. Compaction (`save`/`lock`): serialize → seal → write `path.tmp` → fsync → -atomic rename; journal restarts empty. +atomic rename → fsync of the containing folder (POSIX), so the rename itself +survives power loss; journal restarts empty. ### Manifest (signed vaults) diff --git a/README.md b/README.md index 4a5eb76..4da0c26 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ network, no telemetry. [![mcpservers.org](https://img.shields.io/badge/mcpservers.org-listed-8b5cf6)](https://mcpservers.org/servers/maxfreedompollard/compartment) [![LobeHub](https://lobehub.com/badge/mcp/maxfreedompollard-compartment)](https://lobehub.com/mcp/maxfreedompollard-compartment) -**One-click install** (after `pip install compartment && compartment init`): +**One-click install** (after [installing](#install) and running `compartment init`):

Add to Cursor @@ -381,7 +381,10 @@ sized to the requested number of results could be emptied by them while matching memories sit just past the cut. The pool starts at 200 per channel and widens up to three times when filtering leaves too few. Below 20,000 records the vector search is exact (SIMD matrix math, recall 1.0); above -that, HNSW at about 99% recall. +that, HNSW at about 99% recall, unless usearch can use no SIMD on the +machine - an agent's sandbox can hide the CPU features it probes - in which +case the exact index stays, because rebuilding HNSW without SIMD on every +open cost from 50 seconds at 25,000 records. ## Security and the lock model @@ -504,7 +507,8 @@ own size. ## Install -**No command line needed.** On a Mac, download **Compartment.pkg** from the +**No command line needed.** On a Mac, download **Compartment-<version>.pkg** +(Apple silicon) or **Compartment-<version>-intel.pkg** (Intel) from the [latest release](https://github.com/MaxFreedomPollard/Compartment/releases/latest) and open it. Python, the embedding model and every dependency are inside it. It asks you to choose a passphrase, creates the vault, and puts Compartment @@ -514,14 +518,63 @@ From the command line, on any system: | | | |---|---| -| **pip** (macOS, Linux, Windows) | `pip install compartment && compartment init` | -| **pipx / uv** | `pipx install compartment` or `uv tool install compartment`, then `compartment init` | -| **Claude Code plugin** | after `pip install compartment && compartment init`: `/plugin marketplace add MaxFreedomPollard/Compartment`, then `/plugin install compartment@maxfreedompollard`. Codex reads the same marketplace file | +| **pipx** (recommended) | `pipx install compartment && compartment init` | +| **uv** | `uv tool install compartment && compartment init` | +| **pip** | `pip install compartment && compartment init` inside a virtual environment. Debian 12+, Ubuntu 23.04+, Fedora, Arch and Homebrew refuse a system-wide `pip install` with "externally-managed-environment"; pipx and uv avoid that. On Windows, `py -m pip install compartment` | +| **Install script** | from a checkout: `bash install/install-linux.sh` (Linux, WSL, ChromeOS), `bash install/install-macos.sh`, or `powershell -ExecutionPolicy Bypass -File install\install-windows.ps1`. Each picks a Python that works on the machine, handles the externally-managed case, creates the vault and connects the agents it finds | | **Docker** | `docker build -t compartment .` from a checkout; see [Wiring each agent](#wiring-each-agent) | -The pip route needs Python 3.11 or newer. The app runs on macOS 13 or -newer, on Windows with the Microsoft Visual C++ runtime installed, and on -any Linux desktop. +Agents can also install it from their own stores - the Claude Desktop +extension (`compartment-.mcpb` on each release), the Claude Code and +Codex plugin marketplaces, Gemini CLI extensions, Hermes Agent plugins and +more. Each still needs the vault from `compartment init`, because the +passphrase is typed by you. [docs/PLUGINS.md](docs/PLUGINS.md) lists every +store and its install command. + +### Where it runs + +| System | CPU | Python for pip / pipx / uv | The app | +|---|---|---|---| +| macOS 14 or newer | Apple silicon | 3.11 - 3.14 | menu bar; `.pkg` / `.dmg` | +| macOS 13.4 or newer, and every Intel Mac | Apple silicon, Intel | 3.11 - 3.13 | menu bar; `-intel.pkg` / `-intel.dmg` on Intel | +| Windows 10 / 11 | x64 | 3.11 - 3.14 | tray icon | +| Windows 11 on ARM | ARM64 | 3.12 - 3.14 | tray icon | +| Linux with glibc 2.28+ (Ubuntu 20.04+, Debian 11+, Fedora, RHEL 8+, Arch, openSUSE), WSL 2, ChromeOS Linux | x86_64, aarch64 (incl. Raspberry Pi 5) | 3.11 - 3.14 | a window (needs Tk: `python3-tk` or your distro's equivalent) | +| Docker | amd64, arm64 | in the image | headless | + +The limits come from onnxruntime, the one dependency without wheels +everywhere: its macOS builds need macOS 14 from 1.24 on, and its last Intel +Mac build (1.23.2) stops at Python 3.13. CI runs the suite on every row and +checks the whole matrix for a dependency that would have to be compiled +(`tools/wheel_coverage.py`). On Windows, onnxruntime also needs the +[Microsoft Visual C++ runtime](https://aka.ms/vs/17/release/vc_redist.x64.exe) +([ARM64](https://aka.ms/vs/17/release/vc_redist.arm64.exe)); Compartment says +so if it is missing. + +**Elsewhere, with the system's onnxruntime.** Alpine (musl), FreeBSD and +Android (Termux) package onnxruntime themselves; install it from the system +and give Compartment a virtual environment that can see it. The install +script does this for you. + +```bash +# Alpine +apk add python3 py3-onnxruntime +# FreeBSD (default Python 3.12; the compiled dependencies come from packages) +pkg install py312-onnxruntime py312-tokenizers py312-pydantic-core py312-pynacl py312-argon2-cffi py312-numpy py312-cryptography +# Termux +pkg install python onnxruntime python-numpy rust + +# then, on any of them +python3 -m venv --system-site-packages ~/.compartment-venv +~/.compartment-venv/bin/pip install compartment && ~/.compartment-venv/bin/compartment init +``` + +On Termux there is no shared memory for the +unlock-until-restart credential, so set `COMPARTMENT_PASSPHRASE` or use +`compartment unlock --once`. A glibc older than 2.28, 32-bit systems, armv7 +and riscv64 are not supported. On WSL 2, the VM shutting down after it has +been idle clears the unlock credential, so the vault relocks between +sessions. `init` asks you to choose a passphrase, creates the vault, loads the reference facts, connects Claude Code, Hermes Agent or OpenClaw if they are @@ -534,7 +587,7 @@ To connect an agent later, or any other client: compartment integrate claude # Claude Code + Claude Desktop compartment integrate hermes # Hermes Agent compartment integrate openclaw # OpenClaw -compartment integrate --list # the 28 MCP clients it can wire: Cursor, VS Code, Cline, Roo Code, Zed, OpenCode, Codex CLI, Gemini CLI, Oh My Pi, LM Studio, AnythingLLM, BoltAI ... +compartment integrate --list # the 33 MCP clients it can wire: Cursor, VS Code, Windsurf, Cline, Roo Code, Kilo Code, Zed, OpenCode, Codex CLI, Gemini CLI, Kiro, Amazon Q, Junie, Goose, LM Studio, Continue ... compartment integrate --all # every one of them that is installed here ``` @@ -546,6 +599,11 @@ installed in their skills directories. Any other MCP client uses this block { "mcpServers": { "compartment": { "command": "compartment", "args": ["serve"] } } } ``` +An app started from the Dock, Finder or the Start menu does not see your +shell's PATH, so a bare `compartment` can fail there with "spawn compartment +ENOENT". Use the full path that `which compartment` (`where compartment` on +Windows) prints; `integrate` always writes that full path. + ## Wiring each agent None of this needs a terminal: the **Connect an agent** buttons in the app @@ -607,7 +665,7 @@ backup. Then run `openclaw gateway restart` and check with **Any MCP client** -`compartment integrate ` wires any of the 28 clients in `--list`. +`compartment integrate ` wires any of the 33 clients in `--list`. Each config write takes a byte-exact backup first, merges rather than replaces, writes atomically, and refuses to touch a file it cannot parse (it prints the block to paste instead). To do it by hand, use the block in @@ -620,10 +678,19 @@ Client-by-client walkthroughs are in **Docker** -`docker build -t compartment .` from a checkout builds a headless image: -stdio only, no port, unprivileged user, vault on a bind mount at `/data`. -Create the vault on the host first with `compartment init`, because that -step prompts for the passphrase. +`docker build -t compartment .` from a checkout builds a headless image +(amd64 and arm64): stdio only, no port, unprivileged user, the offline guard +on, vault on a bind mount at `/data`. Create the vault on the host first +with `compartment init`, because that step prompts for the passphrase, then +give the client this command: + +```bash +docker run -i --rm --network none --user "$(id -u):$(id -g)" \ + -v "$HOME/.compartment:/data" -e COMPARTMENT_PASSPHRASE compartment +``` + +`--user` lets the container read your 0600 vault on a Linux host; Docker +Desktop on macOS and Windows does not need it. ## Configuration @@ -748,6 +815,7 @@ is off unless you turn it on above. | | | |---|---| | [docs/MEMORY.md](docs/MEMORY.md) | how memory is stored, what gets remembered, and the ranking design | +| [docs/PLUGINS.md](docs/PLUGINS.md) | every plugin store and listing: how users install from it, what the maintainer does | | [docs/INTEGRATIONS.md](docs/INTEGRATIONS.md) | selecting Compartment in Hermes Agent, OpenClaw, Claude, everything else | | [docs/COMPARISON.md](docs/COMPARISON.md) | other memory servers, with sources | | [SECURITY.md](SECURITY.md) | the full threat model and its limits | diff --git a/RELEASING.md b/RELEASING.md index 14c6ea8..ca7aac9 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -7,14 +7,20 @@ most convenient one. Someone arriving at the release page should not have to know which artifact they need, go hunting through PyPI, or find that the only link is for a platform they are not on. -Attach all four, and list them together in the notes: +Attach all of them, and list them together in the notes. The Release +workflow (`.github/workflows/release.yml`) builds every one from the tag on +GitHub's machines and attaches them to a draft, so nobody has to remember: | Artifact | For | |---|---| -| `Compartment-.pkg` | macOS, one click, with the optional menu bar utility | -| `Compartment-.dmg` | macOS, drag to Applications | +| `Compartment-.pkg` | macOS on Apple silicon, one click, with the optional menu bar utility | +| `Compartment-.dmg` | macOS on Apple silicon, drag to Applications | +| `Compartment--intel.pkg` | macOS on Intel, one click | +| `Compartment--intel.dmg` | macOS on Intel, drag to Applications | | `compartment--py3-none-any.whl` | pip / offline installs, any OS | | `compartment-.tar.gz` | source distribution, packagers, air-gapped builds | +| `SHA256SUMS.txt` | the checksums of everything above | +| `compartment-.mcpb` | Claude Desktop extension (attached by publish.yml once PyPI serves the version, because its lock file pins that release) | GitHub adds "Source code (zip/tar.gz)" on its own. The notes should also carry the `pip install compartment` line for people who want the CLI and MCP @@ -33,6 +39,12 @@ server without the app. # server.json version = "2.1.0" the SEMVER form # plugin.json version = "2.1.0" Agent Plugins manifest # gemini-extension.json version = "2.1.0" Gemini CLI extension manifest +# .claude-plugin/plugin.json version = "2.1.0" Claude Code plugin (Codex, +# OpenClaw read it too) +# packaging/mcpb/manifest.json version = "2.1.0" Claude Desktop extension +# packaging/mcpb/pyproject.toml version = "2.1" and its pin, +# "compartment==2.1" which is what it installs +# packaging/continue/compartment.yaml version: 2.1 Continue hub block # # The two server.json fields are deliberately NOT the same value. Step 5 # explains why; setting them both to "2.1" is what published a registry @@ -49,22 +61,33 @@ server without the app. # plugin.yaml pair sat at 1.7.0 while the package shipped 4.9.2. # # tests/test_version_lockstep.py now fails when any of them disagree, so -# this list is a gate rather than a thing to remember. +# this list is a gate rather than a thing to remember. It also fails when +# a file anywhere in the tree carries a version it does not check, so a +# new manifest cannot join the list silently. docs/PLUGINS.md says what +# each store needs after the bump. python -m pytest -q # must be green -# 2. python artifacts -rm -rf dist && python -m build - -# 3. macOS app + installers (needs a NON-framework Python; see the script) -python tools/build_macos_app.py --pkg --dmg +# 2. merge the release PR, then tag the merge commit and push the tag. +# That is the whole build: release.yml checks the tag against every +# version field, builds the wheel and sdist, builds the macOS app, .pkg +# and .dmg on an Apple silicon runner AND an Intel one, and attaches all +# of them plus SHA256SUMS.txt to a DRAFT release. Release notes come from +# .github/release-notes/vX.Y.Z.md when that file exists. +git tag vX.Y.Z && git push origin vX.Y.Z + +# 3. review the draft on the Releases page, then press Publish. Publishing +# triggers publish.yml, which uploads the release's own wheel and sdist to +# PyPI (the same bytes as the downloads), then builds the Claude Desktop +# bundle against that PyPI release and attaches it; and publish-registry.yml. +# A rebuild of an existing tag: Actions -> Release -> Run workflow -> tag. +# +# Building by hand still works when it has to, from a Mac: +# rm -rf dist && python -m build +# python tools/build_macos_app.py --pkg --dmg # NON-framework Python; see the script +# gh release create vX.Y.Z --draft --notes-file NOTES.md build/*.pkg build/*.dmg dist/* -# 4. tag, publish, attach EVERYTHING -gh release create vX.Y.Z --title "…" --notes-file NOTES.md \ - build/Compartment-X.Y.Z.pkg \ - build/Compartment-X.Y.Z.dmg \ - dist/compartment-X.Y.Z-py3-none-any.whl \ - dist/compartment-X.Y.Z.tar.gz +# 4. nothing to do by hand for PyPI or the registry; see "Verify" below. # 5. the MCP registry entry tracks the version too. Three things bite here. # @@ -80,8 +103,10 @@ gh release create vX.Y.Z --title "…" --notes-file NOTES.md \ # c) Published versions are immutable. A wrong pin cannot be edited, only # superseded, so it is worth getting right the first time. # -# The registry token expires within minutes, so log in immediately before -# every publish or status call, even if you logged in one command ago. +# publish-registry.yml does this on Publish, authenticated by OIDC. By +# hand only when that workflow failed: the registry token expires within +# minutes, so log in immediately before every publish or status call, +# even if you logged in one command ago. mcp-publisher login github --token "$(gh auth token)" \ && mcp-publisher validate && mcp-publisher publish ``` diff --git a/_config.yml b/_config.yml index 916a7ea..71116cd 100644 --- a/_config.yml +++ b/_config.yml @@ -42,3 +42,12 @@ exclude: - server.json - glama.json - RELEASING.md + # The repository root is also a plugin for Claude Code, Codex, Gemini CLI, + # OpenClaw and Hermes. Their manifests are not pages, and skills/ holds a + # SKILL.md whose YAML front matter Jekyll would otherwise render as one. + - skills + - plugin.json + - mcp.json + - gemini-extension.json + - GEMINI.md + - Dockerfile diff --git a/add/lmstudio.html b/add/lmstudio.html index 8d8a493..153965e 100644 --- a/add/lmstudio.html +++ b/add/lmstudio.html @@ -7,20 +7,20 @@ - + Add Compartment to LM Studio

Add Compartment to LM Studio

-

The app should open now with Compartment ready to add. If it did not, open it by hand.

+

The app should open now with Compartment ready to add. If it did not, open it by hand.

Before the first run, install the package and create the vault once: pip install compartment && compartment init. The vault is encrypted under a passphrase you choose, and nothing leaves your machine.

Manual setup: in LM Studio open the MCP configuration (Program, then Install, then Edit mcp.json) and add:

{
   "mcpServers": {
     "compartment": {
       "command": "compartment",
-      "args": ["--caller", "lm-studio", "serve"]
+      "args": ["--caller", "lmstudio", "serve"]
     }
   }
 }
diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 186f3a8..650976a 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -28,11 +28,37 @@ Claude Desktop (`claude_desktop_config.json`): ```json { "mcpServers": { "compartment": { - "command": "compartment", + "command": "/Users/you/Library/Python/3.12/bin/compartment", "args": ["--vault", "/Users/you/.compartment/memory.vault", "--caller", "claude-desktop", "serve"] } } } ``` +Use the absolute path of the `compartment` command (`which compartment`, +or `where compartment` on Windows), not the bare name: an app started from +the Dock or the Start menu does not get your shell's PATH, and a bare +`compartment` fails there with "spawn compartment ENOENT". `integrate` +writes the absolute path for you: the script pip installed (including the +`pip install --user` directory the install scripts use), or the link a +package manager keeps for it (Homebrew, pipx, uv), which survives +upgrades. + +Where Claude Desktop keeps that file: + +| Platform | File | +|---|---| +| macOS | `~/Library/Application Support/Claude/claude_desktop_config.json` | +| Windows (MSIX: the Store build and the current direct download) | `%LOCALAPPDATA%\Packages\Claude_pzs8sxrjxfjjc\LocalCache\Roaming\Claude\claude_desktop_config.json` (Store: `AnthropicPBC.Claude_fnn82j28hfe8t`) | +| Windows (older installer) | `%APPDATA%\Claude\claude_desktop_config.json` | +| Linux (official beta, and community builds) | `~/.config/Claude/claude_desktop_config.json` | + +An MSIX app reads its private copy under `Packages` once it has saved its +settings, and ignores `%APPDATA%\Claude` from then on; `integrate claude` +writes whichever one the installed app is reading. Quit Claude Desktop +completely before you run it (on Windows, from the tray icon), or quit +and reopen it straight after: a running Claude Desktop can write the +config it loaded at startup back over the file, dropping a server added +while it ran (anthropics/claude-ai-mcp#1150). + ### Claude Code already has a memory - Compartment takes it over Claude Code keeps its own memory as Markdown files in a per-project @@ -51,9 +77,14 @@ and neither is complete. `compartment integrate claude` resolves it in three way in the system prompt, which otherwise outranks anything a tool says. 3. **It stops relying on the model.** `integrate claude` installs a - `PostToolUse` hook into `~/.claude/settings.json`. When Claude Code writes - a file into its memory directory, the hook stores it in the vault - no - tool call required, so capture is deterministic instead of discretionary. + `PostToolUse` hook into `~/.claude/settings.json` (`%USERPROFILE%\.claude` + on Windows, or `$CLAUDE_CONFIG_DIR/settings.json` when that is set). When + Claude Code writes a file into its memory directory (under `projects/`, or + wherever its `autoMemoryDirectory` setting points), the hook stores it in + the vault - no tool call required, so capture is deterministic instead of + discretionary. Claude Code runs hooks through bash, Git Bash on Windows, + so the hook names `compartment` by an absolute path written with forward + slashes there. Your existing hooks are preserved, settings.json is backed up first, and re-installing updates in place rather than duplicating. The hook always exits 0 (a memory tool must never break an edit) and does nothing when the @@ -212,11 +243,52 @@ shutdown and before compaction, and recalled memory is injected into the compaction context so summaries do not drop prior decisions. `memory_search` and `memory_store` are also registered as tools, independent of MCP wiring. +## Every other MCP client - `compartment integrate ` + +`compartment integrate --list` shows every client below and whether it is +installed and connected here; `compartment integrate --all` wires every one +it finds. Each write keeps the client's other servers, takes a byte-exact +backup first (`.compartment-backup`), follows a symlinked config to +the file it points at, and refuses - printing the block to paste instead - +anything it cannot read or edit safely. Files saved with a byte order mark +or as UTF-16 (Notepad, PowerShell 5) are read and written back as UTF-8. + +| `integrate …` | File it writes (macOS / Linux; Windows where it differs) | Notes | +|---|---|---| +| `cursor` | `~/.cursor/mcp.json` | | +| `windsurf` | `~/.codeium/windsurf/mcp_config.json` | | +| `vscode`, `vscode-insiders` | `/mcp.json`, key `servers` | comments kept | +| `zed` | `~/.config/zed/settings.json` (Linux: `$XDG_CONFIG_HOME/zed`; Windows: `%APPDATA%\Zed`), key `context_servers` | comments kept | +| `cline` | `~/.cline/data/settings/cline_mcp_settings.json` (`CLINE_DIR`, `CLINE_DATA_DIR`, `CLINE_MCP_SETTINGS_PATH`); older Cline: its editor's globalStorage | found in VS Code, Insiders, VSCodium, Cursor, Windsurf | +| `roo` | `/User/globalStorage/rooveterinaryinc.roo-cline/settings/mcp_settings.json` | | +| `kilo` | `~/.config/kilo/kilo.jsonc` (or the kilo/opencode file already there), key `mcp` | comments kept | +| `continue` | `~/.continue/mcpServers/compartment.json` (`CONTINUE_GLOBAL_DIR`) | a file of its own; config.yaml untouched | +| `codex` | `~/.codex/config.toml` (`CODEX_HOME`), `[mcp_servers.compartment]` | re-running refreshes a stale entry | +| `gemini` | `~/.gemini/settings.json` (`$GEMINI_CLI_HOME/.gemini`) | comments kept | +| `qwen` | `~/.qwen/settings.json` (`QWEN_HOME`) | comments kept | +| `copilot-cli` | `~/.copilot/mcp-config.json` (`COPILOT_HOME`) | | +| `opencode` | `~/.config/opencode/opencode.json` or `.jsonc` (`OPENCODE_CONFIG_DIR`), key `mcp` | comments kept | +| `crush` | `~/.config/crush/crush.json` (`CRUSH_GLOBAL_CONFIG`), key `mcp` | | +| `kiro` | `~/.kiro/settings/mcp.json` (`KIRO_HOME`) | | +| `amazon-q` | `~/.aws/amazonq/mcp.json` | | +| `junie` | `~/.junie/mcp/mcp.json` | | +| `factory` | `~/.factory/mcp.json` | | +| `omp` | `~/.omp/agent/mcp.json` | | +| `lmstudio` | `/mcp.json`, from `~/.lmstudio-home-pointer` (default `~/.lmstudio`) | | +| `anythingllm`, `boltai`, `5ire`, `trae` | each app's own `mcp.json` | | +| `goose` | prints an `extensions:` block for `~/.config/goose/config.yaml` (Windows: `%APPDATA%\Block\goose\config\config.yaml`; `GOOSE_PATH_ROOT`) | paste, not written | +| `librechat`, `jan`, `cherry-studio`, `chatbox`, `msty`, `witsy`, `open-webui` | prints the block and where it goes | paste, not written | + +"Comments kept" means the client reads its file as JSON with comments, so a +file holding `//` comments or trailing commas is edited as text: the entry +is spliced in, the result is read back and must hold exactly the old +settings plus the entry, and every comment stays where it was. + ## Everything else | Agent kind | Mechanism | What to do | |---|---|---| -| Any MCP-capable host (Cursor, Windsurf, custom SDK agents, …) | MCP stdio | the same one-block config as Claude, with its own `--caller` name | +| Any other MCP-capable host (custom SDK agents, …) | MCP stdio | the same one-block config as Claude, with its own `--caller` name | | Python frameworks (LangChain/LlamaIndex-style) | vector-store adapter | import `compartment.vault.Vault` directly, or use the CLI; a drop-in `VectorStore` adapter class is on the roadmap | | Anything that can run a subprocess | CLI/JSON | `compartment search "query" --json`, `compartment store "text" --source "how you know"` | diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md new file mode 100644 index 0000000..d01b2d7 --- /dev/null +++ b/docs/PLUGINS.md @@ -0,0 +1,320 @@ +# Plugins, extensions and store listings + +Compartment is one Python package on PyPI and one MCP server +(`compartment serve`, stdio, 14 memory tools). Everything below is a way for a +particular host or catalogue to find that server and start it. This page says, +store by store, what the listing is, how a user installs from it, and what the +maintainer has to do to publish or update it. The formats were checked against +each host's own repository in October 2026; the source is named for each. + +`tests/test_plugin_manifests.py` checks every manifest here against the parts +of its host's format that matter, and against each other (same command, same +subcommand, one caller per host, no wrong tool count). `tests/test_version_lockstep.py` +fails the build when any manifest carrying a version disagrees with the package, +or when a file anywhere in the tree carries a version it does not check. + +## At a glance + +| Store / host | Listing in this repo | User installs with | Maintainer, each release | Maintainer, once | +|---|---|---|---|---| +| PyPI | `pyproject.toml` | `pip install compartment` | automatic (`publish.yml`) | - | +| Official MCP Registry | `server.json` | their client's registry browser, `uvx compartment serve` | automatic (`publish-registry.yml`) | - | +| Claude Code plugins | `.claude-plugin/` | `/plugin marketplace add MaxFreedomPollard/Compartment` | version is gated; nothing else | submit to Anthropic's plugin directory | +| Claude Desktop extension | `packaging/mcpb/` | the `.mcpb` from the release | automatic (`publish.yml` builds and attaches it once PyPI has the version) | submit to Anthropic's extension directory | +| Codex plugins | root `plugin.json` + `.claude-plugin/marketplace.json` | `codex plugin marketplace add MaxFreedomPollard/Compartment` | nothing | - | +| Gemini CLI extensions | `gemini-extension.json`, `GEMINI.md` | `gemini extensions install https://github.com/MaxFreedomPollard/Compartment` | nothing (crawled daily) | - (the `gemini-cli-extension` topic is set) | +| Hermes Agent | `integrations/hermes/`, root `plugin.json` + `mcp.json` | `compartment integrate hermes` or `hermes plugins install MaxFreedomPollard/Compartment` | nothing | upstream PR (see below) | +| OpenClaw | `.claude-plugin/` (as a bundle), `skills/` | `compartment integrate openclaw` | nothing | publish the skill on ClawHub | +| Smithery | the `.mcpb` | Smithery's install command | `smithery mcp publish` | `smithery auth login` | +| Glama | `glama.json`, `Dockerfile` | Glama's page | nothing | - | +| Docker MCP Catalog | `packaging/docker-mcp/server.yaml` | Docker Desktop, MCP Toolkit | nothing (Docker bumps the pin) | PR to docker/mcp-registry | +| Cline marketplace | `llms-install.md` | Cline's MCP marketplace | nothing | issue at cline/mcp-marketplace | +| Continue hub | `packaging/continue/compartment.yaml` | hub.continue.dev, or copy the file | republish the block | publish the block | +| goose | `add/goose.html` | the Install button | nothing | PR to goose's extension directory | +| LM Studio | `add/lmstudio.html` | the Add button | nothing | - | +| Cursor | deeplink in the README | the Add to Cursor button | nothing | - | +| VS Code, Copilot | deeplinks in the README, `server.json` | the Install buttons, VS Code's MCP gallery | nothing | - | +| Oh My Pi | `integrations/omp/extension.ts` | copy the file; `compartment integrate omp` | nothing | - | + +## The command a host runs, and PATH + +Most listings start the server as the bare command `compartment`, because their +formats allow nothing else portable: Agent Plugins (`mcp.json`, read by Codex, +Hermes and OpenClaw) requires a single bare executable name or a path inside +the plugin, and a Claude Code, Gemini, Continue or LM Studio entry has no way +to discover where pip put the script. A bare name works when the host inherits +the shell's PATH, which terminal programs (Claude Code, Codex, Gemini CLI, +Oh My Pi) do. GUI apps started from the Dock, the Start menu or a desktop +launcher often do not, and then the server fails with `command not found` or +`spawn compartment ENOENT`. + +Where a format allows something safer, the listing uses it: + +- The Claude Desktop bundle runs the server through `uv`, which Claude Desktop + supplies, in an environment it builds from the bundle; no PATH lookup of + `compartment` at all. +- goose runs `uvx compartment` (goose only accepts its own launchers from a + link, and ships uvx). +- The Docker catalogue entry runs the image's own entrypoint. +- `compartment integrate ` writes the absolute path of the executable + into every config it edits, which is the reliable route for GUI clients. + +For a hand-written config in a GUI client, use the absolute path: +`which compartment` (macOS, Linux) or `where compartment` (Windows). + +## The caller + +Every listing passes `--caller `, which names the client in the audit +log and is the key per-caller ACL entries are written against. Where +`compartment integrate` wires the same host, the listing uses the same name +(`claude-code`, `claude-desktop`, `gemini`, `lmstudio`, `goose`, `continue`), +so one app never shows up as two clients. The Agent Plugins package uses +`agent-plugin`, because Codex, Hermes and OpenClaw all read it; the Docker +image uses `docker`. A test enforces all of this. + +## Using two routes at once + +A host wired two ways starts two servers, and the model sees every tool twice: + +- `compartment init` connects Claude Code when it is installed (the same as + `compartment integrate claude`). Installing the Claude Code plugin as well + adds a second server. Claude Code deduplicates a plugin server only when its + command is identical to a configured one, and integrate writes an absolute + path, so it does not here. Use one: integrate is the fuller route (capture + hook, CLAUDE.md block, Claude Desktop). +- The same holds for the Claude Desktop extension next to + `compartment integrate claude`. +- Gemini CLI is the exception: a server in `settings.json` with the same name + as an extension's replaces it, so `integrate gemini` and the extension do not + double up. + +## Store by store + +### Official MCP Registry + +- **Listing:** `server.json`, schema `2025-12-11` (the newest released schema + in modelcontextprotocol/registry), name `io.github.MaxFreedomPollard/compartment`, + one PyPI package run as `uvx compartment serve`, plus `title`, `websiteUrl` + and `icons`. Validates against the schema with `mcp-publisher validate`. +- **Users:** clients with a registry browser install it from there. VS Code's + MCP gallery and GitHub's MCP registry draw on this registry. +- **Each release:** automatic. `publish-registry.yml` publishes on every GitHub + release after the version reaches PyPI. The two version fields are + deliberately different (RELEASING.md step 5 explains why). +- The registry proves PyPI ownership by finding `mcp-name: ` in the + README PyPI serves; a test keeps the two in sync. +- Possible later: an `mcpb` package entry pointing at the release's `.mcpb`. + It needs the bundle's SHA-256 in `server.json` (`fileSha256`), so the release + workflow would have to write it in before publishing. + +### Claude Code plugin marketplace + +- **Listing:** `.claude-plugin/marketplace.json` (marketplace `maxfreedompollard`, + one plugin with `source: "./"`) and `.claude-plugin/plugin.json` (name, + display name, version, the MCP server). Because the plugin root is the + repository root, Claude Code also loads `skills/compartmentalize`, which + appears as `/compartment:compartmentalize`. Format: anthropics/claude-code + and anthropics/claude-plugins-official, schemas on SchemaStore. +- **Users:** after `pip install compartment && compartment init`: + `/plugin marketplace add MaxFreedomPollard/Compartment`, then + `/plugin install compartment@maxfreedompollard`. Updates arrive with + `/plugin marketplace update maxfreedompollard`. See "Using two routes at + once" first. +- **Each release:** the `version` in `.claude-plugin/plugin.json` moves with + the package (gated); Claude Code compares it to decide what is an update. +- **Once:** to appear in `/plugin` > Discover for everyone, submit the plugin + to Anthropic's directory (anthropics/claude-plugins-official, through the + submission form its README links, `clau.de/plugin-directory-submission`). +- **Not shipped: the capture hook.** `integrate claude` installs a + `PostToolUse` hook into `~/.claude/settings.json`, and `compartment init` + runs that on any machine with Claude Code. A plugin hook would run a second + copy of `compartment hook capture` on every Write and Edit for nearly every + plugin user. It cannot live at the root as `hooks/hooks.json` either, because + Gemini CLI and OpenClaw read that path in their own formats. It can ship once + the capture command can tell it is the plugin's copy and stand down when + settings.json already has one (for example a flag that `hook capture` + accepts and checks with `claude_hooks.is_installed()`); then the hook goes + inline in `.claude-plugin/plugin.json`. + +### Claude Desktop extension (MCP Bundle) + +- **Listing:** `packaging/mcpb/` - `manifest.json` (MCPB `manifest_version` + 0.4, server type `uv`), `pyproject.toml` pinning `compartment==`, + `.python-version`, and `server/main.py`. Format: modelcontextprotocol/mcpb + (formerly anthropics/dxt). Type `uv` is the only one that can carry + compiled dependencies (onnxruntime, PyNaCl, argon2-cffi, numpy) to every + platform: Claude Desktop has uv build the environment from PyPI once, at + install. The server it starts makes no network connection. +- **Users:** download `compartment-.mcpb` from the release and open + it, or Claude Desktop > Settings > Extensions > Install Extension. The vault + still comes from `compartment init` in a terminal (the passphrase has to be + typed somewhere). +- **Each release:** automatic. Publishing the release runs `publish.yml`, + whose `claude-desktop-bundle` job waits for the version on PyPI, then + runs `python tools/build_mcpb.py` (dist/compartment-.mcpb, with + uv.lock) and attaches it to the release. By hand, the same two commands + after the version is on PyPI: `pip install uv`, then the build. The + build refuses a manifest or pin that is not the package version, and is + reproducible byte for byte. `uv lock` needs the release on PyPI, so this + runs after `publish.yml`, not before. +- **Once:** Claude Desktop's curated extension directory is Anthropic's; + submission is through Anthropic, not a GitHub repository. +- The interpreter is pinned (3.12, and `requires-python` below 3.15) because uv + otherwise picks the newest Python, and onnxruntime has no wheels for 3.15 + yet. Raise both when it does. + +### Codex + +- **Listing:** Codex reads `.claude-plugin/marketplace.json` (it looks for + `.agents/plugins/`, then `.claude-plugin/`, then `.cursor-plugin/`). For the + plugin at the root it loads the Agent Plugins `plugin.json` first, then + `mcp.json` and `skills/`, and takes its listing (display name, category, + prompts, logo) from `extensions["com.openai"]` in that manifest. Format: + openai/codex `codex-rs/core-plugins`. +- **Users:** `codex plugin marketplace add MaxFreedomPollard/Compartment`, then + `codex plugin add compartment@maxfreedompollard`. +- **Each release:** nothing beyond the gated version. +- OpenAI's curated directory (openai/plugins) has no public submission + process as of October 2026. + +### Gemini CLI + +- **Listing:** `gemini-extension.json` (name, version, description, + `contextFileName: GEMINI.md`, the MCP server). Gemini also loads `skills/` + from the extension. `GEMINI.md` is the server's handshake instructions, + verbatim (tested). Format: google-gemini/gemini-cli `docs/extensions/`. +- **Users:** `gemini extensions install https://github.com/MaxFreedomPollard/Compartment`. + Gemini only starts stdio servers in trusted folders (`gemini trust`). +- **Each release:** nothing; the gallery crawler indexes tagged releases daily. + Installs from a release use the source archive as long as the release has + more than one asset. A release with exactly one asset would make Gemini + download that asset as the extension, so keep the full set attached. +- **Once:** the `gemini-cli-extension` topic on the GitHub repository + (About > Topics) is the gallery's only requirement beyond the manifest at + the root; it is set. + +### Hermes Agent + +- **Listing:** the native memory provider in `integrations/hermes/` (the + fuller route) and the portable Agent Plugins package at the root + (`plugin.json` 1.0.0, `mcp.json`, `skills/`). Agent Plugins 1.0.0 is still + the published release; 1.1.0 is a draft no host accepts. +- **Users:** `compartment integrate hermes`, or + `hermes plugins install MaxFreedomPollard/Compartment`. +- **Once:** the upstream PR described in `integrations/hermes/upstream/PR.md`. + Its `plugin.yaml` there is stale (version 1.7.0, `compartment>=2.3`); the + lockstep test pins it as a known stale copy until it is refreshed. + +### OpenClaw + +- **Listing:** OpenClaw installs Agent Plugins, Claude, Codex and Cursor + bundles. This repository is detected as a Claude bundle (a client marker + wins over the root `plugin.json`): it gets `skills/` and the MCP server from + `.claude-plugin/plugin.json`. Format: openclaw/openclaw `docs/plugins/bundles.md`. +- **Users:** `compartment integrate openclaw` (writes the server with caller + `openclaw` and installs the skill), or + `openclaw plugins install git:github.com/MaxFreedomPollard/Compartment` + (then the caller is `claude-code`, the bundle's). +- **Once:** publish `skills/compartmentalize` to ClawHub with the `clawhub` + CLI (needs a ClawHub account). A native memory-slot plugin + (`openclaw.plugin.json`) needs in-process TypeScript and is not written; + adding that manifest at the root would make OpenClaw stop treating the + repository as a bundle. + +### Smithery + +- **Listing:** Smithery no longer builds local servers from a `smithery.yaml`; + local servers are published as MCP Bundles. Use the release's `.mcpb` + (Smithery's limit is 25 MB; ours is about 40 KB). +- **Each release / once:** `smithery auth login`, then + `smithery mcp publish dist/compartment-.mcpb -n maxfreedompollard/compartment`. + +### Glama + +- **Listing:** `glama.json` (schema and maintainers); Glama builds the + `Dockerfile` to inspect the server. Already listed; nothing to do. + +### Docker MCP Catalog + +- **Listing:** `packaging/docker-mcp/server.yaml`, the entry docker/mcp-registry + wants at `servers/compartment/server.yaml`. Docker builds and signs + `mcp/compartment` from the `Dockerfile`, mounts the user's vault folder at + `/data`, takes the passphrase from Docker Desktop's secret store as + `COMPARTMENT_PASSPHRASE`, and runs the container with no network. +- **Once:** fork docker/mcp-registry, copy the file to + `servers/compartment/server.yaml` (the header comment can go), run + `task validate -- --name compartment` and `task build -- --tools compartment`, + and open the PR with their template. The server lists its tools without a + vault, so no `tools.json` is needed; add one only if their build cannot + start the image. After the merge, Docker's pin-update workflow moves + `source.commit` with each release. +- **Users:** create the vault on the host first (`compartment init`). On + Linux the image's user (uid 10001) must be able to read the mounted vault, + whose files are mode 0600; Docker Desktop on macOS and Windows maps + ownership for you. + +### Cline marketplace + +- **Listing:** `llms-install.md`, the instructions Cline's agent follows to + install (including PEP 668 systems, Windows and PATH problems). +- **Once:** open an issue at github.com/cline/mcp-marketplace with the + repository URL, `docs/images/logo-400.png` (400x400 PNG) and the reason; + test that Cline can install from `llms-install.md` first. + +### Continue hub + +- **Listing:** `packaging/continue/compartment.yaml`, a Continue block + (`schema: v1`). Users can also copy it into `.continue/mcpServers/` (one + workspace) or `~/.continue/mcpServers/` (all of them). +- **Each release / once:** publish the block on hub.continue.dev (needs an + account). Its version is gated with the package. + +### goose + +- **Listing:** `add/goose.html`, a `goose://extension` link running + `uvx compartment --caller goose serve`. +- **Once:** goose's extension directory is `documentation/static/servers.json` + in block/goose. A PR adding: + + ```json + { + "id": "compartment", + "name": "Compartment", + "description": "Encrypted, fully offline long-term memory for AI agents", + "command": "uvx compartment --caller goose serve", + "link": "https://github.com/MaxFreedomPollard/Compartment", + "installation_notes": "Create the encrypted vault once first: pip install compartment && compartment init", + "is_builtin": false, + "endorsed": false, + "environmentVariables": [] + } + ``` + +### LM Studio, Cursor, VS Code, Kiro + +- **LM Studio:** `add/lmstudio.html`, an `lmstudio://add_mcp` link with the + entry base64-encoded (format: lmstudio-ai/docs). No directory. +- **Cursor:** the README's `cursor.com/install-mcp` button, and the existing + Cursor Directory listing. Cursor's own plugin marketplace + (`.cursor-plugin/`) is curated by Cursor and submitted through the Cursor + team; a `.cursor-plugin` manifest at the root would also read the root + `mcp.json`, which is in Agent Plugins format, and would make OpenClaw treat + the repository as a Cursor bundle, so none is shipped. +- **VS Code / Copilot:** the README's `vscode.dev/redirect/mcp/install` + buttons; the gallery side comes from the MCP Registry. +- **Kiro:** the README's `kiro.dev/launch/mcp/add` button. + +### Oh My Pi + +- **Listing:** `integrations/omp/extension.ts`, an omp extension module + (`ExtensionAPI`): automatic recall and capture, and `memory_search` / + `memory_store` tools declared with omp's approval tiers (read, write) and + loaded as essential. Format: can1357/oh-my-pi `docs/extensions.md`. +- **Users:** copy it to `~/.omp/agent/extensions/compartment.ts`; + `compartment integrate omp` adds the MCP server as well. + +### Directories that scrape + +mcp.so, MCP Market, LobeHub, mcpservers.org, PulseMCP, MCP Toplist, the +toolsdk registry and the awesome lists read GitHub, PyPI or the MCP Registry. +Nothing in this repository is specific to them. diff --git a/gemini-extension.json b/gemini-extension.json index 3f0986a..e15ca56 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "compartment", - "version": "4.12.0", + "version": "4.13.0", "description": "Encrypted, fully offline long-term memory for Gemini CLI. One vault on your machine, shared by every MCP client. Requires `pip install compartment && compartment init` first.", "contextFileName": "GEMINI.md", "mcpServers": { @@ -8,7 +8,7 @@ "command": "compartment", "args": [ "--caller", - "gemini-cli", + "gemini", "serve" ] } diff --git a/install/install-linux.sh b/install/install-linux.sh index fecb2f1..184a777 100755 --- a/install/install-linux.sh +++ b/install/install-linux.sh @@ -1,41 +1,250 @@ #!/usr/bin/env bash -# Compartment installer - Linux -# Installs the package, creates + unlocks a vault, -# and prints the Hermes selection step. Fully offline after this script. +# Compartment installer - Linux, WSL and ChromeOS (Crostini). On Alpine, +# Termux and the BSDs it says what to install first. +# +# Installs the package, creates and unlocks a vault, and connects the agents +# it finds. Fully offline after this script. +# +# bash install/install-linux.sh from a checkout +# PYTHON=python3.12 bash install-linux.sh to choose the interpreter +# +# Running it again upgrades Compartment and leaves an existing vault alone. +# Written for bash 3.2 and up, so the same constructs run on macOS's bash. set -euo pipefail echo "Compartment installer (Linux)" -PY="${PYTHON:-python3}" -command -v "$PY" >/dev/null || { echo "python3 not found. Install via your package manager (apt/dnf/pacman)."; exit 1; } -if [ -f "$(dirname "$0")/../pyproject.toml" ]; then - "$PY" -m pip install --user "$(cd "$(dirname "$0")/.." && pwd)" +# --- 1. A Python that can run Compartment ------------------------------------ +# +# Compartment runs on Python 3.11 to 3.14 with x86_64 or aarch64 and glibc +# 2.28 or newer: that is where every compiled dependency publishes wheels, +# onnxruntime above all, which Compartment cannot embed anything without +# (tools/wheel_coverage.py checks the matrix in CI). Elsewhere - musl +# (Alpine), Android (Termux), the BSDs, 32-bit, armv7, riscv64, ppc64le, +# s390x, an older glibc, a Python newer than its wheels - pip fails with a +# bare "No matching distribution found for onnxruntime". So this is checked +# first, and where the operating system packages onnxruntime instead, that +# copy is used. +# +# The probe runs inside each candidate. First line: "ok", or why this +# platform has no onnxruntime wheel. Second line: whether onnxruntime is +# importable already. Exit 0: usable. Exit 1: not Python 3.11 to 3.14. +# Exit 2: no wheel and no onnxruntime. +PROBE=' +import os, platform, sys +if not (3, 11) <= sys.version_info[:2] <= (3, 14): + print("Python %d.%d" % sys.version_info[:2]) + sys.exit(1) +reason = "ok" +machine = platform.machine().lower() +if hasattr(sys, "getandroidapilevel") or sys.platform == "android": + reason = "android" +elif platform.system() != "Linux": + reason = platform.system() or "unknown" +elif machine not in ("x86_64", "amd64", "aarch64", "arm64"): + reason = "arch " + machine +else: + try: + libc = os.confstr("CS_GNU_LIBC_VERSION") or "" + except (ValueError, OSError): + libc = "" + if not libc.startswith("glibc "): + reason = "musl" + else: + parts = libc.split()[1].split(".") + if (int(parts[0]), int(parts[1])) < (2, 28): + reason = "glibc " + libc.split()[1] +try: + import importlib.util + present = importlib.util.find_spec("onnxruntime") is not None +except Exception: + present = False +print(reason) +print("present" if present else "absent") +sys.exit(0 if reason == "ok" or present else 2) +' + +PY="" +PLATFORM="" +NO_WHEEL="" +NO_WHEEL_PY="" +TOO_OLD="" +if [ -n "${PYTHON:-}" ]; then + set -- "$PYTHON" +else + set -- python3 python3.14 python3.13 python3.12 python3.11 +fi +for cand in "$@"; do + command -v "$cand" >/dev/null 2>&1 || continue + rc=0 + out="$("$cand" -c "$PROBE" 2>/dev/null)" || rc=$? + first="$(printf '%s\n' "$out" | sed -n 1p)" + case "$rc" in + 0) PY="$(command -v "$cand")"; PLATFORM="$first"; break ;; + 2) if [ -z "$NO_WHEEL" ]; then NO_WHEEL="$first"; NO_WHEEL_PY="$cand"; fi ;; + *) TOO_OLD="${TOO_OLD:+$TOO_OLD, }$cand is ${first:-not runnable}" ;; + esac +done + +if [ -z "$PY" ] && [ -n "$NO_WHEEL" ]; then + pyxy="$("$NO_WHEEL_PY" -c 'import sys; print("%d%d" % sys.version_info[:2])')" + echo + echo "onnxruntime, which Compartment needs, publishes no build for this system ($NO_WHEEL)." + case "$NO_WHEEL" in + android) + echo "On Termux, install it from Termux's own packages, then run this script again:" + echo " pkg install python onnxruntime python-numpy rust" + echo "(rust lets pip build tokenizers and pydantic-core, which have no Android wheels.)" ;; + musl) + echo "On Alpine, install it from the community repository, then run this script again:" + echo " apk add python3 py3-onnxruntime" + echo "Every other dependency has musllinux wheels." ;; + FreeBSD) + echo "On FreeBSD, install it and the other compiled dependencies from packages," + echo "then run this script again:" + echo " pkg install py${pyxy}-onnxruntime py${pyxy}-tokenizers py${pyxy}-pydantic-core \\" + echo " py${pyxy}-pynacl py${pyxy}-argon2-cffi py${pyxy}-numpy" ;; + arch\ *) + echo "onnxruntime ships wheels for x86_64 and aarch64 only. On a Raspberry Pi," + echo "install the 64-bit OS. Elsewhere, install onnxruntime for $NO_WHEEL_PY from" + echo "your distribution if it packages it, then run this script again." ;; + glibc\ *) + echo "Its wheels need glibc 2.28 or newer (Debian 10, Ubuntu 20.04, RHEL 8" + echo "and later). Use a newer distribution, or a container of one." + exit 1 ;; + *) + echo "Build onnxruntime's Python package for $NO_WHEEL_PY from source" + echo "(https://github.com/microsoft/onnxruntime), then run this script again." ;; + esac + echo "This script then installs Compartment in its own environment that uses" + echo "that onnxruntime." + exit 1 +fi + +if [ -z "$PY" ]; then + echo + if [ -n "$TOO_OLD" ]; then + echo "Compartment needs Python 3.11 to 3.14; found: $TOO_OLD." + else + echo "Compartment needs Python 3.11 to 3.14, and none was found." + fi + echo "Install one with your package manager, for example" + echo " sudo apt install python3.12 python3.12-venv (Ubuntu 22.04: python3.11)" + echo " sudo dnf install python3.12" + echo "then run this script again, or name it: PYTHON=python3.12 bash ${BASH_SOURCE[0]:-install-linux.sh}" + exit 1 +fi +echo "Using $PY" + +# --- 2. Install --------------------------------------------------------------- +# +# This checkout when the script is run from one, otherwise the release on +# PyPI. Only a checkout whose pyproject names compartment counts: run as +# `curl ... | bash` there is no script file, and `dirname "$0"` named the +# directory the user happened to be in, so the pyproject one level up was +# somebody else's project, which got installed instead. +SRC="compartment" +SCRIPT="${BASH_SOURCE[0]:-}" +if [ -n "$SCRIPT" ] && [ -f "$SCRIPT" ]; then + ROOT="$(cd "$(dirname "$SCRIPT")/.." && pwd)" + if [ -f "$ROOT/pyproject.toml" ] && grep -q '^name = "compartment"$' "$ROOT/pyproject.toml"; then + SRC="$ROOT" + fi +fi + +# Where to put it. `pip install --user` into the system Python fails on +# Debian 12+, Ubuntu 23.04+, Arch, Crostini and Homebrew with +# "externally-managed-environment" (PEP 668), which under `set -e` ended the +# script before anything was installed. So: +# - inside an activated virtualenv: into it (--user is refused there); +# - onnxruntime from the OS (Alpine, Termux, FreeBSD): a private venv that +# can see the system site-packages, so pip uses that copy; +# - a Python that allows it: `pip install --user`, as before; +# - an externally managed one, or one without pip (minimal containers): +# pipx, then uv, then a private venv, which brings its own pip. +KIND="$("$PY" -c ' +import importlib.util, os, sys, sysconfig +if sys.prefix != sys.base_prefix: + print("venv") +elif os.path.isfile(os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")): + print("managed") +elif importlib.util.find_spec("pip") is None: + print("nopip") +else: + print("plain") +')" +if [ "$KIND" = venv ]; then + METHOD=venv-active +elif [ "$PLATFORM" != ok ]; then + METHOD=private +elif [ "$KIND" = plain ]; then + METHOD=user +elif command -v pipx >/dev/null 2>&1; then + METHOD=pipx +elif command -v uv >/dev/null 2>&1; then + METHOD=uv else - "$PY" -m pip install --user compartment + METHOD=private fi -# `pip install --user` puts the console script in the interpreter's own user -# bin directory (usually ~/.local/bin), which is not on PATH on a clean -# machine. Ask the interpreter where that is and use it, so the rest of this -# script runs instead of dying on "compartment: command not found". -USER_BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" +LOCAL_BIN="$HOME/.local/bin" +case "$METHOD" in + venv-active) + "$PY" -m pip install --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" ;; + user) + # `pip install --user` puts the console script in the interpreter's own + # user bin directory (usually ~/.local/bin), which is not on PATH on a + # clean machine, so the interpreter is asked where that is. + "$PY" -m pip install --user --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" ;; + pipx) + pipx install --force --python "$PY" "$SRC" + BIN="$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + uv) + uv tool install --force --upgrade --python "$PY" "$SRC" + BIN="$(uv tool dir --bin 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + private) + VENV="${XDG_DATA_HOME:-$HOME/.local/share}/compartment/venv" + mkdir -p "$(dirname "$VENV")" + SITE_FLAG="" + if [ "$PLATFORM" != ok ]; then SITE_FLAG="--system-site-packages"; fi + # No --clear: a re-run upgrades in place, under a status bar app that + # may be running from this very environment. + if ! "$PY" -m venv $SITE_FLAG "$VENV"; then + echo + echo "Could not create a virtual environment with $PY." + echo "On Debian, Ubuntu and Crostini install the venv module first:" + echo " sudo apt install python3-venv (or python3.X-venv for $PY)" + echo "or install pipx (sudo apt install pipx), then run this script again." + exit 1 + fi + "$VENV/bin/python" -m pip install --upgrade "$SRC" + mkdir -p "$LOCAL_BIN" + for cmd in compartment cmpt engram; do + if [ -x "$VENV/bin/$cmd" ]; then ln -sf "$VENV/bin/$cmd" "$LOCAL_BIN/$cmd"; fi + done + BIN="$LOCAL_BIN" ;; +esac + ON_PATH_ALREADY=no -if command -v compartment >/dev/null 2>&1; then ON_PATH_ALREADY=yes; fi -if [ -d "$USER_BIN" ]; then - PATH="$USER_BIN:$PATH" +case ":$PATH:" in *":$BIN:"*) ON_PATH_ALREADY=yes ;; esac +if [ -d "$BIN" ]; then + PATH="$BIN:$PATH" export PATH fi -if [ -x "$USER_BIN/compartment" ]; then - COMPARTMENT="$USER_BIN/compartment" +if [ -x "$BIN/compartment" ]; then + COMPARTMENT="$BIN/compartment" elif command -v compartment >/dev/null 2>&1; then COMPARTMENT="$(command -v compartment)" elif "$PY" -c 'import compartment.cli' >/dev/null 2>&1; then COMPARTMENT="" # no console script on PATH, but the module is importable else echo "compartment was installed but cannot be found." - echo "Expected the command in: $USER_BIN" - echo "Try: \"$PY\" -m pip install --user --force-reinstall compartment" + echo "Expected the command in: $BIN" exit 1 fi @@ -47,26 +256,81 @@ run_compartment() { fi } +# --- 3. The vault --------------------------------------------------------------- +# +# Where `compartment` looks by default (home.py): COMPARTMENT_VAULT, then +# the legacy ENGRAM_VAULT, then memory.vault in ~/.compartment, or in +# ~/.engram when only that exists. A vault that is there is never touched: +# `init` refuses to overwrite one, and a re-run should not fail on that. +if [ -n "${COMPARTMENT_VAULT:-}" ]; then + VAULT="$COMPARTMENT_VAULT" +elif [ -n "${ENGRAM_VAULT:-}" ]; then + VAULT="$ENGRAM_VAULT" +elif [ ! -e "$HOME/.compartment" ] && [ -e "$HOME/.engram" ]; then + VAULT="$HOME/.engram/memory.vault" +else + VAULT="$HOME/.compartment/memory.vault" +fi + +# Staying unlocked until the next restart keeps a per-boot secret in POSIX +# shared memory. Termux has none, and glibc keeps it in /dev/shm, which some +# containers and chroots lack; there `init` would create the vault and then +# fail. Such a vault is created locked and opened with the passphrase. +NO_SESSION="" +if [ -n "${TERMUX_VERSION:-}" ] || case "${PREFIX:-}" in *com.termux*) true ;; *) false ;; esac \ + || [ ! -d /dev/shm ] || [ ! -w /dev/shm ]; then + NO_SESSION="--no-session" +fi + echo -echo "Creating your encrypted vault..." -run_compartment init +INIT_RAN=no +if [ -e "$VAULT" ]; then + echo "Your vault at $VAULT is kept as it is." + echo "Open it with: compartment unlock" +else + INIT_RAN=yes + echo "Creating your encrypted vault..." + # `init` asks for a passphrase on the terminal. Run as `curl ... | bash`, + # stdin is the script itself, so the terminal is handed over explicitly. + if [ -t 0 ]; then + run_compartment init $NO_SESSION + elif (: /dev/null; then + run_compartment init $NO_SESSION /dev/null || { echo "python3 not found. Install from python.org or 'brew install python'."; exit 1; } -# From a source checkout, install this repo; otherwise install from PyPI. -if [ -f "$(dirname "$0")/../pyproject.toml" ]; then - "$PY" -m pip install --user "$(cd "$(dirname "$0")/.." && pwd)" +# --- 1. A Python that can run Compartment ------------------------------------ +# +# Compartment runs on macOS 13 or newer, with a Python its compiled +# dependencies publish wheels for (tools/wheel_coverage.py checks the +# matrix in CI). onnxruntime, which it cannot embed anything without, +# decides it: +# - Apple silicon, macOS 14 or newer: Python 3.11 to 3.14. +# - Apple silicon on macOS 13: Python 3.11 to 3.13 - onnxruntime's +# builds for 3.14 need macOS 14. +# - Intel Macs (and an x86_64 Python under Rosetta): Python 3.11 to 3.13 - +# onnxruntime stopped building for Intel after 1.23.2, which stops at +# 3.13. +# Any other Python fails in pip with a bare "No matching distribution found +# for onnxruntime", so each candidate is checked first and the first one +# that fits is used. +# +# The probe runs inside each candidate. Exit 0: usable. Exit 1: older than +# Python 3.11. Exit 2: Python 3.11+, but this Mac has no wheels for it; its +# output names the newest Python that would do, or "none" for a macOS +# older than 13. +MACOS_VERSION="$(sw_vers -productVersion 2>/dev/null || true)" +export CMPT_MACOS_VERSION="$MACOS_VERSION" +PROBE=' +import os, platform, sys +v = sys.version_info[:2] +if v < (3, 11): + print("Python %d.%d" % v) + sys.exit(1) +text = os.environ.get("CMPT_MACOS_VERSION") or platform.mac_ver()[0] or "0" +parts = [int(x) for x in text.split(".")[:2] if x.isdigit()] + [0, 0] +mac = (parts[0], parts[1]) +if mac == (10, 16): + mac = (11, 0) +if mac < (13, 0): + print("none") + sys.exit(2) +intel = platform.machine().lower() not in ("arm64", "aarch64") +newest = (3, 13) if (intel or mac < (14, 0)) else (3, 14) +if v > newest: + print("%d.%d %s" % (newest + (platform.machine(),))) + sys.exit(2) +' + +# /usr/bin/python3 on a Mac without the Command Line Tools is a stub that +# opens an installer dialog instead of running, so it is only tried where +# the tools are installed. +usable_name() { + case "$(command -v "$1" 2>/dev/null)" in + "") return 1 ;; + /usr/bin/python3) xcode-select -p >/dev/null 2>&1 ;; + *) return 0 ;; + esac +} + +PY="" +NEWEST="" +PYARCH="" +TOO_OLD="" +# Where Homebrew and the python.org installer put interpreters that are not +# on PATH. CMPT_EXTRA_PYTHONS replaces the list when it is set, even to +# nothing: the test suite sets it empty, so the stand-in machine it builds +# can never reach the real interpreters of the Mac running the tests (which +# then installed into one of them and sat waiting for a passphrase). +EXTRA_PYTHONS="${CMPT_EXTRA_PYTHONS-/opt/homebrew/bin/python3.13 /opt/homebrew/bin/python3.12 /opt/homebrew/bin/python3.11 /usr/local/bin/python3.13 /usr/local/bin/python3.12 /usr/local/bin/python3.11 /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 /Library/Frameworks/Python.framework/Versions/3.12/bin/python3 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3}" +if [ -n "${PYTHON:-}" ]; then + set -- "$PYTHON" +else + # shellcheck disable=SC2086 # one word per path, none has a space + set -- python3 python3.13 python3.12 python3.11 python3.14 $EXTRA_PYTHONS +fi +for cand in "$@"; do + usable_name "$cand" || continue + rc=0 + out="$("$cand" -c "$PROBE" 2>/dev/null)" || rc=$? + case "$rc" in + 0) PY="$(command -v "$cand")"; break ;; + 2) if [ -z "$NEWEST" ]; then NEWEST="${out%% *}"; PYARCH="${out#* }"; fi + TOO_OLD="${TOO_OLD:+$TOO_OLD, }$cand has no onnxruntime build here" ;; + *) TOO_OLD="${TOO_OLD:+$TOO_OLD, }$cand is ${out:-not runnable}" ;; + esac +done + +if [ -z "$PY" ]; then + echo + if [ "$NEWEST" = none ]; then + echo "Compartment needs macOS 13 or newer; this Mac has macOS $MACOS_VERSION," + echo "for which onnxruntime and the menu bar app's libraries publish no builds." + exit 1 + elif [ -n "$NEWEST" ]; then + echo "For an $PYARCH Python on macOS $MACOS_VERSION, onnxruntime, which" + echo "Compartment needs, publishes builds only for Python 3.11 to $NEWEST." + echo "Found: $TOO_OLD." + echo "Install Python $NEWEST, then run this script again:" + echo " brew install python@$NEWEST (or the installer from python.org)" + echo " PYTHON=python$NEWEST bash ${BASH_SOURCE[0]:-install-macos.sh}" + else + if [ -n "$TOO_OLD" ]; then + echo "Compartment needs Python 3.11 or newer; found: $TOO_OLD." + else + echo "Compartment needs Python 3.11 or newer, and none was found." + fi + echo "Install it from python.org or with Homebrew (brew install python@3.13)," + echo "then run this script again." + fi + exit 1 +fi +echo "Using $PY" + +# --- 2. Install --------------------------------------------------------------- +# +# This checkout when the script is run from one, otherwise the release on +# PyPI. Only a checkout whose pyproject names compartment counts: run as +# `curl ... | bash` there is no script file, and `dirname "$0"` named the +# directory the user happened to be in, so the pyproject one level up was +# somebody else's project, which got installed instead. +SRC="compartment" +SCRIPT="${BASH_SOURCE[0]:-}" +if [ -n "$SCRIPT" ] && [ -f "$SCRIPT" ]; then + ROOT="$(cd "$(dirname "$SCRIPT")/.." && pwd)" + if [ -f "$ROOT/pyproject.toml" ] && grep -q '^name = "compartment"$' "$ROOT/pyproject.toml"; then + SRC="$ROOT" + fi +fi + +# Where to put it. Homebrew's Python is externally managed (PEP 668), so +# `pip install --user` there fails with "externally-managed-environment", +# which under `set -e` ended the script before anything was installed. So: +# - inside an activated virtualenv: into it (--user is refused there); +# - a Python that allows it (python.org's): `pip install --user`, as before; +# - an externally managed one, or one without pip (minimal containers): +# pipx, then uv, then a private venv, which brings its own pip. +KIND="$("$PY" -c ' +import importlib.util, os, sys, sysconfig +if sys.prefix != sys.base_prefix: + print("venv") +elif os.path.isfile(os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")): + print("managed") +elif importlib.util.find_spec("pip") is None: + print("nopip") +else: + print("plain") +')" +if [ "$KIND" = venv ]; then + METHOD=venv-active +elif [ "$KIND" = plain ]; then + METHOD=user +elif command -v pipx >/dev/null 2>&1; then + METHOD=pipx +elif command -v uv >/dev/null 2>&1; then + METHOD=uv else - "$PY" -m pip install --user compartment + METHOD=private fi -# `pip install --user` puts the console script in the interpreter's own user -# bin directory (on macOS ~/Library/Python/3.X/bin), which is not on PATH on a -# clean machine. Ask the interpreter where that is and use it, so the rest of -# this script runs instead of dying on "compartment: command not found". -USER_BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" +LOCAL_BIN="$HOME/.local/bin" +case "$METHOD" in + venv-active) + "$PY" -m pip install --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" ;; + user) + # `pip install --user` puts the console script in the interpreter's own + # user bin directory (~/Library/Python/3.X/bin), which is not on PATH on + # a clean machine, so the interpreter is asked where that is. + "$PY" -m pip install --user --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" ;; + pipx) + pipx install --force --python "$PY" "$SRC" + BIN="$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + uv) + uv tool install --force --upgrade --python "$PY" "$SRC" + BIN="$(uv tool dir --bin 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + private) + VENV="${XDG_DATA_HOME:-$HOME/.local/share}/compartment/venv" + mkdir -p "$(dirname "$VENV")" + # No --clear: a re-run upgrades in place, under a menu bar app that may + # be running from this very environment. + if ! "$PY" -m venv "$VENV"; then + echo + echo "Could not create a virtual environment with $PY." + echo "Install pipx (brew install pipx) and run this script again." + exit 1 + fi + "$VENV/bin/python" -m pip install --upgrade "$SRC" + mkdir -p "$LOCAL_BIN" + for cmd in compartment cmpt engram; do + if [ -x "$VENV/bin/$cmd" ]; then ln -sf "$VENV/bin/$cmd" "$LOCAL_BIN/$cmd"; fi + done + BIN="$LOCAL_BIN" ;; +esac + ON_PATH_ALREADY=no -if command -v compartment >/dev/null 2>&1; then ON_PATH_ALREADY=yes; fi -if [ -d "$USER_BIN" ]; then - PATH="$USER_BIN:$PATH" +case ":$PATH:" in *":$BIN:"*) ON_PATH_ALREADY=yes ;; esac +if [ -d "$BIN" ]; then + PATH="$BIN:$PATH" export PATH fi -if [ -x "$USER_BIN/compartment" ]; then - COMPARTMENT="$USER_BIN/compartment" +if [ -x "$BIN/compartment" ]; then + COMPARTMENT="$BIN/compartment" elif command -v compartment >/dev/null 2>&1; then COMPARTMENT="$(command -v compartment)" elif "$PY" -c 'import compartment.cli' >/dev/null 2>&1; then COMPARTMENT="" # no console script on PATH, but the module is importable else echo "compartment was installed but cannot be found." - echo "Expected the command in: $USER_BIN" - echo "Try: \"$PY\" -m pip install --user --force-reinstall compartment" + echo "Expected the command in: $BIN" exit 1 fi @@ -48,26 +233,62 @@ run_compartment() { fi } +# --- 3. The vault --------------------------------------------------------------- +# +# Where `compartment` looks by default (home.py): COMPARTMENT_VAULT, then +# the legacy ENGRAM_VAULT, then memory.vault in ~/.compartment, or in +# ~/.engram when only that exists. A vault that is there is never touched: +# `init` refuses to overwrite one, and a re-run should not fail on that. +if [ -n "${COMPARTMENT_VAULT:-}" ]; then + VAULT="$COMPARTMENT_VAULT" +elif [ -n "${ENGRAM_VAULT:-}" ]; then + VAULT="$ENGRAM_VAULT" +elif [ ! -e "$HOME/.compartment" ] && [ -e "$HOME/.engram" ]; then + VAULT="$HOME/.engram/memory.vault" +else + VAULT="$HOME/.compartment/memory.vault" +fi + echo -echo "Creating your encrypted vault..." -run_compartment init +INIT_RAN=no +if [ -e "$VAULT" ]; then + echo "Your vault at $VAULT is kept as it is." + echo "Open it with: compartment unlock (or from the menu bar)" +else + INIT_RAN=yes + echo "Creating your encrypted vault..." + # `init` asks for a passphrase on the terminal. Run as `curl ... | bash`, + # stdin is the script itself, so the terminal is handed over explicitly. + if [ -t 0 ]; then + run_compartment init + elif (: /dev/null; then + run_compartment init $null -if ($LASTEXITCODE -ne 0) { Write-Error "Python not found. Install from python.org (check 'Add to PATH')."; exit 1 } -$repoRoot = Split-Path -Parent $PSScriptRoot -if (Test-Path (Join-Path $repoRoot "pyproject.toml")) { - & $py -m pip install --user $repoRoot +function Stop-Install([string]$message) { + Write-Host "" + Write-Host $message + # `exit` would close the PowerShell window of someone who ran this as + # `irm ... | iex`, where there is no script file to exit from. + if ($PSCommandPath) { exit 1 } else { throw "Compartment was not installed." } +} + +# --- 1. A Python that can run Compartment ------------------------------------ +# +# Compartment's compiled dependencies publish Windows wheels for 64-bit +# Python 3.11 to 3.14 on x64, and 3.12 to 3.14 on ARM64 (no pyyaml wheel for +# 3.11 there); tools/wheel_coverage.py checks that matrix in CI. 32-bit +# Python has no onnxruntime build at all. An x64 Python on an ARM64 machine +# runs under emulation and takes the x64 wheels. Anything else fails in pip +# with a bare "No matching distribution found", so each candidate is probed +# first and the first one inside the matrix is used. +# +# The probe is a file rather than `-c`: Windows PowerShell 5.1 passes an +# argument with double quotes in it to a native program mangled. +$probe = @' +import sys, sysconfig +v = sys.version_info[:2] +plat = sysconfig.get_platform() +if plat == 'win32': + print('32-bit Python %d.%d' % v) + sys.exit(2) +low = (3, 12) if plat == 'win-arm64' else (3, 11) +if not (low <= v <= (3, 14)): + print('Python %d.%d (%s)' % (v[0], v[1], plat)) + sys.exit(1) +print('ok') +'@ +$probeFile = Join-Path ([System.IO.Path]::GetTempPath()) "compartment-probe-$PID.py" +Set-Content -LiteralPath $probeFile -Value $probe -Encoding ASCII + +function Invoke-Probe($cand) { + # Local to this function: a candidate that is missing, or the Microsoft + # Store's python.exe stand-in, writes to stderr, and Windows PowerShell + # turns redirected stderr into a terminating error under "Stop". + $ErrorActionPreference = "Continue" + if (-not (Get-Command $cand.Exe -ErrorAction SilentlyContinue)) { return $null } + try { + $a = $cand.Args + $out = & $cand.Exe @a $probeFile 2>$null + return @{ Code = $LASTEXITCODE; Out = ((@($out) -join " ").Trim()) } + } catch { + return $null + } +} + +$os = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE } +$candidates = @() +if ($env:PYTHON) { + $candidates += @{ Exe = $env:PYTHON; Args = @() } } else { - & $py -m pip install --user compartment + if ($os -eq "ARM64") { + foreach ($v in "3.14", "3.13", "3.12") { $candidates += @{ Exe = "py"; Args = @("-$v-arm64") } } + } + foreach ($v in "3", "3.14", "3.13", "3.12", "3.11") { $candidates += @{ Exe = "py"; Args = @("-$v") } } + $candidates += @{ Exe = "python"; Args = @() } + $candidates += @{ Exe = "python3"; Args = @() } +} + +$exe = $null +$pyArgs = @() +$found = @() +try { + foreach ($cand in $candidates) { + $r = Invoke-Probe $cand + if ($null -eq $r) { continue } + $label = (@($cand.Exe) + $cand.Args) -join " " + if ($r.Code -eq 0) { $exe = $cand.Exe; $pyArgs = $cand.Args; break } + $found += "$label is $(if ($r.Out) { $r.Out } else { 'not runnable' })" + } +} finally { + Remove-Item -LiteralPath $probeFile -ErrorAction SilentlyContinue +} + +if (-not $exe) { + $range = if ($os -eq "ARM64") { "3.12 to 3.14 (ARM64), or 3.11 to 3.14 (x64)" } else { "3.11 to 3.14" } + $what = if ($found.Count) { "Found: " + ($found -join "; ") + "." } else { "None was found." } + Stop-Install ("Compartment needs 64-bit Python $range. $what`n" + + "Install it from python.org, or: winget install Python.Python.3.13`n" + + "then run this script again, or name it with `$env:PYTHON.") +} +Write-Host "Using $((@($exe) + $pyArgs) -join ' ')" + +function Get-PyText([string]$code) { + # UTF-8 both ways, so a profile path with a non-ASCII user name (C:\Users\ + # José) comes back as itself rather than in the console's OEM code page. + $saved = $null + try { $saved = [Console]::OutputEncoding; [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch { } + try { + $out = & $exe @pyArgs -X utf8 -c $code + } finally { + if ($saved) { try { [Console]::OutputEncoding = $saved } catch { } } + } + return ((@($out) -join "`n").Trim()) +} + +# --- 2. Install --------------------------------------------------------------- +# +# This checkout when the script is run from one, otherwise the release on +# PyPI. Run as `irm ... | iex` there is no script file and $PSScriptRoot is +# empty, which Split-Path refused, ending the script under "Stop". +$src = "compartment" +if ($PSScriptRoot) { + $repoRoot = Split-Path -Parent $PSScriptRoot + $pyproject = Join-Path $repoRoot "pyproject.toml" + if ((Test-Path -LiteralPath $pyproject) -and + (Select-String -LiteralPath $pyproject -Pattern '^name = "compartment"$' -Quiet)) { + $src = $repoRoot + } +} + +# pip refuses --user inside a virtualenv, so an activated one is installed +# into directly; otherwise --user, as before. Windows Pythons are not +# externally managed. +$kind = Get-PyText "import sys; print('venv' if sys.prefix != sys.base_prefix else 'plain')" +$pipArgs = @("-m", "pip", "install", "--upgrade") +if ($kind -ne "venv") { $pipArgs += "--user" } +& $exe @pyArgs @pipArgs $src +if ($LASTEXITCODE -ne 0) { + Stop-Install "pip could not install Compartment (exit code $LASTEXITCODE); its reason is above." } # `pip install --user` puts compartment.exe in the interpreter's own user @@ -20,8 +145,11 @@ if (Test-Path (Join-Path $repoRoot "pyproject.toml")) { # on a clean machine. Ask the interpreter where that is and use it, so the # rest of this script runs instead of dying on "compartment is not # recognized". -$userBin = & $py -c "import sysconfig; print(sysconfig.get_path('scripts', scheme='nt_user'))" -$userBin = "$userBin".Trim() +if ($kind -eq "venv") { + $userBin = Get-PyText "import sysconfig; print(sysconfig.get_path('scripts'))" +} else { + $userBin = Get-PyText "import sysconfig; print(sysconfig.get_path('scripts', scheme='nt_user'))" +} $onPathAlready = [bool](Get-Command compartment -ErrorAction SilentlyContinue) if ($userBin -and (Test-Path -LiteralPath $userBin)) { $env:Path = "$userBin;$env:Path" @@ -37,43 +165,73 @@ if ($userBin -and (Test-Path -LiteralPath (Join-Path $userBin "compartment.exe") # PowerShell turns a native command's redirected stderr into an error # under "Stop", which here means only that the import failed. try { - & $py -c "import compartment.cli" *> $null + & $exe @pyArgs -c "import compartment.cli" *> $null $importable = ($LASTEXITCODE -eq 0) } catch { $importable = $false } if (-not $importable) { - Write-Host "compartment was installed but cannot be found." - Write-Host "Expected the command in: $userBin" - Write-Host "Try: $py -m pip install --user --force-reinstall compartment" - exit 1 + Stop-Install "compartment was installed but cannot be found. Expected the command in: $userBin" } } function Invoke-Compartment { - if ($compartment) { & $compartment @args } else { & $py -m compartment.cli @args } + if ($compartment) { & $compartment @args } else { & $exe @pyArgs -m compartment.cli @args } +} + +# --- 3. The vault --------------------------------------------------------------- +# +# Where `compartment` looks by default (home.py): COMPARTMENT_VAULT, then +# the legacy ENGRAM_VAULT, then memory.vault in %USERPROFILE%\.compartment, +# or in .engram when only that exists. A vault that is there is never +# touched: `init` refuses to overwrite one, and a re-run should not fail on +# that. +$profileDir = $env:USERPROFILE +if ($env:COMPARTMENT_VAULT) { + $vault = $env:COMPARTMENT_VAULT +} elseif ($env:ENGRAM_VAULT) { + $vault = $env:ENGRAM_VAULT +} elseif (-not (Test-Path -LiteralPath (Join-Path $profileDir ".compartment")) -and + (Test-Path -LiteralPath (Join-Path $profileDir ".engram"))) { + $vault = Join-Path $profileDir ".engram\memory.vault" +} else { + $vault = Join-Path $profileDir ".compartment\memory.vault" } Write-Host "" -Write-Host "Creating your encrypted vault..." -Invoke-Compartment init +$initRan = $false +if (Test-Path -LiteralPath $vault) { + Write-Host "Your vault at $vault is kept as it is." + Write-Host "Open it with: compartment unlock (or from the tray icon)" +} else { + Write-Host "Creating your encrypted vault..." + Invoke-Compartment init + if ($LASTEXITCODE -ne 0) { + Stop-Install "compartment init stopped (exit code $LASTEXITCODE); its reason is above." + } + $initRan = $true +} if (-not $onPathAlready) { + $quoted = $userBin -replace "'", "''" Write-Host "" Write-Host "Note: $userBin is not on your PATH." Write-Host "Run this once in PowerShell so 'compartment' works in new terminals:" - Write-Host " [Environment]::SetEnvironmentVariable('Path', '$userBin;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User')" + Write-Host " [Environment]::SetEnvironmentVariable('Path', '$quoted;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User')" } +Write-Host "" +Write-Host "Done. Compartment is installed." +if ($initRan) { + Write-Host "``init`` connected the agents it found (Claude Code, Hermes Agent," + Write-Host "OpenClaw); restart them to load it." +} Write-Host @" -Done. Compartment is installed and your vault is unlocked (it stays unlocked -through logins until the next restart, then asks for your passphrase once). - -To use it with Hermes: - 1. python -m pip install --user compartment # into the Hermes venv if separate - 2. Copy-Item -Recurse integrations\hermes\compartment `$env:USERPROFILE\.hermes\plugins\compartment - 3. hermes memory setup # pick "compartment" in the list +To connect an agent: + compartment integrate claude # Claude Code + Claude Desktop + compartment integrate hermes # Hermes Agent + compartment integrate --list # every MCP client it can wire Verify anytime: compartment selftest "@ diff --git a/integrations/hermes/compartment/plugin.yaml b/integrations/hermes/compartment/plugin.yaml index 8e338f1..3874ec7 100644 --- a/integrations/hermes/compartment/plugin.yaml +++ b/integrations/hermes/compartment/plugin.yaml @@ -1,5 +1,5 @@ name: compartment -version: 4.12.0 +version: 4.13.0 description: "Compartment - high-security, fully offline, encrypted vector memory. AEAD-encrypted vault, locked-by-default after restart, native Hermes provider, no API key." # The engine. `hermes plugins install compartment` (the plugin catalog) and # `hermes memory setup` install it into the Hermes environment from this diff --git a/integrations/hermes/upstream/plugin.yaml b/integrations/hermes/upstream/plugin.yaml index 3633e8e..54e1e44 100644 --- a/integrations/hermes/upstream/plugin.yaml +++ b/integrations/hermes/upstream/plugin.yaml @@ -3,10 +3,10 @@ # `hermes memory setup` auto-installs compartment when the user selects # Compartment in the picker. Use THIS file in the upstream PR. name: compartment -version: 1.7.0 +version: 4.13.0 description: "Compartment - high-security, fully offline, encrypted vector memory. AEAD-encrypted vault, locked-by-default after restart, native Hermes provider, no API key." pip_dependencies: - - "compartment>=2.3" + - "compartment>=4.9.3" requires_env: [] hooks: - on_session_switch diff --git a/integrations/omp/README.md b/integrations/omp/README.md index 5a4466b..d48a5c1 100644 --- a/integrations/omp/README.md +++ b/integrations/omp/README.md @@ -62,5 +62,9 @@ The extension works with or without the MCP entry; both share the same vault. state or user instructions. - User-turn collection deduplicates by text; flushes are capped (20 turns, 4000 chars) per store call. +- The two tools declare their omp approval tier, `memory_search` a read and + `memory_store` a write, so an approval mode that passes reads does not + stop every recall as if it ran code, and both load as `essential`, in the + model's top-level tool list rather than behind tool search. - Vault locked? The CLI exits non-zero: recall adds nothing, a flush logs one line saying the turns were not stored, and nothing blocks the session. diff --git a/integrations/omp/extension.ts b/integrations/omp/extension.ts index b74a0c0..226534c 100644 --- a/integrations/omp/extension.ts +++ b/integrations/omp/extension.ts @@ -126,11 +126,19 @@ export default function compartmentOmp(pi: ExtensionAPI) { }); // --- Explicit tools (model-driven, MCP-independent) --------------------- + // omp treats a tool without an `approval` tier as "exec" (running code) + // and an extension tool without a `loadMode` as "discoverable", hidden + // behind tool search. Recall is a read and a store is a write, so approval + // modes that pass reads stop asking before every search; and a tool the + // model must call BEFORE answering cannot sit where it has to be found + // first, so both stay top-level. const z = pi.zod; pi.registerTool({ name: "memory_search", label: "Memory Search", + approval: "read", + loadMode: "essential", description: "Recall from the user's persistent encrypted memory vault BEFORE answering " + "anything that may depend on past work, decisions, preferences, or project " @@ -148,6 +156,8 @@ export default function compartmentOmp(pi: ExtensionAPI) { pi.registerTool({ name: "memory_store", label: "Memory Store", + approval: "write", + loadMode: "essential", description: "Store one claim into the encrypted memory vault, one per call, dated " + "automatically. " diff --git a/llms-install.md b/llms-install.md index 5f38818..385af05 100644 --- a/llms-install.md +++ b/llms-install.md @@ -16,8 +16,9 @@ Compartment needs **Python 3.11 or newer**. python3 --version ``` -If that reports anything below 3.11, stop and tell the user to install a newer -Python first. Do not try to work around it. +On Windows the command is `py --version` (or `python --version`). If it +reports anything below 3.11, stop and tell the user to install a newer Python +first. Do not try to work around it. ## 2. Install the package @@ -25,16 +26,30 @@ Python first. Do not try to work around it. pip install compartment ``` -`pipx install compartment` and `uv tool install compartment` both work too, and -are better if the user keeps their global Python clean. All three put a -`compartment` executable on the PATH. Confirm it landed: +On Windows: `py -m pip install compartment`. + +If pip refuses with `error: externally-managed-environment` (Debian 12 and +newer, Ubuntu 23.04 and newer, Homebrew's Python, Arch and others mark the +system Python as managed by the OS), do not pass `--break-system-packages`. +Use one of these instead, which install Compartment into its own environment: + +```bash +pipx install compartment +uv tool install compartment +``` + +They are also the better choice when the user keeps their global Python +clean. All three routes put a `compartment` executable on the PATH. Confirm it +landed: ```bash compartment --version ``` If the shell cannot find `compartment`, the install directory is not on PATH. -On most systems that directory is `~/.local/bin`. +On most systems that directory is `~/.local/bin` (`pipx ensurepath` or +`uv tool update-shell` adds it); on Windows it is the `Scripts` folder of the +Python installation, which `py -m pip show compartment` reveals. ## 3. Create the vault (the user must do this step, not you) @@ -100,7 +115,8 @@ Then confirm the MCP server itself starts: compartment serve --help ``` -Cline launches it as `compartment serve` over stdio. You do not need to run +Cline launches the server itself over stdio, using the full path to the +`compartment` executable that `integrate` wrote. You do not need to run `compartment serve` yourself, and you should not leave a copy running. ## Manual configuration (any MCP client) @@ -121,14 +137,23 @@ If a client is not in `--list`, add this block to its MCP config by hand: VS Code uses the key `servers` and wants `"type": "stdio"`. Zed uses `context_servers`. Codex uses TOML under `[mcp_servers.compartment]`. +A client started from the Dock, the Start menu or a desktop launcher does not +see the PATH the user's shell sets up, so a bare `compartment` that works in +the terminal can fail inside it. When writing the block by hand, prefer the +absolute path: `which compartment` prints it on macOS and Linux, +`where compartment` on Windows. `compartment integrate` always writes the +absolute path. + ## Troubleshooting **Tools return a locked error.** The vault is locked. `compartment unlock`, or `compartment unlock --keychain` on macOS so it survives the next reboot. -**`compartment: command not found` inside the client but not in the shell.** -The client is not inheriting the user's PATH. Put the absolute path to the -executable in the `command` field instead. `which compartment` prints it. +**`compartment: command not found` (or `spawn compartment ENOENT`) inside the +client but not in the shell.** The client is not inheriting the user's PATH. +Put the absolute path to the executable in the `command` field instead, or run +`compartment integrate `, which does. `which compartment` prints the +path on macOS and Linux, `where compartment` on Windows. **The client shows the server but lists no tools.** Reload the client window. Most MCP clients only read their server list at startup. diff --git a/packaging/continue/compartment.yaml b/packaging/continue/compartment.yaml new file mode 100644 index 0000000..ccc39de --- /dev/null +++ b/packaging/continue/compartment.yaml @@ -0,0 +1,23 @@ +# Compartment as a Continue block: the MCP server entry Continue's hub +# (hub.continue.dev) publishes, and a file Continue reads as it is. +# +# Without the hub, copy this file into a workspace's .continue/mcpServers/ +# folder (or ~/.continue/mcpServers/ for every workspace) and Continue picks +# it up; `compartment integrate continue` prints the same server for pasting +# under `mcpServers:` in config.yaml instead. Either way the vault must exist +# first: `pip install compartment && compartment init`. +# +# The caller is `continue`, the name `integrate continue` registers. The +# version moves with the package (tests/test_version_lockstep.py), so the +# hub shows the release the block was published from. +name: Compartment +version: 4.13.0 +schema: v1 +mcpServers: + - name: Compartment + type: stdio + command: compartment + args: + - --caller + - continue + - serve diff --git a/packaging/docker-mcp/server.yaml b/packaging/docker-mcp/server.yaml new file mode 100644 index 0000000..5f2645f --- /dev/null +++ b/packaging/docker-mcp/server.yaml @@ -0,0 +1,67 @@ +# Compartment's entry for the Docker MCP Catalog (Docker Desktop's MCP +# Toolkit, hub.docker.com/mcp). It belongs in github.com/docker/mcp-registry +# as servers/compartment/server.yaml; it is kept here so it is reviewed and +# tested with the code it describes. docs/PLUGINS.md has the submission +# steps. This repository does not read the file itself. +# +# Docker builds the image from the Dockerfile at `source.commit` and signs it +# under mcp/compartment. That Dockerfile runs the server as an unprivileged +# user over stdio, with the vault at /data/memory.vault and no port. The +# catalog adds the one thing a container cannot do for itself: mount the +# user's vault folder. disableNetwork runs the container with no network at +# all, which is the offline guarantee enforced by Docker rather than +# promised by us. +# +# The vault has to exist before the first run (`compartment init` on the +# host), because creating it asks for the passphrase and a catalog server +# has no terminal. Inside the container there is no Keychain and no boot +# session to inherit, so the passphrase arrives the way the Dockerfile +# documents, as COMPARTMENT_PASSPHRASE, kept in Docker Desktop's secret store. +# +# source.commit is the current main; Docker's pin-update workflow moves it +# with each release once the entry is merged. +name: compartment +image: mcp/compartment +type: server +meta: + category: productivity + tags: + - memory + - agent-memory + - encryption + - offline + - local-first + - privacy +about: + title: Compartment + description: | + Encrypted, fully offline long-term memory for AI agents. One vault on + your own machine, AEAD-encrypted at rest including the vector index, + shared by every MCP client: what one agent stores, the others recall. + Embedding runs locally inside the container, which runs with no + network. Create the vault once on the host with + `pip install compartment && compartment init`, then mount its folder. + icon: https://raw.githubusercontent.com/MaxFreedomPollard/Compartment/main/docs/images/logo-400.png +source: + project: https://github.com/MaxFreedomPollard/Compartment + commit: be47c40ce1e7fc62ff213a926fe5405e571d34a8 +run: + volumes: + - "{{compartment.vault_dir}}:/data" + disableNetwork: true +config: + description: Mount the folder that holds your vault and give its passphrase + secrets: + - name: compartment.passphrase + env: COMPARTMENT_PASSPHRASE + example: + description: The passphrase you chose at `compartment init` + parameters: + type: object + properties: + vault_dir: + type: string + description: Folder on this machine that holds memory.vault (created by `compartment init`, usually ~/.compartment) + default: /Users/local-test/.compartment + required: + - vault_dir diff --git a/packaging/mcpb/.python-version b/packaging/mcpb/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/packaging/mcpb/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/packaging/mcpb/manifest.json b/packaging/mcpb/manifest.json new file mode 100644 index 0000000..9c8e177 --- /dev/null +++ b/packaging/mcpb/manifest.json @@ -0,0 +1,70 @@ +{ + "manifest_version": "0.4", + "name": "compartment", + "display_name": "Compartment", + "version": "4.13.0", + "description": "Encrypted, fully offline memory for Claude: one vault on your machine, shared with every MCP client. No network, no API key.", + "long_description": "Compartment is persistent memory for AI agents, stored on your own computer and encrypted at rest. What Claude learns in one conversation is available in every later one, and to Claude Code, Cursor, Codex, Gemini CLI, Hermes Agent, OpenClaw and any other MCP client on the same machine. Nothing leaves it: the embedding model runs locally, the server speaks stdio only and opens no ports, and there is no account.\n\n**Before the first run**, create the vault once in a terminal: `pip install compartment && compartment init` (or `pipx install compartment`, or `uv tool install compartment`). It asks for a passphrase you choose; that passphrase is the only key and Compartment cannot recover it. If the vault is locked after a restart, run `compartment unlock`.\n\nIf you already connected Claude Desktop with `compartment integrate claude` or the app's Connect button, you do not need this extension as well: both would start the same server and Claude would see every tool twice.", + "author": { + "name": "Max Freedom Pollard", + "url": "https://github.com/MaxFreedomPollard" + }, + "repository": { + "type": "git", + "url": "https://github.com/MaxFreedomPollard/Compartment" + }, + "homepage": "https://maxfreedompollard.github.io/Compartment/", + "documentation": "https://github.com/MaxFreedomPollard/Compartment#readme", + "support": "https://github.com/MaxFreedomPollard/Compartment/issues", + "icon": "icon.png", + "server": { + "type": "uv", + "entry_point": "server/main.py", + "mcp_config": { + "command": "uv", + "args": [ + "run", + "--directory", + "${__dirname}", + "server/main.py" + ] + } + }, + "tools": [ + {"name": "memory_store", "description": "Save one claim to the encrypted memory, with its source and date."}, + {"name": "memory_store_many", "description": "Save several separate facts in one call, one memory each."}, + {"name": "memory_search", "description": "Recall from memory before answering: hybrid vector and keyword search."}, + {"name": "memory_link", "description": "Record a relationship between two entities in the memory graph."}, + {"name": "memory_relations", "description": "Query the memory graph, optionally as of a date."}, + {"name": "memory_unlink", "description": "Remove one relation from the memory graph."}, + {"name": "memory_get", "description": "Fetch one memory by id."}, + {"name": "memory_forget", "description": "Delete a memory, optionally crypto-shredding it."}, + {"name": "memory_list_namespaces", "description": "List namespaces and their record counts."}, + {"name": "memory_recent", "description": "The most recently stored memories."}, + {"name": "memory_status", "description": "Vault status: lock state, counts, model, index."}, + {"name": "memory_selftest", "description": "Health check against the built-in reference facts."}, + {"name": "memory_lock", "description": "Panic lock: seal the vault and drop the key now."}, + {"name": "memory_unlock", "description": "Unlock from the agent; disabled unless the vault config enables it."} + ], + "tools_generated": false, + "keywords": [ + "memory", + "agent-memory", + "encryption", + "offline", + "local-first", + "privacy", + "vector-search" + ], + "license": "Apache-2.0", + "compatibility": { + "platforms": [ + "darwin", + "win32", + "linux" + ], + "runtimes": { + "python": ">=3.11" + } + } +} diff --git a/packaging/mcpb/pyproject.toml b/packaging/mcpb/pyproject.toml new file mode 100644 index 0000000..4e241e8 --- /dev/null +++ b/packaging/mcpb/pyproject.toml @@ -0,0 +1,31 @@ +# The environment Claude Desktop builds for the Compartment extension. +# +# The bundle is an MCP Bundle of server type "uv" (manifest_version 0.4): +# it ships no Python packages of its own. Claude Desktop reads this file and +# has uv install the pinned release from PyPI, with the right interpreter and +# the right compiled wheels for the machine, once, when the extension is +# installed. That is the same network use as `pip install compartment`, and +# the only one: the server it then starts makes no connection at all. +# +# Not a package. It has a different name from its one dependency because a +# project may not depend on itself, and uv is told not to build it. +# +# The pin moves with every release (tests/test_version_lockstep.py), so the +# bundle attached to a release installs exactly that release. + +[project] +name = "compartment-claude-desktop" +version = "4.13.0" +description = "Compartment as a Claude Desktop extension (MCP Bundle)" +# uv runs the newest interpreter it can find or download unless told +# otherwise, and the compiled dependencies trail new Python releases: +# onnxruntime 1.31 has wheels for 3.11 to 3.14 and none for 3.15, so an +# unpinned bundle installed after 3.15 shipped would fail to build its +# environment. .python-version picks 3.12 (what CI runs); the upper bound +# here makes a host that overrides it fail with a clear message instead of +# a missing-wheel error. Raise both when onnxruntime ships wheels for more. +requires-python = ">=3.11,<3.15" +dependencies = ["compartment==4.13.0"] + +[tool.uv] +package = false diff --git a/packaging/mcpb/server/main.py b/packaging/mcpb/server/main.py new file mode 100644 index 0000000..9f78ddb --- /dev/null +++ b/packaging/mcpb/server/main.py @@ -0,0 +1,19 @@ +"""Compartment's MCP server, as the Claude Desktop extension starts it. + +Claude Desktop runs this file with `uv run` inside the environment it built +from the bundle's pyproject.toml, so `compartment` here is the pinned +release in that environment, not whatever may or may not be on the PATH a +GUI application was started with. That is the reason the bundle exists: +every other Claude Desktop route needs the `compartment` command findable, +and a Dock-launched app does not inherit the shell's PATH. + +The caller is `claude-desktop`, the same name `compartment integrate +claude` registers, so the audit log and any per-caller ACL treat the two +routes as one client. The vault is Compartment's default (COMPARTMENT_HOME +or ~/.compartment, or the ~/.engram an older install left), which is where +`compartment init` creates it. +""" +from compartment.cli import main + +if __name__ == "__main__": + main(["--caller", "claude-desktop", "serve"]) diff --git a/plugin.json b/plugin.json index ea770df..183385d 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "compartment", - "version": "4.12.0", + "version": "4.13.0", "description": "High-security, fully offline, encrypted vector memory for AI agents. AEAD-encrypted vault, locked by default after restart, no API key and no network at runtime.", "author": { "name": "Max Freedom Pollard", @@ -19,5 +19,27 @@ "vector-search", "privacy", "mcp" - ] + ], + "extensions": { + "com.openai": { + "interface": { + "displayName": "Compartment", + "shortDescription": "Encrypted, offline memory for agents", + "longDescription": "One encrypted memory vault on your own machine, shared by Codex, Claude, Gemini CLI, Hermes, OpenClaw and every other MCP client. Recall before answering, store what is worth keeping; no network, no API key, no account. Requires `pip install compartment && compartment init` first.", + "developerName": "Max Freedom Pollard", + "category": "Productivity", + "capabilities": [ + "Read", + "Write" + ], + "websiteURL": "https://maxfreedompollard.github.io/Compartment/", + "privacyPolicyURL": "https://maxfreedompollard.github.io/Compartment/privacy", + "defaultPrompt": [ + "What do you remember about this project?", + "Remember the decisions we made today." + ], + "logo": "./docs/images/logo-400.png" + } + } + } } diff --git a/pyproject.toml b/pyproject.toml index 798e873..0fb848a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "compartment" -version = "4.12.0" +version = "4.13.0" description = "Compartment - high-security, fully offline, encrypted vector memory for AI agents (MCP)" readme = "README.md" license = { text = "Apache-2.0" } @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Security :: Cryptography", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Database :: Database Engines/Servers", @@ -41,8 +42,16 @@ dependencies = [ # Both majors. mcp 2.0 (2026-07-28) removed mcp.server.fastmcp in favour # of mcp.server.mcpserver; server.py imports whichever is present, so the # same wheel runs on 1.x and 2.x and single-version distros are not - # stranded on either side of the rename. - "mcp>=1.0,<3", + # stranded on either side of the rename. The floor is 1.21.2, not 1.0: + # nothing before 1.10 can register the tools at all (FastMCP arrived in + # 1.2, tool titles in 1.10), and 1.10 through 1.21.1 break against the + # pydantic 2.12+ that pip installs beside them - an import error inside + # mcp, or a TypeError on this module's postponed annotations - so an + # older mcp already in the environment meant a server that never + # started while pip called the install satisfied. With this floor pip + # upgrades it instead. CI's lowest-dependencies job runs the suite at + # exactly these floors. + "mcp>=1.21.2,<3", # The app ships with every install. It is not an add-on: unlocking, # locking and changing the passphrase are things people do through a # panel, not through a terminal, so an install that only lays down a CLI @@ -53,6 +62,21 @@ dependencies = [ "pyobjc-framework-Cocoa>=10.0; sys_platform == 'darwin'", "pystray>=0.19; sys_platform == 'win32'", "pillow>=10; sys_platform == 'win32'", + # Two platforms the newest releases of two indirect dependencies left + # without wheels. pip takes the newest version whether or not it has a + # wheel for the machine, and then tries to compile it: cryptography (via + # mcp -> pyjwt[crypto]) needs a Rust toolchain, so `pip install + # compartment` simply failed on a Snapdragon Windows laptop and on an + # Intel Mac, and the release build of the Intel app failed with it. + # These caps hold each one at its last release with a wheel there, on + # that platform only; everywhere else nothing changes. Compartment + # itself never imports cryptography - mcp needs it only for its HTTP + # transports' auth, which a stdio server does not use. Lift a cap when + # the project ships those wheels again; CI's windows-11-arm and + # macos-15-intel jobs show it. + "cryptography<46.0.4; sys_platform == 'win32' and platform_machine == 'ARM64'", + "cryptography<49; sys_platform == 'darwin' and platform_machine == 'x86_64'", + "argon2-cffi-bindings<26; sys_platform == 'darwin' and platform_machine == 'x86_64'", ] # PyPI is the repo's declared homepage, so it is a real front door - and @@ -87,7 +111,7 @@ hnsw = ["usearch>=2.9"] # failing on an unknown extra. They add nothing. menubar = [] tray = [] -dev = ["pytest>=8", "pytest-cov>=5", "usearch>=2.9"] +dev = ["pytest>=8", "pytest-cov>=5", "pytest-timeout>=2.3", "usearch>=2.9"] [tool.setuptools.packages.find] where = ["src"] diff --git a/registry.html b/registry.html index 5f106d1..536fda8 100644 --- a/registry.html +++ b/registry.html @@ -15,7 +15,7 @@

Compartment in the official MCP Registry

Compartment is published in the Model Context Protocol registry, the shared catalogue that MCP clients read, under the name io.github.MaxFreedomPollard/compartment. Each release is published there by the same workflow that publishes to PyPI, so the registry's latest version is always the one pip install compartment serves.

Install

pip install compartment && compartment init
-

Then wire your client: compartment integrate claude, hermes, openclaw, cursor, or any of the 28 clients that compartment integrate --list names. Clients that install straight from the registry run it as uvx compartment serve.

+

Then wire your client: compartment integrate claude, hermes, openclaw, cursor, or any other client that compartment integrate --list names. Clients that install straight from the registry run it as uvx compartment serve.

Verify it yourself

The registry has no web page of its own; it is an API. The current entry, as JSON: every published version, or a search for "compartment". The version flagged isLatest names the PyPI release it installs.

Compartment on GitHub · on PyPI

diff --git a/server.json b/server.json index 0baeef5..95a0ad8 100644 --- a/server.json +++ b/server.json @@ -3,18 +3,28 @@ "name": "io.github.MaxFreedomPollard/compartment", "title": "Compartment", "description": "Durable agentic memory, encrypted at rest. Fully offline: no network, no API key, no cloud.", + "websiteUrl": "https://maxfreedompollard.github.io/Compartment/", + "icons": [ + { + "src": "https://raw.githubusercontent.com/MaxFreedomPollard/Compartment/main/docs/images/logo-400.png", + "mimeType": "image/png", + "sizes": [ + "400x400" + ] + } + ], "repository": { "url": "https://github.com/MaxFreedomPollard/Compartment", "source": "github", "id": "1306651494" }, - "version": "4.12.0", + "version": "4.13.0", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "compartment", - "version": "4.12.0", + "version": "4.13.0", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/compartment/__init__.py b/src/compartment/__init__.py index 6b639b0..ef89784 100644 --- a/src/compartment/__init__.py +++ b/src/compartment/__init__.py @@ -1,6 +1,6 @@ """Compartment - high-security, fully offline, encrypted vector memory for AI agents.""" -__version__ = "4.12.0" +__version__ = "4.13.0" from . import offline_guard as _og diff --git a/src/compartment/acl.py b/src/compartment/acl.py index 7534f36..7b7a16a 100644 --- a/src/compartment/acl.py +++ b/src/compartment/acl.py @@ -54,6 +54,16 @@ class AclError(CryptoError): pass +class ConfigError(ValueError): + """The config file exists and cannot be used. + + A ValueError, as the json error it replaces was, and deliberately not a + CryptoError: callers read a CryptoError out of an unlock as "the vault + is locked" (the capture hook does, and says so), and a damaged settings + file is not that. Every caller that coped with the old error copes with + this one; what changes is that it says which file and what to do.""" + + @dataclass class VaultConfig: # Deep copies: a shallow copy would hand every VaultConfig in the process @@ -73,13 +83,42 @@ def load(cls, vault_path: str) -> "VaultConfig": p = cls.path_for(vault_path) if not os.path.exists(p): return cls() - with open(p, encoding="utf-8") as f: - data = json.load(f) + # Bytes, not text: json.loads detects the encoding itself, so a file + # saved with a UTF-8 byte order mark (Notepad's "UTF-8 with BOM", + # Windows PowerShell 5.1's Set-Content -Encoding UTF8) or as UTF-16 + # (that PowerShell's `>` redirection) opens like any other. Read as + # UTF-8 text, the first of those failed every unlock with + # "Unexpected UTF-8 BOM", after a user had done nothing but edit + # their ACLs in the editor Windows ships. + with open(p, "rb") as f: + raw = f.read() + try: + data = json.loads(raw) + except (ValueError, UnicodeDecodeError) as exc: + raise ConfigError( + cls._unreadable(p, f"it is not valid JSON ({exc})")) from None + if not isinstance(data, dict) or not isinstance( + data.get("callers", {}), dict) or not isinstance( + data.get("settings", {}), dict): + raise ConfigError(cls._unreadable( + p, 'it must be a JSON object whose "callers" and "settings" ' + 'are objects')) cfg = cls() cfg.callers = data.get("callers", cfg.callers) cfg.settings = {**cfg.settings, **data.get("settings", {})} return cfg + @staticmethod + def _unreadable(path: str, why: str) -> str: + """The refusal for a config that cannot be used. It holds the access + rules, so the vault does not open on a guess, and the message says + what moving it aside would mean, since the defaults grant every + caller read-write everywhere.""" + return (f"{path} cannot be used: {why}. It holds this vault's access " + "rules and settings, so the vault is not opened without it. " + "Fix the file, or move it aside to fall back to the defaults " + "(every caller reads and writes every namespace).") + def save(self, vault_path: str) -> None: # Every unlock reads this file, so it is never rewritten in place: a # reader would meet it empty, and a write that fails (a full disk) diff --git a/src/compartment/agent_skill.py b/src/compartment/agent_skill.py index db67abe..edb8584 100644 --- a/src/compartment/agent_skill.py +++ b/src/compartment/agent_skill.py @@ -15,9 +15,9 @@ `/skills//SKILL.md`, YAML frontmatter, markdown body - so one packaged file serves all three and the only thing that differs is the root: - claude ~/.claude/skills/compartmentalize/SKILL.md + claude $CLAUDE_CONFIG_DIR or ~/.claude/skills/compartmentalize/SKILL.md hermes $HERMES_HOME or ~/.hermes/skills/compartmentalize/SKILL.md - openclaw $OPENCLAW_HOME or ~/.openclaw/skills/compartmentalize/SKILL.md + openclaw /skills/compartmentalize/SKILL.md Writing into somebody's skills directory is a larger claim on their machine than registering an MCP server, so this module never does it silently and never @@ -35,12 +35,46 @@ #: Written next to a skill we are about to replace, when it is not ours. BACKUP_SUFFIX = ".compartment-backup" -#: Every agent known to read the Agent Skills layout, and the environment -#: variable that relocates its home. Claude Code has no such variable. +def _openclaw_dir() -> Path | None: + """OpenClaw's config directory when the environment moves it, resolved + as OpenClaw resolves it (src/infra/config-dir.ts): OPENCLAW_STATE_DIR is + the directory; else the one holding OPENCLAW_CONFIG_PATH; else + OPENCLAW_HOME, which stands in for the home directory, with .openclaw + under it. OPENCLAW_HOME used to be taken for the directory itself, so + with it set the skill went one level above the skills/ OpenClaw loads + managed skills from.""" + state = os.environ.get("OPENCLAW_STATE_DIR", "").strip() + if state: + return Path(state).expanduser() + config = os.environ.get("OPENCLAW_CONFIG_PATH", "").strip() + if config: + return Path(config).expanduser().parent + home = os.environ.get("OPENCLAW_HOME", "").strip() + if home: + return Path(home).expanduser() / ".openclaw" + return None + + +def _claude_dir() -> Path | None: + """Claude Code's directory when CLAUDE_CONFIG_DIR moves it, by the same + rule the hook and the import use (an absolute path only).""" + if not os.environ.get("CLAUDE_CONFIG_DIR", "").strip(): + return None + from .claude_memory import claude_config_dir + return claude_config_dir() + + +#: Every agent known to read the Agent Skills layout, and what relocates +#: its home: the environment variable naming it, or a function working it +#: out. Claude Code's is CLAUDE_CONFIG_DIR, which moves its skills/ with +#: everything else. The default home is a function too, called when it is +#: needed: Path.home() raises where HOME is unset and the uid has no passwd +#: entry, and evaluated here it killed every `compartment` command at +#: import in such a container. SKILL_TARGETS = { - "claude": (None, Path.home() / ".claude"), - "hermes": ("HERMES_HOME", Path.home() / ".hermes"), - "openclaw": ("OPENCLAW_HOME", Path.home() / ".openclaw"), + "claude": (_claude_dir, lambda: Path.home() / ".claude"), + "hermes": ("HERMES_HOME", lambda: Path.home() / ".hermes"), + "openclaw": (_openclaw_dir, lambda: Path.home() / ".openclaw"), } @@ -55,9 +89,13 @@ def agent_home(target: str) -> Path | None: if entry is None: return None var, default = entry - if var and os.environ.get(var): + if callable(var): + override = var() + if override: + return override + elif var and os.environ.get(var): return Path(os.environ[var]).expanduser() - return default + return default() if callable(default) else default def skill_path(target: str) -> Path | None: diff --git a/src/compartment/claude_desktop.py b/src/compartment/claude_desktop.py index dea524d..afa1c01 100644 --- a/src/compartment/claude_desktop.py +++ b/src/compartment/claude_desktop.py @@ -20,21 +20,87 @@ NAME = "compartment" FILENAME = "claude_desktop_config.json" +#: The MSIX package families Claude Desktop is published under on Windows: +#: the direct download (whose Claude Setup.exe has been an MSIX bootstrapper +#: since 2026, removing any older install) and the Microsoft Store build. +#: Taken from `Get-AppxPackage` output and the app's own main.log quoted in +#: anthropics/claude-ai-mcp#1150 and shraga100/claude-desktop-rtl-patch#45. +MSIX_FAMILIES = ("Claude_pzs8sxrjxfjjc", "AnthropicPBC.Claude_fnn82j28hfe8t") + + +def _windows() -> bool: + return os.name == "nt" + + +def _local_appdata() -> Path: + return Path(os.environ.get("LOCALAPPDATA") + or Path.home() / "AppData" / "Local") + + +def _roaming_config() -> Path: + """%APPDATA%\\Claude\\claude_desktop_config.json, the real one.""" + base = os.environ.get("APPDATA") or str(Path.home() / "AppData" + / "Roaming") + return Path(base) / "Claude" / FILENAME + + +def _msix_configs() -> list[Path]: + """The private config of each MSIX package of Claude Desktop installed + for this user, installed meaning its package directory exists.""" + packages = _local_appdata() / "Packages" + out = [] + for family in MSIX_FAMILIES: + try: + if (packages / family).is_dir(): + out.append(packages / family / "LocalCache" / "Roaming" + / "Claude" / FILENAME) + except OSError: + pass + return out + def config_path() -> Path: """Claude Desktop's config file on this platform. - The three locations are the app's own, not ours: macOS keeps it under - Application Support, Windows under %APPDATA%, Linux under the XDG config - directory. + The locations are the app's own, not ours: macOS keeps it under + Application Support, Linux (the official beta since June 2026, and the + community builds before it) under the XDG config directory, and Windows + under %APPDATA% - except that an MSIX app does not write %APPDATA%. + + Claude Desktop on Windows is an MSIX package now, from the Store or + from the direct download, and Windows redirects a packaged app's + writes under %APPDATA% into the package's own + %LOCALAPPDATA%\\Packages\\\\LocalCache\\Roaming. The app reads + that private copy first. Compartment is not inside the package, so it + wrote the real %APPDATA%\\Claude file, which the app had stopped + reading the first time it saved its settings: `integrate claude` + reported Claude Desktop connected and the app never saw the entry. + + So with a package installed, its private copy is the file, once it + exists. Until it does, the app still reads the real file through + Windows' merged view, and that is the one written: a new private file + holding only our server would hide every server already in the real + one. With neither, the private path is created, which the app will + read first. Of two packages with a private copy each, the one saved + last is the one in use. """ if sys.platform == "darwin": return (Path.home() / "Library" / "Application Support" / "Claude" / FILENAME) - if os.name == "nt": - base = os.environ.get("APPDATA") or str(Path.home() / "AppData" - / "Roaming") - return Path(base) / "Claude" / FILENAME + if _windows(): + roaming = _roaming_config() + packaged = _msix_configs() + if not packaged: + return roaming + written = [] + for p in packaged: + try: + written.append((p.stat().st_mtime, p)) + except OSError: + pass + if written: + return max(written)[1] + return roaming if roaming.exists() else packaged[0] base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") return Path(base) / "Claude" / FILENAME @@ -46,6 +112,11 @@ def present(path: Path | None = None) -> bool: copy that has never been opened is recognised by its application directory instead. Writing a config for an app that is not installed would leave litter behind and connect nothing. + + On Windows that is an MSIX package directory, or the older installer's + %LOCALAPPDATA%\\AnthropicClaude. A package that has never been opened + has no %APPDATA%\\Claude to find, so it was reported as not installed + and `integrate claude` skipped it. """ p = Path(path) if path else config_path() if p.exists() or p.parent.is_dir(): @@ -53,6 +124,12 @@ def present(path: Path | None = None) -> bool: if sys.platform == "darwin": return (Path("/Applications/Claude.app").exists() or (Path.home() / "Applications" / "Claude.app").exists()) + if _windows() and path is None: + try: + return (bool(_msix_configs()) + or (_local_appdata() / "AnthropicClaude").is_dir()) + except OSError: + return False return False @@ -66,16 +143,10 @@ def is_registered(path: Path | None = None) -> bool: def _read(path: Path) -> dict: - if not path.exists(): - return {} - try: - text = path.read_text(encoding="utf-8").strip() - data = json.loads(text) if text else {} - except (json.JSONDecodeError, OSError): - raise ValueError(f"{path} is not valid JSON; refusing to touch it") - if not isinstance(data, dict): # null, a list: not a config - raise ValueError(f"{path} is not a JSON object; refusing to touch it") - return data + """The config as a dict, or ValueError: the same reader every other + client's config goes through, so a file Notepad or PowerShell saved + with a byte order mark, or as UTF-16, is read rather than refused.""" + return clients._read_json(path) def register(command: str, args: list[str], path: Path | None = None) -> dict: diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index f7a4954..d90e423 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -25,7 +25,25 @@ from . import claude_memory, clients -SETTINGS = Path.home() / ".claude" / "settings.json" +# Claude Code's user settings, which move with CLAUDE_CONFIG_DIR, are +# SETTINGS: worked out when asked for rather than at import, since +# Path.home() raises where HOME is unset and the uid has no passwd entry +# (a container run as an arbitrary uid), and an import-time lookup killed +# every `compartment` command there. Assigning SETTINGS still pins it. + + +def settings_path() -> Path: + """Claude Code's user settings.json: SETTINGS when it has been + assigned, otherwise /settings.json, now.""" + pinned = globals().get("SETTINGS") + return Path(pinned) if pinned is not None else ( + claude_memory.claude_config_dir() / "settings.json") + + +def __getattr__(name: str): + if name == "SETTINGS": + return claude_memory.claude_config_dir() / "settings.json" + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") # Identifies OUR entries. Both tokens, not one literal string: the command # carries `--vault ...` between them when a vault is pinned, so a single # substring would stop matching and re-installing would duplicate the hook. @@ -42,16 +60,50 @@ def _entry(command: str) -> dict: "timeout": TIMEOUT}]} +#: Characters a word can hold and still mean itself to a POSIX shell. +_SHELL_SAFE = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789_./:@%+=,-") + + +def _windows() -> bool: + return os.name == "nt" + + +def _shell_word(s: str, always: bool = False) -> str: + """`s` as one word for the shell Claude Code runs hooks in. + + That shell is bash on every platform, Git Bash on Windows (Claude Code + 2.1.47 and later), and an unquoted backslash is an escape character to + it. The hook used to quote the executable only when its path held a + space, so the usual Windows path (C:\\Users\\ana\\...\\compartment.exe) + reached bash as C:Usersana...compartment.exe: "command not found" after + every Write and Edit, and nothing was ever captured. On Windows the path is + written with forward slashes, which Git Bash, cmd and PowerShell all + take as a path and none of them treats as an escape; it is still the + same file to Windows, and the vault it names opens the same session, + since the session store normalises the path. Anything still special + inside double quotes ($, `, ", \\) is escaped, so a path with one of + those in it is not expanded either. A word that needs no quoting is + left bare unless `always`: a bare executable path is also one + PowerShell runs, where a quoted one is a string expression to it. + """ + if _windows(): + s = s.replace("\\", "/") + if not always and s and all(c in _SHELL_SAFE for c in s): + return s + for ch in ("\\", '"', "$", "`"): + s = s.replace(ch, "\\" + ch) + return f'"{s}"' + + def hook_command(compartment_bin: str | None = None, vault: str | None = None) -> str: """The exact shell command the hook runs. The compartment beside this interpreter, as every MCP registration names it: a bare name only works where `compartment` is on the PATH Claude Code's shell gets, and the .app install puts it on none.""" - exe = compartment_bin or clients.executable() - if " " in exe: - exe = f'"{exe}"' - vault_part = f' --vault "{vault}"' if vault else "" + exe = _shell_word(compartment_bin or clients.executable()) + vault_part = f" --vault {_shell_word(vault, always=True)}" if vault else "" return f"{exe}{vault_part} hook capture" @@ -62,35 +114,66 @@ def is_ours(command: object) -> bool: def is_installed(settings: Path | None = None) -> bool: - data = _read(settings or SETTINGS) - for group in (data.get("hooks", {}) or {}).get(EVENT, []) or []: - for h in group.get("hooks", []) or []: + """Whether compartment's capture hook is in settings.json. Raises + ValueError when the file cannot be read as settings.""" + path = Path(settings) if settings else settings_path() + for group in _groups(_read(path), path): + for h in group.get("hooks") or []: if is_ours(h.get("command")): return True return False def _read(path: Path) -> dict: - if not path.exists(): - return {} - try: - text = path.read_text(encoding="utf-8").strip() - data = json.loads(text) if text else {} - except (json.JSONDecodeError, OSError): - raise ValueError(f"{path} is not valid JSON; refusing to touch it") - if not isinstance(data, dict): # null, a list: not settings - raise ValueError(f"{path} is not a JSON object; refusing to touch it") - return data + """settings.json as a dict, or ValueError, through the reader every + client config goes through (a byte order mark or UTF-16 is decoded).""" + return clients._read_json(path) + + +def _groups(data: dict, path: Path) -> list: + """The PostToolUse matcher groups, checked for the shape Claude Code + reads: `hooks` an object, the event a list of objects, each one's + `hooks` a list of objects. + + Anything else is somebody's file in a shape this does not understand, + and it is refused as a malformed one is. It used to be walked as if it + were right: a list where the object belongs died with AttributeError, + which `integrate claude`, `hook status` and the uninstaller do not + catch, so the user got a traceback instead of the file's name and what + to fix, and an event that was an object was replaced by a list.""" + hooks = data.get("hooks") + if hooks is None: + return [] + if not isinstance(hooks, dict): + raise ValueError(f"{path}: \"hooks\" is not an object; refusing to " + "touch it") + groups = hooks.get(EVENT) + if groups is None: + return [] + if not isinstance(groups, list): + raise ValueError(f"{path}: \"hooks.{EVENT}\" is not a list; " + "refusing to touch it") + for group in groups: + inner = group.get("hooks") if isinstance(group, dict) else None + if not isinstance(group, dict) or not ( + inner is None or (isinstance(inner, list) and all( + isinstance(h, dict) for h in inner))): + raise ValueError(f"{path}: an entry in \"hooks.{EVENT}\" is not " + "a matcher with a list of hooks; refusing to " + "touch it") + return groups def install(compartment_bin: str | None = None, vault: str | None = None, settings: Path | None = None) -> dict: """Add (or refresh) compartment's capture hook. Never removes anyone else's.""" - path = Path(settings) if settings else SETTINGS + path = Path(settings) if settings else settings_path() data = _read(path) # raises on malformed + groups = _groups(data, path) # and on a bad shape - hooks = data.setdefault("hooks", {}) - groups = hooks.setdefault(EVENT, []) + if data.get("hooks") is None: # absent, or null + data["hooks"] = {} + hooks = data["hooks"] # drop only our own previous entries, so re-installing updates in place kept = [] for group in groups: @@ -111,9 +194,9 @@ def install(compartment_bin: str | None = None, vault: str | None = None, def uninstall(settings: Path | None = None) -> bool: - path = Path(settings) if settings else SETTINGS + path = Path(settings) if settings else settings_path() data = _read(path) - groups = (data.get("hooks", {}) or {}).get(EVENT) + groups = _groups(data, path) if not groups: return False kept, removed = [], False @@ -153,12 +236,29 @@ def _file_from_payload(payload: dict) -> str | None: def is_memory_file(path: str | Path) -> bool: """True for Claude Code's own memory notes - and nothing else. The index - is a table of contents, so it is skipped like the importer skips it.""" + is a table of contents, so it is skipped like the importer skips it. + + A memory directory under the CLAUDE_CONFIG_DIR Claude Code was started + with counts as well: that directory need not be called .claude, and + when it was not, every memory write was passed over. So does the + directory the autoMemoryDirectory setting names.""" p = Path(path) if p.suffix.lower() != ".md" or p.name == claude_memory.INDEX_NAME: return False + custom = claude_memory.auto_memory_dir() + if custom is not None and os.path.abspath(p.parent) == os.path.abspath( + custom): + return True # autoMemoryDirectory's notes parts = [x.lower() for x in p.parts] - return "memory" in parts and ".claude" in parts + if "memory" not in parts: + return False + if ".claude" in parts: + return True + try: + return Path(os.path.abspath(p)).is_relative_to( + os.path.abspath(claude_memory.claude_config_dir())) + except (OSError, ValueError): + return False def _earlier_versions(v, tag: str, ns: str, text: str) -> list[str] | None: @@ -248,4 +348,5 @@ def capture(stream=None, vault_path: str | None = None) -> dict: __all__ = ["install", "uninstall", "is_installed", "is_ours", "capture", "hook_command", - "is_memory_file", "SETTINGS", "MARKER", "EVENT", "MATCHER"] + "is_memory_file", "SETTINGS", "settings_path", "MARKER", "EVENT", + "MATCHER"] diff --git a/src/compartment/claude_memory.py b/src/compartment/claude_memory.py index ec3eb31..b02384c 100644 --- a/src/compartment/claude_memory.py +++ b/src/compartment/claude_memory.py @@ -27,8 +27,59 @@ import re from pathlib import Path -# Claude Code's per-project memory lives under //memory. -DEFAULT_ROOT = Path.home() / ".claude" / "projects" + + +def claude_config_dir() -> Path: + """Claude Code's own directory: $CLAUDE_CONFIG_DIR when it is set, + ~/.claude otherwise. + + Claude Code keeps everything there - settings.json, the per-project + memory under projects/, skills/, the user CLAUDE.md - and moves all of + it when CLAUDE_CONFIG_DIR is set ("Respect CLAUDE_CONFIG_DIR + everywhere", Claude Code 1.0.6), which is how people run a second + account or keep it in a dotfiles tree. Compartment only ever looked in + ~/.claude, so for them the capture hook went into a settings.json + Claude Code never read, the import found no memories, and the skill + landed where no session would load it. Like Claude Code's own VS Code + extension, only an absolute path counts; an empty or relative value is + ignored.""" + raw = os.environ.get("CLAUDE_CONFIG_DIR", "").strip() + if raw: + p = Path(raw).expanduser() + if p.is_absolute(): + return p + return Path.home() / ".claude" + + +# Claude Code's per-project memory lives under //memory, +# and is DEFAULT_ROOT. It is worked out when it is asked for, not at +# import: in a container run as an arbitrary uid, with HOME unset and no +# passwd entry, Path.home() raises, and computing it here killed every +# `compartment` command at import, including the ones that never touch +# Claude Code. A caller may still pin DEFAULT_ROOT by assigning it. + + +def _projects() -> Path: + try: + return claude_config_dir() / "projects" + except RuntimeError: + # No home directory to be found, so no Claude Code memory either: + # a path that names nothing, for discovery to find nothing in and + # `import-claude --help` to show, rather than an error. + return Path("~") / ".claude" / "projects" + + +def default_root() -> Path: + """Where Claude Code keeps per-project memory: DEFAULT_ROOT when it has + been assigned, otherwise /projects, now.""" + pinned = globals().get("DEFAULT_ROOT") + return Path(pinned) if pinned is not None else _projects() + + +def __getattr__(name: str): + if name == "DEFAULT_ROOT": + return _projects() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") INDEX_NAME = "MEMORY.md" # a table of contents, not a fact # Frontmatter `type` -> Compartment importance. Mirrors the vault's own tiers: @@ -44,9 +95,42 @@ _FRONTMATTER = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", re.DOTALL) +def auto_memory_dir() -> Path | None: + """The directory Claude Code's `autoMemoryDirectory` setting moves + auto memory to, from the user settings.json beside DEFAULT_ROOT, or + None when it is not set. + + Since Claude Code 2.1.74 that one setting replaces + projects//memory/ with any directory the user names (~/ + expanded; it is ignored in a project's checked-in settings, so only + the user's own file is read). Memory written there was invisible to + both the import and the capture hook, which looked only under + projects/. A relative value is ignored: there is no project here to + resolve it against.""" + try: + from .clients import _read_json + value = _read_json(default_root().parent / "settings.json").get( + "autoMemoryDirectory") + except (OSError, ValueError): + return None + if not isinstance(value, str) or not value.strip(): + return None + d = Path(value.strip()).expanduser() + return d if d.is_absolute() else None + + def discover(root: Path | str | None = None) -> list[Path]: - """Every Claude Code memory file, sorted, index files excluded.""" - root = Path(root) if root else DEFAULT_ROOT + """Every Claude Code memory file, sorted, index files excluded: under + `root`, or by default under DEFAULT_ROOT and in the directory + autoMemoryDirectory names.""" + if root is None: + out = discover(default_root()) + extra = auto_memory_dir() + if extra is not None and extra.is_dir(): + out += [p for p in sorted(extra.glob("*.md")) + if p.name != INDEX_NAME and p.is_file() and p not in out] + return out + root = Path(root) if not root.exists(): return [] if root.name == "memory" and root.is_dir(): # a memory dir directly @@ -88,7 +172,10 @@ def _parse_frontmatter(raw: str) -> tuple[dict, str]: def parse(path: Path) -> dict: """One memory file -> the record Compartment should store.""" - raw = path.read_text(encoding="utf-8", errors="replace") + # utf-8-sig: a note saved by a Windows editor with a byte order mark + # still opens with its frontmatter, instead of the mark hiding it and + # the whole block being stored as the fact. + raw = path.read_text(encoding="utf-8-sig", errors="replace") fields, body = _parse_frontmatter(raw) name = fields.get("name") or path.stem description = fields.get("description", "") @@ -195,8 +282,9 @@ def pending(vault, root: Path | str | None = None) -> int: def default_root_exists() -> bool: - return DEFAULT_ROOT.exists() and bool(discover()) + return default_root().exists() and bool(discover()) -__all__ = ["DEFAULT_ROOT", "discover", "parse", "import_files", "pending", - "default_root_exists", "IMPORTANCE", "file_tag"] +__all__ = ["DEFAULT_ROOT", "default_root", "discover", "parse", "import_files", "pending", + "default_root_exists", "IMPORTANCE", "file_tag", + "claude_config_dir", "auto_memory_dir"] diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 9fc9224..615bcbd 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -10,6 +10,7 @@ import datetime import getpass import hashlib +import io import json import os import shutil @@ -22,13 +23,48 @@ from . import (__version__, agent_skill, audit, claude_desktop, claude_hooks, claude_memory, clients, gate, offline_guard, packs, selftest, session) -from .acl import VaultConfig +from .acl import ConfigError, VaultConfig from .crypto import CryptoError from .embed import DEFAULT_MODEL, OPTIONAL_MODELS, Embedder, get_embedder, user_model_dir from .vault import Vault, keychain_clear, keychain_get, keychain_store from .vaultfile import read_vault_file, verify_manifest -DEFAULT_VAULT = env("VAULT", str(home() / "memory.vault")) +def _default_vault() -> str | None: + """COMPARTMENT_VAULT (or ENGRAM_VAULT) with ~ expanded, else + memory.vault in Compartment's home; None where there is no home to look + in. + + Asked when a command runs, not when this module is imported. It was a + module constant, so with HOME unset and no password-database entry for + the uid, as for a container started with an arbitrary --user, every + command died at import with "Could not determine home directory", even + `compartment --vault /data/memory.vault serve`, which needs no home. + And the variable was taken literally, so a "~/..." an MCP client config + passes (no shell expands it there) named a folder called "~" in + whatever directory the client started the server in.""" + explicit = env("VAULT") + if explicit: + return os.path.expanduser(explicit) + try: + return str(home() / "memory.vault") + except RuntimeError: + return None + + +def __getattr__(name: str): + # DEFAULT_VAULT stays importable for anything that read it, computed + # when it is read. + if name == "DEFAULT_VAULT": + return _default_vault() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +#: What a command says when there is neither --vault nor a home to find the +#: default vault in. +NO_HOME = ("no home directory to find the vault in (HOME is not set and " + "this user has no entry in the password database, as for a " + "container started with an arbitrary --user): pass --vault PATH, " + "or set COMPARTMENT_VAULT") def _utf8_console() -> None: @@ -89,6 +125,11 @@ def _can_prompt() -> bool: def _open_vault(args) -> Vault: + if not os.path.exists(args.vault): + # Before the credential: with none stored, a missing vault was + # reported as a locked one, and `compartment unlock` cannot help. + _die(f"no vault at {args.vault} (run `compartment init`, or pass " + "--vault PATH)") try: pw, key = Vault.resolve_credential(args.vault) except CryptoError: @@ -207,9 +248,21 @@ def cmd_init(args) -> None: keychain_store(path, v._master) print(" keychain credential stored (persists across reboots)") elif not args.no_session: - session.store(path, v._master) - print(" unlocked: stays open until the next restart/power loss or " - "`compartment lock`") + try: + session.store(path, v._master) + print(" unlocked: stays open until the next restart/power loss " + "or `compartment lock`") + except CryptoError as exc: + # No volatile holder for the per-boot secret here (no shared + # memory: Termux, OpenBSD, a container without /dev/shm). The + # vault is made and seeded; dying at this point left the rest of + # init undone (the save, the agent wiring, the app) over a + # credential the vault does not need to exist. + print(f" the vault is created and stays locked: this machine " + f"cannot hold an unlock credential ({exc}).") + print(" Give agents and commands the passphrase in " + "COMPARTMENT_PASSPHRASE, or check it with " + "`compartment unlock --once`.") st = v.status() v.save() # Before the app appears, so the first thing anyone sees is a panel whose @@ -305,6 +358,25 @@ def _linux_gui_available() -> bool: } +def _status_bar_argv(app, vault: str) -> list[str]: + """The command that starts the status bar app now. + + On Windows the program start at sign-in uses (systray._autostart_parts): + pythonw.exe beside this interpreter where there is one. compartment.exe + is a console program, and DETACHED_PROCESS only keeps it off this + terminal's console: the Python it starts could still open a console + window of its own behind a tray app that has no window.""" + if sys.platform == "win32": + parts = getattr(app, "_autostart_parts", None) + exe = parts(vault)[0] if parts else None + if exe and Path(exe).name.lower() == "pythonw.exe": + return [exe, "-m", "compartment.cli", "--vault", vault, "tray"] + exe = shutil.which("compartment") + return ([exe, "--vault", vault, "menubar"] if exe else + [sys.executable, "-m", "compartment.cli", "--vault", vault, + "menubar"]) + + def _start_status_bar_app(vault: str) -> None: """Put the status bar app up now, and again at every login. @@ -326,6 +398,24 @@ def _start_status_bar_app(vault: str) -> None: print(" The CLI and the MCP server are ready, and `compartment " "dash` opens the vault in a browser.") return + if linux and not _linux_gui_available(): + # Before start at login is registered, not after: an autostart + # entry or a systemd user unit for a panel that cannot import + # tkinter fails at every login from then on, with nothing on screen + # to say why. + hint = getattr(app, "tk_install_hint", None) + print(" the panel needs tkinter, which this Python does not have.") + print(" Quickest fix, a Python that includes it:") + print(" uv tool install compartment") + print(" or give this Python tkinter:") + print(f" {hint()}" if hint else + " install your distribution's Tk package for Python " + "(often python3-tk or python3-tkinter)") + print(" then turn start at login on with `compartment panel " + "--login on`.") + print(" Everything else works now: the CLI, the MCP server, and " + "`compartment dash`.") + return state = "" try: state = app.set_login(True, vault) @@ -340,15 +430,6 @@ def _start_status_bar_app(vault: str) -> None: print(" applications menu entry: on") except Exception as exc: # noqa: BLE001 print(f" could not register the app ({exc})") - if linux and not _linux_gui_available(): - print(" the panel needs tkinter, which this Python does not have.") - print(" Quickest fix, a Python that includes it:") - print(" uv tool install compartment") - print(" or install your distribution's package (Debian/Ubuntu: " - "python3-tk, Fedora: python3-tkinter).") - print(" Everything else works now: the CLI, the MCP server, and " - "`compartment dash`.") - return # Let the supervisor start it where there is one, so the copy that comes # up is the copy that gets restarted if it dies. On macOS bootstrapping # the agent a moment ago already did it (RunAtLoad); on Linux and Windows @@ -368,10 +449,7 @@ def _start_status_bar_app(vault: str) -> None: print(" start at login is registered, but no icon appeared; " "starting one now") try: - exe = shutil.which("compartment") - argv = ([exe, "--vault", vault, "menubar"] if exe else - [sys.executable, "-m", "compartment.cli", "--vault", vault, - "menubar"]) + argv = _status_bar_argv(app, vault) # stdin included. Without it the app inherits this process's stdin, # sees a terminal on the other end, decides it was typed at a prompt # and relaunches itself detached - so the copy the installer started @@ -428,10 +506,26 @@ def _wait_for_status_bar_app(timeout: float = 15.0) -> bool: def _install_kind() -> tuple[str, list[str]]: - """How this copy was installed, and the command that upgrades it.""" - exe = Path(sys.argv[0]).resolve() - if "uv/tools" in str(exe) or shutil.which("uv") and "uv/tools" in str( - Path(shutil.which("compartment") or "").resolve()): + """How this copy was installed, and the command that upgrades it. + + Asked of the environment this interpreter runs in, which each tool marks + the same way on every platform: `uv tool install` leaves uv-receipt.toml + at its root and `pipx install` leaves pipx_metadata.json. The path of the + running script, which is all this looked at before, says "uv/tools" only + where the launcher is a symlink into uv's directory and the separators + are forward slashes: on Windows, where uv's launcher is an .exe of its + own in ~/.local/bin, a uv install was taken for pip, and `update` ran + `python -m pip` in an environment uv builds without pip. A pipx install + was taken for pip everywhere: `update` left pipx's record of it stale, + and the last step `uninstall` printed left pipx a broken install.""" + prefix = Path(sys.prefix) + if (prefix / "uv-receipt.toml").is_file(): + return "uv", ["uv", "tool", "upgrade", "compartment"] + if (prefix / "pipx_metadata.json").is_file(): + return "pipx", ["pipx", "upgrade", "compartment"] + exe = Path(sys.argv[0]).resolve().as_posix() + if "uv/tools" in exe or shutil.which("uv") and "uv/tools" in Path( + shutil.which("compartment") or "").resolve().as_posix(): return "uv", ["uv", "tool", "upgrade", "compartment"] return "pip", [sys.executable, "-m", "pip", "install", "--upgrade", "compartment"] @@ -443,6 +537,7 @@ def cmd_update(args) -> None: if args.source: target = "git+https://github.com/MaxFreedomPollard/Compartment@main" cmd = (["uv", "tool", "install", "--force", target] if kind == "uv" + else ["pipx", "install", "--force", target] if kind == "pipx" else [sys.executable, "-m", "pip", "install", "--upgrade", "--force-reinstall", target]) print(f"Compartment {__version__} - updating ({kind} install)") @@ -604,26 +699,25 @@ def cmd_uninstall(args) -> None: f"compartment from {client.config_path()} by hand") except Exception as exc: # noqa: BLE001 print(f" {client.display}: {exc}") - openclaw = Path(os.environ.get("OPENCLAW_HOME", - Path.home() / ".openclaw")) / "openclaw.json" + openclaw = None try: + from . import menubar + openclaw = menubar._openclaw_config() cfg = (json.loads(openclaw.read_text(encoding="utf-8")) if openclaw.is_file() else {}) - servers = cfg.get("mcpServers") if isinstance(cfg, dict) else None - if isinstance(servers, dict) and "compartment" in servers: - del servers["compartment"] + if isinstance(cfg, dict) and _openclaw_unregister(cfg): # As integrate wrote it: backed up, then replaced whole. clients.write_config(openclaw, json.dumps(cfg, indent=2)) print(f" OpenClaw registration removed from {openclaw}") except Exception as exc: # noqa: BLE001 - print(f" OpenClaw: {openclaw} left as it is ({exc})") + print(f" OpenClaw: {openclaw or 'its config'} left as it is ({exc})") try: md = _remove_managed_claude_md() if md: print(f" compartment block removed from {md}") except Exception as exc: # noqa: BLE001 print(f" CLAUDE.md: {exc}") - hermes_home = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) + hermes_home = _hermes_home() plugin = hermes_home / "plugins" / "compartment" try: files = ("__init__.py", "plugin.yaml") @@ -665,6 +759,7 @@ def cmd_uninstall(args) -> None: print(" delete it yourself, or re-run with --purge, once you are sure") kind, _ = _install_kind() final = ("uv tool uninstall compartment" if kind == "uv" + else "pipx uninstall compartment" if kind == "pipx" else f"{sys.executable} -m pip uninstall compartment") print("\nOne step left, which cannot remove itself while it is running:") print(f" {final}") @@ -730,17 +825,28 @@ def cmd_unlock(args) -> None: v.save() +def _signing_identity(path: str | None, creator: str) -> dict: + """The Ed25519 identity that signs a sealed vault or a pack, made on + first use. + + The file holds the private seed, so it is written readable by its owner + alone: written plainly it took the umask's 0644, and any account on the + machine that could read it could sign a pack this user's keys trust, or + seal a vault `compartment verify` reports as signed by them.""" + ident_path = Path(path) if path else home() / "identity.json" + if ident_path.exists(): + return json.loads(ident_path.read_text(encoding="utf-8")) + identity = packs.new_identity(creator) + ident_path.parent.mkdir(parents=True, exist_ok=True) + _write_private(str(ident_path), json.dumps(identity, indent=2)) + print(f"generated signing identity → {ident_path} (keep it private)") + return identity + + def cmd_lock(args) -> None: if args.sign: v = _open_vault(args) - ident_path = Path(args.identity) - if ident_path.exists(): - identity = json.loads(ident_path.read_text(encoding="utf-8")) - else: - identity = packs.new_identity(args.creator) - ident_path.parent.mkdir(parents=True, exist_ok=True) - ident_path.write_text(json.dumps(identity, indent=2), encoding="utf-8") - print(f"generated signing identity → {ident_path} (keep it private)") + identity = _signing_identity(args.identity, args.creator) v.lock(signing_key=packs.load_signing_key(identity)) print(f"vault sealed + signed by {identity['signer']} " f"(pub {identity['pub_hex'][:16]}…); verify with `compartment verify`") @@ -777,6 +883,25 @@ def cmd_status(args) -> None: _print(v.status()) +def _write_private(path: str, text: str) -> None: + """Write decrypted memory text where only this user can read it. + + Readable by its owner alone, as the vault file is (VAULT_MODE): an + export is every memory the vault holds, credentials included, in the + clear, and a plain write left it to the umask, 0644 on most systems, so + any local account could read one made into /tmp or a shared home. + fchmod for a file that already existed, whose mode O_CREAT leaves as it + was.""" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as f: + if hasattr(os, "fchmod"): + try: + os.fchmod(fd, 0o600) + except OSError: + pass + f.write(text) + + def cmd_store(args) -> None: v = _open_vault(args) supersedes = getattr(args, "supersedes", None) @@ -829,8 +954,7 @@ def cmd_atomize(args) -> None: lines = "".join(json.dumps(r, sort_keys=True) + "\n" for r in rows) if args.out: try: - with open(args.out, "w", encoding="utf-8") as f: - f.write(lines) + _write_private(args.out, lines) except OSError as exc: _die(f"cannot write {args.out}: {exc}") else: @@ -1005,11 +1129,29 @@ def cmd_menubar(args) -> None: sys.exit(app.run(args.vault, show=args.show, render_to=args.render)) +def _hook_payload() -> io.StringIO: + """The hook's JSON from stdin, decoded as the UTF-8 Claude Code writes. + + Python decodes a piped stdin with the locale's encoding, which on + Windows is the ANSI code page: JSON.stringify leaves non-ASCII as it + is, so the path of a memory note under C:\\Users\\José\\.claude arrived + as C:\\Users\\José\\.claude, which does not exist, and every capture on + such a machine was dropped as "file no longer exists".""" + buf = getattr(sys.stdin, "buffer", None) + try: + raw = buf.read() if buf is not None else sys.stdin.read() + except (OSError, ValueError): + raw = b"" + if isinstance(raw, bytes): + raw = raw.decode("utf-8-sig", errors="replace") + return io.StringIO(raw) + + def cmd_hook(args) -> None: """Deterministic capture, so remembering does not depend on the model choosing to call a tool.""" if args.hook_cmd == "capture": - res = claude_hooks.capture(vault_path=args.vault) + res = claude_hooks.capture(_hook_payload(), vault_path=args.vault) if args.json: _print(res) sys.exit(0) # never fail a user's edit over a memory write @@ -1026,12 +1168,22 @@ def cmd_hook(args) -> None: print(" Claude Code memory writes now land in the vault " "automatically (restart Claude Code to load it)") elif args.hook_cmd == "uninstall": - print("capture hook removed" if claude_hooks.uninstall() + # A settings.json that is not JSON (hand-edited, cut short) is + # refused with ValueError naming the file, as install already said. + try: + removed = claude_hooks.uninstall() + except ValueError as exc: + _die(str(exc)) + print("capture hook removed" if removed else "no compartment hook was installed") else: # No subcommand given (bare `compartment hook`) behaves like `status`, # then points at the rest of the group instead of failing. - print("installed" if claude_hooks.is_installed() else "not installed") + try: + installed = claude_hooks.is_installed() + except ValueError as exc: + _die(str(exc)) + print("installed" if installed else "not installed") if getattr(args, "hook_cmd", None) is None: print(" subcommands: status | install | uninstall | capture") @@ -1145,7 +1297,7 @@ def cmd_export(args) -> None: data = v.export_jsonl() if args.plaintext: print("WARNING: exporting PLAINTEXT memories to disk", file=sys.stderr) - Path(args.out).write_text(data, encoding="utf-8") + _write_private(args.out, data) # Count the lines actually written rather than newline characters: # export_jsonl ends with a trailing newline only when it is non-empty, # so a newline count is one off the moment that changes. @@ -1156,9 +1308,49 @@ def cmd_export(args) -> None: _save_reads(v) +def _read_text_file(path: str) -> str: + """A text file the user handed the CLI, as text, whatever Windows tool + saved it. + + PowerShell 5.1 writes UTF-8 with a byte-order mark (`Out-File -Encoding + utf8`) and UTF-16 for a plain `>` redirect; Notepad wrote a BOM for + years. Read as bare UTF-8, the first gave JSONDecodeError "Unexpected + UTF-8 BOM" and the second UnicodeDecodeError, both as tracebacks. The + encoding is found as json finds it (UTF-8 with or without a BOM, + UTF-16, UTF-32), and anything that does not decode is refused by + name.""" + raw = Path(path).read_bytes() + try: + # json's own detection: the BOM where there is one, else the zero + # bytes an ASCII "{" leaves in UTF-16 or UTF-32, else UTF-8. + enc = json.detect_encoding(raw) + return raw.decode("utf-8-sig" if enc == "utf-8" else enc) + except UnicodeDecodeError as exc: + _die(f"{path} is not UTF-8 text (byte {exc.start}): save it as " + "UTF-8 and import it again") + + +def _check_jsonl(path: str, text: str) -> None: + """Refuse a JSONL file by line number before anything is imported: a + line that is not JSON, or a record with no text, ended the import in a + traceback that named neither the file's line nor the field.""" + for no, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + rec = json.loads(line) + except ValueError as exc: + _die(f"{path}, line {no}: not JSON ({exc.msg}). Each line must " + "be one record, as `compartment export` writes them") + if not isinstance(rec, dict) or not isinstance(rec.get("text"), str): + _die(f"{path}, line {no}: a record needs a \"text\" string") + + def cmd_import(args) -> None: + text = _read_text_file(args.file) + _check_jsonl(args.file, text) v = _open_vault(args) - n = v.import_jsonl(Path(args.file).read_text(encoding="utf-8"), namespace=args.namespace) + n = v.import_jsonl(text, namespace=args.namespace) print(f"imported {n} {'record' if n == 1 else 'records'}") @@ -1389,8 +1581,15 @@ def cmd_2fa(args) -> None: from . import crypto as _crypto kf = _secrets.token_bytes(_crypto.KEYFILE_LEN) p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(kf) - os.chmod(p, 0o600) + # Created 0600 and exclusively: written first and chmod'ed after, + # the second factor sat at the umask's mode (0644 on most systems) + # for that moment, readable by every local account, and a file + # that appeared at the path in between would have been written + # through. O_BINARY keeps Windows from translating its bytes. + fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL + | getattr(os, "O_BINARY", 0), 0o600) + with os.fdopen(fd, "wb") as f: + f.write(kf) print(f"keyfile written → {p}") print("BACK THIS FILE UP (it is random bytes, not derived from " "anything).\nLose it and the passphrase alone will NOT " @@ -1451,13 +1650,7 @@ def cmd_pack_build(args) -> None: _die("source must be a .jsonl, .csv, or a directory of .md files") if not records: _die("no records found in source") - ident_path = Path(args.identity) - if ident_path.exists(): - identity = json.loads(ident_path.read_text(encoding="utf-8")) - else: - identity = packs.new_identity(args.creator) - ident_path.write_text(json.dumps(identity, indent=2), encoding="utf-8") - print(f"generated new signing identity → {ident_path} (keep it private)") + identity = _signing_identity(args.identity, args.creator) emb = get_embedder(DEFAULT_MODEL) vectors = emb.embed_passages([r["text"] for r in records]) pw = None @@ -1548,13 +1741,25 @@ def cmd_pack_export(args) -> None: "`memory_search` does not return, store it into compartment.") +def _claude_md_path() -> Path: + """The user CLAUDE.md Claude Code loads: CLAUDE_MD when set (an + override for tests and unusual setups), else CLAUDE.md in Claude + Code's own directory, which CLAUDE_CONFIG_DIR moves. The block was + always written to ~/.claude/CLAUDE.md, which a Claude Code run with + CLAUDE_CONFIG_DIR (a second account, a dotfiles tree) never reads.""" + explicit = os.environ.get("CLAUDE_MD") + if explicit: + return Path(explicit) + return claude_memory.claude_config_dir() / "CLAUDE.md" + + def _write_managed_claude_md() -> Path: """Write an idempotent, sentinel-fenced compartment block into the user's CLAUDE.md. Only ever rewrites BETWEEN the markers - the user's own text above and below is never touched; re-running updates the block in place. (Belt-and-suspenders: the authoritative behavioral instruction rides the MCP `initialize` handshake; this covers hosts that under-weight it.)""" - md = Path(os.environ.get("CLAUDE_MD", Path.home() / ".claude" / "CLAUDE.md")) + md = _claude_md_path() md.parent.mkdir(parents=True, exist_ok=True) block = f"{_CLAUDE_MD_BEGIN}\n{_CLAUDE_MD_BODY}\n{_CLAUDE_MD_END}" text = md.read_text(encoding="utf-8") if md.exists() else "" @@ -1574,7 +1779,7 @@ def _remove_managed_claude_md() -> Path | None: """Take the fenced block back out of the user's CLAUDE.md: the inverse of _write_managed_claude_md, leaving everything outside the markers as it is. Returns the file, or None when it holds no block.""" - md = Path(os.environ.get("CLAUDE_MD", Path.home() / ".claude" / "CLAUDE.md")) + md = _claude_md_path() if not md.is_file(): return None text = md.read_text(encoding="utf-8") @@ -1703,7 +1908,7 @@ def _refresh_installed() -> list[str]: the user had turned off), no registration, no import. An edited skill is backed up before it is replaced, as `integrate` does.""" done: list[str] = [] - md = Path(os.environ.get("CLAUDE_MD", Path.home() / ".claude" / "CLAUDE.md")) + md = _claude_md_path() try: before = md.read_text(encoding="utf-8") if md.is_file() else "" if _CLAUDE_MD_BEGIN in before and _CLAUDE_MD_END in before: @@ -1728,7 +1933,7 @@ def _refresh_installed() -> list[str]: line += f" (the previous copy is kept at {r['backup']})" done.append(line) - hermes_home = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) + hermes_home = _hermes_home() plug_src = _data_dir() / "hermes-plugin" plug_dst = hermes_home / "plugins" / "compartment" if (plug_dst / "plugin.yaml").is_file(): @@ -1879,6 +2084,78 @@ def _integrate_client(client, args, brief: bool = False) -> None: #: small step for that program; one busy updating itself is no reason to #: hold an install for as long as that takes. AGENT_TOOL_TIMEOUT = 120 +def _quit_claude_desktop(platform: str | None = None) -> str: + """How to make Claude Desktop load a config just written. + + Quit, not close: closing its window leaves Claude Desktop running (in + the notification area on Windows), so it never rereads the file, and a + copy still running can write the config it holds back over this one. + "Restart Claude Desktop" read as closing the window.""" + platform = platform or sys.platform + where = ("right-click its icon in the notification area and choose " + "Quit" if platform == "win32" else + "Claude > Quit Claude, or Cmd-Q" if platform == "darwin" + else "quit it") + return (f"Quit Claude Desktop completely ({where}; closing its window " + "leaves it running), then open it again to load it.") + + +def _hermes_home() -> Path: + """Hermes Agent's home, found the way Hermes finds it + (hermes_constants.get_hermes_home): HERMES_HOME, with ~ and variables + expanded, else the platform default. + + On Windows that default is %LOCALAPPDATA%\\hermes, not ~/.hermes, and the + Windows installer does not export HERMES_HOME. Looking only in ~/.hermes, + `integrate hermes` on native Windows put the provider plugin where + Hermes never reads it and reported it installed, never found the + Hermes venv to put compartment into, and uninstall and the refresh + after an update looked in the same wrong place.""" + from . import menubar # one rule for the CLI and the panel + return menubar.hermes_home() + + +def _openclaw_register(cfg: dict, entry: dict) -> None: + """Put compartment's entry where current OpenClaw reads MCP servers, + `mcp.servers`, and take back one an earlier release wrote at the top + level as `mcpServers`. + + OpenClaw's root schema rejects keys it does not know + (src/config/zod-schema.ts), so that top-level entry was never loaded as + a server, and could keep the whole config from loading.""" + mcp = cfg.setdefault("mcp", {}) + if not isinstance(mcp, dict): + raise ValueError('its "mcp" is not a JSON object') + servers = mcp.setdefault("servers", {}) + if not isinstance(servers, dict): + raise ValueError('its "mcp.servers" is not a JSON object') + servers["compartment"] = entry + _drop_legacy_openclaw_entry(cfg) + + +def _drop_legacy_openclaw_entry(cfg: dict) -> bool: + legacy = cfg.get("mcpServers") + if not isinstance(legacy, dict) or "compartment" not in legacy: + return False + del legacy["compartment"] + if not legacy: + del cfg["mcpServers"] # a key OpenClaw would refuse + return True + + +def _openclaw_unregister(cfg: dict) -> bool: + """Take compartment out of an OpenClaw config, from `mcp.servers` and + from the top-level `mcpServers` earlier releases wrote. True if + anything was removed.""" + removed = _drop_legacy_openclaw_entry(cfg) + mcp = cfg.get("mcp") + servers = mcp.get("servers") if isinstance(mcp, dict) else None + if isinstance(servers, dict) and "compartment" in servers: + del servers["compartment"] + removed = True + return removed + + #: The `import compartment` check in Hermes's own interpreter. HERMES_IMPORT_TIMEOUT = 30 #: pip installing compartment into Hermes's own environment. @@ -1892,7 +2169,7 @@ def _integrate_target(args) -> None: vault = args.vault if target == "hermes": - hermes_home = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) + hermes_home = _hermes_home() plug_src = _data_dir() / "hermes-plugin" plug_dst = hermes_home / "plugins" / "compartment" plug_dst.mkdir(parents=True, exist_ok=True) @@ -1900,7 +2177,8 @@ def _integrate_target(args) -> None: shutil.copy2(plug_src / f, plug_dst / f) print(f"✓ provider plugin installed → {plug_dst}") # compartment must be importable from Hermes's own venv - hermes_py = hermes_home / "hermes-agent" / "venv" / "bin" / "python" + hermes_py = hermes_home / "hermes-agent" / "venv" / ( + "Scripts/python.exe" if sys.platform == "win32" else "bin/python") if hermes_py.exists(): # No stdin for any program run here: a question one of them asks # must fail, not wait on the user's keyboard, and a failure or a @@ -1954,8 +2232,8 @@ def _integrate_target(args) -> None: compartment_bin = clients.executable() entry = {"command": compartment_bin, "args": ["--vault", vault, "--caller", "openclaw", "serve"]} - cfg_path = Path(os.environ.get("OPENCLAW_HOME", - Path.home() / ".openclaw")) / "openclaw.json" + from . import menubar + cfg_path = menubar._openclaw_config() wrote = False # An OpenClaw that has not written its config yet is a normal first # install, not a reason to hand the JSON back for pasting: it reads @@ -1967,7 +2245,7 @@ def _integrate_target(args) -> None: if existed else {}) if not isinstance(cfg, dict): # null, a list: not a config raise ValueError("not a JSON object") - cfg.setdefault("mcpServers", {})["compartment"] = entry + _openclaw_register(cfg, entry) # A byte-exact backup first, then the file replaced whole. backup = clients.write_config(cfg_path, json.dumps(cfg, indent=2)) @@ -1979,9 +2257,10 @@ def _integrate_target(args) -> None: except (ValueError, OSError) as exc: print(f" could not edit {cfg_path} automatically ({exc});") if not wrote: - print(" add this under \"mcpServers\" in ~/.openclaw/openclaw.json, " + print(f" add this to {cfg_path}, merged into what is there, " "then run `openclaw gateway restart`:") - print(json.dumps({"compartment": entry}, indent=2)) + print(json.dumps({"mcp": {"servers": {"compartment": entry}}}, + indent=2)) if not os.path.exists(vault): print(f"\n! no vault at {vault} - run `compartment init` to create one") @@ -2027,7 +2306,7 @@ def _integrate_target(args) -> None: print(f" ✓ registered in {reg['config']}") if reg["backup"]: print(f" (previous config backed up to {reg['backup']})") - print(" Restart Claude Desktop to load it.") + print(f" {_quit_claude_desktop()}") except ValueError as exc: # malformed config - never guess print(f" ! {exc}") print(" fix the file, then run `compartment integrate " @@ -2130,7 +2409,9 @@ def cmd_setup(args) -> None: hashes = {} for fname, url in spec["files"].items(): print(f" downloading {fname} …") - with urllib.request.urlopen(url) as r: + # Bounded: a stalled connection otherwise holds the command for + # ever, with nothing on screen after "downloading". + with urllib.request.urlopen(url, timeout=120) as r: data = r.read() (d / fname).write_bytes(data) hashes[fname] = hashlib.sha256(data).hexdigest() @@ -2187,8 +2468,10 @@ def main(argv: list[str] | None = None) -> None: ap = argparse.ArgumentParser( prog="compartment", description="Compartment - high-security offline vector memory for AI agents") - ap.add_argument("--vault", default=DEFAULT_VAULT, - help=f"vault path (default {DEFAULT_VAULT})") + default_vault = _default_vault() + ap.add_argument("--vault", default=default_vault, + help="vault path (default " + f"{default_vault or 'none: no home directory'})") ap.add_argument("--caller", default="user") ap.add_argument("--keyfile", help="second-factor keyfile (only needed if `compartment 2fa " @@ -2232,7 +2515,9 @@ def main(argv: list[str] | None = None) -> None: p = sub.add_parser("lock", help="clear stored credential (vault stays sealed)") p.add_argument("--sign", action="store_true", help="seal with an Ed25519 signed manifest before locking") - p.add_argument("--identity", default=str(home() / "identity.json")) + p.add_argument("--identity", default=None, + help="signing identity (default identity.json in " + "Compartment's home)") p.add_argument("--creator", default="vault-owner") p.set_defaults(fn=cmd_lock) @@ -2535,7 +2820,9 @@ def main(argv: list[str] | None = None) -> None: p.add_argument("--version", default="1.0.0") p.add_argument("--description", default="") p.add_argument("--creator", default="pack-author") - p.add_argument("--identity", default=str(home() / "identity.json")) + p.add_argument("--identity", default=None, + help="signing identity (default identity.json in " + "Compartment's home)") p.add_argument("--encrypt", action="store_true") p.add_argument("--out") p.set_defaults(fn=cmd_pack_build) @@ -2615,13 +2902,104 @@ def main(argv: list[str] | None = None) -> None: p.set_defaults(fn=cmd_setup) ps.set_defaults(fn=cmd_setup, setup_cmd=None) + _accept_global_options_anywhere(ap) args = ap.parse_args(argv) + if args.vault is not None: + # "~/..." from an MCP client config or a quoted shell argument: + # nothing between there and here expands it. + args.vault = os.path.expanduser(args.vault) + elif args.fn not in (cmd_embed_daemon, cmd_setup): + _die(NO_HOME) if args.assert_offline: offline_guard.activate() try: args.fn(args) - except CryptoError as exc: + # Inside the try: buffered output reaches a pipe here, and a reader + # that has gone away fails this flush, not one at interpreter exit. + # A stdout already closed is left alone: mcp 1.x's stdio transport + # closes the buffer under sys.stdout when `serve` ends. + if not sys.stdout.closed: + sys.stdout.flush() + except (CryptoError, ConfigError) as exc: + # ConfigError: a .config.json that cannot be used, which + # names the file and what to do; a ValueError, so it was a + # traceback. _die(str(exc)) + except BrokenPipeError: + # `compartment recent --all | head`: the reader took what it wanted + # and closed the pipe. Not a failure of the command, and not a + # traceback; stdout goes to the null device so that the flush at + # exit does not raise the same error again. + try: + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + except (OSError, ValueError): + pass + sys.exit(1) + except KeyboardInterrupt: + print("\ninterrupted", file=sys.stderr) + sys.exit(130) + except OSError as exc: + # A file named on the command line that is not there or cannot be + # read, a folder that cannot be written, a full disk: said in one + # line naming the path, instead of a traceback an agent running + # the CLI has to read to the bottom of to find the same thing. + _die(_os_error_message(exc, args)) + + +#: The options of `compartment` itself, which every subcommand also takes +#: after its own name. Help text only: the value comes from the top level. +_GLOBAL_OPTIONS = ( + (("--vault",), {"help": "vault path (also accepted before the " + "subcommand)"}), + (("--caller",), {"help": "who is asking, for the ACL and the audit log"}), + (("--keyfile",), {"help": "second-factor keyfile, if 2FA moved it"}), + (("--assert-offline",), {"action": "store_true", + "help": "abort if anything attempts network " + "access"}), +) + + +def _accept_global_options_anywhere(parser: argparse.ArgumentParser) -> None: + """Let `--vault`, `--caller`, `--keyfile` and `--assert-offline` follow + the subcommand as well as precede it. + + argparse takes a parent's options only before the subcommand, so + `compartment serve --vault ~/v.vault` died with "unrecognized arguments: + --vault". That is the order MCP client configs are written in by hand + and by registries ("command": "compartment", "args": ["serve", + "--vault", ...]), and the order people type, and `compartment serve + --help` did not mention --vault at all. Each subcommand that does not + define one of these itself gets it with a SUPPRESS default, so it sets + the value only when it is given there, and the top-level value or + default stands otherwise.""" + for action in parser._actions: + if not isinstance(action, argparse._SubParsersAction): + continue + seen = set() + for child in action.choices.values(): + if id(child) in seen: # an alias: the same parser + continue + seen.add(id(child)) + for flags, kw in _GLOBAL_OPTIONS: + if not any(f in child._option_string_actions for f in flags): + child.add_argument(*flags, default=argparse.SUPPRESS, + **kw) + _accept_global_options_anywhere(child) + + +def _os_error_message(exc: OSError, args) -> str: + """One line for an OSError that ended a command, naming the file.""" + where = exc.filename + vault = getattr(args, "vault", None) + if (isinstance(exc, FileNotFoundError) and where is not None and vault + and os.path.abspath(os.fsdecode(where)) == os.path.abspath(vault)): + return (f"no vault at {vault} (run `compartment init`, or pass " + "--vault PATH)") + if where is None: + return str(exc) + if exc.filename2 is not None: + where = f"{os.fsdecode(where)} -> {os.fsdecode(exc.filename2)}" + return f"{exc.strerror or exc}: {os.fsdecode(where)}" if __name__ == "__main__": diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 4cd2b3e..c4043b6 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -34,9 +34,11 @@ """ from __future__ import annotations +import codecs import errno import json import os +import re import secrets import shutil import sys @@ -80,16 +82,132 @@ def _app_support(name: str) -> Path: def _vscode_user(app: str = "Code") -> Path: """VS Code's User directory, which several extensions nest inside. - Cline, Roo Code and Kilo Code are VS Code extensions, so their settings - live under VS Code's storage rather than anywhere of their own. + Roo Code, and older Cline and Kilo Code, are VS Code extensions whose + settings live under the editor's storage rather than anywhere of their + own. """ return _app_support(app) / "User" +#: The editors an extension may be installed in, by the name of their user +#: data folder: VS Code, its Insiders build, VSCodium, and the forks that +#: run VS Code extensions (vscode-docs, VSCodium's prepare_vscode.sh). +_EDITORS = ("Code", "Code - Insiders", "VSCodium", "Cursor", "Windsurf") + + +def _editor_users() -> list[Path]: + """Every editor User directory an extension could keep storage in, + the Flathub builds of VS Code and VSCodium included (Flatpak sets their + XDG_CONFIG_HOME to ~/.var/app//config).""" + users = [_vscode_user(app) for app in _EDITORS] + if sys.platform != "darwin" and os.name != "nt": + flat = Path.home() / ".var" / "app" + users += [flat / "com.visualstudio.code" / "config" / "Code" / "User", + flat / "com.vscodium.codium" / "config" / "VSCodium" + / "User"] + return users + + +def _vscode_mcp(app: str, flatpak_id: str) -> Callable[[], Path]: + """VS Code's user-level mcp.json, in the Flathub build's sandboxed + config (~/.var/app//config//User) when that is the only one + on this Linux machine. The native path alone used to be written, which + the Flathub VS Code never reads.""" + def path() -> Path: + native = _vscode_user(app) + if sys.platform != "darwin" and os.name != "nt": + flat = (Path.home() / ".var" / "app" / flatpak_id / "config" + / app / "User") + try: + if not native.is_dir() and flat.is_dir(): + return flat / "mcp.json" + except OSError: + pass + return native / "mcp.json" + return path + + +def _ext_storage(ext: str) -> Path | None: + """The globalStorage directory of extension `ext` in the first editor + that has one, or None when no editor has run it.""" + for user in _editor_users(): + d = user / "globalStorage" / ext + try: + if d.is_dir(): + return d + except OSError: + pass + return None + + def _ext_settings(ext: str, filename: str) -> Callable[[], Path]: - """A VS Code extension's own settings file, by extension id.""" - return lambda: (_vscode_user() / "globalStorage" / ext / "settings" - / filename) + """A VS Code extension's own settings file, by extension id, in the + editor it is installed in. + + Only VS Code's own storage used to be looked at, so the extension in + Cursor, Windsurf, VSCodium or VS Code Insiders was reported as not + installed, and wiring it wrote a file in a VS Code that might not even + be there. VS Code is still the answer when no editor has run it.""" + def path() -> Path: + storage = _ext_storage(ext) or (_vscode_user() / "globalStorage" + / ext) + return storage / "settings" / filename + return path + + +def _env_dir(var: str) -> Path | None: + """The directory an environment variable names, or None when it is + unset or blank.""" + raw = os.environ.get(var, "").strip() + return Path(raw).expanduser() if raw else None + + +def _cline_dir() -> Path: + return _env_dir("CLINE_DIR") or Path.home() / ".cline" + + +def _cline_settings() -> Path: + """cline_mcp_settings.json, where the Cline that is installed reads it. + + Cline 4 keeps one file for the VS Code extension, the CLI and the + JetBrains plugin, ~/.cline/data/settings/cline_mcp_settings.json + (CLINE_MCP_SETTINGS_PATH, CLINE_DATA_DIR and CLINE_DIR move it; + sdk/packages/shared/src/storage/paths.ts), and imports the old + globalStorage file once, the first time it runs. A server written to + the old file after that is never read. An older Cline, one that has + not created ~/.cline, still reads the old file in its editor.""" + explicit = _env_dir("CLINE_MCP_SETTINGS_PATH") + if explicit: + return explicit + data = _env_dir("CLINE_DATA_DIR") + current = ((data or _cline_dir() / "data") / "settings" + / "cline_mcp_settings.json") + try: + if data or current.exists() or _cline_dir().is_dir(): + return current + except OSError: + return current + legacy = _ext_storage("saoudrizwan.claude-dev") + if legacy is not None: + return legacy / "settings" / "cline_mcp_settings.json" + return current + + +def _kilo_config() -> Path: + """Kilo Code's global config. + + Kilo Code 7 is built on OpenCode and reads MCP servers from + ~/.config/kilo (XDG_CONFIG_HOME, on every platform): the first of + kilo.jsonc, kilo.json, opencode.jsonc, opencode.json and config.json + that exists, kilo.jsonc when none does (packages/opencode/src/config/ + config.ts). The extension's old mcp_settings.json is read only from VS + Code's own storage, only for migration, and never by the Kilo CLI.""" + d = _xdg() / "kilo" + for name in ("kilo.jsonc", "kilo.json", "opencode.jsonc", + "opencode.json", "config.json"): + if (d / name).exists(): + return d / name + return d / "kilo.jsonc" def _home(*parts: str) -> Callable[[], Path]: @@ -97,6 +215,102 @@ def _home(*parts: str) -> Callable[[], Path]: return lambda: Path.home().joinpath(*parts) +def _home_from(var: str, *default: str) -> Path: + """The directory `var` names, or ~/ when it is unset.""" + return _env_dir(var) or Path.home().joinpath(*default) + + +def _zed_settings() -> Path: + """Zed's settings.json (crates/paths/src/paths.rs): %APPDATA%\\Zed on + Windows, the XDG config directory on Linux, and ~/.config/zed on macOS + whatever XDG_CONFIG_HOME says, since Zed does not consult it there. + Taking XDG_CONFIG_HOME on a Mac that sets it wrote a file Zed never + read.""" + if os.name == "nt": + return _appdata() / "Zed" / "settings.json" + if sys.platform == "darwin": + return Path.home() / ".config" / "zed" / "settings.json" + return _xdg() / "zed" / "settings.json" + + +def _opencode_config() -> Path: + """OpenCode's global config: OPENCODE_CONFIG_DIR, or opencode under the + XDG config directory on every platform, and in it opencode.json or + opencode.jsonc, both of which it loads. The one that exists is + written; a new opencode.json beside an opencode.jsonc would split the + user's config in two.""" + d = _env_dir("OPENCODE_CONFIG_DIR") or _xdg() / "opencode" + if not (d / "opencode.json").exists() and (d / "opencode.jsonc").exists(): + return d / "opencode.jsonc" + return d / "opencode.json" + + +def _lmstudio_home() -> Path: + """LM Studio's home, as LM Studio finds it (lmstudio-js, + findLMStudioHome.ts): the path in ~/.lmstudio-home-pointer, which LM + Studio writes and which is how a home moved in its settings is found; + else ~/.cache/lm-studio, where older versions lived; else ~/.lmstudio.""" + h = Path.home() + try: + pointer = (h / ".lmstudio-home-pointer").read_text( + encoding="utf-8").strip() + if pointer: + return Path(pointer).expanduser() + except (OSError, UnicodeDecodeError): + pass + legacy = h / ".cache" / "lm-studio" + return legacy if legacy.is_dir() else h / ".lmstudio" + + +def _goose_config() -> Path: + """Goose's config.yaml (crates/goose/src/config/paths.rs): + $GOOSE_PATH_ROOT/config when that is an absolute path; otherwise the + XDG config directory on macOS and Linux, and on Windows + %APPDATA%\\Block\\goose\\config, where the XDG path that was printed + named a file Goose never reads.""" + root = _env_dir("GOOSE_PATH_ROOT") + if root is not None and root.is_absolute(): + return root / "config" / "config.yaml" + if os.name == "nt": + return _appdata() / "Block" / "goose" / "config" / "config.yaml" + return _xdg() / "goose" / "config.yaml" + + +def _yaml_str(v: str) -> str: + """A YAML double-quoted scalar. YAML 1.2 is a superset of JSON, so a + JSON string is one, escapes and all.""" + return json.dumps(v, ensure_ascii=False) + + +def _goose_block(entry: dict) -> str: + """The extension as Goose's config.yaml spells it: under `extensions:`, + with `cmd` rather than `command` and a `type` (ExtensionConfig::Stdio in + crates/goose/src/agents/extension.rs). The `mcpServers` JSON that used + to be printed for Goose is not a shape Goose reads at all.""" + return "\n".join([ + "extensions:", + f" {NAME}:", + " enabled: true", + " type: stdio", + f" name: {NAME}", + " description: Encrypted offline memory", + f" cmd: {_yaml_str(entry['command'])}", + " args: [" + ", ".join(_yaml_str(a) for a in entry["args"]) + "]", + " timeout: 300", + ]) + + +def _librechat_block(entry: dict) -> str: + """The server under librechat.yaml's `mcpServers:` mapping, as YAML.""" + return "\n".join([ + "mcpServers:", + f" {NAME}:", + " type: stdio", + f" command: {_yaml_str(entry['command'])}", + " args: [" + ", ".join(_yaml_str(a) for a in entry["args"]) + "]", + ]) + + # --- the shape each client wants an entry in ------------------------------- def _entry_plain(command: str, args: list[str]) -> dict: @@ -128,6 +342,15 @@ class Client: writes: bool = True #: whether we are willing to edit it note: str = "" #: anything the user needs to know after aliases: tuple[str, ...] = field(default=()) + #: Whether the client reads its file as JSON with comments, so a file + #: holding comments or trailing commas is edited as text, not refused. + jsonc: bool = False + #: Whether the client is installed, when the config file's directory is + #: not the proof (it may not exist until the first server is added). + installed: Callable[[], bool] | None = None + #: The block to print, from the entry, for a client whose file is not + #: JSON (Goose's YAML); JSON under `root` when None. + block: Callable[[dict], str] | None = None def config_path(self) -> Path: return self.path() @@ -143,7 +366,14 @@ def present(self) -> bool: p = self.config_path() if not p.is_absolute(): return p.exists() - return p.exists() or p.parent.is_dir() + if p.exists() or p.parent.is_dir(): + return True + if self.installed is not None: + try: + return bool(self.installed()) + except OSError: + return False + return False # --- the table ------------------------------------------------------------- @@ -159,23 +389,51 @@ def present(self) -> bool: Client("cursor", "Cursor", _home(".cursor", "mcp.json")), Client("windsurf", "Windsurf", _home(".codeium", "windsurf", "mcp_config.json")), - Client("cline", "Cline", - _ext_settings("saoudrizwan.claude-dev", "cline_mcp_settings.json"), - note="reload the VS Code window to pick it up"), + Client("cline", "Cline", _cline_settings, + installed=lambda: (_cline_dir().is_dir() or _ext_storage( + "saoudrizwan.claude-dev") is not None), + note="reload the editor window, or restart the Cline CLI, to " + "pick it up"), + # Roo Code's repository was archived in May 2026; the extension still + # reads globalStorage/rooveterinaryinc.roo-cline/settings. Client("roo", "Roo Code", _ext_settings("rooveterinaryinc.roo-cline", "mcp_settings.json"), aliases=("roo-code",), - note="reload the VS Code window to pick it up"), - Client("kilo", "Kilo Code", - _ext_settings("kilocode.kilo-code", "mcp_settings.json"), - aliases=("kilo-code",), - note="reload the VS Code window to pick it up"), - Client("gemini", "Gemini CLI", _home(".gemini", "settings.json"), - aliases=("gemini-cli",)), - Client("qwen", "Qwen Code CLI", _home(".qwen", "settings.json"), - aliases=("qwen-code",)), + installed=lambda: _ext_storage( + "rooveterinaryinc.roo-cline") is not None, + note="reload the editor window to pick it up"), + Client("kilo", "Kilo Code", _kilo_config, root="mcp", + entry=_entry_opencode, jsonc=True, aliases=("kilo-code",), + installed=lambda: ((_xdg() / "kilo").is_dir() or _ext_storage( + "kilocode.kilo-code") is not None), + note="reload the editor window, or restart the Kilo CLI, to " + "pick it up"), + # GEMINI_CLI_HOME stands in for the home directory, .gemini under it; + # QWEN_HOME and COPILOT_HOME are the directory itself. Both CLIs strip + # comments before parsing settings.json. + Client("gemini", "Gemini CLI", + lambda: (_env_dir("GEMINI_CLI_HOME") or Path.home()) / ".gemini" + / "settings.json", jsonc=True, aliases=("gemini-cli",)), + Client("qwen", "Qwen Code CLI", + lambda: _home_from("QWEN_HOME", ".qwen") / "settings.json", + jsonc=True, aliases=("qwen-code",)), Client("copilot-cli", "GitHub Copilot CLI", - _home(".copilot", "mcp-config.json")), + lambda: _home_from("COPILOT_HOME", ".copilot") / "mcp-config.json"), + # Kiro (IDE and CLI) and the Amazon Q Developer CLI it replaced; Kiro's + # directory moves with KIRO_HOME (kirodotdev/KiroCrew, config/paths.py), + # Amazon Q's is fixed (crates/chat-cli/src/util/paths.rs). + Client("kiro", "Kiro", lambda: _home_from("KIRO_HOME", ".kiro") + / "settings" / "mcp.json", aliases=("kiro-cli",)), + Client("amazon-q", "Amazon Q Developer CLI", + _home(".aws", "amazonq", "mcp.json"), aliases=("amazonq",)), + # Junie's user-level file, from JetBrains' own skill for driving it + # (JetBrains/junie, hermes-plugin/skills/junie/SKILL.md). + Client("junie", "Junie", _home(".junie", "mcp", "mcp.json"), + installed=lambda: (Path.home() / ".junie").is_dir()), + # Factory's droid; stdio is the default type (docs/cli/configuration/ + # mcp.mdx in Factory-AI/factory). + Client("factory", "Factory Droid", _home(".factory", "mcp.json"), + aliases=("droid",)), # Oh My Pi reads a project-level .omp/mcp.json as well, but that one is # relative to whatever repository you happen to be in, so the user-level # file is the only one an integrate command can honestly write. @@ -184,27 +442,34 @@ def present(self) -> bool: note="run `/mcp reload` in omp to pick it up without restarting"), # -- coding agents that spell the key differently ---------------------- - Client("vscode", "VS Code", lambda: _vscode_user() / "mcp.json", - root="servers", entry=_entry_vscode), + Client("vscode", "VS Code", + _vscode_mcp("Code", "com.visualstudio.code"), + root="servers", entry=_entry_vscode, jsonc=True), Client("vscode-insiders", "VS Code Insiders", - lambda: _vscode_user("Code - Insiders") / "mcp.json", - root="servers", entry=_entry_vscode), - Client("zed", "Zed", - lambda: (_appdata() / "Zed" / "settings.json" if os.name == "nt" - else _xdg() / "zed" / "settings.json"), - root="context_servers", - note="Zed allows comments in settings.json; if yours has any, " - "the block is printed rather than written"), - Client("opencode", "OpenCode", - lambda: _xdg() / "opencode" / "opencode.json", - root="mcp", entry=_entry_opencode), + _vscode_mcp("Code - Insiders", "com.visualstudio.code.insiders"), + root="servers", entry=_entry_vscode, jsonc=True), + # Zed writes its settings.json with a block of // comments on first + # run, so it is edited as JSON with comments: comments kept. + Client("zed", "Zed", _zed_settings, root="context_servers", jsonc=True), + # Crush: $CRUSH_GLOBAL_CONFIG/crush.json, else crush/crush.json under + # XDG_CONFIG_HOME or ~/.config on every platform, servers under "mcp" + # with a required type (internal/config/load.go, config.go). + Client("crush", "Crush", + lambda: (_env_dir("CRUSH_GLOBAL_CONFIG") or _xdg() / "crush") + / "crush.json", root="mcp", entry=_entry_vscode), + # The legacy "mcp": {name: entry} shape, which every release reads and + # which current ones merge with the newer "mcp": {"servers": ...}. + Client("opencode", "OpenCode", _opencode_config, + root="mcp", entry=_entry_opencode, jsonc=True), # -- Codex keeps TOML -------------------------------------------------- - Client("codex", "Codex CLI", _home(".codex", "config.toml"), + # CODEX_HOME moves the whole directory (codex-rs/utils/home-dir). + Client("codex", "Codex CLI", + lambda: _home_from("CODEX_HOME", ".codex") / "config.toml", root="mcp_servers", kind="toml", aliases=("codex-cli",)), # -- desktop and self-hosted chat apps --------------------------------- - Client("lmstudio", "LM Studio", _home(".lmstudio", "mcp.json"), + Client("lmstudio", "LM Studio", lambda: _lmstudio_home() / "mcp.json", aliases=("lm-studio",), note="LM Studio reloads mcp.json on save, no restart needed"), Client("anythingllm", "AnythingLLM", @@ -220,21 +485,33 @@ def present(self) -> bool: if sys.platform == "darwin" or os.name == "nt" else _xdg() / "trae" / "mcp.json")), + # Continue loads every *.json under its global mcpServers folder, in the + # Claude Desktop shape (core/context/mcp/json/loadJsonMcpConfigs.ts), so + # it gets a file of its own: nothing of the user's is edited, and its + # config.yaml, whose comments a rewrite would lose, is never touched. + Client("continue", "Continue", + lambda: (_home_from("CONTINUE_GLOBAL_DIR", ".continue") + / "mcpServers" / "compartment.json"), + installed=lambda: _home_from("CONTINUE_GLOBAL_DIR", + ".continue").is_dir(), + jsonc=True, + note="Continue loads it from its mcpServers folder; reload the " + "editor window to pick it up"), + # -- known, documented, but not ours to write -------------------------- # # Each of these keeps its configuration somewhere a merge cannot be made # safely: a YAML document whose comments and anchors we would flatten, or # an application database, or a settings blob with no stable on-disk # shape. Printing the exact block is honest; writing a guess is not. - Client("continue", "Continue", _home(".continue", "config.yaml"), - kind="manual", writes=False, - note="paste under `mcpServers:` in config.yaml"), - Client("goose", "Goose", lambda: _xdg() / "goose" / "config.yaml", - kind="manual", writes=False, - note="or run `goose configure` and add it as an extension"), + Client("goose", "Goose", _goose_config, kind="manual", writes=False, + block=_goose_block, + note="add this to config.yaml (merge it into `extensions:` if " + "you have one), or run `goose configure` and add it as a " + "command-line extension"), Client("librechat", "LibreChat", lambda: Path("librechat.yaml"), - kind="manual", writes=False, - note="paste under `mcpServers:` in your librechat.yaml, then " + kind="manual", writes=False, block=_librechat_block, + note="merge this into `mcpServers:` in your librechat.yaml, then " "restart LibreChat"), Client("jan", "Jan", lambda: _app_support("Jan") / "data" / "mcp_config.json", kind="manual", writes=False, @@ -291,16 +568,134 @@ def executable() -> str: `Compartment`, macOS filesystems are case-insensitive by default, and a config pointed at it would start the menu bar app every time a client tried to open a memory server. + + `pip install --user`, which is what the install scripts for all three + platforms run, puts the script in the user scheme's directory + (~/Library/Python/3.X/bin, ~/.local/bin, or on Windows + %APPDATA%\\Python\\PythonXY\\Scripts), beside no interpreter and on no + PATH a GUI app or the panel's child process has. Looking only beside the interpreter and then on PATH + found nothing there and wrote the bare name: every client then failed + to start the server with "spawn compartment ENOENT". So the scripts + directory of the scheme this package was actually installed under is + asked first; it is the one pip wrote the script into, by definition. + + A script found inside a package manager's versioned tree (Homebrew's + Cellar/compartment//libexec/bin) is dead after the next + upgrade and cleanup, while the link the package manager keeps on PATH + for it (/opt/homebrew/bin/compartment, and pipx's and uv's + ~/.local/bin links) is repointed by every upgrade. So when such a link + resolves to the very script found, the link is what is written. """ - prefix = Path(sys.prefix) - for c in (prefix / "bin" / NAME, prefix / "Scripts" / f"{NAME}.exe", - Path(sys.executable).parent / NAME): + found = _installed_script() + if found is None: + which = shutil.which(NAME) + return which or NAME + return str(_stable_link(found)) + + +def _script_in(directory: Path) -> Path | None: + """The compartment console script in `directory`, if there is one and it + is not an app bundle's launcher.""" + for c in (directory / NAME, directory / f"{NAME}.exe"): try: if c.is_file() and c.parent.name != "MacOS": - return str(c) + return c except OSError: pass - return shutil.which(NAME) or NAME + return None + + +def _package_root() -> Path: + """The site-packages directory this copy of compartment was imported + from. An editable install's is the source tree, which no scheme + names.""" + return Path(__file__).resolve().parent.parent + + +def _scheme_scripts() -> list[Path]: + """The scripts directory of each install scheme whose library directory + holds this package: where pip put the console script for this copy. + + The schemes are expanded against the running interpreter's own prefix, + the venv's inside a venv, and the user scheme is the one pip uses for + `--user` on this platform (osx_framework_user for python.org's macOS + builds, nt_user on Windows, posix_user elsewhere).""" + import sysconfig + try: + here = _package_root() + except OSError: + return [] + names = [] + for get in (sysconfig.get_default_scheme, + lambda: sysconfig.get_preferred_scheme("prefix"), + lambda: sysconfig.get_preferred_scheme("user")): + try: + names.append(get()) + except (KeyError, ValueError, AttributeError): + pass + out = [] + for scheme in dict.fromkeys(names): + try: + paths = sysconfig.get_paths(scheme, vars={ + "base": sys.prefix, "platbase": sys.exec_prefix}) + libs = {Path(paths["purelib"]).resolve(), + Path(paths["platlib"]).resolve()} + except (KeyError, ValueError, OSError): + continue + if here in libs: + out.append(Path(paths["scripts"])) + return out + + +def _installed_script() -> Path | None: + """The console script that belongs to the running copy of compartment, + or None when there is none to be found (a source checkout run with + `python -m`).""" + prefix = Path(sys.prefix) + for d in (*_scheme_scripts(), prefix / "bin", prefix / "Scripts"): + hit = _script_in(d) + if hit is not None: + return hit + beside = Path(sys.executable).parent / NAME + try: + if beside.is_file() and beside.parent.name != "MacOS": + return beside + except OSError: + pass + return None + + +def _link_dirs() -> list[Path]: + """Where package managers keep the links they repoint on upgrade.""" + h = Path.home() + dirs = [h / ".local" / "bin", Path("/opt/homebrew/bin"), + Path("/usr/local/bin"), Path("/home/linuxbrew/.linuxbrew/bin"), + h / ".linuxbrew" / "bin", h / ".nix-profile" / "bin", + Path("/run/current-system/sw/bin"), h / "bin"] + user = os.environ.get("USER") + if user: + dirs.append(Path("/etc/profiles/per-user") / user / "bin") + return dirs + + +def _stable_link(script: Path) -> Path: + """A link on PATH, or in a package manager's bin directory, that + resolves to `script` itself; `script` when there is none.""" + try: + real = os.path.realpath(script) + except (OSError, ValueError): + return script + which = shutil.which(NAME) + candidates = ([Path(which)] if which else []) + [d / NAME + for d in _link_dirs()] + for c in candidates: + try: + if (c.is_absolute() and c != script and c.is_symlink() + and os.path.realpath(c) == real): + return c + except (OSError, ValueError): + continue + return script def server_args(vault: str, caller: str) -> list[str]: @@ -318,12 +713,53 @@ def snippet(client: Client, vault: str, caller: str | None = None) -> str: """What to print when we are not writing the file ourselves.""" if client.kind == "toml": return _toml_block(entry_for(client, vault, caller)) + if client.block is not None: + return client.block(entry_for(client, vault, caller)) return json.dumps({client.root: {NAME: entry_for(client, vault, caller)}}, indent=2) # --- reading and writing --------------------------------------------------- +#: Byte order marks, longest first: UTF-32's little-endian mark begins +#: with UTF-16's. +_BOMS = ((codecs.BOM_UTF8, "utf-8-sig"), (codecs.BOM_UTF32_LE, "utf-32"), + (codecs.BOM_UTF32_BE, "utf-32"), (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16")) + + +def read_config_text(path: Path) -> str: + """A config file's text, in whichever Unicode encoding it was saved. + + Windows editors do not all save UTF-8 without a mark. Notepad before + Windows 10 1903 and PowerShell 5's `Out-File -Encoding utf8` put a UTF-8 + byte order mark first; PowerShell 5's `>` and `Out-File` write UTF-16. + Those are the tools the Claude Desktop and Cursor guides send Windows + users to, and a file saved by them was refused as "not valid JSON" (the + mark is not whitespace to json.loads) or failed to decode at all, so + the user got a block to paste into the very file that had just been + refused. A marked file is decoded by its mark; anything unmarked must + be UTF-8, as before. What is written back is always UTF-8 without a + mark, which every client reads: Node's JSON.parse, which Electron + clients use, rejects the mark and cannot read UTF-16 at all, so a + marked file was already broken for most of them. + + Line endings come back as \\n; write_config puts back the ones the file + had. + + Raises OSError when the file cannot be read and UnicodeDecodeError (a + ValueError) when it is not text in any of these encodings. + """ + raw = Path(path).read_bytes() + for bom, codec in _BOMS: + if raw.startswith(bom): + text = raw.decode(codec) + break + else: + text = raw.decode("utf-8") + return text.lstrip("\ufeff").replace("\r\n", "\n").replace("\r", "\n") + + def _read_json(path: Path) -> dict: """The file as a dict, or a refusal. @@ -334,7 +770,7 @@ def _read_json(path: Path) -> dict: if not path.exists(): return {} try: - text = path.read_text(encoding="utf-8").strip() + text = read_config_text(path).strip() data = json.loads(text) if text else {} except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc: raise ValueError(f"{path} is not valid JSON ({exc}); " @@ -344,6 +780,330 @@ def _read_json(path: Path) -> dict: return data +# --- JSONC: the comments and trailing commas some clients allow ------------ +# +# VS Code (and Insiders and VSCodium), Zed, Gemini CLI, Qwen Code and +# OpenCode read their config as JSON with comments: // and /* */, and a +# comma after the last member. Zed writes its own settings.json that way on +# first run, with a block of // lines above the opening brace, so a strict +# JSON reader refused every Zed user's file and printed the block instead. +# +# Re-serialising would parse, but it would delete every comment the user +# wrote. So the edit is textual: the document is parsed with the position +# of every object member, our member is spliced in (or out) as text, and +# the result is parsed again and must equal the original data with exactly +# our change applied. Anything this cannot do cleanly - a duplicate key, a +# root that is not an object, a result that does not read back - is +# refused, and the user gets the block to paste, as before. + +_JSONC_SCALAR = re.compile( + r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?|true|false|null") + + +class _JsoncObject(dict): + """A parsed object that remembers where it is in the text: its braces, + and for each member the start of its key and the span of its value.""" + + def __init__(self, open_at: int): + super().__init__() + self.open = open_at + self.close = -1 + self.spans: dict[str, tuple[int, int, int]] = {} + + +class _JsoncParser: + def __init__(self, text: str): + self.t, self.i = text, 0 + + def fail(self, what: str): + line = self.t.count("\n", 0, self.i) + 1 + raise ValueError(f"{what} at line {line}") + + def skip(self) -> None: + t, n = self.t, len(self.t) + while self.i < n: + c = t[self.i] + if c in " \t\r\n": + self.i += 1 + elif t.startswith("//", self.i): + j = t.find("\n", self.i) + self.i = n if j < 0 else j + 1 + elif t.startswith("/*", self.i): + j = t.find("*/", self.i + 2) + if j < 0: + self.fail("unterminated comment") + self.i = j + 2 + else: + return + + def at(self, ch: str) -> bool: + return self.t.startswith(ch, self.i) + + def document(self): + self.skip() + value = self.value() + self.skip() + if self.i != len(self.t): + self.fail("text after the end of the document") + return value + + def value(self): + self.skip() + if self.at("{"): + return self.object() + if self.at("["): + return self.array() + if self.at('"'): + return self.string() + m = _JSONC_SCALAR.match(self.t, self.i) + if not m: + self.fail("unexpected character") + self.i = m.end() + return json.loads(m.group()) + + def string(self) -> str: + try: + s, self.i = json.decoder.scanstring(self.t, self.i + 1, True) + except json.JSONDecodeError as exc: + raise ValueError(str(exc)) from exc + return s + + def object(self) -> _JsoncObject: + obj = _JsoncObject(self.i) + self.i += 1 + while True: + self.skip() + if self.at("}"): + break + if not self.at('"'): + self.fail("expected a quoted key") + key_at = self.i + key = self.string() + self.skip() + if not self.at(":"): + self.fail("expected ':'") + self.i += 1 + self.skip() + value_at = self.i + value = self.value() + if key in obj: + # Which one a client honours is the client's business; an + # edit that guessed could change the one it reads. + self.fail(f"duplicate key {key!r}") + obj[key] = value + obj.spans[key] = (key_at, value_at, self.i) + self.skip() + if self.at(","): + self.i += 1 + continue + if not self.at("}"): + self.fail("expected ',' or '}'") + break + obj.close = self.i + self.i += 1 + return obj + + def array(self) -> list: + out = [] + self.i += 1 + while True: + self.skip() + if self.at("]"): + break + out.append(self.value()) + self.skip() + if self.at(","): + self.i += 1 + continue + if not self.at("]"): + self.fail("expected ',' or ']'") + break + self.i += 1 + return out + + +def _jsonc_parse(text: str): + """The data in a JSONC document. Raises ValueError.""" + return _JsoncParser(text).document() + + +def _plain(value): + """`value` as plain dicts and lists, for comparing and copying.""" + if isinstance(value, dict): + return {k: _plain(v) for k, v in value.items()} + if isinstance(value, list): + return [_plain(v) for v in value] + return value + + +def _line_start(text: str, i: int) -> int: + return text.rfind("\n", 0, i) + 1 + + +def _own_line_indent(text: str, i: int) -> str | None: + """The indentation before position `i`, when nothing else is before it + on its line.""" + lead = text[_line_start(text, i):i] + return lead if not lead.strip() else None + + +def _line_indent(text: str, i: int) -> str: + """The indentation of the line holding position `i`.""" + start = _line_start(text, i) + line = text[start:] + return line[:len(line) - len(line.lstrip(" \t"))] + + +def _indent_unit(text: str, root: _JsoncObject) -> str: + """One level of indentation, as the document's own first member has.""" + for key_at, _, _ in sorted(root.spans.values()): + lead = _own_line_indent(text, key_at) + if lead: + return lead + break + return " " + + +def _jsonc_dump(value, indent: str, unit: str) -> str: + """`value` as JSON, continuation lines indented to sit at `indent`.""" + return json.dumps(value, indent=unit).replace("\n", "\n" + indent) + + +def _jsonc_insert(text: str, obj: _JsoncObject, key: str, value, + unit: str) -> str: + """`text` with `"key": value` added as the last member of `obj`.""" + if not obj.spans: + close_indent = _line_indent(text, obj.open) + member_indent = close_indent + unit + member = f"{json.dumps(key)}: {_jsonc_dump(value, member_indent, unit)}" + inside = text[obj.open + 1:obj.close] + if not inside.strip(): + return (text[:obj.open + 1] + "\n" + member_indent + member + + "\n" + close_indent + text[obj.close:]) + return (text[:obj.open + 1] + "\n" + member_indent + member + + text[obj.open + 1:]) + + first_key = min(k for k, _, _ in obj.spans.values()) + _, _, last_end = max(obj.spans.values()) + member_indent = _own_line_indent(text, first_key) + if member_indent is None: # a one-line object + compact = json.dumps(value) + return (text[:last_end] + f", {json.dumps(key)}: {compact}" + + text[last_end:]) + member = f"{json.dumps(key)}: {_jsonc_dump(value, member_indent, unit)}" + eol = text.find("\n", last_end) + tail = text[last_end:] if eol < 0 else text[last_end:eol] + after = tail.lstrip(" \t") + trailing_comma = after.startswith(",") + if trailing_comma: + after = after[1:].lstrip(" \t") + if eol >= 0 and (not after or after.startswith("//")): + # The last member ends its line, perhaps with a comma or a comment + # after it: the new member goes on a line of its own below. + comma = "" if trailing_comma else "," + return (text[:last_end] + comma + text[last_end:eol] + "\n" + + member_indent + member + ("," if trailing_comma else "") + + text[eol:]) + compact = json.dumps(value) + return (text[:last_end] + f", {json.dumps(key)}: {compact}" + + text[last_end:]) + + +def _jsonc_set(text: str, root: str, name: str, entry: dict) -> str: + """`text` with root[name] set to `entry`, every comment and every other + byte left where it was. Raises ValueError when that cannot be done and + proven.""" + doc = _jsonc_parse(text) + if not isinstance(doc, _JsoncObject): + raise ValueError("not a JSON object") + unit = _indent_unit(text, doc) + expected = _plain(doc) + if root in doc: + servers = doc[root] + if not isinstance(servers, _JsoncObject): + raise ValueError(f"{root} is not an object") + if name in servers: + key_at, value_at, value_end = servers.spans[name] + indent = _line_indent(text, key_at) + new = (text[:value_at] + _jsonc_dump(entry, indent, unit) + + text[value_end:]) + else: + new = _jsonc_insert(text, servers, name, entry, unit) + expected[root][name] = entry + else: + new = _jsonc_insert(text, doc, root, {name: entry}, unit) + expected[root] = {name: entry} + if _plain(_jsonc_parse(new)) != expected: + raise ValueError("the edited file would not read back as intended") + return new + + +def _skip_blank(text: str, i: int) -> int: + """The first position at or after `i` that is not whitespace or a + comment.""" + p = _JsoncParser(text) + p.i = i + p.skip() + return p.i + + +def _jsonc_delete(text: str, root: str, name: str) -> str | None: + """`text` with root[name] taken out and nothing else changed, or None + when it is not there. Raises ValueError when it cannot be taken out + cleanly.""" + doc = _jsonc_parse(text) + if not isinstance(doc, _JsoncObject): + raise ValueError("not a JSON object") + servers = doc.get(root) + if not isinstance(servers, _JsoncObject) or name not in servers: + return None + order = sorted(servers.spans, key=lambda k: servers.spans[k][0]) + idx = order.index(name) + key_at, _, value_end = servers.spans[name] + cuts: list[tuple[int, int]] = [] + + def member_span(start: int, end: int) -> tuple[int, int]: + """Widen [start, end) to whole lines when the member has its lines + to itself.""" + eol = text.find("\n", end) + rest = text[end:] if eol < 0 else text[end:eol] + if _own_line_indent(text, start) is not None and not rest.strip(): + return _line_start(text, start), (len(text) if eol < 0 + else eol + 1) + return start, end + + if idx < len(order) - 1: + comma = _skip_blank(text, value_end) + if not text.startswith(",", comma): + raise ValueError("no comma after the entry") + cuts.append(member_span(key_at, comma + 1)) + elif idx > 0: + prev_end = servers.spans[order[idx - 1]][2] + comma = _skip_blank(text, prev_end) + if not text.startswith(",", comma): + raise ValueError("no comma before the entry") + end = value_end + trailing = _skip_blank(text, value_end) + if text.startswith(",", trailing): # it had a trailing comma + end = trailing + 1 + cuts.append((comma, comma + 1)) + cuts.append(member_span(key_at, end)) + else: + end = value_end + trailing = _skip_blank(text, value_end) + if text.startswith(",", trailing): + end = trailing + 1 + cuts.append(member_span(key_at, end)) + new = text + for start, end in sorted(cuts, reverse=True): + new = new[:start] + new[end:] + expected = _plain(doc) + del expected[root][name] + if _plain(_jsonc_parse(new)) != expected: + raise ValueError("the edited file would not read back as intended") + return new + + def write_config(path: Path, text: str) -> Path | None: """Replace a file the user or another program owns with `text`. Returns the byte-exact backup taken first, beside it, or None when there was no @@ -367,12 +1127,32 @@ def write_config(path: Path, text: str) -> Path | None: raise PermissionError(errno.EACCES, os.strerror(errno.EACCES), str(target)) backup = None + crlf = False if existed: backup = path.with_suffix(path.suffix + BACKUP_SUFFIX) shutil.copy2(target, backup) + try: + raw = target.read_bytes() + for bom, codec in _BOMS: # CRLF inside UTF-16 too + if raw.startswith(bom): + raw = raw.decode(codec, "replace").encode("utf-8") + break + crlf = b"\r\n" in raw + except OSError: + pass + # The file keeps its own line endings, on every system. Text comes in + # with "\n" (the reader normalises), and a text-mode write used to turn + # every "\n" into "\r\n" on Windows only, so the same `integrate` gave + # a different file depending on where it ran, and a CRLF file that kept + # a "\r\n" through some other path came out with "\r\r\n", which is + # not valid TOML. Written verbatim: CRLF if the file had CRLF, LF + # otherwise and for a new file. + text = text.replace("\r\n", "\n") + if crlf: + text = text.replace("\n", "\r\n") tmp = target.with_name(f"{target.name}.{secrets.token_hex(8)}.tmp") try: - with open(tmp, "w", encoding="utf-8") as f: + with open(tmp, "w", encoding="utf-8", newline="") as f: if existed and os.name != "nt": shutil.copymode(target, tmp) # before any text is in it f.write(text) @@ -388,18 +1168,75 @@ def write_config(path: Path, text: str) -> Path | None: return backup +#: TOML's short escapes; every other control character is \\uXXXX. +_TOML_ESCAPES = {'"': '\\"', "\\": "\\\\", "\b": "\\b", "\t": "\\t", + "\n": "\\n", "\f": "\\f", "\r": "\\r"} + + +def _toml_string(s: str) -> str: + """A TOML basic string. + + Not json.dumps, whose escaping looks like TOML's and is not: it writes a + character outside the Basic Multilingual Plane as a UTF-16 surrogate + pair, \\ud83d\\ude00, and TOML allows only Unicode scalar values in an + escape. A vault in a folder named with an emoji, or with a CJK Extension + B character in a user name, appended a table that made the whole of + config.toml unreadable, and Codex refuses to start on a config it cannot + read. Here only what TOML requires is escaped, and the rest is written + as itself, in the UTF-8 the file is. A lone surrogate (an undecodable + byte in a POSIX path) cannot be written in UTF-8 at all and raises + ValueError. + """ + out = [] + for ch in s: + o = ord(ch) + if ch in _TOML_ESCAPES: + out.append(_TOML_ESCAPES[ch]) + elif o < 0x20 or o == 0x7F: + out.append(f"\\u{o:04x}") + elif 0xD800 <= o <= 0xDFFF: + raise ValueError(f"{s!r} cannot be written as UTF-8 TOML") + else: + out.append(ch) + return '"' + "".join(out) + '"' + + +def _toml_key(k: str) -> str: + """A key, bare when TOML allows it bare.""" + return k if k and all(c.isascii() and (c.isalnum() or c in "-_") + for c in k) else _toml_string(k) + + def _toml_value(v) -> str: - """Just enough TOML to emit one table of strings and string arrays.""" - if isinstance(v, list): - return "[" + ", ".join(_toml_value(x) for x in v) + "]" + """A TOML value for anything tomllib reads back: our own strings and + string arrays, and whatever else the user put in our table (a + startup_timeout_sec, an env table, enabled = false), which a refresh + carries over.""" if isinstance(v, bool): return "true" if v else "false" - return json.dumps(str(v)) # JSON string escaping is TOML's too + if isinstance(v, int): + return str(v) + if isinstance(v, float): + if v != v: + return "nan" + if v in (float("inf"), float("-inf")): + return "inf" if v > 0 else "-inf" + return repr(v) + if isinstance(v, list): + return "[" + ", ".join(_toml_value(x) for x in v) + "]" + if isinstance(v, dict): + if not v: + return "{}" + return "{ " + ", ".join(f"{_toml_key(k)} = {_toml_value(x)}" + for k, x in v.items()) + " }" + if hasattr(v, "isoformat"): # datetime, date, time + return v.isoformat() + return _toml_string(str(v)) def _toml_block(entry: dict) -> str: lines = [f"[mcp_servers.{NAME}]"] - lines += [f"{k} = {_toml_value(v)}" for k, v in entry.items()] + lines += [f"{_toml_key(k)} = {_toml_value(v)}" for k, v in entry.items()] return "\n".join(lines) @@ -470,6 +1307,13 @@ def _toml_without_ours(client: Client, text: str) -> str | None: j += 1 for k in range(i, last + 1): keep[k] = False + # The blank line that set our table off goes with it when another + # follows, so taking it out (or refreshing it) leaves one, not two. + after = next((k for k in range(last + 1, len(stmts)) if keep[k]), + None) + if (i > 0 and keep[i - 1] and not stmts[i - 1][0].strip() + and (after is None or not stmts[after][0].strip())): + keep[i - 1] = False i = j out = "".join(s for (s, _), k in zip(stmts, keep) if k) try: @@ -485,7 +1329,7 @@ def is_registered(client: Client, path: Path | None = None) -> bool: return False if client.kind == "toml": try: - return _toml_has_ours(client, p.read_text(encoding="utf-8")) + return _toml_has_ours(client, read_config_text(p)) except (OSError, ValueError): # unreadable, not UTF-8, not TOML return False if client.kind != "json": @@ -493,6 +1337,13 @@ def is_registered(client: Client, path: Path | None = None) -> bool: try: return NAME in (_read_json(p).get(client.root) or {}) except ValueError: + if not client.jsonc: + return False + try: + doc = _jsonc_parse(read_config_text(p)) + servers = doc.get(client.root) if isinstance(doc, dict) else None + return isinstance(servers, dict) and NAME in servers + except (OSError, ValueError): return False @@ -521,6 +1372,9 @@ def register(client: Client, vault: str, caller: str | None = None, try: data = _read_json(p) except ValueError as exc: + if client.jsonc: + return _register_jsonc(client, p, entry, out, str(exc), + snippet(client, vault, caller)) out["reason"] = str(exc) out["snippet"] = snippet(client, vault, caller) return out @@ -542,18 +1396,85 @@ def register(client: Client, vault: str, caller: str | None = None, return out +def _register_jsonc(client: Client, p: Path, entry: dict, out: dict, + strict_error: str, block: str) -> dict: + """Register in a file that is not strict JSON but that this client + reads as JSON with comments: spliced in as text, every comment kept, + and proven by reading it back. When it cannot be, the strict reader's + refusal stands and the block is printed.""" + try: + new = _jsonc_set(read_config_text(p), client.root, NAME, entry) + except (OSError, ValueError) as exc: + out["reason"] = (f"{strict_error}; nor could it be edited as JSON " + f"with comments ({exc})") + out["snippet"] = block + return out + try: + backup = write_config(p, new) + except OSError as exc: + out["reason"] = f"cannot write {p} ({exc})" + out["snippet"] = block + return out + out["backup"] = str(backup) if backup else None + out["written"] = True + return out + + +def _toml_breaks(client: Client, before: str, after: str, + entry: dict) -> str | None: + """Why `after` must not be written in place of `before`, or None. + + Appending a table is the whole edit, and an appended table is not + always a valid one: a config that declares `mcp_servers = { ... }` + inline, or as an array, cannot be extended by a [mcp_servers.x] header, + and the result is a file Codex will not start with. So the document is + read back as Codex will read it: it has to parse, hold exactly our + entry, and hold everything else exactly as it did before.""" + try: + doc = tomllib.loads(after) + was = tomllib.loads(before) + except tomllib.TOMLDecodeError as exc: + return f"unreadable ({exc})" + servers = doc.get(client.root) + if not isinstance(servers, dict) or servers.get(NAME) != entry: + return f"without a [{client.root}.{NAME}] table Codex can read" + rest = dict(doc) + rest[client.root] = {k: v for k, v in servers.items() if k != NAME} + if not rest[client.root] and client.root not in was: + del rest[client.root] + return None if rest == was else "with its other settings changed" + + def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: """Codex keeps TOML, and TOML tables are order-independent. So the whole edit is appending one table, which leaves every comment, every ordering and every hand-made formatting decision in the file - exactly as its owner left it. Re-registering rewrites nothing: an entry - that is already there is reported, not duplicated. A file that does - not parse is refused, as a JSON one is. + exactly as its owner left it. Re-registering with the same command and + arguments rewrites nothing. A file that does not parse is refused, as a + JSON one is. + + Re-registering with a different command or vault refreshes the entry, + as it does for every JSON client. It used to report "already + registered; left as it is" whatever the entry said, so a config that + named a bare `compartment` Codex could not find, a console script that + an upgrade had moved, or last month's vault could not be put right by + running `integrate codex` again: the user had to remove it first, + which nothing told them. Our table is taken out exactly as + `unregister` takes it out and appended again with the new command and + arguments, and whatever else the user had put in it (a + startup_timeout_sec, an env table, enabled = false) is carried over. + When the table cannot be taken out cleanly (an inline entry, dotted + keys), the file is left as it is and the block is printed. """ - block = _toml_block(entry) try: - existing = p.read_text(encoding="utf-8") if p.exists() else "" + block = _toml_block(entry) + except ValueError as exc: + out["reason"] = f"cannot be written as TOML ({exc})" + out["snippet"] = json.dumps({client.root: {NAME: entry}}, indent=2) + return out + try: + existing = read_config_text(p) if p.exists() else "" registered = _toml_has_ours(client, existing) except (OSError, UnicodeDecodeError) as exc: out["reason"] = f"cannot read {p} ({exc})" @@ -565,13 +1486,43 @@ def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: return out if registered: - out["reason"] = "already registered; left as it is" + old = tomllib.loads(existing)[client.root][NAME] + if not isinstance(old, dict) or all(old.get(k) == v + for k, v in entry.items()): + out["reason"] = "already registered; left as it is" + out["snippet"] = block + return out + rest = _toml_without_ours(client, existing) + if rest is None: + out["reason"] = ("registered with another command, written in a " + "form that cannot be updated without breaking " + f"{p}; left as it is. Replace the entry with " + "this one") + out["snippet"] = block + return out + entry = {**entry, **{k: v for k, v in old.items() if k not in entry}} + try: + refreshed = _toml_block(entry) + except ValueError as exc: + out["reason"] = f"cannot be written as TOML ({exc})" + out["snippet"] = block + return out + before, verb = rest, "updating" + else: + before, refreshed, verb = existing, block, "adding" + + # One blank line before the table, however many times it is refreshed. + joiner = ("" if not before or before.endswith("\n\n") + else "\n" if before.endswith("\n") else "\n\n") + text = before + joiner + refreshed + "\n" + broken = _toml_breaks(client, before, text, entry) + if broken: + out["reason"] = (f"{verb} the table would leave {p} {broken}; " + "refusing to touch it") out["snippet"] = block return out - - joiner = "" if not existing else ("\n" if existing.endswith("\n") else "\n\n") try: - backup = write_config(p, existing + joiner + block + "\n") + backup = write_config(p, text) except OSError as exc: out["reason"] = f"cannot write {p} ({exc})" out["snippet"] = block @@ -589,7 +1540,7 @@ def unregister(client: Client, path: Path | None = None) -> bool: if client.kind == "toml": try: - text = p.read_text(encoding="utf-8") + text = read_config_text(p) if not _toml_has_ours(client, text): return False except ValueError: # not UTF-8 or not TOML: not ours to guess at @@ -603,7 +1554,16 @@ def unregister(client: Client, path: Path | None = None) -> bool: try: data = _read_json(p) except ValueError: - return False + if not client.jsonc: + return False + try: + rest = _jsonc_delete(read_config_text(p), client.root, NAME) + except (OSError, ValueError): # left for the user to take out + return False + if rest is None: + return False + write_config(p, rest) + return True servers = data.get(client.root) or {} if not isinstance(servers, dict) or NAME not in servers: return False diff --git a/src/compartment/curate.py b/src/compartment/curate.py index b3ae06a..7bea550 100644 --- a/src/compartment/curate.py +++ b/src/compartment/curate.py @@ -87,7 +87,9 @@ def parse_plan(text: str) -> list[dict]: ensure_ascii=False leaves U+2028 and U+2029 unescaped inside strings, and splitlines() would cut a valid plan line in half at them.""" plan = [] - for n, line in enumerate(text.split("\n"), 1): + # A plan saved by a Windows editor may start with a byte order mark, + # which json.loads refuses as line 1 not being JSON. + for n, line in enumerate(text.lstrip("\ufeff").split("\n"), 1): if not line.strip(): continue try: diff --git a/src/compartment/dash.py b/src/compartment/dash.py index 80a764a..734055e 100644 --- a/src/compartment/dash.py +++ b/src/compartment/dash.py @@ -418,9 +418,18 @@ def start(ref: _VaultRef, host: str = "127.0.0.1", return httpd, url +def _open_browser(url: str) -> None: + """Open the page, or say how to reach it when nothing here can.""" + from .menubar import open_in_browser + if not open_in_browser(url): + port = urllib.parse.urlsplit(url).port + print(" no browser to open here: open the address above in one. " + "From another machine, forward the port first: " + f"ssh -L {port}:127.0.0.1:{port} ", flush=True) + + def run(path: str, vault: Vault) -> None: """CLI entry: serve until Ctrl-C. Zero flags, zero configuration.""" - import webbrowser ref = _VaultRef(path, vault) httpd, url = start(ref) # Flushed, and before the browser: the panel learns where the page is @@ -430,10 +439,11 @@ def run(path: str, vault: Vault) -> None: print(" serving from RAM, 127.0.0.1 only, GET only - viewing the vault " "changes nothing in it", flush=True) print(" Ctrl-C to stop", flush=True) - try: - webbrowser.open(url) - except Exception: - pass + # Beside the server, not before it. A browser that runs in the + # foreground - a text browser in this terminal - was waited for here, + # while the page it was asking for could not be served until it exited. + threading.Thread(target=_open_browser, args=(url,), daemon=True, + name="compartment-dash-browser").start() try: httpd.serve_forever() except KeyboardInterrupt: diff --git a/src/compartment/data/hermes-plugin/plugin.yaml b/src/compartment/data/hermes-plugin/plugin.yaml index 8e338f1..3874ec7 100644 --- a/src/compartment/data/hermes-plugin/plugin.yaml +++ b/src/compartment/data/hermes-plugin/plugin.yaml @@ -1,5 +1,5 @@ name: compartment -version: 4.12.0 +version: 4.13.0 description: "Compartment - high-security, fully offline, encrypted vector memory. AEAD-encrypted vault, locked-by-default after restart, native Hermes provider, no API key." # The engine. `hermes plugins install compartment` (the plugin catalog) and # `hermes memory setup` install it into the Hermes environment from this diff --git a/src/compartment/embed.py b/src/compartment/embed.py index b82f15d..6b59bcc 100644 --- a/src/compartment/embed.py +++ b/src/compartment/embed.py @@ -12,6 +12,9 @@ import functools import json import os +import re +import sys +import sysconfig import threading from pathlib import Path @@ -86,12 +89,37 @@ class ModelError(CryptoError): pass +#: Every code point the tokenizer cannot take. In a Python str a code point +#: in U+D800..U+DFFF is always a lone surrogate - a pair is one code point +#: above U+FFFF - and it gets there as half of a cut emoji: a JSON "\\ud83d" +#: escape an MCP client may legally send, or a string sliced mid-pair. +_LONE_SURROGATE = re.compile("[\ud800-\udfff]") + + +def well_formed(text: str) -> str: + """`text` with each lone surrogate replaced by U+FFFD. + + tokenizers 0.16 and later refuse a str holding one with TypeError + ("TextInputSequence must be str"), from every embedding call, so a + store or a search with half an emoji in it failed outright, in the + shared daemon and in-process alike, while 0.15 let it through. Every + path into the tokenizer comes through here, so the daemon and this + process embed such a text identically, and as the replacement + character any UTF-8 decoder would have put in its place.""" + return _LONE_SURROGATE.sub("\ufffd", text) if text else text + + def bundled_model_dir() -> Path: return Path(__file__).resolve().parent / "models" / DEFAULT_MODEL def user_model_dir() -> Path: - return Path(env("MODEL_DIR", home() / "models")) + # COMPARTMENT_MODEL_DIR first, and the home only without it: the default + # was computed even when the variable was set, so with no home directory + # (HOME unset and no password-database entry) naming the folder did not + # help, and a downloaded model could not be found. + explicit = env("MODEL_DIR") + return Path(explicit) if explicit else home() / "models" def resolve_model_dir(name: str) -> Path: @@ -179,11 +207,126 @@ def _missing_runtime_error(exc: ImportError) -> ModelError | None: return ModelError( "onnxruntime cannot load: this Windows machine is missing the " "Microsoft Visual C++ runtime it is built against. Install " - "https://aka.ms/vs/17/release/vc_redist.x64.exe (one small " - "installer, no restart needed), then run this command again.") + f"https://aka.ms/vs/17/release/vc_redist.{_vc_redist_arch()}.exe " + "(one small installer, no restart needed), then run this " + "command again.") return None +def _vc_redist_arch() -> str: + """Which Visual C++ runtime installer this Python needs: the one for + the architecture the interpreter was BUILT for, not the machine's. On + Windows on ARM an ARM64 Python needs vc_redist.arm64.exe, and the x64 + installer the message used to name installs a runtime it cannot load; + an x64 Python running under emulation there needs the x64 one.""" + plat = sysconfig.get_platform() + if plat == "win-arm64": + return "arm64" + if plat == "win32": + return "x86" + return "x64" + + +def _runtime_not_installed(exc: ModuleNotFoundError) -> ModelError | None: + """The actionable form of onnxruntime or tokenizers simply not being + importable, or None for any other missing module. + + pip installs both with Compartment wherever it can, but onnxruntime + publishes no wheel for some platforms (musl/Alpine, 32-bit ARM, the + BSDs, Android), where an install made with --no-deps or from a + distribution's packages can lack it. The bare error was a traceback in + the CLI, and in the MCP server, which passes the model only the text of + its own errors, "Error executing tool" with no reason at all.""" + if exc.name not in ("onnxruntime", "tokenizers"): + return None + return ModelError( + f"{exc.name} is not installed for this Python ({sys.executable}), " + "and embedding needs it. Install it with: " + f"{sys.executable} -m pip install {exc.name}. Where pip has no " + f"{exc.name} build for this platform ({sysconfig.get_platform()}), " + "install your system's own package of it, or run Compartment on a " + "glibc-based Python (a Debian or Ubuntu image rather than Alpine).") + + +def _cgroup_cpu_limit(root: str = "/sys/fs/cgroup", + own: str = "/proc/self/cgroup") -> float | None: + """The CPUs a cgroup quota grants this process (2.0 for `docker run + --cpus 2`), or None where no quota applies. + + cgroup v2 keeps " " in cpu.max, at every level of the + hierarchy, and the tightest one on the way up from this process's own + group is the one that binds; inside a container the namespace makes + that group the root. cgroup v1 keeps the two numbers in separate files + of the cpu controller's mount.""" + limits: list[float] = [] + rel = None + try: + with open(own, encoding="utf-8") as f: + for line in f: + if line.startswith("0::"): + rel = line[3:].strip() + except OSError: + pass + if rel is not None: + top = Path(root) + here = top / rel.lstrip("/") + while here == top or top in here.parents: + try: + quota, period = (here / "cpu.max").read_text( + encoding="ascii").split()[:2] + if quota != "max" and int(period) > 0: + limits.append(int(quota) / int(period)) + except (OSError, ValueError): + pass + if here == top: + break + here = here.parent + for d in ("cpu", "cpu,cpuacct", "cpuacct,cpu"): + try: + quota = int((Path(root) / d / "cpu.cfs_quota_us").read_text( + encoding="ascii")) + period = int((Path(root) / d / "cpu.cfs_period_us").read_text( + encoding="ascii")) + except (OSError, ValueError): + continue + if quota > 0 and period > 0: + limits.append(quota / period) + break + return min(limits) if limits else None + + +def intra_op_threads() -> int: + """Threads for one inference, or 0 for onnxruntime's own choice. + + onnxruntime sizes its pool from the machine's cores. It does not see a + container's CPU quota, so under `docker run --cpus 2` on a 64-core host + it ran 32 threads that the kernel let use two CPUs between them: the + quota was spent in the first few milliseconds of each period and every + search then waited, throttled, for the next one. Nor, before Python + 3.13 made it easy to ask, a CPU set narrowed with taskset or cpuset. + So when either leaves this process at most half of the machine, the + pool is sized to what it may use. Half, because onnxruntime's own + choice is the PHYSICAL cores, half the logical ones where each core + runs two threads: above half, its default already fits on such a + machine, and setting more threads than that would only add contention. + """ + total = os.cpu_count() or 1 + usable = total + count = getattr(os, "process_cpu_count", None) + try: + if count is not None: + usable = count() or total + elif hasattr(os, "sched_getaffinity"): + usable = len(os.sched_getaffinity(0)) or total + except OSError: + pass + if sys.platform.startswith("linux"): + limit = _cgroup_cpu_limit() + if limit is not None: + usable = min(usable, max(1, int(limit))) + return usable if usable <= total // 2 else 0 + + class Embedder: """CLS-pooled, L2-normalized sentence embeddings from a local ONNX model.""" @@ -200,12 +343,17 @@ def __init__(self, model_name: str = DEFAULT_MODEL): os.environ.setdefault("ORT_DISABLE_TELEMETRY", "1") try: import onnxruntime as ort # local import: keeps CLI startup fast + from tokenizers import Tokenizer + except ModuleNotFoundError as exc: + better = _runtime_not_installed(exc) + if better is not None: + raise better from exc + raise except ImportError as exc: better = _missing_runtime_error(exc) if better is not None: raise better from exc raise - from tokenizers import Tokenizer # Errors only, and for the DEFAULT logger, not just the session's: # device discovery runs under the default logger and its warnings go @@ -243,6 +391,9 @@ def __init__(self, model_name: str = DEFAULT_MODEL): # the arena the same work returns to under 90 MB when it is done, at # the same speed for the batch sizes BATCH_TOKENS allows. so.enable_cpu_mem_arena = False + threads = intra_op_threads() + if threads: + so.intra_op_num_threads = threads self.sess = ort.InferenceSession(str(info.onnx), so, providers=["CPUExecutionProvider"]) self._needs_token_type = any( @@ -250,6 +401,7 @@ def __init__(self, model_name: str = DEFAULT_MODEL): def _encode(self, texts: list[str]) -> list[list[int]]: """Token ids per text, truncated to the model's 512, never padded.""" + texts = [well_formed(t) for t in texts] with self._tok_lock: return [e.ids for e in self.tok.encode_batch(texts)] @@ -319,6 +471,9 @@ def chunk(self, text: str, window: int = CHUNK_WINDOW, """ if not text: return [text] + # The windows are cut from the text the tokenizer saw: replacing a + # surrogate is one code point for one, so its offsets index this. + text = well_formed(text) with self._tok_lock: self.tok.no_truncation() try: diff --git a/src/compartment/embed_daemon.py b/src/compartment/embed_daemon.py index ad06edf..8767692 100644 --- a/src/compartment/embed_daemon.py +++ b/src/compartment/embed_daemon.py @@ -31,6 +31,9 @@ Windows has no AF_UNIX in Python, so there every process keeps its own model as it always did; `enabled()` says no and `get_embedder` never comes here. +Neither does a process the operating system refuses Unix domain sockets, as +an agent's sandbox does: after the first refusal it keeps its own model too, +without starting a daemon that could never listen (see `_refused`). """ from __future__ import annotations @@ -115,6 +118,31 @@ class Unavailable(RuntimeError): """The shared daemon cannot be used; the caller embeds locally.""" +#: Why the operating system will not let this process use a Unix domain +#: socket at all, once it has said so; None until then. +#: +#: An agent's own shell is often sandboxed: on macOS, Claude Code and Codex +#: run their shell commands under a Seatbelt profile that refuses bind() on +#: an AF_UNIX path with EPERM, and other sandboxes refuse connect() or +#: starting a process as well. There, every `compartment search` and `store` an agent ran started a +#: daemon that could only fail to bind and exit, and waited for it to do so: +#: a fresh interpreter, numpy and onnxruntime imported, a quarter of a +#: second or more, at every command, and again at every reconnect in a +#: server, before the in-process model it was always going to use. A +#: sandbox does not lift while a process runs, so the answer is kept for +#: the life of the process: the first refusal costs one failed system call, +#: and every later embedder goes straight to its own model. +_refused: str | None = None + + +def _note_refused(what: str, exc: OSError) -> None: + global _refused + if _refused is None: + _refused = (f"this process may not {what} ({exc.strerror or exc})," + " as inside an agent's sandbox: it embeds with a model " + "of its own") + + # --------------------------------------------------------------- where and if def supported() -> bool: @@ -216,7 +244,16 @@ def _peer_uid(conn: socket.socket) -> int | None: if hasattr(socket, "SO_PEERCRED"): raw = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("iII")) - _pid, uid, _gid = struct.unpack("iII", raw) + if sys.platform.startswith("openbsd"): + # OpenBSD has SO_PEERCRED too, but its struct sockpeercred + # is { uid_t uid; gid_t gid; pid_t pid; }, not Linux's + # struct ucred { pid; uid; gid; }. Read as Linux's, the + # "uid" was the peer's gid, so the daemon turned away every + # client of a user whose group id is not their user id, and + # would have let in one whose group id happened to be ours. + uid, _gid, _pid = struct.unpack("IIi", raw) + else: + _pid, uid, _gid = struct.unpack("iII", raw) return uid if sys.platform == "darwin" or "bsd" in sys.platform: # struct xucred { u_int cr_version; uid_t cr_uid; short @@ -232,6 +269,32 @@ def _peer_uid(conn: socket.socket) -> int | None: return None +def _open_private(name: str, create: bool = True) -> int: + """A descriptor on this user's own regular file `name`, made 0600 if + it is not there, for the daemon's lock and spawn files. + + They sit beside the socket, and a home too deep for an AF_UNIX path puts + the socket in the shared temp directory (see socket_path), where the + names are predictable from the home path. Opened plainly, a symlink or a + hard link another user left at the lock file's name led the daemon to + truncate whatever it pointed at and write its pid there: the user's own + shell profile, say. Linux refuses to follow a stranger's link in /tmp + (fs.protected_symlinks), the BSDs and others do not. So a link is never + followed, and a file that is not this user's alone is refused.""" + flags = os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + fd = os.open(name, flags | (os.O_CREAT if create else 0), 0o600) + try: + st = os.fstat(fd) + except OSError: + os.close(fd) + raise + if (not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() + or st.st_nlink != 1): + os.close(fd) + raise PermissionError(f"{name} is not a file of this user's own") + return fd + + def _owned_socket(p: Path) -> bool: """True if `p` is a socket file that belongs to this user.""" try: @@ -260,8 +323,14 @@ def __init__(self, path: Path, idle: int): # -- lifecycle --------------------------------------------------------- def run(self) -> int: - self.path.parent.mkdir(parents=True, exist_ok=True) - if not self._take_lock(): + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + locked = self._take_lock() + except OSError as exc: + print(f"compartment: the embedding daemon cannot take its lock " + f"file beside {self.path}: {exc}", file=sys.stderr) + return 1 + if not locked: return 0 # another daemon owns this socket try: self.path.unlink() # a socket file a dead daemon left @@ -272,7 +341,14 @@ def run(self) -> int: srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: srv.bind(str(self.path)) - except OSError: + except OSError as exc: + # Said for `compartment embed-daemon run`; a daemon a client + # started has no stderr, and the client goes on without it. + print(f"compartment: the embedding daemon cannot listen at " + f"{self.path}: {exc.strerror or exc}", file=sys.stderr) + srv.close() + fd, self._lock_fd = self._lock_fd, None + os.close(fd) # releases the flock return 1 os.chmod(self.path, 0o600) self._inode = os.stat(self.path).st_ino @@ -303,7 +379,7 @@ def run(self) -> int: return 0 def _take_lock(self) -> bool: - fd = os.open(str(self.path) + ".lock", os.O_RDWR | os.O_CREAT, 0o600) + fd = _open_private(str(self.path) + ".lock") try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: @@ -513,7 +589,14 @@ def run(path: Path | None = None, idle: int | None = None) -> int: print("compartment: the shared embedding daemon needs a POSIX system " "with Unix domain sockets", file=sys.stderr) return 2 - return _Daemon(path or socket_path(), idle or idle_seconds()).run() + try: + path = path or socket_path() + except (OSError, RuntimeError) as exc: + print(f"compartment: no session directory to hold the embedding " + f"daemon's socket ({exc}); pass --socket PATH", + file=sys.stderr) + return 1 + return _Daemon(path, idle or idle_seconds()).run() # ------------------------------------------------------------ starting it up @@ -521,10 +604,23 @@ def run(path: Path | None = None, idle: int | None = None) -> int: def _connect(path: Path, timeout: float = 5.0) -> socket.socket | None: if not _owned_socket(path): return None - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except PermissionError as exc: + _note_refused("create a Unix domain socket", exc) + return None + except OSError: + return None s.settimeout(timeout) try: s.connect(str(path)) + except PermissionError as exc: + # The socket is this user's own (checked above) in this user's own + # directory, so a refusal is not its permissions: it is a sandbox + # that will not let this process reach any socket file. + s.close() + _note_refused("connect to a Unix domain socket", exc) + return None except OSError: s.close() return None @@ -535,12 +631,56 @@ def _connect(path: Path, timeout: float = 5.0) -> socket.socket | None: return s +def _bind_refused(path: Path) -> bool: + """Whether the operating system refuses this process a listening Unix + domain socket beside `path`, which is what the daemon it would start + needs: the daemon inherits whatever sandbox this process is in. + + Asked with one bind() on a name of its own, which is removed again, + instead of with a daemon: a refusal is EPERM or EACCES at once, while a + daemon has to be started, import numpy and onnxruntime, and exit before + anyone learns the same thing. Any other error (a path too long, a name + in use) is left for the daemon to meet, as before.""" + if _refused is not None: + return True + probe = str(path) + ".p" + try: + os.unlink(probe) # one a killed probe left behind + except OSError: + pass + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except PermissionError as exc: + _note_refused("create a Unix domain socket", exc) + return True + except OSError: + return False + try: + s.bind(probe) + except PermissionError as exc: + _note_refused("bind a Unix domain socket", exc) + return True + except OSError: + return False + finally: + s.close() + try: + os.unlink(probe) + except OSError: + pass + return False + + def _spawn(path: Path, idle: int) -> subprocess.Popen: """Start the daemon detached: its own session, so neither the MCP client's process-group cleanup nor a closing terminal takes it down, and no inherited stdio, because stdout of an MCP server IS the transport.""" - cmd = [sys.executable, "-m", "compartment.embed_daemon", + # -P: `-m` puts the working directory first on sys.path, and that is the + # home directory, so a random.py, json.py or compartment.py anyone keeps + # there was imported in place of the real one and the daemon died at + # start, every time, leaving every agent to load a model of its own. + cmd = [sys.executable, "-P", "-m", "compartment.embed_daemon", "--socket", str(path), "--idle", str(idle)] # The daemon must run the same code as the client that starts it, so it # imports the package from wherever this module was imported from - a @@ -570,12 +710,17 @@ def connect_or_spawn(path: Path, idle: int, *, spawn: bool = True, daemon, the others wait on the lock and connect to it. Whoever spawned it polls until the socket accepts, which is a few hundred milliseconds - the daemon binds first and loads the model on the first request. + + Where the operating system refuses this process Unix domain sockets + (see `_refused`), nothing is started and None comes back at once. """ + if _refused is not None: + return None s = _connect(path) - if s is not None or not spawn: + if s is not None or not spawn or _refused is not None: return s path.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(str(path) + ".spawn", os.O_RDWR | os.O_CREAT, 0o600) + fd = _open_private(str(path) + ".spawn") try: deadline = time.monotonic() + wait while True: @@ -587,10 +732,17 @@ def connect_or_spawn(path: Path, idle: int, *, spawn: bool = True, return None time.sleep(0.05) s = _connect(path) - if s is not None: + if s is not None or _refused is not None: return s + if _bind_refused(path): + return None try: child = _spawn(path, idle) + except PermissionError as exc: + # A sandbox that forbids starting processes forbids it every + # time: one try per process, not one per vault opened. + _note_refused("start the embedding daemon", exc) + return None except OSError: return None while time.monotonic() < deadline: @@ -654,9 +806,12 @@ def connect(cls, model_name: str = DEFAULT_MODEL, *, path: Path | None = None, return None try: return cls(model_name, path or socket_path(), spawn=spawn, wait=wait) - except (Unavailable, OSError): + except (Unavailable, OSError, RuntimeError): # OSError: the session directory or the lock files cannot be # made here (a read-only home), which is no daemon either. + # RuntimeError: there is no home directory at all (HOME unset + # and no password-database entry, as for a container's + # arbitrary uid), so no session directory to put a socket in. return None # -- properties the vault reports -------------------------------------- @@ -826,14 +981,25 @@ def embed_record(self, text: str) -> np.ndarray: # ------------------------------------------------------- status, stop, main def status(path: Path | None = None) -> dict: - """What the daemon at `path` reports, or that nothing is running.""" - path = path or socket_path() + """What the daemon at `path` reports, or that nothing is running, and + when nothing can run for this process, why.""" if not supported(): - return {"running": False, "socket": str(path), - "reason": "not supported on this platform"} + # Before socket_path(): its fallback names the file after the uid, + # which Windows does not have. + return {"running": False, "socket": None, + "reason": "not supported on this platform: every process " + "embeds with a model of its own"} + try: + path = path or socket_path() + except (OSError, RuntimeError) as exc: + return {"running": False, "socket": None, + "reason": f"no session directory to hold its socket ({exc})"} s = _connect(path, timeout=2.0) if s is None: - return {"running": False, "socket": str(path)} + out = {"running": False, "socket": str(path)} + if _bind_refused(path): + out["reason"] = _refused + return out try: w, r = s.makefile("wb"), s.makefile("rb") _write(w, {"op": "status"}) @@ -853,9 +1019,12 @@ def stop(path: Path | None = None, wait: float = 5.0) -> dict: gets SIGTERM. Stale files a crashed daemon left are removed so the next client starts cleanly. """ - path = path or socket_path() if not supported(): - return {"stopped": False, "socket": str(path)} + return {"stopped": False, "socket": None} + try: + path = path or socket_path() + except (OSError, RuntimeError): + return {"stopped": False, "socket": None} pid = None s = _connect(path, timeout=2.0) if s is not None: @@ -900,7 +1069,7 @@ def _connect_probe(path: Path) -> bool: def _locked_pid(path: Path) -> int | None: """The pid written into the lock file, if a live daemon holds the lock.""" try: - fd = os.open(str(path) + ".lock", os.O_RDWR) + fd = _open_private(str(path) + ".lock", create=False) except OSError: return None try: diff --git a/src/compartment/home.py b/src/compartment/home.py index 1ac95ca..b435fc2 100644 --- a/src/compartment/home.py +++ b/src/compartment/home.py @@ -50,4 +50,17 @@ def env(suffix: str, default: str | None = None) -> str | None: or default) -__all__ = ["home", "file", "env", "NAME", "LEGACY_NAME"] +def session_dir() -> Path: + """Where the unlock credential and the per-boot markers live: + COMPARTMENT_SESSION_DIR, else home()/session. + + The variable is read first and home() only when it is unset. Writing + env("SESSION_DIR", home() / "session") evaluated home() every time, and + in a container run as a uid with no passwd entry and no HOME, + Path.home() raises: setting COMPARTMENT_SESSION_DIR, the documented way + to put the session somewhere writable, could not get past it.""" + given = env("SESSION_DIR") + return Path(given) if given else home() / "session" + + +__all__ = ["home", "file", "env", "session_dir", "NAME", "LEGACY_NAME"] diff --git a/src/compartment/longmemeval.py b/src/compartment/longmemeval.py index 8ff4bb2..ae00526 100644 --- a/src/compartment/longmemeval.py +++ b/src/compartment/longmemeval.py @@ -42,6 +42,7 @@ from .home import env, home import hashlib import json +import os import sqlite3 import time from pathlib import Path @@ -92,15 +93,30 @@ def download(variant: str = "s") -> Path: print(f"downloading {url}") h = hashlib.sha256() done = 0 - with urllib.request.urlopen(url) as r, open(dest, "wb") as f: - while True: - chunk = r.read(1 << 22) - if not chunk: - break - f.write(chunk) - h.update(chunk) - done += len(chunk) - print(f"\r {done // (1 << 20)} MB", end="", flush=True) + # Into a file beside it, renamed into place only once the whole body has + # arrived. Written straight to `dest`, a download cut short (Ctrl-C, a + # dropped connection, a full disk; the m variant is gigabytes) left a + # truncated file that run() trusted because it existed, and every run + # after failed parsing it as "Unterminated string" with nothing to say + # the download was the problem or that deleting it was the fix. + part = dest.with_name(dest.name + ".part") + try: + with urllib.request.urlopen(url) as r, open(part, "wb") as f: + while True: + chunk = r.read(1 << 22) + if not chunk: + break + f.write(chunk) + h.update(chunk) + done += len(chunk) + print(f"\r {done // (1 << 20)} MB", end="", flush=True) + os.replace(part, dest) + except BaseException: + try: + part.unlink() + except OSError: + pass + raise print(f"\n sha256 {h.hexdigest()}") print(f" → {dest}") return dest diff --git a/src/compartment/menubar.py b/src/compartment/menubar.py index 1e0f1aa..6d80dd2 100644 --- a/src/compartment/menubar.py +++ b/src/compartment/menubar.py @@ -324,7 +324,8 @@ def _run(args: list[str], timeout: int = 60) -> tuple[int, str]: try: p = subprocess.run(args, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}) + env={**os.environ, "PATH": user_path()}, + **_no_console()) return p.returncode, (p.stdout or "") + (p.stderr or "") except (OSError, subprocess.SubprocessError) as exc: return 1, str(exc) @@ -350,13 +351,11 @@ def _json_cmd(vault: str, *sub: str) -> dict | None: def _json_run(argv: list[str], timeout: float = 60) -> dict | None: """Run a command and parse the JSON it prints: _json_cmd's runner, for any command line. None if it failed, timed out, or printed no JSON.""" - kwargs = {} - if sys.platform == "win32": - kwargs["creationflags"] = _CREATE_NO_WINDOW try: p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}, **kwargs) + env={**os.environ, "PATH": user_path()}, + **_no_console()) except (OSError, subprocess.SubprocessError): return None if p.returncode != 0: @@ -442,6 +441,17 @@ def set_setting(vault: str, key: str, value) -> dict: #: and the vault view would keep its open for as long as it runs. _CREATE_NO_WINDOW = 0x08000000 + +def _no_console() -> dict: + """subprocess keyword arguments for a child the panel starts: on Windows, + no console window. Every child, not only the two that had it. Unlock, + Lock, Create, Change password and Connect each ran the CLI as a console + program, so from the tray - pythonw.exe, which has no console of its own + - each click flashed a console window, and Dashboard kept one open for + as long as the dashboard ran, where closing it stopped the dashboard.""" + return ({"creationflags": _CREATE_NO_WINDOW} if sys.platform == "win32" + else {}) + NO_VAULT_NOTE = "no vault yet - choose a passphrase below to create one" CREDENTIAL_REFUSED_NOTE = ("The saved unlock no longer opens this vault. " "Enter the passphrase to unlock it again.") @@ -534,15 +544,12 @@ def __init__(self, vault: str): self._next = 0 def _start(self): - kwargs = {} - if sys.platform == "win32": - kwargs["creationflags"] = _CREATE_NO_WINDOW proc = subprocess.Popen( _view_argv(self.vault), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None if env("MENUBAR_DEBUG") else subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace", bufsize=1, - env={**os.environ, "PATH": user_path()}, **kwargs) + env={**os.environ, "PATH": user_path()}, **_no_console()) lines: queue.Queue = queue.Queue() def pump(): @@ -919,6 +926,66 @@ def dashboard_url(line: str) -> str | None: return None +def _is_wsl() -> bool: + """Linux under Windows (WSL 1 or 2), where the user's browser is a + Windows program.""" + if os.environ.get("WSL_DISTRO_NAME") or os.environ.get("WSL_INTEROP"): + return True + try: + with open("/proc/sys/kernel/osrelease", encoding="utf-8", + errors="replace") as fh: + return "microsoft" in fh.read().lower() + except OSError: + return False + + +def open_in_browser(url: str) -> bool: + """Open `url` in the user's own browser. False when there is none to + open here, so the caller can say to open the address by hand. + + webbrowser.open on its own got three places wrong: + + * A Linux machine with no desktop (SSH, a container, WSL without WSLg) + has no graphical browser, and with TERM set webbrowser falls back to + a text browser - w3m, lynx, www-browser - run in the same terminal and + waited for. `compartment dash` called it before serving, so the text + browser sat waiting on a server that would not answer until the + browser exited. Without a display nothing is opened. + * WSL: the browser is on the Windows side, and xdg-open there opens + nothing or a stray Linux one. wslview (wslu) or explorer.exe opens + the Windows default browser, which reaches 127.0.0.1 in WSL 2 through + its localhost forwarding. + * BROWSER, when set, is the user's own choice and is honoured as + webbrowser honours it, display or not. + + Never raises. Can block if the browser it starts is one that runs in + the foreground, so a caller with other work runs it on a thread. + """ + try: + if os.environ.get("BROWSER"): + return bool(webbrowser.open(url)) + if sys.platform not in ("win32", "darwin"): + if _is_wsl(): + for opener in ("wslview", "explorer.exe"): + exe = shutil.which(opener) + if exe: + # explorer.exe exits 1 even when it opened the page, + # so having started it is the answer. + subprocess.Popen([exe, url], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True) + return True + return False + if not (os.environ.get("DISPLAY") + or os.environ.get("WAYLAND_DISPLAY")): + return False + return bool(webbrowser.open(url)) + except Exception: # noqa: BLE001 + return False + + def _drain(proc) -> None: # `dash` says a few more lines over its life; read them so it can never # block on a full pipe. @@ -941,14 +1008,17 @@ def open_dashboard(vault: str) -> str | None: """ proc, url = _DASH["proc"], _DASH["url"] if proc is not None and proc.poll() is None and url: - webbrowser.open(url) + # On a thread: this is the UI thread, and a browser that runs in the + # foreground would hold it for as long as it is open. + threading.Thread(target=open_in_browser, args=(url,), daemon=True, + name="compartment-open-browser").start() return url try: proc = subprocess.Popen( [*_cli_argv(), "--vault", vault, "dash"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}) + env={**os.environ, "PATH": user_path()}, **_no_console()) except (OSError, subprocess.SubprocessError): return None _DASH["proc"], _DASH["url"] = proc, None @@ -1011,7 +1081,7 @@ def unlock_vault(vault: str, passphrase: str) -> tuple[bool, str]: p = subprocess.run( [*_cli_argv(), "--vault", vault, "unlock", "--passphrase-stdin"], input=passphrase + "\n", capture_output=True, text=True, - timeout=180, encoding="utf-8", errors="replace") + timeout=180, encoding="utf-8", errors="replace", **_no_console()) except (OSError, subprocess.SubprocessError) as exc: return False, str(exc) out = " ".join(((p.stdout or "") + (p.stderr or "")).split()) @@ -1064,7 +1134,7 @@ def create_vault(vault: str, passphrase: str, repeat: str) -> tuple[bool, str]: "--no-app"], input=passphrase + "\n", capture_output=True, text=True, timeout=900, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}) + env={**os.environ, "PATH": user_path()}, **_no_console()) except subprocess.TimeoutExpired as exc: if os.path.exists(vault): # init writes the vault in its first seconds. What ran out of @@ -1106,7 +1176,7 @@ def change_passphrase(vault: str, new: str, repeat: str) -> tuple[bool, str]: p = subprocess.run( [*_cli_argv(), "--vault", vault, "rekey", "--new-passphrase-stdin"], input=new + "\n", capture_output=True, text=True, - timeout=180, encoding="utf-8", errors="replace") + timeout=180, encoding="utf-8", errors="replace", **_no_console()) except (OSError, subprocess.SubprocessError) as exc: return False, str(exc) out = " ".join(((p.stdout or "") + (p.stderr or "")).split()) @@ -1165,7 +1235,7 @@ def integration_status(vault: str) -> dict: p = _openclaw_config() if p.is_file(): cfg = json.loads(p.read_text(encoding="utf-8")) - out["openclaw"] = "compartment" in (cfg.get("mcpServers") or {}) + out["openclaw"] = "compartment" in openclaw_servers(cfg) except (OSError, ValueError): pass return out @@ -1175,14 +1245,58 @@ def _claude_code_config() -> Path: return Path.home() / ".claude.json" +def hermes_home() -> Path: + """Hermes Agent's home, found the way Hermes finds it + (hermes_constants.get_hermes_home): HERMES_HOME, with ~ and variables + expanded, else the platform default. + + On Windows that default is %LOCALAPPDATA%\\hermes, not ~/.hermes, and + Hermes's Windows installer does not export HERMES_HOME. Looking only in + ~/.hermes, `integrate hermes` put the provider plugin where Hermes on + Windows never reads it, and the panel could neither see Hermes nor tell + that it was connected.""" + explicit = os.environ.get("HERMES_HOME", "").strip() + if explicit: + return Path(os.path.expanduser(os.path.expandvars(explicit))) + if sys.platform == "win32": + local = os.environ.get("LOCALAPPDATA", "").strip() + return (Path(local) if local + else Path.home() / "AppData" / "Local") / "hermes" + return Path.home() / ".hermes" + + def _hermes_plugin_manifest() -> Path: - return (Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) - / "plugins" / "compartment" / "plugin.yaml") + return hermes_home() / "plugins" / "compartment" / "plugin.yaml" + + +def openclaw_dir() -> Path: + """OpenClaw's config directory, by OpenClaw's own rule + (src/infra/config-dir.ts, as agent_skill._openclaw_dir reads it): + OPENCLAW_STATE_DIR, else the folder of OPENCLAW_CONFIG_PATH, else + .openclaw under OPENCLAW_HOME or the home directory. OPENCLAW_HOME was + taken for the .openclaw folder itself, one level too high.""" + from .agent_skill import _openclaw_dir + return _openclaw_dir() or Path.home() / ".openclaw" def _openclaw_config() -> Path: - return (Path(os.environ.get("OPENCLAW_HOME", Path.home() / ".openclaw")) - / "openclaw.json") + explicit = os.environ.get("OPENCLAW_CONFIG_PATH", "").strip() + if explicit: + return Path(explicit).expanduser() + return openclaw_dir() / "openclaw.json" + + +def openclaw_servers(cfg) -> dict: + """The MCP servers an OpenClaw config declares: `mcp.servers`. + + Current OpenClaw reads its servers there, and its root schema rejects + keys it does not know (src/config/zod-schema.ts), so the top-level + `mcpServers` that `integrate openclaw` used to write was never loaded + and could stop the whole config from loading. An entry left there is + therefore not a connection, and is not counted as one.""" + mcp = cfg.get("mcp") if isinstance(cfg, dict) else None + servers = mcp.get("servers") if isinstance(mcp, dict) else None + return servers if isinstance(servers, dict) else {} def integration_paths() -> list[Path]: @@ -1232,12 +1346,10 @@ def agent_present(target: str, path: str | None = None) -> bool: or claude_desktop.present()) if target == "hermes": return (bool(shutil.which("hermes", path=path)) - or Path(os.environ.get("HERMES_HOME", - Path.home() / ".hermes")).is_dir()) + or hermes_home().is_dir()) if target == "openclaw": return (bool(shutil.which("openclaw", path=path)) - or Path(os.environ.get("OPENCLAW_HOME", - Path.home() / ".openclaw")).is_dir()) + or openclaw_dir().is_dir()) except Exception: # noqa: BLE001 return False return False @@ -1990,8 +2102,13 @@ def run(vault: str | None = None, show: bool = False, from Foundation import (NSBundle, NSDistributedNotificationCenter, NSObject) except ImportError: - print("error: the menu bar app needs PyObjC.\n" - " pip install 'compartment[menubar]'", file=sys.stderr) + # Into this interpreter, by name. The [menubar] extra this used to + # name has been empty since PyObjC became a base dependency, so + # installing it changed nothing. + print("error: the menu bar app needs PyObjC, which this Python does " + "not have.\n" + f' "{sys.executable}" -m pip install pyobjc-framework-Cocoa', + file=sys.stderr) return 1 vault_path = vault or default_vault() diff --git a/src/compartment/offline_guard.py b/src/compartment/offline_guard.py index 34e0a08..ac1212a 100644 --- a/src/compartment/offline_guard.py +++ b/src/compartment/offline_guard.py @@ -4,11 +4,16 @@ that could put a byte of ours on the wire raises OfflineViolation and the process aborts loudly. That is three doors, not one: - 1. Sockets. Creating an AF_INET/AF_INET6 socket raises. AF_UNIX and every - other local family keep working, because local IPC is not networking. - So does socket.socketpair, which on Windows is two AF_INET sockets on - loopback and which asyncio's event loop needs, for the length of that - call and in that thread only. + 1. Sockets. Creating a socket of any family that can reach another + machine raises. That is not only AF_INET/AF_INET6: AF_PACKET writes raw + frames, AF_BLUETOOTH reaches a paired device without any privilege, + AF_VSOCK and AF_HYPERV reach the host of a VM (WSL2's included), AF_CAN, + AF_TIPC and AF_RDS are buses and cluster links. So the families are an + allow list: AF_UNIX, because local IPC is not networking, and the + kernel's own interfaces (netlink/route, AF_ALG, macOS AF_SYSTEM), which + have no far end. socket.socketpair, which on Windows is two AF_INET + sockets on loopback and which asyncio's event loop needs, is let + through for the length of that call and in that thread only. 2. Name resolution. getaddrinfo, gethostbyname, gethostbyname_ex, gethostbyaddr, getnameinfo, getfqdn and create_connection raise for any host that is not already local. A DNS lookup needs no socket object at @@ -47,6 +52,13 @@ _real_raw_socket = _socket.socket _original_socketpair = socket.socketpair _INET_FAMILIES = (socket.AF_INET, socket.AF_INET6) +#: The only families a socket may have while the guard is on: none of them +#: has a far end on another machine. Anything not listed - a family this +#: platform has and this list never heard of included - is refused. +_LOCAL_FAMILIES = frozenset( + getattr(socket, name) + for name in ("AF_UNIX", "AF_NETLINK", "AF_ROUTE", "AF_ALG", "AF_SYSTEM") + if hasattr(socket, name)) _RESOLVERS = ("getaddrinfo", "gethostbyname", "gethostbyname_ex", "gethostbyaddr", "getnameinfo", "getfqdn", "create_connection") @@ -108,11 +120,20 @@ def _reject_lookup(host: object) -> None: def _reject_inet(family: object) -> None: + """Refuse every family but the local ones. -1 is "the default", which is + AF_INET; with a fileno it means "whatever that descriptor is", which the + audit hook cannot see, so it is refused too, as it always was.""" fam = family if family not in (-1, None) else socket.AF_INET - if fam in _INET_FAMILIES and not getattr(_state, "socketpair", 0): - raise _violation( - "something attempted to create a network socket" - ) + try: + if fam in _LOCAL_FAMILIES: + return + except TypeError: # unhashable: not a family at all + pass + if fam in _INET_FAMILIES and getattr(_state, "socketpair", 0): + return + raise _violation( + "something attempted to create a network socket" + ) # ------------------------------------------------------------ socket patches diff --git a/src/compartment/packs.py b/src/compartment/packs.py index 287a88a..df17bc2 100644 --- a/src/compartment/packs.py +++ b/src/compartment/packs.py @@ -207,6 +207,13 @@ def read_pack(blob: bytes, passphrase: str | None = None, *, sections = unpack_sections(body) records = [json.loads(l) for l in sections["records"].decode("utf-8").splitlines() if l.strip()] + # A "\ud83d" escape in a record decodes to a lone surrogate, which the + # encoder refuses when the vectors have to be recomputed and SQLite + # refuses on insert; see vault.clean_text. + from .vault import clean_text + for r in records: + if isinstance(r.get("text"), str): + r["text"] = clean_text(r["text"]) dim = int(header["model"]["dim"]) vectors = np.frombuffer(sections["vectors"], dtype=np.float32).reshape(-1, dim) if vectors.shape[0] != len(records) or vectors.shape[0] != header["records"]: diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index e35aa69..b7194c0 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -7,7 +7,8 @@ """ from __future__ import annotations -from .home import env, home +from .home import env, home, session_dir +import errno import json import os import platform @@ -27,30 +28,52 @@ # Advisory exclusive file lock (context manager over an open file handle) # --------------------------------------------------------------------------- +# Only these errors mean "someone else holds it". Anything else means the +# filesystem cannot lock at all - NFS whose lock service is not running or +# that is mounted `nolock`/`nolocks` (ENOLCK, ENOTSUP), some FUSE and SMB +# mounts (EOPNOTSUPP, EINVAL) - and no amount of waiting changes that. +# Treating those as busy made every write on such a vault wait out the +# timeout and then blame "another process", which sent people hunting for a +# process that did not exist. if IS_WINDOWS: import msvcrt + _BUSY = frozenset({errno.EACCES, getattr(errno, "EDEADLOCK", 36)}) + + # msvcrt.locking locks bytes from the handle's CURRENT position, and the + # lock file is opened "a+", whose position is end of file. Two processes + # agree on which byte that is only while the file stays empty; anything + # that ever wrote to it (a tool, an older version, a user) would put + # them on different bytes and both would "hold" the lock. Byte 0 always. def _lock_nb(fh) -> bool: try: + fh.seek(0) msvcrt.locking(fh.fileno(), msvcrt.LK_NBLCK, 1) return True - except OSError: - return False + except OSError as exc: + if exc.errno in _BUSY: + return False + raise def _unlock(fh) -> None: try: + fh.seek(0) msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1) except OSError: pass else: import fcntl + _BUSY = frozenset({errno.EAGAIN, errno.EWOULDBLOCK, errno.EACCES}) + def _lock_nb(fh) -> bool: try: fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) return True - except OSError: - return False + except OSError as exc: + if exc.errno in _BUSY: + return False + raise def _unlock(fh) -> None: try: @@ -70,15 +93,30 @@ def __init__(self, path: str, timeout: float = 10.0): def __enter__(self): self._fh = open(self.path, "a+") deadline = time.time() + self.timeout - while not _lock_nb(self._fh): + while True: + try: + if _lock_nb(self._fh): + return self + except OSError as exc: + self._fh.close() + self._fh = None + from .crypto import CryptoError + raise CryptoError( + f"Cannot lock {self.path}: the filesystem it is on does " + f"not support file locks ({exc.strerror or exc}). That is " + f"usually a network share - NFS without its lock service " + f"or mounted nolock, or an SMB mount without byte-range " + f"locks. Keep the vault on a local disk (--vault or " + f"COMPARTMENT_VAULT), or enable locking on the " + f"mount.") from exc if time.time() > deadline: self._fh.close() + self._fh = None from .crypto import CryptoError raise CryptoError( f"Vault is busy: another process holds the write lock " f"(waited {self.timeout:.0f}s)") time.sleep(0.05) - return self def __exit__(self, *exc): if self._fh is not None: @@ -133,18 +171,10 @@ def _posix_boot_estimate() -> tuple[float, float] | None: `kern.boottime` reading, so a step moves both ends equally and cancels. """ system = platform.system() - if system == "Darwin": - try: - out = subprocess.run(["sysctl", "-n", "kern.boottime"], - capture_output=True, text=True, check=True, - timeout=_SUBPROCESS_TIMEOUT).stdout - import re - m = re.search(r"sec = (\d+)", out) - if m: - sec = float(m.group(1)) - return sec, time.time() - sec - except (OSError, subprocess.SubprocessError, ValueError): - pass + if system == "Darwin" or system.endswith("BSD") or system == "DragonFly": + est = _sysctl_boot_estimate() + if est is not None: + return est elif system == "Linux": try: btime = None @@ -159,13 +189,96 @@ def _posix_boot_estimate() -> tuple[float, float] | None: return btime, up except (OSError, ValueError): pass + # The kernel's own boot record could not be read. That is the normal + # state inside an agent's sandbox - Claude Code and Codex run shell + # commands under macOS Seatbelt, which refuses kern.boottime - and on + # Android, where SELinux hides /proc/stat from apps such as Termux. A + # clock that counts from boot answers the same question in-process. + return _clock_boot_estimate() + + +def _sysctl_boot_estimate() -> tuple[float, float] | None: + """kern.boottime on macOS and the BSDs: in-process first, so a vault open + does not fork, then through the sysctl binary for a libc without + sysctlbyname (OpenBSD).""" + import struct + try: + import ctypes + # The process's own symbols, which include libc's. Not + # ctypes.util.find_library("c"): on FreeBSD, OpenBSD and DragonFly + # that runs /sbin/ldconfig -r, and on NetBSD it runs the C compiler, + # in a subprocess, on every call - a fork on every vault open, which + # is the thing reading this in-process was meant to avoid. + libc = ctypes.CDLL(None, use_errno=True) + buf = ctypes.create_string_buffer(32) + size = ctypes.c_size_t(len(buf)) + if libc.sysctlbyname(b"kern.boottime", buf, ctypes.byref(size), None, 0) == 0: + # struct timeval: a 64-bit time_t on every 64-bit system, and a + # 32-bit one where the whole struct is 8 bytes. + fmt = "=q" if size.value >= 16 else "=i" + sec = float(struct.unpack_from(fmt, buf.raw)[0]) + if sec > 0: + return sec, time.time() - sec + except (ImportError, OSError, AttributeError, ValueError, TypeError, + struct.error): + pass + if platform.system() == "Darwin": + # macOS always has sysctlbyname, and the binary makes the same call: + # where the call above was refused - Seatbelt, under Claude Code and + # Codex - the binary is refused too. Forking it anyway cost every + # vault open in an agent's shell a process and a sandbox-violation + # report, for an answer that could not come. + return None + try: + out = subprocess.run(["sysctl", "-n", "kern.boottime"], + capture_output=True, text=True, check=True, + timeout=_SUBPROCESS_TIMEOUT).stdout + import re + # FreeBSD: "{ sec = 1700000000, usec = 0 } Tue Nov 14 ...". + # OpenBSD, the one that needs this path, prints only the seconds + # under -n (sbin/sysctl/sysctl.c), which the first pattern missed. + m = (re.search(r"sec = (\d+)", out) + or re.fullmatch(r"\s*(\d+)\s*", out)) + if m: + sec = float(m.group(1)) + return sec, time.time() - sec + except (OSError, subprocess.SubprocessError, ValueError): + pass + return None + + +def _clock_boot_estimate() -> tuple[float, float] | None: + """(boot instant, uptime) from a clock that starts at boot, or None. + + Only clocks that keep counting through sleep qualify, or every resume + would read as a reboot: CLOCK_BOOTTIME on Linux, Android and OpenBSD, and + CLOCK_MONOTONIC on macOS. There libc computes CLOCK_MONOTONIC as the wall + clock minus the kernel's boot time, read from the commpage, which no + sandbox gates; so wall clock minus it IS kern.boottime, to the + microsecond (checked under Seatbelt: constant across calls, and 8.6 s + away from mach_continuous_time, CLOCK_MONOTONIC_RAW, on a machine up + for two days). The result feeds _pinned_boot_token like any other + estimate, so the first value seen this boot is the one every process + agrees on.""" + names = ("CLOCK_MONOTONIC",) if platform.system() == "Darwin" else ( + "CLOCK_BOOTTIME",) + for name in names: + clock = getattr(time, name, None) + if clock is None: + continue + try: + up = time.clock_gettime(clock) + except OSError: + continue + if up > 0: + return time.time() - up, up return None def _posix_marker() -> Path: """The POSIX pin. Windows keeps its own `.winboot` - renaming a marker that live installs already rely on would re-key their credentials.""" - base = Path(env("SESSION_DIR", home() / "session")) + base = session_dir() return base / ".boottime" @@ -177,8 +290,8 @@ def _pinned_boot_token(computed: float, uptime_s: float, marker: Path) -> str: mid-session in three ways this avoids: 1. sub-second int() truncation flapping between two processes; 2. wall-clock steps and NTP slew moving the derived boot instant; - 3. a jump of the whole sleep duration where the uptime source - excludes sleep (Windows GetTickCount64). + 3. a jump of the whole sleep duration, where an uptime source + excludes sleep (none of those used here does). A real reboot resets uptime and moves the boot instant far beyond the tolerance below, so it is detected and the token changes, as intended.""" try: @@ -203,7 +316,7 @@ def _pinned_boot_token(computed: float, uptime_s: float, marker: Path) -> str: def _win_marker() -> Path: - base = Path(env("SESSION_DIR", home() / "session")) + base = session_dir() return base / ".winboot" @@ -219,13 +332,27 @@ def _windows_boot_token() -> str: credential and relocks the vault mid-session in three ways this avoids: 1. sub-second int() truncation flapping between two processes; 2. wall-clock drift after NTP slew accumulating past a second boundary; - 3. a jump of the whole sleep duration after every sleep/resume - (GetTickCount64 excludes sleep; time.time() does not). + 3. a jump of the whole sleep duration after every sleep/resume, were + the uptime source to skip sleep. GetTickCount64 does not (it + includes time suspended; PEP 418), and the tolerance relies on + that: QueryUnbiasedInterruptTime, which skips sleep, would read + every sleep longer than five minutes as a reboot. A real reboot resets uptime and moves the boot instant far beyond the - tolerance below, so it is detected and the token changes, as intended.""" + tolerance below, so it is detected and the token changes, as intended. + + GetTickCount64 returns a 64-bit count, and ctypes assumes a C int for + any function it is not told about. Called bare, the count was cut to 32 + signed bits: past 2**31 ms of uptime - 24.9 days, which a Windows + machine that only ever sleeps, or shuts down with Fast Startup, reaches + easily - it went negative, the derived boot instant jumped 49.7 days, + the pin read that as a reboot, and the session credential was deleted; + then again every 49.7 days.""" import ctypes - uptime_s = ctypes.windll.kernel32.GetTickCount64() / 1000.0 # excludes sleep - computed = time.time() - uptime_s # ~boot instant + get_tick_count = ctypes.windll.kernel32.GetTickCount64 + get_tick_count.restype = ctypes.c_uint64 + get_tick_count.argtypes = [] + uptime_s = get_tick_count() / 1000.0 # counts through sleep + computed = time.time() - uptime_s # ~boot instant return _pinned_boot_token(computed, uptime_s, _win_marker()) @@ -262,7 +389,7 @@ def _windows_boot_token() -> str: def _machine_marker() -> Path: - base = Path(env("SESSION_DIR", home() / "session")) + base = session_dir() return base / ".machineid" diff --git a/src/compartment/server.py b/src/compartment/server.py index 22ed303..c3f15ad 100644 --- a/src/compartment/server.py +++ b/src/compartment/server.py @@ -102,7 +102,12 @@ mcp = MCPServer("compartment", instructions=COMPARTMENT_INSTRUCTIONS, version=__version__) except TypeError: # mcp 1.x - mcp = MCPServer("compartment", instructions=COMPARTMENT_INSTRUCTIONS) + # WARNING, not the 1.x default of INFO, which logged "Processing request + # of type CallToolRequest" to stderr for every request: Cursor shows + # each stderr line of a server as an [error], and the root logger it + # configures at that level passed every library's INFO lines through. + mcp = MCPServer("compartment", instructions=COMPARTMENT_INSTRUCTIONS, + log_level="WARNING") try: mcp._mcp_server.version = __version__ except Exception: # noqa: BLE001 @@ -193,6 +198,10 @@ def _credential_gone(v: Vault) -> bool: return False +class VaultMissingError(CryptoError): + """The vault this server was pointed at does not exist.""" + + def _vault() -> Vault: v = _state["vault"] if v is not None and not v._locked and _credential_gone(v): @@ -211,6 +220,13 @@ def _vault() -> Vault: _state["vault"] = None stale, v = v, None if v is None or v._locked: + if not os.path.exists(_state["path"]): + # Not "locked": `compartment unlock` cannot make a vault that is + # not there, and that is what the locked message sent people to. + raise VaultMissingError( + f"No vault at {_state['path']}. Create one with `compartment " + "init` on this machine, or point --vault in this client's " + "MCP config at the vault that exists.") # try silent re-unlock via keychain/env (user intent persists until # `compartment lock` clears the credential) sig = _credential_signature() @@ -404,11 +420,35 @@ def _offload(fn): async def wrapper(*args, **kwargs): def _call(): with _toolgate: - return fn(*args, **kwargs) + try: + return fn(*args, **kwargs) + except ToolError: + raise + except Exception as exc: # noqa: BLE001 + # mcp 2.x passes the model the text of a ToolError + # only; anything else arrives as "Error executing tool + # " and nothing more. A full disk, a vault file + # this user cannot read, the shared embedding process + # refusing a text: each says what it is, so the model + # can tell the user, as mcp 1.x always let it. + raise _fail(exc) from exc return await anyio.to_thread.run_sync(_call) return wrapper +def _hints(title: str, **hints) -> ToolAnnotations: + """A tool's annotations, with openWorldHint=False on every one. + + The spec's value for an absent openWorldHint is true: "this tool may + interact with an open world of external entities", the way a web search + does. Left out, every memory tool told the client it might reach beyond + this machine, which is the one thing Compartment never does: clients + that weigh the hint (to group, flag or confirm such tools) treated a + fully offline vault as an internet tool. Every tool works on the local + vault alone, so all of them say so.""" + return ToolAnnotations(title=title, openWorldHint=False, **hints) + + IMPORTANCE_DEFAULT = 0.5 # Records per slice of a background retag pass. Sized so the gate is held for @@ -433,8 +473,8 @@ def _importance(value: float) -> float: return min(1.0, max(0.0, v)) -@mcp.tool(title="Store a memory", annotations=ToolAnnotations( - title="Store a memory", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Store a memory", annotations=_hints( + "Store a memory", readOnlyHint=False, destructiveHint=False)) @_offload def memory_store(text: str, source: str, namespace: str | None = None, tags: list[str] | None = None, importance: float = 0.5, @@ -560,8 +600,8 @@ def _prevalidate_facts(v: Vault, facts: list, namespace: str | None = None, raise MemoryToolError(f"facts[{i}]: {exc}") from None -@mcp.tool(title="Store several memories", annotations=ToolAnnotations( - title="Store several memories", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Store several memories", annotations=_hints( + "Store several memories", readOnlyHint=False, destructiveHint=False)) @_offload def memory_store_many(facts: list[dict], source: str, namespace: str | None = None, @@ -636,8 +676,8 @@ def _store_all(v): return json.dumps({"stored": results, "count": len(results)}) -@mcp.tool(title="Search memory", annotations=ToolAnnotations( - title="Search memory", readOnlyHint=True)) +@mcp.tool(title="Search memory", annotations=_hints( + "Search memory", readOnlyHint=True)) @_offload def memory_search(query: str, namespace: str | None = None, tags: list[str] | None = None, top_k: int = 8, @@ -668,8 +708,9 @@ def memory_search(query: str, namespace: str | None = None, raise _fail(exc) from exc -@mcp.tool(title="Link two entities", annotations=ToolAnnotations( - title="Link two entities", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Link two entities", annotations=_hints( + "Link two entities", readOnlyHint=False, destructiveHint=False, + idempotentHint=True)) @_offload def memory_link(subject: str, predicate: str, object: str, src_id: str | None = None, valid_from: float | None = None, @@ -691,8 +732,8 @@ def memory_link(subject: str, predicate: str, object: str, raise _fail(exc) from exc -@mcp.tool(title="Query the memory graph", annotations=ToolAnnotations( - title="Query the memory graph", readOnlyHint=True)) +@mcp.tool(title="Query the memory graph", annotations=_hints( + "Query the memory graph", readOnlyHint=True)) @_offload def memory_relations(entity: str | None = None, subject: str | None = None, predicate: str | None = None, object: str | None = None, @@ -719,8 +760,9 @@ def memory_relations(entity: str | None = None, subject: str | None = None, return json.dumps(out) -@mcp.tool(title="Remove a relation", annotations=ToolAnnotations( - title="Remove a relation", readOnlyHint=False, destructiveHint=True)) +@mcp.tool(title="Remove a relation", annotations=_hints( + "Remove a relation", readOnlyHint=False, destructiveHint=True, + idempotentHint=True)) @_offload def memory_unlink(relation_id: str) -> str: """Remove one relation from the memory graph (memories stay untouched).""" @@ -731,8 +773,8 @@ def memory_unlink(relation_id: str) -> str: raise _fail(exc) from exc -@mcp.tool(title="Fetch a memory", annotations=ToolAnnotations( - title="Fetch a memory", readOnlyHint=True)) +@mcp.tool(title="Fetch a memory", annotations=_hints( + "Fetch a memory", readOnlyHint=True)) @_offload def memory_get(record_id: str) -> str: """Fetch one memory by id.""" @@ -743,8 +785,9 @@ def memory_get(record_id: str) -> str: raise _fail(exc) from exc -@mcp.tool(title="Forget a memory", annotations=ToolAnnotations( - title="Forget a memory", readOnlyHint=False, destructiveHint=True)) +@mcp.tool(title="Forget a memory", annotations=_hints( + "Forget a memory", readOnlyHint=False, destructiveHint=True, + idempotentHint=True)) @_offload def memory_forget(record_id: str, shred: bool = False) -> str: """Delete a memory. shred=True crypto-shreds it (unrecoverable from this vault).""" @@ -755,8 +798,8 @@ def memory_forget(record_id: str, shred: bool = False) -> str: raise _fail(exc) from exc -@mcp.tool(title="List namespaces", annotations=ToolAnnotations( - title="List namespaces", readOnlyHint=True)) +@mcp.tool(title="List namespaces", annotations=_hints( + "List namespaces", readOnlyHint=True)) @_offload def memory_list_namespaces() -> str: """List namespaces and record counts.""" @@ -766,8 +809,8 @@ def memory_list_namespaces() -> str: raise _fail(exc) from exc -@mcp.tool(title="Recent memories", annotations=ToolAnnotations( - title="Recent memories", readOnlyHint=True)) +@mcp.tool(title="Recent memories", annotations=_hints( + "Recent memories", readOnlyHint=True)) @_offload def memory_recent(limit: int = 20, namespace: str | None = None, include_seeded: bool = False) -> str: @@ -785,8 +828,8 @@ def memory_recent(limit: int = 20, namespace: str | None = None, raise _fail(exc) from exc -@mcp.tool(title="Vault status", annotations=ToolAnnotations( - title="Vault status", readOnlyHint=True)) +@mcp.tool(title="Vault status", annotations=_hints( + "Vault status", readOnlyHint=True)) @_offload def memory_status() -> str: """Vault status: lock state, counts, packs, model, index, RAM, audit head.""" @@ -801,12 +844,15 @@ def memory_status() -> str: # genuinely locked: say so, do not fail the call return json.dumps({"vault": _state["path"], "locked": True, "message": str(exc)}) + except VaultMissingError as exc: + return json.dumps({"vault": _state["path"], "exists": False, + "message": str(exc)}) except CryptoError as exc: raise _fail(exc) from exc -@mcp.tool(title="Memory self-test", annotations=ToolAnnotations( - title="Memory self-test", readOnlyHint=True)) +@mcp.tool(title="Memory self-test", annotations=_hints( + "Memory self-test", readOnlyHint=True)) @_offload def memory_selftest() -> str: """Health check: canned queries against the built-in seed pack, with latencies.""" @@ -834,8 +880,9 @@ def _drop_key(v: Vault) -> None: v._id_by_ikey = {} -@mcp.tool(title="Lock the vault", annotations=ToolAnnotations( - title="Lock the vault", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Lock the vault", annotations=_hints( + "Lock the vault", readOnlyHint=False, destructiveHint=False, + idempotentHint=True)) @_offload def memory_lock() -> str: """PANIC LOCK: flush, seal, and drop key material now. Always available. @@ -882,8 +929,9 @@ def memory_lock() -> str: return json.dumps(out) -@mcp.tool(title="Unlock the vault", annotations=ToolAnnotations( - title="Unlock the vault", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Unlock the vault", annotations=_hints( + "Unlock the vault", readOnlyHint=False, destructiveHint=False, + idempotentHint=True)) @_offload def memory_unlock(passphrase: str) -> str: """Unlock the vault for this session so the other memory tools can use it @@ -941,11 +989,11 @@ def _save_on_exit() -> None: def _stop(signum, frame) -> None: - """SIGTERM: the save a closed stdin gets, then the end SIGTERM always - meant. Not SystemExit raised through the event loop: anyio tells its - idle worker threads to stop only when the server's task ends, and the - interpreter waits for them on the way out, so a server ended that way - never exited. A second SIGTERM during the save is ignored: the process + """SIGTERM or SIGHUP: the save a closed stdin gets, then the end the + signal always meant. Not SystemExit raised through the event loop: + anyio tells its idle worker threads to stop only when the server's task + ends, and the interpreter waits for them on the way out, so a server + ended that way never exited. A second signal during the save is ignored: the process ends as soon as the save is done.""" signal.signal(signum, signal.SIG_IGN) try: @@ -963,6 +1011,9 @@ def main(argv: list[str] | None = None) -> None: args = ap.parse_args(argv) if args.assert_offline: offline_guard.activate() + # `python -m compartment.server --vault ~/...` from a client config: no + # shell expands the ~ there. + args.vault = os.path.expanduser(args.vault) _state["path"] = args.vault _state["caller"] = args.caller sig = _credential_signature() @@ -975,9 +1026,24 @@ def main(argv: list[str] | None = None) -> None: _read_settings() except CryptoError: _state["vault"] = None # start locked; tools will say so + except OSError as exc: + # The file is missing or unreadable. Starting anyway gives the + # client a server whose tools say what is wrong; dying here gave it + # a server that "failed to start", with the reason in a log file. + print(f"compartment: cannot open the vault at {args.vault}: {exc}", + file=sys.stderr, flush=True) + _state["vault"] = None threading.Thread(target=_autolock_loop, daemon=True).start() threading.Thread(target=_retag_loop, daemon=True).start() - signal.signal(signal.SIGTERM, _stop) + # SIGHUP too, where there is one: a client running in a terminal, as + # Claude Code, Codex and Gemini CLI do, starts its servers in its own + # process group, and closing that terminal hangs them all up. Its + # default action ended the server on the spot, and the reads it had + # audited only in RAM with it. + for name in ("SIGTERM", "SIGHUP"): + sig = getattr(signal, name, None) + if sig is not None: + signal.signal(sig, _stop) try: mcp.run() # stdio transport finally: diff --git a/src/compartment/session.py b/src/compartment/session.py index 5494c66..28919ed 100644 --- a/src/compartment/session.py +++ b/src/compartment/session.py @@ -21,7 +21,9 @@ uid, mode 0600. It outlives the process that created it, is shared by every later process of this user, and dies with the kernel that holds it. On Linux that object is a file on the /dev/shm tmpfs; on macOS it has no - filesystem presence at all. + filesystem presence at all. FreeBSD keeps it in the kernel and NetBSD on + the /var/shm tmpfs. OpenBSD is refused: its shm_open is a plain file in + /tmp, which is on disk. * Windows: a REG_OPTION_VOLATILE key under HKEY_CURRENT_USER. Volatile keys are never written to the hive on disk and vanish when the hive unloads, which is at logoff and at every restart. Windows therefore also relocks @@ -69,7 +71,7 @@ """ from __future__ import annotations -from .home import env, home +from .home import env, home, session_dir import ctypes import ctypes.util import errno @@ -106,9 +108,20 @@ def _session_dir() -> Path: - d = Path(env("SESSION_DIR", home() / "session")) + """The session store, created 0700 if it is not there yet. + + The mode is narrowed only when it is wrong, not on every call. A chmod + is a write, and the processes that most need to READ a stored credential + are often not allowed to write here: an agent's shell under Claude Code + or Codex runs in a macOS Seatbelt profile whose write allowlist is the + project and the temp directory, and there chmod on the user's own 0700 + session directory fails with EPERM. Calling it unconditionally made + every `compartment search` the agent ran die with PermissionError + before it could look at the credential the user had unlocked.""" + d = session_dir() d.mkdir(parents=True, exist_ok=True) - os.chmod(d, 0o700) + if stat.S_IMODE(d.stat().st_mode) != 0o700: + os.chmod(d, 0o700) return d @@ -119,9 +132,35 @@ def _boot_time() -> str: def _uid() -> str: - # os.getuid() is POSIX-only; on Windows fall back to the username. + """Who this is, for the holder's name and the wrap key's context. + + os.getuid() is POSIX-only; Windows uses the account name. USERNAME is + where that has always been read from, and it stays first so every + existing credential keeps its holder. But it is only an environment + variable, and a process can be started without it: an MCP client that + replaces the environment with the `env` block of its config, a service + wrapper, a scheduler. Such a process used to call itself "user", derive + a different holder name, find no secret under it, and delete the one + credential file every other process of the user was opening. The + account name the variable normally carries is asked of Windows + instead.""" getuid = getattr(os, "getuid", None) - return str(getuid()) if getuid else os.environ.get("USERNAME", "user") + if getuid: + return str(getuid()) + return os.environ.get("USERNAME") or _windows_account_name() or "user" + + +def _windows_account_name() -> str | None: + """GetUserNameW: the logon name Windows itself puts in USERNAME.""" + try: + from ctypes import wintypes + size = wintypes.DWORD(257) # UNLEN + 1 + buf = ctypes.create_unicode_buffer(size.value) + if ctypes.windll.advapi32.GetUserNameW(buf, ctypes.byref(size)): + return buf.value or None + except (AttributeError, OSError, ValueError): + pass + return None # --------------------------------------------------------------------------- @@ -136,6 +175,11 @@ def _uid() -> str: _libc_cache = None +#: What to do instead, for every error that means "no holder here". +_ALTERNATIVES = ( + "Use COMPARTMENT_PASSPHRASE, `compartment unlock --once`, `compartment " + "unlock --keychain` (macOS), or `compartment init --no-session`.") + def _libc(): """libc/librt with shm_open, argtypes set for the variadic call. @@ -148,6 +192,18 @@ def _libc(): global _libc_cache if _libc_cache is not None: return _libc_cache + if sys.platform.startswith("openbsd"): + # OpenBSD's libc implements shm_open as an ordinary file, + # /tmp/.shm, on whatever /tmp is - a disk + # partition on a default install (lib/libc/gen/shm_open.c). The + # secret would then be on the same disk as the session file, and + # stay there after a power cut until the next boot prunes /tmp: a + # copy of that disk opens the credential, which is the one thing + # this holder exists to prevent. + raise CryptoError( + "OpenBSD keeps POSIX shared memory as a file in /tmp, on disk, " + "so it cannot hold a per-boot secret that a copy of the disk " + "would not carry. " + _ALTERNATIVES) for name in (None, "rt", "c"): try: lib = ctypes.CDLL(ctypes.util.find_library(name) if name else None, @@ -166,9 +222,18 @@ def _libc(): "No POSIX shared memory (shm_open) on this system, so there is " "nowhere to hold a per-boot secret that a restart destroys. " "Compartment will not fall back to a credential that anyone holding " - "the session file could recompute. Use `compartment unlock " - "--keychain` (macOS), COMPARTMENT_PASSPHRASE, `compartment unlock " - "--once`, or `compartment init --no-session`.") + "the session file could recompute. " + _ALTERNATIVES) + + +#: Why shm_open fails where it exists at all. glibc keeps its objects in +#: /dev/shm and fails ENOENT where that is not mounted (chroots, some build +#: and CI containers, Android); a container's /dev/shm can be read-only or +#: full; and sandboxes refuse names outside their own prefix (snap, the +#: macOS App Sandbox) with EACCES or EPERM. +_NO_SHM_HERE = ( + "POSIX shared memory is not usable for this process: /dev/shm may be " + "missing, read-only or full (some containers and chroots), or a " + "sandbox may refuse it.") def _shm_open(name: str, flags: int, mode: int | None = None) -> int: @@ -195,7 +260,8 @@ def _posix_read(name: str) -> tuple[str, bytes | None]: if exc.errno == errno.ENOENT: return "absent", None raise CryptoError( - f"Cannot open this boot's secret ({name}): {exc}") from exc + f"Cannot open this boot's secret ({name}): {exc}. " + f"{_NO_SHM_HERE} {_ALTERNATIVES}") from exc try: st = os.fstat(fd) if st.st_uid != os.getuid(): @@ -226,7 +292,8 @@ def _posix_create(name: str) -> bytes | None: if exc.errno == errno.EEXIST: return None raise CryptoError( - f"Cannot create this boot's secret ({name}): {exc}") from exc + f"Cannot create this boot's secret ({name}): {exc}. " + f"{_NO_SHM_HERE} {_ALTERNATIVES}") from exc try: os.ftruncate(fd, SECRET_LEN) secret = secrets.token_bytes(SECRET_LEN) @@ -393,7 +460,9 @@ def _forget_boot_secret() -> bool: try: name = _holder() return _win_forget(name) if IS_WINDOWS else _posix_forget(name) - except CryptoError: + except (CryptoError, OSError): + # OSError: the session directory, which names the holder, cannot be + # made here. Then no credential was ever stored in it either. return False @@ -450,7 +519,7 @@ def credential_path(vault_path: str) -> Path: unlocked in a terminal by looking at the file's modification time - the one check cheap enough to make on every click. Unlike `_file_for`, it never creates the session directory or changes its mode.""" - return Path(env("SESSION_DIR", home() / "session")) / _name_for(vault_path) + return session_dir() / _name_for(vault_path) def watch_signature(vault_path: str) -> tuple: @@ -540,8 +609,15 @@ def get(vault_path: str) -> bytes | None: None means locked. A CryptoError means something is wrong that deleting the credential would not fix - an unreadable file, no boot clock - and in - that case nothing is deleted.""" - p = _file_for(vault_path) + that case nothing is deleted. + + Looking costs nothing on disk: with no credential there, nothing is + created. Every vault open starts here, before the Keychain and before + COMPARTMENT_PASSPHRASE are consulted, so a lookup that insisted on + making the session directory first took down every open where that + directory cannot be made - a container run as an arbitrary uid, whose + HOME is `/`, or a read-only root - even with the passphrase exported.""" + p = credential_path(vault_path) if not p.is_file(): return None try: @@ -590,10 +666,10 @@ def clear(vault_path: str) -> bool: If that was the last one in this store, destroy the per-boot secret too: keeping it would leave key material in memory that nothing on disk can use, and a panic-lock should not leave that lying about.""" - p = _file_for(vault_path) + p = credential_path(vault_path) existed = p.is_file() if existed: p.unlink() - if not any(_session_dir().glob("*.session")): + if not p.parent.is_dir() or not any(p.parent.glob("*.session")): _forget_boot_secret() return existed diff --git a/src/compartment/store.py b/src/compartment/store.py index c16c6ab..7efdbb0 100644 --- a/src/compartment/store.py +++ b/src/compartment/store.py @@ -10,6 +10,7 @@ import json import math import sqlite3 +import sys import time import uuid @@ -112,6 +113,27 @@ class StoreError(CryptoError): pass +def _no_fts5_error() -> StoreError: + """The refusal for a Python whose SQLite was built without FTS5. + + Rare, but real: Python compiled against the SQLite of an older + enterprise distribution (CentOS/RHEL 7, Amazon Linux 2) and some + minimal or embedded builds. Keyword search is half of every recall and + the full-text table is part of the vault's image, so there is no + degraded mode to fall back to; what the user needs is to know which + interpreter is the problem and where a working one comes from.""" + return StoreError( + f"This Python's SQLite ({sqlite3.sqlite_version}, used by " + f"{sys.executable or 'this interpreter'}) was built without FTS5, " + "the full-text index Compartment's keyword search and vault format " + "require. Run Compartment under a Python whose SQLite has it: the " + "python.org installers, Homebrew, conda, `uv python install 3.12`, " + "and the python3 of any current Linux distribution all do. " + "Reinstall Compartment with that interpreter ( -m pip " + "install compartment, or uv tool install --python 3.12 " + "compartment); your vault is untouched.") + + class Store: def __init__(self, image: bytes | None = None): # Autocommit: the DB lives only in RAM - durability comes from the @@ -154,21 +176,23 @@ def __init__(self, image: bytes | None = None): # GROUP BY over every record. Dropped by whatever adds a record, # removes one or moves one to another namespace. self._ns_cache: list[dict] | None = None - if image is not None: - self.conn.deserialize(image) - # idempotent schema upgrade: vaults sealed by older versions gain - # any new tables (e.g. relations) the moment they are opened - self.conn.executescript(SCHEMA) - else: - self.conn.executescript(SCHEMA) - self._migrate_columns() try: + if image is not None: + self.conn.deserialize(image) + # Idempotent schema upgrade: vaults sealed by older versions gain + # any new tables (e.g. relations) the moment they are opened. + # Creating the full-text table is also where a SQLite without + # FTS5 first fails, for a new vault: the friendly refusal below + # used to come after it and was reached only when an existing + # vault was opened, so `compartment init` died on a bare + # "no such module: fts5". + self.conn.executescript(SCHEMA) + self._migrate_columns() self.conn.execute("SELECT count(*) FROM fts") except sqlite3.OperationalError as exc: - raise StoreError( - "This Python's SQLite lacks FTS5, which Compartment requires " - "for hybrid search. Install a Python built with full SQLite." - ) from exc + if "fts5" in str(exc).lower(): + raise _no_fts5_error() from exc + raise # New COLUMNS on an existing table, unlike new tables, do not arrive for # free: `CREATE TABLE IF NOT EXISTS` is a no-op the moment the table diff --git a/src/compartment/systray.py b/src/compartment/systray.py index dff85ca..40a0ba5 100644 --- a/src/compartment/systray.py +++ b/src/compartment/systray.py @@ -175,11 +175,12 @@ def _winreg(): def _autostart_command(vault: str | None = None) -> str: """What Windows runs at sign-in. - Two things this has to get right. It launches through the console script - or pythonw.exe rather than python.exe, because python.exe opens a console - window behind a tray app that has no window, at every single sign-in. And - it carries the vault path, because a user running a non-default vault - would otherwise silently get the default one back after a reboot. + Two things this has to get right. It launches through pythonw.exe, the + one interpreter that opens no console, because anything else opens a + console window behind a tray app that has no window, at every single + sign-in. And it carries the vault path, because a user running a + non-default vault would otherwise silently get the default one back + after a reboot. """ exe, args = _autostart_parts(vault) return f'"{exe}" {args}' @@ -188,14 +189,25 @@ def _autostart_command(vault: str | None = None) -> str: def _autostart_parts(vault: str | None = None) -> tuple[str, str]: """The same command, split. A scheduled task wants the program and its arguments in separate XML elements, and re-parsing a quoted command line - to get them back would be a second place for the quoting to be wrong.""" + to get them back would be a second place for the quoting to be wrong. + + pythonw.exe beside this interpreter comes first, and the console script + only when there is none. compartment.exe, which pip and uv write for the + console script, is a console program like python.exe: started at + sign-in it opened a console window that stayed for the whole session, + since that process IS the tray app, and closing the window quit it. It + used to come first. Every venv, uv tool environment and python.org + install has a pythonw.exe beside python.exe, and it runs the same + environment's compartment. + """ vault = vault or env("VAULT") or str(home() / "memory.vault") + pyw = Path(sys.executable).with_name("pythonw.exe") + if pyw.is_file(): + return str(pyw), f'-m compartment.cli --vault "{vault}" tray' exe = shutil.which("compartment") if exe: return exe, f'--vault "{vault}" tray' - pyw = Path(sys.executable).with_name("pythonw.exe") - runner = str(pyw) if pyw.is_file() else sys.executable - return runner, f'-m compartment.cli --vault "{vault}" tray' + return sys.executable, f'-m compartment.cli --vault "{vault}" tray' # --- the Linux application entry -------------------------------------------- @@ -364,6 +376,22 @@ def systemd_available() -> bool: or "no medium" in low or not out) +def graphical_session_active() -> bool: + """Does this login's desktop start graphical-session.target? + + The unit is WantedBy that target, so it starts at login only on a + desktop whose session manager activates it: GNOME, KDE Plasma, and + compositors set up for it. Xfce, MATE, Cinnamon (Linux Mint's default), + LXQt and a bare window manager never do, and there an enabled unit is a + unit that never starts - while the autostart entry it replaced, which + every XDG desktop reads, was deleted as the redundant second starter. + Asked while the desktop is up (install time, from its terminal); asked + from SSH or a text console it is inactive too, and the autostart entry, + which works everywhere, is the right answer there as well. + """ + return _systemctl("is-active", "graphical-session.target")[0] == 0 + + def systemd_unit_text(vault: str | None = None) -> str: return ( "[Unit]\n" @@ -708,6 +736,32 @@ def _delete_run_value() -> None: pass +#: Present at the root of every Flatpak sandbox. +_FLATPAK_INFO = "/.flatpak-info" + + +def _app_sandbox() -> str | None: + """The Flatpak or Snap this process is confined in, if any. + + Inside one, XDG_CONFIG_HOME and XDG_DATA_HOME (Flatpak) or HOME itself + (Snap) point into the app's private directory, which the desktop never + reads, and the Python an entry would name is the sandbox's own, which + the desktop cannot run. `compartment init` typed into the terminal of a + Flatpak editor wrote its autostart entry there and reported start at + login as on.""" + flatpak = os.environ.get("FLATPAK_ID") + if flatpak or os.path.isfile(_FLATPAK_INFO): + return f"a Flatpak sandbox ({flatpak or 'unknown app'})" + # Strict confinement only, which is what moves HOME to the snap's own + # SNAP_USER_DATA. A classic snap - VS Code's, among others - sets + # SNAP_NAME for every terminal it opens and confines nothing. + snap = os.environ.get("SNAP_NAME") + if (snap and os.environ.get("SNAP_USER_DATA") + and os.environ.get("HOME") == os.environ.get("SNAP_USER_DATA")): + return f"a Snap sandbox ({snap})" + return None + + def set_login(enabled: bool, vault: str | None = None) -> str: """Register or drop start at sign-in. Returns what actually happened. @@ -722,6 +776,12 @@ def set_login(enabled: bool, vault: str | None = None) -> str: remove_systemd_unit() remove_autostart_entry() return "off" if remove_desktop_entry() else "error" + sandbox = _app_sandbox() + if sandbox: + return (f"error: this is running inside {sandbox}, where an " + "entry would be written somewhere your desktop never " + "reads. Run `compartment panel --login on` from a " + "terminal outside it.") # Both, and they are not the same thing: the menu entry is how the # app is found, the autostart entry is how it comes up at login. out = install_desktop_entry(vault) @@ -730,7 +790,8 @@ def set_login(enabled: bool, vault: str | None = None) -> str: # Supervised where the machine can be. A unit that enables but never # starts is not proof of anything, so it has to be active too, and # anything short of that is undone rather than left half installed. - if systemd_available(): + # Only on a desktop that starts the unit's target at login at all. + if systemd_available() and graphical_session_active(): if install_systemd_unit(vault) == "on" and systemd_active(): remove_autostart_entry() # exactly one starter return "on" @@ -779,6 +840,155 @@ def set_login(enabled: bool, vault: str | None = None) -> str: return f"error: {exc}" +# --- when the panel cannot draw --------------------------------------------- +# pip cannot install tkinter: it comes with the Python, or with a package of +# the distribution's. "Install your distribution's Python Tk package" left +# the user to find out what that is called, which differs on every family, +# and is the wrong advice altogether for a Python the distribution did not +# provide (pyenv, asdf, conda, one built from source), where the package +# changes nothing. + +#: Per distribution family: the package that adds Tk to the distribution's +#: own Python, and the headers a Python built from source needs to build it. +_TK_BY_FAMILY = { + "debian": ("sudo apt install python3-tk", "sudo apt install tk-dev"), + "fedora": ("sudo dnf install python3-tkinter", + "sudo dnf install tk-devel"), + "suse": ("sudo zypper install python3-tk", "sudo zypper install tk-devel"), + "arch": ("sudo pacman -S tk", "sudo pacman -S tk"), + "alpine": ("sudo apk add python3-tkinter", "sudo apk add tk-dev"), + "void": ("sudo xbps-install python3-tkinter", + "sudo xbps-install tk-devel"), + "gentoo": ("enable the tk USE flag for dev-lang/python, then: " + "sudo emerge --oneshot dev-lang/python", + "sudo emerge dev-lang/tk"), +} + +#: os-release ID and ID_LIKE words, to a family above. +_TK_FAMILY_OF = { + "debian": "debian", "ubuntu": "debian", "linuxmint": "debian", + "pop": "debian", "raspbian": "debian", "kali": "debian", + "elementary": "debian", "zorin": "debian", "neon": "debian", + "fedora": "fedora", "rhel": "fedora", "centos": "fedora", + "rocky": "fedora", "almalinux": "fedora", "amzn": "fedora", + "ol": "fedora", "nobara": "fedora", + "suse": "suse", "opensuse": "suse", "sles": "suse", + "arch": "arch", "manjaro": "arch", "endeavouros": "arch", + "cachyos": "arch", "garuda": "arch", "steamos": "arch", + "alpine": "alpine", "postmarketos": "alpine", + "void": "void", "gentoo": "gentoo", +} + + +def _os_release() -> str: + for path in ("/etc/os-release", "/usr/lib/os-release"): + try: + with open(path, encoding="utf-8", errors="replace") as fh: + return fh.read() + except OSError: + continue + return "" + + +def _distro_family(os_release: str) -> str | None: + fields = {} + for line in os_release.splitlines(): + key, sep, value = line.partition("=") + if sep: + fields[key.strip()] = value.strip().strip("\"'").lower() + words = [fields.get("ID", "")] + fields.get("ID_LIKE", "").split() + for word in words: + if not word: + continue + # opensuse-tumbleweed, opensuse-leap, sles_sap and the like + for name in (word, word.split("-")[0], word.split("_")[0]): + if name in _TK_FAMILY_OF: + return _TK_FAMILY_OF[name] + return None + + +def tk_install_hint(os_release: str | None = None, + base_prefix: str | None = None, + platform: str | None = None) -> str: + """The command that gives this Python tkinter, as exactly as this + machine allows. Pure given its arguments, which default to this + machine's, so every family is testable anywhere.""" + platform = platform or sys.platform + prefix = base_prefix if base_prefix is not None else sys.base_prefix + ver = f"{sys.version_info[0]}{sys.version_info[1]}" + if "com.termux" in prefix: + return "pkg install x11-repo && pkg install python-tkinter" + if platform.startswith("freebsd"): + return f"sudo pkg install py{ver}-tkinter" + if platform.startswith("openbsd"): + return "doas pkg_add python-tkinter" + if platform.startswith("netbsd"): + return f"sudo pkgin install py{ver}-Tk" + if os.path.isdir(os.path.join(prefix, "conda-meta")): + return "conda install tk" + if "linuxbrew" in prefix or "homebrew" in prefix.lower(): + return ("brew install " + f"python-tk@{sys.version_info[0]}.{sys.version_info[1]}") + if "/uv/python/" in prefix.replace(os.sep, "/"): + # uv's Pythons carry Tk; one without it is a damaged install. + return ("uv python install --reinstall " + f"{sys.version_info[0]}.{sys.version_info[1]}") + family = _distro_family(_os_release() if os_release is None + else os_release) + system, headers = _TK_BY_FAMILY.get(family, (None, None)) + if prefix.rstrip("/") not in ("/usr", ""): + # Not the distribution's Python: pyenv, asdf, a source build in + # /usr/local. Its _tkinter is built when the Python is, against + # Tk's headers, so the headers go in first and the Python is + # built again. + build = ("then build this Python again (pyenv: " + "pyenv install --force " + f"{sys.version_info[0]}.{sys.version_info[1]}." + f"{sys.version_info[2]})") + return (f"{headers}, {build}" if headers else + f"install Tk's development headers, {build}") + return system or ("install your distribution's Tk package for Python " + "(often python3-tk or python3-tkinter)") + + +def tk_missing_message(what: str = "panel") -> str: + return ("error: this Python has no tkinter, which the " + f"{what} needs.\n" + f" This Python is {sys.executable}. To give it tkinter:\n" + f" {tk_install_hint()}\n" + " Or install Compartment with uv, which brings its own Python " + "with tkinter in it:\n" + " uv tool install compartment\n" + " Everything else works without it: the CLI, the MCP server, " + "and `compartment dash`.") + + +def no_display_message(exc: Exception) -> str: + """Why Tk could not open a window, in words that say what to do.""" + text = str(exc) + low = text.lower() + if "init.tcl" in low or "tk.tcl" in low: + return (f"error: this Python's Tcl/Tk files are missing or not where " + f"it looks for them ({text}).\n" + " Reinstall this Python, or install Compartment with uv, " + "whose Python carries its own:\n" + " uv tool install compartment") + if (sys.platform not in ("win32", "darwin") + and os.environ.get("WAYLAND_DISPLAY") + and not os.environ.get("DISPLAY")): + # Tk draws through X11 only. A Wayland desktop runs XWayland for + # programs like it, usually on demand; this one has none running. + return ("error: the panel is drawn with Tk, which needs X11, and " + "this Wayland session has no X server (DISPLAY is not set).\n" + " Turn XWayland on in your compositor (sway: `xwayland " + "enable`; GNOME and KDE start it by themselves when it is\n" + " installed: package xwayland or xorg-x11-server-Xwayland), " + "or use `compartment dash` to read the vault in a browser.") + return (f"error: no graphical display to draw the panel on ({text}).\n" + " On a headless machine use the CLI, or `compartment dash` to\n" + " read the vault in a browser.") + + def panel_geometry(content: int, maximum: int) -> tuple[int, bool]: """How tall to draw the panel, and whether it needs a scrollbar. @@ -844,13 +1054,7 @@ def run(vault: str | None = None, show: bool = False, import tkinter as tk from tkinter import ttk except ImportError: - print("error: this Python has no tkinter, which the " - f"{'tray panel' if tray else 'panel'} needs.\n" - " The quickest fix is to install Compartment with uv, which\n" - " brings its own Python with tkinter already in it:\n" - " uv tool install compartment\n" - " Otherwise install your distribution's Python Tk package\n" - " (Debian/Ubuntu: python3-tk, Fedora: python3-tkinter).", + print(tk_missing_message("tray panel" if tray else "panel"), file=sys.stderr) return 3 pystray = Image = None @@ -859,8 +1063,13 @@ def run(vault: str | None = None, show: bool = False, import pystray from PIL import Image except ImportError: - print("error: the tray app needs pystray and Pillow.\n" - " pip install 'compartment[tray]'", file=sys.stderr) + # Into this interpreter, by name. The [tray] extra this used to + # name has been empty since the two became base dependencies, + # so installing it changed nothing. + print("error: the tray app needs pystray and Pillow, which this " + "Python does not have.\n" + f' "{sys.executable}" -m pip install pystray pillow', + file=sys.stderr) return 3 # Read the DPI (and declare awareness) before the first window exists. @@ -873,11 +1082,10 @@ def run(vault: str | None = None, show: bool = False, try: root = tk.Tk() except tk.TclError as exc: - # A machine with no desktop: SSH, a container, a CI runner. Say so - # in one line rather than showing a Tk stack trace. - print(f"error: no graphical display to draw the panel on ({exc}).\n" - " On a headless machine use the CLI, or `compartment dash` to\n" - " read the vault in a browser.", file=sys.stderr) + # A machine with no desktop (SSH, a container, a CI runner), a + # Wayland session with no XWayland, or a Python whose Tcl files are + # not where it looks. Say which, rather than show a Tk stack trace. + print(no_display_message(exc), file=sys.stderr) return 4 if S != 1.0: # Tk sizes fonts in points; this is what turns a point into a pixel. @@ -1362,11 +1570,28 @@ def from_tray(fn): return 0 -#: Every process on Windows with the command line that started it, as CSV. +#: Every process on Windows with the command line that started it and the +#: sign-in session it runs in, as CSV. SessionId is last, so the first three +#: columns are where they always were. _WINDOWS_PROCESSES = ("Get-CimInstance Win32_Process | Select-Object " - "ProcessId, ParentProcessId, CommandLine | " + "ProcessId, ParentProcessId, CommandLine, SessionId | " "ConvertTo-Csv -NoTypeInformation") + +def _windows_session() -> str | None: + """The Windows session this process runs in (one per signed-in user, + fast user switching included), or None if it cannot be asked.""" + try: + import ctypes + from ctypes import wintypes + sid = wintypes.DWORD() + if ctypes.windll.kernel32.ProcessIdToSessionId(os.getpid(), + ctypes.byref(sid)): + return str(sid.value) + except Exception: # noqa: BLE001 + pass + return None + #: A Windows command line that is the panel: the CLI, then its subcommand. _WINDOWS_APP = re.compile(r"compartment.*\s(menubar|panel|tray)(\s|$)", re.IGNORECASE) @@ -1395,12 +1620,19 @@ def running_pids() -> list[int]: except (OSError, subprocess.SubprocessError): return [] mine, pids = {os.getpid(), os.getppid()}, [] + # Only this session's. With fast user switching, a second person + # signed in on the same PC has a tray app of their own, and from an + # elevated prompt their command lines are readable too: `init` then + # took theirs as this user's app coming up, and started none. + session = _windows_session() # Anything that is not a row - the header, a message instead of a # table - has no pid in its first field to misread. for row in csv.reader((out or "").splitlines()): if (len(row) > 2 and row[0].isdigit() and int(row[0]) not in mine - and _WINDOWS_APP.search(row[2])): + and _WINDOWS_APP.search(row[2]) + and (session is None or len(row) < 4 + or row[3] == session)): pids.append(int(row[0])) return pids # The app's own command line and this user's processes, as on macOS: @@ -1462,6 +1694,7 @@ def quit_running(vault: str | None = None, timeout: float = 10.0) -> bool: "autostart_entry_path", "install_autostart_entry", "autostart_is_enabled", "remove_autostart_entry", "systemd_unit_path", "systemd_unit_text", "systemd_available", + "graphical_session_active", "systemd_enabled", "install_systemd_unit", "remove_systemd_unit", "scheduled_task_xml", "install_scheduled_task", "remove_scheduled_task", "scheduled_task_registered", diff --git a/src/compartment/vault.py b/src/compartment/vault.py index e04790c..b571dc7 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -87,10 +87,59 @@ def local_stamp(created: float | None) -> str: try: return datetime.datetime.fromtimestamp( float(created)).strftime("%Y-%m-%d %H:%M") - except (TypeError, ValueError, OSError): + except (TypeError, ValueError, OSError, OverflowError): + # OverflowError: an infinite or vast value, which a JSONL import + # could carry in, raised past every reader of the record, so one + # such memory failed every search that would have returned it. return "" +#: The span a stored instant may take: what a calendar date can be made of +#: (years 1 to 9999, in UTC), checked as a number so the answer does not +#: depend on what the platform's C library will convert. +_MIN_INSTANT = -62135596800.0 +_MAX_INSTANT = 253402300799.0 + + +def instant(value) -> float | None: + """A moment as unix seconds, or None when `value` does not name one. + + `created` and `affirmed` reach the vault from outside on the restore + paths - a JSONL import, a curation plan - and whatever a file holds was + stored as it came: an ISO string such as "2024-05-01T09:30:00Z" went + into the REAL column as text, and the next search failed converting it + for the recency prior. Every search, for good, since every search reads + the whole population. So the value is read the way a person would read + it: a number of seconds; numeric text; an ISO date or datetime (one + without a zone is local time, as a person writing it means); or a count + of milliseconds, which is what JavaScript's Date.now() writes and which + as seconds would be a date thousands of years away. Anything else, + including NaN and infinity, is None, and the caller falls back to now + rather than keep a date nothing can read.""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, (int, float)): + t = float(value) + else: + s = str(value).strip() + if not s: + return None + try: + t = float(s) + except ValueError: + try: + t = datetime.datetime.fromisoformat(s).timestamp() + except (ValueError, OverflowError, OSError): + return None + if not math.isfinite(t): + return None + if _MIN_INSTANT <= t <= _MAX_INSTANT: + return t + if _MIN_INSTANT <= t / 1000.0 <= _MAX_INSTANT: + return t / 1000.0 + return None + + # A memory's own account of where it came from, written at the end of its text # as "[method, YYYY-MM-DD]". Recognised on the way back IN so that re-importing # an exported memory does not stack a second clause onto the first. @@ -108,8 +157,18 @@ def discovery_date(value=None) -> str: time and lives in `created`.""" if value in (None, ""): return datetime.date.today().isoformat() - if isinstance(value, (int, float)): - return datetime.date.fromtimestamp(float(value)).isoformat() + if isinstance(value, (int, float)) and not isinstance(value, bool): + # Through instant, so milliseconds (what a JavaScript client sends) + # read as the day they name, and a number that names no day is kept + # as written below instead of raising "year 56759 is out of range" + # out of the store. + t = instant(value) + if t is not None: + try: + return datetime.date.fromtimestamp(t).isoformat() + except (ValueError, OverflowError, OSError): + pass + return str(value)[:40] s = str(value).strip() try: return datetime.date.fromisoformat(s[:10]).isoformat() @@ -120,6 +179,33 @@ def discovery_date(value=None) -> str: return s[:40] +def clean_text(value): + """Text as UTF-8 can carry it: every lone surrogate replaced by U+FFFD, + and a surrogate pair that arrived as two code points joined into the + character it encodes. Anything that is not a str (None, a number) and a + list of strings come back the same shape. + + Python's json decoder turns "\\ud83d" into a lone surrogate without + complaint, and lone surrogates are common in what agents send: a + JavaScript client that cuts a string to length cuts emoji in half, and + Windows hands back unpaired UTF-16 from file names and the clipboard. + Such a string cannot be encoded as UTF-8, so the tokenizer refused it + ("TextInputSequence must be str") and SQLite would have refused it next: + the store or search failed outright, with an error that said nothing + about why. One replacement character is a far smaller loss than the + memory, and it is what every UTF-8 decoder shows for the same damage.""" + if isinstance(value, list): + return [clean_text(v) for v in value] + if not isinstance(value, str): + return value + try: + value.encode("utf-8") + return value + except UnicodeEncodeError: + return value.encode("utf-16", "surrogatepass").decode("utf-16", + "replace") + + def strip_provenance(text: str) -> str: """The claim on its own, without its provenance clause. @@ -204,6 +290,11 @@ def expiry_date(value, today: datetime.date | None = None, if value in (None, ""): return None today = today or datetime.date.today() + if isinstance(value, float) and not math.isfinite(value): + raise CryptoError( + f"expires={value!r} is not a date or a duration. Give the last " + f"day it is true (2026-09-03), or how long it lasts (14d, 2w, " + f"3m, 1y).") if isinstance(value, (int, float)) and not isinstance(value, bool): s = str(int(value)) else: @@ -217,12 +308,20 @@ def expiry_date(value, today: datetime.date | None = None, raise CryptoError( f"expires={value!r}: a duration has to be at least one day") unit = (m.group(2) or "d").lower() - if unit == "m": - return _add_months(today, n).isoformat() - if unit == "y": - return _add_months(today, n * 12).isoformat() - return (today + datetime.timedelta( - days=n * (7 if unit == "w" else 1))).isoformat() + try: + if unit == "m": + return _add_months(today, n).isoformat() + if unit == "y": + return _add_months(today, n * 12).isoformat() + return (today + datetime.timedelta( + days=n * (7 if unit == "w" else 1))).isoformat() + except (ValueError, OverflowError): + # `9999y` lands past the year 9999, the last a date can name. + # A fact meant to outlast that is a fact with no expiry. + raise CryptoError( + f"expires={value!r} runs past the year 9999. A fact that " + f"lasts that long does not expire: leave the expiry off." + ) from None try: day = datetime.date.fromisoformat(s[:10]) except ValueError: @@ -255,6 +354,11 @@ class VaultStaleError(CryptoError): """Another process wrote the vault; the caller should reopen and retry.""" +#: The disk state of a handle whose RAM holds a change its journal append +#: failed to write. Matches no file, so is_stale() is True from then on. +_DIVERGED = (-1, -1, -1, "") + + def _synchronized(fn): """Serialize vault operations: the in-RAM SQLite connection and index are shared with background threads (Hermes provider writer, auto-lock).""" @@ -289,7 +393,7 @@ def __init__(self, path: str, header: vaultfile.VaultHeader, store: Store, #: When the expiry sweep last ran. Zero means never, so the first read #: after opening always sweeps. self._last_expiry_sweep = 0.0 - self._disk_state: tuple[int, int, int] | None = None + self._disk_state: tuple[int, int, int, str] | None = None if os.path.exists(path): self._disk_state = self._stat_disk() self.index = None @@ -298,22 +402,25 @@ def __init__(self, path: str, header: vaultfile.VaultHeader, store: Store, # -------------------------------------------------- multi-process safety - def _stat_disk(self) -> tuple[int, int, int]: + def _stat_disk(self) -> tuple[int, int, int, str]: # The inode as well, as in platforms.file_signature: another # process's compaction can leave a file of the same length with, on a - # coarse clock, the same time, and only the inode shows it is new. - st = os.stat(self.path) - return (st.st_ino, st.st_size, st.st_mtime_ns) + # coarse clock, the same time, and only the inode shows it is new - + # where the filesystem does not hand the new file an old number. The + # identity read with them is what shows it everywhere else; see + # vaultfile.disk_state. + return vaultfile.disk_state(self.path) def is_stale(self) -> bool: """True if the vault file changed under us since we read it. A file that has been DELETED counts as changed. It used to count as unchanged, because the existence test was ANDed into the answer, and - the consequence was silent and permanent: _journal appends with - open(path, "ab"), which recreates the file with journal frames and no + the consequence was silent and permanent: _journal appended with + open(path, "ab"), which recreated the file with journal frames and no header, so every later write reported success into something no - version of this program can ever open again. + version of this program can ever open again. (The append no longer + creates a missing file either; this check is what refuses first.) """ if self._disk_state is None: return False @@ -321,11 +428,23 @@ def is_stale(self) -> bool: return True return self._stat_disk() != self._disk_state + def _refuse_if_diverged(self) -> None: + """Refuse to write from a handle whose journal append failed (see + _journal). Checked at the door of the writes that can answer from RAM + without journalling - a store matched as a duplicate, a link that + already exists - as well as under the write lock.""" + if self._disk_state == _DIVERGED: + raise VaultStaleError( + "An earlier write from this handle did not reach the vault " + "file, so what it holds in memory no longer matches it. " + "Reopen the vault and retry.") + def _with_file_lock(self, fn, timeout: float = 10.0): """Advisory single-writer lock: serializes journal appends and saves across processes sharing one vault (Hermes + Claude + CLI). Cross-platform via platforms.FileLock (POSIX flock / Windows msvcrt).""" with FileLock(self.path + ".flock", timeout=timeout): + self._refuse_if_diverged() if self.is_stale(): if not os.path.exists(self.path): raise VaultStaleError( @@ -355,18 +474,23 @@ def create(cls, path: str, passphrase: str, creator: str = "user", raise CryptoError("Empty passphrase refused - the user sets it") master = crypto.new_key() slot_pw = crypto.make_passphrase_slot(master, passphrase) - emb = get_embedder(model_name) + # The model's pinned description is all a new vault records. Building + # an embedder here loaded the model (or started the machine's shared + # embedding process) before anything needed a vector, and ignored the + # embed_daemon setting the vault's own embedder property honours; it + # is built on first use instead, like an opened vault's. + emb = model_info(model_name) header = vaultfile.VaultHeader( vault_id=uuid.uuid4().hex, created=datetime.datetime.now(datetime.UTC).isoformat(), keyslots=[slot_pw], payload_len=0, - model={"name": model_name, "sha256": emb.model_sha256, "dim": emb.dim}, + model={"name": model_name, "sha256": emb.sha256, "dim": emb.dim}, extra={"creator": creator, "wire": wire.WIRE_FORMAT}, ) db = Store() db.set_meta("model_name", model_name) - db.set_meta("model_sha256", emb.model_sha256) + db.set_meta("model_sha256", emb.sha256) audit.append(db.conn, creator, "init", f"vault created (model {model_name})") # Anchor at creation, not just on save: the anchor is mandatory, so a # vault has to carry one from its very first entry onwards or its own @@ -376,9 +500,7 @@ def create(cls, path: str, passphrase: str, creator: str = "user", vaultfile.write_vault_file(path, header, {"sqlite": db.serialize()}, master) config.save(path) - v = cls(path, header, db, master, config) - v._embedder = emb - return v + return cls(path, header, db, master, config) # ---------------------------------------------------------------- unlock @@ -439,7 +561,9 @@ def _unlock_once(cls, path: str, passphrase: str | None, v._replay(e) merged = v._merge_legacy_starter() rewired = v._migrate_wire_format() - if entries or loaded.truncated_tail or merged or rewired or slots_migrated: + repaired = v._repair_instants() + if (entries or loaded.truncated_tail or merged or rewired + or slots_migrated or repaired): if loaded.truncated_tail: # Reaching here means the framing proved this was an # interrupted append: an intact, checksummed length prefix @@ -512,6 +636,37 @@ def _migrate_wire_format(self) -> int: self.header.extra["wire"] = wire.WIRE_FORMAT return n or 1 # truthy even for an empty vault: the marker must save + def _repair_instants(self) -> int: + """Give every record a `created` (and `affirmed`) that is a number. + + Builds before `instant` stored an imported date as whatever the file + held, so a vault that took one line with an ISO string, a NaN or a + word in it failed every search from then on, and nothing in it could + be fixed from the outside because finding the record needed a + search. Each bad value is read as `instant` reads it; one that names + no moment becomes the time the row was written (`accessed` starts + there, and a record no search could return was never touched since), + and a bad `affirmed` is dropped. Writes nothing itself: the caller's + save commits it, as with the other migrations here.""" + rows = self.db.conn.execute( + "SELECT id, created, affirmed, accessed FROM records" + " WHERE typeof(created) NOT IN ('integer', 'real')" + " OR (affirmed IS NOT NULL" + " AND typeof(affirmed) NOT IN ('integer', 'real'))").fetchall() + for r in rows: + created = (instant(r["created"]) or instant(r["accessed"]) + or time.time()) + self.db.conn.execute( + "UPDATE records SET created = ?, affirmed = ? WHERE id = ?", + (created, instant(r["affirmed"]), r["id"])) + if rows: + # Direct SQL, so the cache of the population's times is stale. + self.db._invalidate_recency() + self._audit_and_capture( + "system", "repair", + f"{len(rows)} records given a readable stored date") + return len(rows) + def _merge_legacy_starter(self) -> int: """One-time reorganization: fold the legacy read-only packs/starter section into "main", so starting memories are ordinary memories - @@ -636,7 +791,18 @@ def find_credential(path: str, passphrase: str | None = None from . import session if passphrase: return passphrase, None - key = session.get(path) + # A session lookup that fails (no shared memory in a container, a + # session directory that cannot be read, a boot that cannot be + # identified inside a sandbox) says nothing about the credentials + # after it. Raised on the spot, it hid a COMPARTMENT_PASSPHRASE set + # precisely because the session store does not work there - which is + # what that error's own message tells the user to do. It is kept and + # raised only when nothing else opens the vault. + session_error: Exception | None = None + try: + key = session.get(path) + except Exception as exc: # noqa: BLE001 + session_error, key = exc, None if key is not None: return None, key key = keychain_get(path) @@ -645,6 +811,8 @@ def find_credential(path: str, passphrase: str | None = None from_env = env("PASSPHRASE") if from_env: return from_env, None + if session_error is not None: + raise session_error return None @staticmethod @@ -812,7 +980,21 @@ def _append(): vaultfile.append_journal_entries(self.path, self.header, self._journal_seq, list(entries), self._master) - self._with_file_lock(_append) + try: + self._with_file_lock(_append) + except BaseException: + # Every caller changes the rows and the index in RAM before it + # journals, so a journal that did not land (the write lock busy + # past its timeout, a full disk, a failed sync) leaves this + # handle holding a change the file does not have. Kept, it went + # to disk with the next save, a memory the caller had been told + # was not stored; and a retry of the same store found it in RAM, + # returned it as a duplicate, and journalled nothing, so it was + # gone at the next reopen. The handle is marked as no longer + # matching the file, so every later write refuses and the server + # reopens from disk, which is the only state worth trusting. + self._disk_state = _DIVERGED + raise self._journal_seq += len(entries) def _replay(self, e: dict) -> None: @@ -910,6 +1092,11 @@ def store(self, text: str, caller: str, namespace: str | None = None, _expiry_strict: bool = True, _gate: bool = True, _split_hint: str | None = None) -> dict: self._require_open() + self._refuse_if_diverged() + text, source, discovered, namespace = ( + clean_text(text), clean_text(source), clean_text(discovered), + clean_text(namespace)) + tags = clean_text(tags) ns = namespace or self.config.default_namespace(caller) if pack is None: self.config.check(caller, ns, write=True) @@ -967,6 +1154,9 @@ def store(self, text: str, caller: str, namespace: str | None = None, # excluded so the vector represents the claim itself. vec = self.embedder.embed_record(strip_provenance(text)) vec = np.atleast_2d(np.asarray(vec, dtype=np.float32)) + # Read as moments, or dropped: see instant. Stored as they came, an + # ISO string or a NaN from an import broke every later search. + created, affirmed = instant(created), instant(affirmed) # Kept to 0..1 as the server's tools keep it, so a value on another # scale (1-10, say) lands in a dashboard bucket; a NaN is the default. importance = float(importance) @@ -1016,7 +1206,11 @@ def store(self, text: str, caller: str, namespace: str | None = None, # being missed. Walk a small window instead, stopping as soon as # the scores drop below the threshold (results are descending). for ikey, score in self.index.search(vec[0], DEDUP_CANDIDATES): - if score < thr: + # `not >=` rather than `<`: a NaN score (a vector from a + # broken model run) compares false both ways, and `< thr` + # read it as a match, so every store came back as a + # "duplicate" of whatever happened to be nearest. + if not score >= thr: break rid = self._id_by_ikey.get(ikey) if not rid: @@ -1090,7 +1284,7 @@ def _similar_opinions(self, qvec, ns: str, exclude: set[str], best-first. One entry per record (best window), organic only.""" best: dict[str, float] = {} for ikey, score in self.index.search(qvec, OPINION_CANDIDATES): - if score < floor: + if not score >= floor: # NaN: never a match break # results are descending rid = self._id_by_ikey.get(ikey) if rid is None or rid in exclude: @@ -1280,6 +1474,10 @@ def search(self, query: str, caller: str, namespace: str | None = None, An explicit `top_k` still means exactly that many, unchanged, because callers that page or benchmark need a fixed window.""" self._require_open() + query, namespace, tags = (clean_text(query), clean_text(namespace), + clean_text(tags)) + discovered_since, discovered_until = (clean_text(discovered_since), + clean_text(discovered_until)) self._maybe_expire() if namespace is not None: self.config.grant_for(caller, namespace) # raises if no access @@ -1695,7 +1893,13 @@ def _maybe_expire(self) -> None: if not self.expiry_enabled(): return now = time.time() - if now - self._last_expiry_sweep < self.EXPIRY_SWEEP_SECONDS: + # A clock set back - a VM or a laptop restored from a snapshot, a + # clock that ran ahead and was corrected - leaves the last sweep in + # the future. Read as "swept moments ago", that held every sweep + # off until the clock caught up again, which can be months in a + # server that stays open. A negative interval says nothing about + # when the vault was last tidied, so it sweeps. + if 0 <= now - self._last_expiry_sweep < self.EXPIRY_SWEEP_SECONDS: return try: self.expire() @@ -1723,6 +1927,10 @@ def link(self, subject: str, predicate: str, obj: str, caller: str, host model (or the user), exactly like memory curation. Idempotent: re-linking the same triple returns the existing relation.""" self._require_open() + self._refuse_if_diverged() + subject, predicate, obj, namespace = ( + clean_text(subject), clean_text(predicate), clean_text(obj), + clean_text(namespace)) ns = namespace or self.config.default_namespace(caller) self.config.check(caller, ns, write=True) for label, val in (("subject", subject), ("predicate", predicate), @@ -1773,6 +1981,9 @@ def relations(self, caller: str, entity: str | None = None, subject OR object (case-insensitive); `as_of` keeps relations whose validity window covers that instant.""" self._require_open() + entity, subject, predicate, obj, namespace = ( + clean_text(entity), clean_text(subject), clean_text(predicate), + clean_text(obj), clean_text(namespace)) if namespace is not None: self.config.grant_for(caller, namespace) allowed = {namespace} @@ -2024,7 +2235,22 @@ def export_jsonl(self, caller: str = "user") -> str: def import_jsonl(self, text: str, caller: str = "user", namespace: str | None = None) -> int: self._require_open() - records = [json.loads(l) for l in text.splitlines() if l.strip()] + # A file saved with a byte order mark (Notepad, Windows PowerShell's + # Out-File) arrives with U+FEFF in front of its first line, which + # json.loads refuses, and the whole import failed on line one. + # Split on newlines only, never splitlines(), for the reason + # curate.parse_plan gives: JSONL written with ensure_ascii=False + # (most tools other than this one) carries U+2028, U+2029 and U+0085 + # unescaped inside strings, and splitlines() cut such a line in two, + # failing the whole import on a memory that merely contained one. A + # trailing \r from CRLF is whitespace to json.loads. + records = [json.loads(l) for l in text.lstrip("\ufeff").split("\n") + if l.strip()] + # Cleaned here, not only in store(): the vectors below are computed + # before store() sees the text, and the tokenizer refuses a lone + # surrogate exactly as it does there. + for r in records: + r["text"] = clean_text(r["text"]) # What store() embeds: the claim without its provenance clause, every # window of it. All of them go in one batched call, and each record # is handed its own block of rows. A claim of at most CHUNK_WINDOW diff --git a/src/compartment/vaultfile.py b/src/compartment/vaultfile.py index a1f4347..ff24383 100644 --- a/src/compartment/vaultfile.py +++ b/src/compartment/vaultfile.py @@ -7,9 +7,10 @@ Invariant I2: plaintext never touches disk. The payload and every journal entry are sealed before any byte is written. lock/save rewrites the file -atomically (temp file → fsync → rename). Journal appends are fsync'd, so an -acknowledged write survives kill -9; a truncated *final* entry is an -unacknowledged write and is discarded on open with a notice - any other +atomically (temp file → fsync → rename → fsync of the folder). Journal +appends are fsync'd, so an acknowledged write survives kill -9 and power +loss (see _sync for what that takes on macOS); a truncated *final* entry is +an unacknowledged write and is discarded on open with a notice - any other malformed byte is a tamper error. """ from __future__ import annotations @@ -27,6 +28,15 @@ from . import crypto, wire from .crypto import CryptoError, TamperError +try: + import fcntl +except ImportError: # Windows + fcntl = None + +#: The fcntl that makes a write durable on Apple's platforms, or None where +#: Python does not define it (everywhere else). See _sync. +_FULLFSYNC = getattr(fcntl, "F_FULLFSYNC", None) + MAGIC = b"NUCV" FORMAT_VERSION = 1 @@ -116,6 +126,10 @@ def is_int(x): ) +#: How long _atomic_replace keeps retrying a Windows sharing violation. +_REPLACE_PATIENCE_S = 5.0 + + def _atomic_replace(tmp: str, path: str) -> None: """os.replace, tolerant of the transient sharing violations Windows raises when another process momentarily holds a handle to `path`. @@ -130,17 +144,87 @@ def _atomic_replace(tmp: str, path: str) -> None: if os.name != "nt": os.replace(tmp, path) return + # Backing off to ~5 s in all: a reader's handle goes in milliseconds, but + # OneDrive's sync engine and some antivirus scanners hold a file they + # are looking at for a second or more, and a save that gives up then + # fails a store the user was told succeeded once it is retried. last: OSError | None = None - for _ in range(20): # ~1s worst case + delay, waited = 0.02, 0.0 + while waited < _REPLACE_PATIENCE_S: try: os.replace(tmp, path) return except PermissionError as exc: last = exc - time.sleep(0.05) + time.sleep(delay) + waited += delay + delay = min(delay * 2, 0.5) + try: + os.replace(tmp, path) + return + except PermissionError as exc: + last = exc raise last if last is not None else OSError(f"could not replace {path}") +def _sync(fd: int) -> None: + """Make what was written to `fd` durable: on the disk itself, not in a + cache that a power cut empties. + + On macOS (and iOS) fsync only hands the data to the drive, which is free + to hold it in its own volatile cache and to write it out in any order; + Apple documents F_FULLFSYNC as the call that actually flushes it, and + SSDs in current Macs are measured to lose fsync'd data on power loss + without it. Two promises rest on this. An acknowledged store is meant to + survive power loss, not only kill -9. And a journal entry whose length + reached the disk before its body did reads back as a complete entry that + fails authentication, which an open has to treat as tampering: the vault + refused to open after a power cut, for no fault of the user's. The + compaction's copy is synced the same way, or the rename could land + before the bytes it points at. + + F_FULLFSYNC costs a few milliseconds per call (about 4 ms measured on an + Apple SSD, against roughly 40 ms for the embedding a store already + pays), and stores are not a hot loop, so it is paid on every append. A + filesystem that does not support it (SMB, some FUSE and USB media) + refuses it, and plain fsync is the best that disk offers.""" + if _FULLFSYNC is not None: + try: + fcntl.fcntl(fd, _FULLFSYNC) + return + except OSError: + pass + os.fsync(fd) + + +def _sync_dir(path: str) -> None: + """Make the rename that just put `path` in place durable. + + A rename is a change to the FOLDER, and on POSIX it is the folder that + has to be synced for it to survive a power cut: syncing the file makes + its bytes durable, not the name pointing at them. Lose the rename and + the old vault comes back, and with it every store acknowledged since the + compaction is gone, because each was appended to the new file. Best + effort: some filesystems (certain FUSE and network mounts) refuse to + sync a directory, and the save has already succeeded by then. + + Not on Windows, where a directory cannot be opened for this at all and + NTFS journals the rename itself.""" + if os.name == "nt": + return + try: + fd = os.open(os.path.dirname(os.path.abspath(path)), + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + except OSError: + return + try: + _sync(fd) + except OSError: + pass + finally: + os.close(fd) + + def _payload_aad(vault_id: str) -> bytes: return wire.payload(vault_id)[0] @@ -197,10 +281,11 @@ class LoadedVaultFile: #: decrypting the whole payload again. journal_start: int = 0 journal_end: int = 0 - #: (inode, size, mtime in ns) of the file as it was read - taken before the - #: read, from the same open file. What an opener compares against to learn - #: that another process wrote while it was reading. See Vault.unlock. - disk_state: tuple[int, int, int] | None = None + #: (inode, size, mtime in ns, identity) of the file as it was read - taken + #: before the read, from the same open file. What an opener compares + #: against to learn that another process wrote while it was reading. See + #: Vault.unlock and disk_state below. + disk_state: tuple[int, int, int, str] | None = None def parse_journal(raw: bytes, pos: int) -> tuple[list[bytes], int, bool, int]: @@ -256,6 +341,53 @@ def parse_journal(raw: bytes, pos: int) -> tuple[list[bytes], int, bool, int]: return journal_cts, pos, truncated_tail, tail_bytes +#: The most header an identity reads. A real header is a few kilobytes; this +#: only bounds the read on a file whose length field is garbage. +_IDENTITY_HEADER_CAP = 1 << 20 + + +def _identity_span(head: bytes) -> int: + """How many leading bytes identify one write of the file: the header and + the payload's nonce, given at least the first ten bytes.""" + if len(head) >= 10 and head[:4] == MAGIC: + (hlen,) = struct.unpack(">I", head[6:10]) + return 10 + min(hlen, _IDENTITY_HEADER_CAP) + crypto.NONCE_LEN + return min(len(head), 10) + + +def _identity(prefix: bytes) -> str: + return crypto.sha256(prefix[:_identity_span(prefix)]) + + +def disk_state(path: str) -> tuple[int, int, int, str]: + """(inode, size, mtime in ns, identity) of the vault file as it is now, + all read from one open file. + + The first three are what a stat can say, and between them they miss a + rewrite on exactly the disks people carry vaults on. A compaction leaves + a file of the same size often - the payload is a whole number of SQLite + pages - and FAT and exFAT keep modification times to two seconds or ten + milliseconds, hand a new file an inode number derived from where its + directory entry or first cluster landed (which a later save reuses), and + a clock stepped back can stamp a newer file with an older time anywhere. + Missed, another process's save is overwritten by this one's, or this + process appends entries sealed to a journal position the new file does + not have, and the next open refuses the vault as tampered. + + The identity closes that: every save reseals the payload under a fresh + random nonce at its front, so the header plus those bytes differ between + any two writes of the file, whatever the filesystem reports. It costs + one read of a few kilobytes. The read-only panel helper (vaultview) has + told rewrites apart the same way.""" + with open(path, "rb") as f: + st = os.fstat(f.fileno()) + head = f.read(10) + span = _identity_span(head) + if span > len(head): + head += f.read(span - len(head)) + return (st.st_ino, st.st_size, st.st_mtime_ns, _identity(head)) + + def read_vault_file(path: str) -> LoadedVaultFile: with open(path, "rb") as f: # Before the read, so an append that lands during it makes the file @@ -264,7 +396,8 @@ def read_vault_file(path: str) -> LoadedVaultFile: st = os.fstat(f.fileno()) raw = f.read() loaded = parse_vault_bytes(raw, path) - loaded.disk_state = (st.st_ino, st.st_size, st.st_mtime_ns) + loaded.disk_state = (st.st_ino, st.st_size, st.st_mtime_ns, + _identity(raw)) return loaded @@ -361,7 +494,7 @@ def write_vault_file( f.write(hjson) f.write(payload_ct) f.flush() - os.fsync(f.fileno()) + _sync(f.fileno()) # O_CREAT only sets the mode when it actually creates the file, so a temp # left behind by an earlier run keeps its old mode. Set it either way. os.chmod(tmp, VAULT_MODE) @@ -375,6 +508,7 @@ def write_vault_file( except OSError: pass raise + _sync_dir(path) def append_journal_entry(path: str, header: VaultHeader, seq: int, entry: dict, master_key: bytes) -> None: @@ -393,14 +527,45 @@ def append_journal_entries(path: str, header: VaultHeader, seq: int, crypto.canonical_json(entry), aad=_journal_aad(header.vault_id, seq + i), ) for i, entry in enumerate(entries)] - with open(path, "ab") as f: - for ct in cts: - length = struct.pack(">I", len(ct)) - f.write(length) - f.write(struct.pack(">I", zlib.crc32(length) & 0xFFFFFFFF)) - f.write(ct) - f.flush() - os.fsync(f.fileno()) + frames = bytearray() + for ct in cts: + length = struct.pack(">I", len(ct)) + frames += length + frames += struct.pack(">I", zlib.crc32(length) & 0xFFFFFFFF) + frames += ct + # No O_CREAT: an append is only ever made to a vault that exists, and + # creating one here would write journal frames into a file with no + # header, which nothing can open again. O_BINARY, where it exists, so + # Windows never translates a newline byte inside the ciphertext. + fd = os.open(path, os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0)) + try: + # The caller holds the vault's write lock, so nothing else appends + # between this and the write below: the file ends here. + start = os.fstat(fd).st_size + try: + view = memoryview(frames) + while view: + view = view[os.write(fd, view):] + _sync(fd) + except BaseException: + # A write the disk refused part way (ENOSPC, EDQUOT, an I/O + # error, a sync that failed) is a write nobody acknowledged, and + # its bytes are taken back off the end. Left there they were a + # torn tail, and a torn tail is the one thing an open cannot + # tolerate without compacting: on a disk that was still full, + # every MCP server, `compartment status` and the panel then + # failed to open the vault at all, reads included, until space + # was freed. Truncating frees the space it took, so it succeeds + # on the full disk that caused it. If it cannot, the bytes stay + # a torn tail and the next open discards them, as before. + try: + os.ftruncate(fd, start) + _sync(fd) + except OSError: + pass + raise + finally: + os.close(fd) # --------------------------------------------------------------------------- diff --git a/src/compartment/vindex.py b/src/compartment/vindex.py index dc0be08..f4904df 100644 --- a/src/compartment/vindex.py +++ b/src/compartment/vindex.py @@ -97,11 +97,21 @@ def __init__(self, dim: int, precision: str = "f32"): def build(cls, dim: int, ikeys: list[int], mat: np.ndarray, precision: str = "f32") -> "UsearchIndex": idx = cls(dim, precision) - if len(ikeys): - idx._index.add(np.array(ikeys, dtype=np.uint64), - np.ascontiguousarray(mat, dtype=np.float32)) + idx.add_all(ikeys, mat) return idx + def add_all(self, ikeys: list[int], mat: np.ndarray) -> None: + if len(ikeys): + self._index.add(np.array(ikeys, dtype=np.uint64), + np.ascontiguousarray(mat, dtype=np.float32)) + + @property + def serial(self) -> bool: + """True when usearch found no SIMD kernels it can use on this CPU + and computes every distance one element at a time. An older usearch + that does not say is taken to have them, as it always was.""" + return getattr(self._index, "hardware_acceleration", None) == "serial" + def add(self, ikey: int, vec: np.ndarray) -> None: self._index.add(np.uint64(ikey), vec.astype(np.float32)) @@ -117,6 +127,35 @@ def __len__(self) -> int: _HNSW_WARNED = False +_PRECISION_WARNED: set[str] = set() + +#: What `index_precision` may name: the float widths and int8, each under the +#: spellings usearch itself accepts. The setting lives in a JSON file people +#: edit by hand, and usearch raised KeyError for anything else ("int-8", 8, +#: "INT8 " with a space): above BRUTE_FORCE_LIMIT records the index is built +#: on every open, so one typo meant the vault could not be opened at all. +#: Its bit and unsigned kinds ("b1", "u8") are refused too: under the cosine +#: metric they do not approximate the vectors, they discard them. +PRECISIONS = { + "f32": "f32", "float32": "f32", "f16": "f16", "float16": "f16", + "bf16": "bf16", "bfloat16": "bf16", "f64": "f64", "float64": "f64", + "i8": "i8", "int8": "i8", +} + + +def _precision(value) -> str: + """The usearch dtype `value` names, or f32 with a notice once.""" + name = PRECISIONS.get(str(value).strip().lower()) + if name is not None: + return name + if repr(value) not in _PRECISION_WARNED: + import sys + print(f"notice: index_precision {value!r} is not one of " + f"{', '.join(sorted(set(PRECISIONS.values())))}; using f32. " + "Set it with `compartment reindex --int8` or `--f32`.", + file=sys.stderr) + _PRECISION_WARNED.add(repr(value)) + return "f32" def build_index(dim: int, ikeys: list[int], mat: np.ndarray, @@ -130,7 +169,23 @@ def build_index(dim: int, ikeys: list[int], mat: np.ndarray, if force == "brute" or (force is None and len(ikeys) < BRUTE_FORCE_LIMIT): return BruteForceIndex.build(dim, ikeys, mat) try: - return UsearchIndex.build(dim, ikeys, mat, precision=precision) + idx = UsearchIndex(dim, _precision(precision)) + # The HNSW graph is built from scratch on every open, and without + # SIMD usearch builds it at a few hundred vectors a second: measured + # 50 s for 25,000 vectors and over five minutes for 100,000, per + # open, where the exact index answers a search over 100,000 in about + # 9 ms and costs nothing to build. That is not a rare machine: SIMD + # dispatch asks the kernel what the CPU supports, and inside macOS + # Seatbelt - where Claude Code and Codex run an agent's commands - + # that query is refused, so usearch reports "serial" on every Apple + # Silicon Mac; likewise on CPUs it has no kernels for. A vault past + # the limit there took minutes to open in every CLI call and every + # MCP server reopen, long past the clients' tool-call timeouts. + # Chosen automatically only; an explicit force is honoured. + if force is None and idx.serial: + return BruteForceIndex.build(dim, ikeys, mat) + idx.add_all(ikeys, mat) + return idx except ImportError: global _HNSW_WARNED if not _HNSW_WARNED: diff --git a/tests/test_apps_browser.py b/tests/test_apps_browser.py new file mode 100644 index 0000000..9acea6e --- /dev/null +++ b/tests/test_apps_browser.py @@ -0,0 +1,148 @@ +"""Opening the dashboard in a browser, on every kind of machine. + +`compartment dash` and the panel's Dashboard button open the page +themselves. webbrowser.open alone got headless Linux and WSL wrong: with no +display it falls back to a text browser run in the same terminal and waits +for it, and `dash` did that before serving, so the browser waited on a +server that could not answer until the browser exited. +""" +from __future__ import annotations + +import threading +import time + +import pytest + +from compartment import dash, menubar + +URL = "http://127.0.0.1:5555/tok/" + + +@pytest.fixture() +def linux(monkeypatch): + monkeypatch.setattr(menubar.sys, "platform", "linux") + for name in ("BROWSER", "DISPLAY", "WAYLAND_DISPLAY", "WSL_DISTRO_NAME", + "WSL_INTEROP"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(menubar, "_is_wsl", lambda: False) + opened, started = [], [] + monkeypatch.setattr(menubar.webbrowser, "open", + lambda url, *a, **k: opened.append(url) or True) + monkeypatch.setattr(menubar.subprocess, "Popen", + lambda argv, **k: started.append(argv)) + return opened, started + + +def test_a_headless_linux_box_opens_nothing(linux): + """No display: the only browsers webbrowser could find are text ones + that take over this terminal.""" + opened, started = linux + assert menubar.open_in_browser(URL) is False + assert opened == [] and started == [] + + +def test_a_desktop_session_gets_its_browser(linux, monkeypatch): + opened, _ = linux + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + assert menubar.open_in_browser(URL) is True + assert opened == [URL] + + +def test_browser_is_honoured_even_without_a_display(linux, monkeypatch): + opened, _ = linux + monkeypatch.setenv("BROWSER", "w3m") + assert menubar.open_in_browser(URL) is True + assert opened == [URL] + + +@pytest.mark.parametrize("on_path,expected", [ + ({"wslview", "explorer.exe"}, "wslview"), + ({"explorer.exe"}, "explorer.exe"), +]) +def test_wsl_opens_the_windows_browser(linux, monkeypatch, on_path, expected): + opened, started = linux + monkeypatch.setattr(menubar, "_is_wsl", lambda: True) + monkeypatch.setenv("DISPLAY", ":0") # WSLg: still the Windows one + monkeypatch.setattr(menubar.shutil, "which", + lambda name, *a, **k: (f"/usr/bin/{name}" + if name in on_path else None)) + assert menubar.open_in_browser(URL) is True + assert started == [[f"/usr/bin/{expected}", URL]] + assert opened == [] + + +def test_wsl_without_interop_says_it_could_not(linux, monkeypatch): + opened, started = linux + monkeypatch.setattr(menubar, "_is_wsl", lambda: True) + monkeypatch.setattr(menubar.shutil, "which", lambda *a, **k: None) + assert menubar.open_in_browser(URL) is False + assert opened == [] and started == [] + + +def test_wsl_is_recognised(monkeypatch, tmp_path): + monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") + assert menubar._is_wsl() is True + + +def test_a_browser_that_fails_never_raises(monkeypatch): + monkeypatch.setattr(menubar.sys, "platform", "darwin") + + def boom(*a, **k): + raise OSError("LSOpenURLsWithRole() failed") + + monkeypatch.setattr(menubar.webbrowser, "open", boom) + assert menubar.open_in_browser(URL) is False + + +class _FakeServer: + def __init__(self): + self.served = threading.Event() + + def serve_forever(self): + self.served.set() + + def server_close(self): + pass + + +def test_dash_serves_while_the_browser_is_still_open(monkeypatch, capsys, + tmp_path): + """A text browser in the foreground used to be waited for before the + server started, so it waited on a page nobody would serve.""" + server = _FakeServer() + release = threading.Event() + seen = {} + + def foreground_browser(url, *a, **k): + seen["served_before_return"] = server.served.wait(5) + release.wait(5) + return True + + monkeypatch.setattr(dash, "start", lambda ref: (server, URL)) + monkeypatch.setattr(menubar, "open_in_browser", foreground_browser) + monkeypatch.setattr(menubar.webbrowser, "open", foreground_browser) + began = time.monotonic() + dash.run(str(tmp_path / "v.vault"), vault=None) + assert server.served.is_set() + assert time.monotonic() - began < 4, "serving waited on the browser" + release.set() + deadline = time.monotonic() + 5 + while "served_before_return" not in seen and time.monotonic() < deadline: + time.sleep(0.01) + assert seen.get("served_before_return") is True + + +def test_dash_says_how_to_reach_the_page_when_nothing_opens(monkeypatch, + capsys, tmp_path): + server = _FakeServer() + monkeypatch.setattr(dash, "start", lambda ref: (server, URL)) + monkeypatch.setattr(menubar, "open_in_browser", lambda url: False) + dash.run(str(tmp_path / "v.vault"), vault=None) + deadline = time.monotonic() + 5 + out = "" + while "no browser" not in out and time.monotonic() < deadline: + out += capsys.readouterr().out + time.sleep(0.01) + assert out.splitlines()[0] == f"Compartment dashboard: {URL}" + assert "open the address above" in out + assert "ssh -L 5555:127.0.0.1:5555" in out diff --git a/tests/test_apps_panel_messages.py b/tests/test_apps_panel_messages.py new file mode 100644 index 0000000..02b0296 --- /dev/null +++ b/tests/test_apps_panel_messages.py @@ -0,0 +1,170 @@ +"""When the panel cannot start, it says the one command that fixes it. + +pip cannot install tkinter, and "install your distribution's Python Tk +package" left the user to find out what that is called on their system, +which is different on nearly every family - and is the wrong advice for a +Python the distribution did not provide. The tray and menu bar named +`pip install 'compartment[tray]'` and `[menubar]`, extras that have been +empty since their packages became base dependencies. +""" +from __future__ import annotations + +import sys +import types + +import pytest + +from compartment import menubar, systray + +UBUNTU = 'NAME="Ubuntu"\nID=ubuntu\nID_LIKE=debian\nVERSION_ID="24.04"\n' +MINT = 'ID=linuxmint\nID_LIKE="ubuntu debian"\n' +FEDORA = "ID=fedora\nVERSION_ID=42\n" +ROCKY = 'ID="rocky"\nID_LIKE="rhel centos fedora"\n' +ARCH = "ID=arch\n" +MANJARO = "ID=manjaro\nID_LIKE=arch\n" +TUMBLEWEED = 'ID="opensuse-tumbleweed"\nID_LIKE="opensuse suse"\n' +ALPINE = "ID=alpine\n" +VOID = 'ID="void"\n' + + +@pytest.mark.parametrize("release,command", [ + (UBUNTU, "sudo apt install python3-tk"), + (MINT, "sudo apt install python3-tk"), + (FEDORA, "sudo dnf install python3-tkinter"), + (ROCKY, "sudo dnf install python3-tkinter"), + (ARCH, "sudo pacman -S tk"), + (MANJARO, "sudo pacman -S tk"), + (TUMBLEWEED, "sudo zypper install python3-tk"), + (ALPINE, "sudo apk add python3-tkinter"), + (VOID, "sudo xbps-install python3-tkinter"), +]) +def test_each_family_gets_its_own_package(release, command): + assert systray.tk_install_hint(release, base_prefix="/usr", + platform="linux") == command + + +def test_a_python_the_distribution_did_not_provide_is_told_to_rebuild(): + """The distribution's package does nothing for pyenv's Python: its + _tkinter is built with the Python, against Tk's headers.""" + hint = systray.tk_install_hint(UBUNTU, + base_prefix="/home/u/.pyenv/versions/3.12.4", + platform="linux") + assert hint.startswith("sudo apt install tk-dev") + assert "pyenv install --force" in hint + assert "python3-tk" not in hint + + +def test_conda_and_the_bsds_and_termux(tmp_path): + (tmp_path / "conda-meta").mkdir() + assert systray.tk_install_hint("", base_prefix=str(tmp_path), + platform="linux") == "conda install tk" + ver = f"{sys.version_info[0]}{sys.version_info[1]}" + assert systray.tk_install_hint("", base_prefix="/usr/local", + platform="freebsd14") == \ + f"sudo pkg install py{ver}-tkinter" + assert systray.tk_install_hint("", base_prefix="/usr/local", + platform="openbsd7") == \ + "doas pkg_add python-tkinter" + assert "python-tkinter" in systray.tk_install_hint( + "", base_prefix="/data/data/com.termux/files/usr", platform="linux") + + +def test_an_unknown_distribution_still_gets_a_useful_line(): + hint = systray.tk_install_hint("ID=somethingnew\n", base_prefix="/usr", + platform="linux") + assert "python3-tk" in hint and "python3-tkinter" in hint + + +def _stand_alone(monkeypatch, platform: str) -> None: + """run() up to the point where it imports its toolkit.""" + monkeypatch.setattr(systray.sys, "platform", platform) + monkeypatch.setattr(systray, "acquire_instance_lock", + lambda vault: (None, True)) + monkeypatch.setattr(systray, "hand_over_to_supervisor", lambda vault: False) + + +def test_a_panel_without_tkinter_prints_this_machines_command( + monkeypatch, capsys, tmp_path): + _stand_alone(monkeypatch, "linux") + monkeypatch.setitem(sys.modules, "tkinter", None) + monkeypatch.setattr(systray, "_os_release", lambda: ARCH) + monkeypatch.setattr(systray.sys, "base_prefix", "/usr") + assert systray.run(str(tmp_path / "v.vault")) == 3 + err = capsys.readouterr().err + assert "sudo pacman -S tk" in err + assert sys.executable in err, "say which Python it is about" + assert "uv tool install compartment" in err and "dash" in err + + +def _fake_tk(monkeypatch, error: str): + tk = types.ModuleType("tkinter") + + class TclError(Exception): + pass + + def Tk(): # noqa: N802 + raise TclError(error) + + tk.TclError, tk.Tk = TclError, Tk + tk.ttk = types.ModuleType("tkinter.ttk") + monkeypatch.setitem(sys.modules, "tkinter", tk) + monkeypatch.setitem(sys.modules, "tkinter.ttk", tk.ttk) + + +def test_a_wayland_session_without_xwayland_is_told_so(monkeypatch, capsys, + tmp_path): + """Tk draws through X11 only. "No graphical display" on a desktop that + is plainly running sends the user looking for the wrong problem.""" + _stand_alone(monkeypatch, "linux") + _fake_tk(monkeypatch, "no display name and no $DISPLAY environment " + "variable") + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-1") + assert systray.run(str(tmp_path / "v.vault")) == 4 + err = capsys.readouterr().err + assert "Wayland" in err and "XWayland" in err + assert "Traceback" not in err + + +def test_a_headless_box_still_gets_the_headless_advice(monkeypatch, capsys, + tmp_path): + _stand_alone(monkeypatch, "linux") + _fake_tk(monkeypatch, "no display name and no $DISPLAY environment " + "variable") + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + assert systray.run(str(tmp_path / "v.vault")) == 4 + err = capsys.readouterr().err + assert "no graphical display" in err and "compartment dash" in err + + +def test_a_python_with_broken_tcl_files_is_not_called_headless( + monkeypatch, capsys, tmp_path): + _stand_alone(monkeypatch, "linux") + _fake_tk(monkeypatch, "Can't find a usable init.tcl in the following " + "directories: /install/lib/tcl8.6") + monkeypatch.setenv("DISPLAY", ":0") + assert systray.run(str(tmp_path / "v.vault")) == 4 + err = capsys.readouterr().err + assert "Tcl/Tk files" in err and "no graphical display" not in err + + +def test_the_tray_names_packages_that_install_something(monkeypatch, capsys, + tmp_path): + _stand_alone(monkeypatch, "win32") + _fake_tk(monkeypatch, "unused") + monkeypatch.setitem(sys.modules, "pystray", None) + assert systray.run(str(tmp_path / "v.vault")) == 3 + err = capsys.readouterr().err + assert f'"{sys.executable}" -m pip install pystray pillow' in err + assert "[tray]" not in err, "an empty extra installs nothing" + + +def test_the_menu_bar_names_a_package_that_installs_something(monkeypatch, + capsys): + monkeypatch.setattr(menubar.sys, "platform", "darwin") + monkeypatch.setitem(sys.modules, "objc", None) + assert menubar.run("/tmp/v.vault") == 1 + err = capsys.readouterr().err + assert f'"{sys.executable}" -m pip install pyobjc-framework-Cocoa' in err + assert "[menubar]" not in err diff --git a/tests/test_apps_windows_children.py b/tests/test_apps_windows_children.py new file mode 100644 index 0000000..905c067 --- /dev/null +++ b/tests/test_apps_windows_children.py @@ -0,0 +1,96 @@ +"""On Windows, nothing the panel starts opens a console window. + +The tray app runs under pythonw.exe, which has no console. Every CLI it +starts is a console program, and a console program started without +CREATE_NO_WINDOW from a process with no console gets a window of its own: +a flash for each click on Unlock, Lock, Create, Change password or Connect, +and, for Dashboard, a window that stayed open for as long as the dashboard +ran, where closing it stopped the dashboard. Only the vault view and the +JSON status calls had the flag. +""" +from __future__ import annotations + +import io +import subprocess + +import pytest + +from compartment import menubar + +CREATE_NO_WINDOW = 0x08000000 + + +@pytest.fixture() +def children(monkeypatch, tmp_path): + """Every child the panel starts, with the keyword arguments it got.""" + seen: list[tuple[list, dict]] = [] + + def run(argv, **kw): + seen.append((list(argv), kw)) + return subprocess.CompletedProcess(argv, 0, "", "") + + class Proc: + def __init__(self, argv, **kw): + seen.append((list(argv), kw)) + self.stdout = io.StringIO( + "Compartment dashboard: http://127.0.0.1:1/t/\n") + self.stdin = io.StringIO() + + def poll(self): + return None + + def terminate(self): + pass + + def wait(self, timeout=None): + return 0 + + def kill(self): + pass + + monkeypatch.setattr(menubar.sys, "platform", "win32") + monkeypatch.setattr(menubar.subprocess, "run", run) + monkeypatch.setattr(menubar.subprocess, "Popen", Proc) + monkeypatch.setattr(menubar, "user_path", lambda: "") + monkeypatch.setattr(menubar, "agent_present", lambda target: True) + monkeypatch.setattr(menubar, "integration_status", + lambda vault: {"claude": True}) + yield seen + menubar._DASH.update(proc=None, url=None) + + +def _flags(kw: dict) -> int: + return int(kw.get("creationflags", 0)) + + +@pytest.mark.parametrize("click", [ + lambda v: menubar.unlock_vault(v, "pw"), + lambda v: menubar.lock_vault(v), + lambda v: menubar.create_vault(v + ".new", "pw", "pw"), + lambda v: menubar.change_passphrase(v, "pw", "pw"), + lambda v: menubar.integrate(v, "claude"), + lambda v: menubar.open_dashboard(v), +], ids=["unlock", "lock", "create", "change-password", "connect", + "dashboard"]) +def test_every_button_starts_its_cli_without_a_console(children, tmp_path, + click): + click(str(tmp_path / "memory.vault")) + cli = [(argv, kw) for argv, kw in children if "compartment.cli" in argv] + assert cli, "the click ran the CLI" + for argv, kw in cli: + assert _flags(kw) & CREATE_NO_WINDOW, argv + + +def test_the_vault_view_and_status_calls_keep_theirs(children, tmp_path): + vault = str(tmp_path / "memory.vault") + menubar.ViewProcess(vault)._start() + menubar._json_run(["x"]) + assert all(_flags(kw) & CREATE_NO_WINDOW for _argv, kw in children) + + +def test_elsewhere_no_windows_flag_is_passed(monkeypatch): + """creationflags is Windows-only; passing it on POSIX raises.""" + monkeypatch.setattr(menubar.sys, "platform", "darwin") + assert menubar._no_console() == {} + monkeypatch.setattr(menubar.sys, "platform", "linux") + assert menubar._no_console() == {} diff --git a/tests/test_claude_hooks_and_integrate.py b/tests/test_claude_hooks_and_integrate.py index 18080e4..36177d7 100644 --- a/tests/test_claude_hooks_and_integrate.py +++ b/tests/test_claude_hooks_and_integrate.py @@ -177,7 +177,10 @@ def test_the_capture_hook_names_a_compartment_that_exists( cmd = _installed_command(home.settings) assert cmd == claude_hooks.hook_command(exe, vault_path) assert claude_hooks.hook_command(vault=vault_path) == cmd # hook install - assert f'--vault "{vault_path}"' in cmd + # Windows: written with forward slashes, which Git Bash, cmd and + # PowerShell all read as the same path (see claude_hooks._shell_word). + shown = vault_path.replace("\\", "/") if os.name == "nt" else vault_path + assert f'--vault "{shown}"' in cmd if os.name != "nt": # run it the way Claude Code does edit = json.dumps({"tool_name": "Edit", diff --git a/tests/test_cli_claude_md.py b/tests/test_cli_claude_md.py new file mode 100644 index 0000000..009560e --- /dev/null +++ b/tests/test_cli_claude_md.py @@ -0,0 +1,52 @@ +"""The CLAUDE.md block goes where Claude Code reads its user CLAUDE.md.""" +from compartment import cli + + +def test_the_block_follows_claude_config_dir(tmp_path, monkeypatch): + """With CLAUDE_CONFIG_DIR set, Claude Code reads CLAUDE.md from there; + the block went to ~/.claude/CLAUDE.md, which it never loads.""" + monkeypatch.delenv("CLAUDE_MD", raising=False) + monkeypatch.setattr(cli.Path, "home", + classmethod(lambda cls: tmp_path / "home")) + moved = tmp_path / "dotfiles" / "claude" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(moved)) + md = cli._write_managed_claude_md() + assert md == moved / "CLAUDE.md" + assert cli._CLAUDE_MD_BEGIN in md.read_text(encoding="utf-8") + assert not (tmp_path / "home" / ".claude" / "CLAUDE.md").exists() + assert cli._remove_managed_claude_md() == md + + +def test_claude_md_still_overrides(tmp_path, monkeypatch): + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "moved")) + monkeypatch.setenv("CLAUDE_MD", str(tmp_path / "explicit.md")) + assert cli._claude_md_path() == tmp_path / "explicit.md" + + + +import pytest # noqa: E402 + + +@pytest.mark.parametrize("platform, words", [ + ("win32", "notification area"), ("darwin", "Cmd-Q"), ("linux", "quit")]) +def test_the_desktop_hint_says_to_quit_it_not_close_it(platform, words): + """"Restart Claude Desktop" read as closing its window, which leaves it + running (in the notification area on Windows): it never rereads its + config, and a copy still running can write the old one back.""" + hint = cli._quit_claude_desktop(platform) + assert hint.startswith("Quit Claude Desktop completely") and words in hint + + +def test_connecting_claude_desktop_prints_that_hint(tmp_path, monkeypatch, + capsys): + from compartment import claude_desktop + monkeypatch.setattr(cli.shutil, "which", lambda name, *a, **k: None) + monkeypatch.setattr(claude_desktop, "present", lambda: True) + monkeypatch.setattr(claude_desktop, "register", lambda exe, args: { + "config": str(tmp_path / "claude_desktop_config.json"), + "backup": None}) + monkeypatch.setenv("CLAUDE_MD", str(tmp_path / "CLAUDE.md")) + cli._integrate_target(cli.argparse.Namespace( + target="claude", vault=str(tmp_path / "v.vault"), no_import=True, + no_hooks=True)) + assert cli._quit_claude_desktop() in capsys.readouterr().out diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py new file mode 100644 index 0000000..e510772 --- /dev/null +++ b/tests/test_cli_errors.py @@ -0,0 +1,251 @@ +"""How a CLI command ends when it cannot do what was asked: one line on +stderr that names the file or the remedy, and a non-zero exit, never a +traceback. Agents run the CLI from their own shells and read what it says; +a traceback is forty lines to read to the bottom of for the same fact.""" +import os + +import pytest + +from compartment import cli + +from conftest import PASS + + +@pytest.fixture() +def no_credential(monkeypatch): + from compartment import vault as vault_module + monkeypatch.setattr(vault_module, "keychain_get", lambda path: None) + for name in ("COMPARTMENT_PASSPHRASE", "ENGRAM_PASSPHRASE"): + monkeypatch.delenv(name, raising=False) + + +def _ends(argv, capsys, code=1): + with pytest.raises(SystemExit) as e: + cli.main(argv) + assert e.value.code == code + return capsys.readouterr().err + + +@pytest.mark.parametrize("command", [["search", "anything"], ["recent"], + ["status"], ["verify"], + ["2fa", "status"]]) +def test_a_vault_that_is_not_there_is_said_to_be_missing( + tmp_path, capsys, no_credential, command): + """With no vault at the path, `search` and the rest said the vault was + locked and to run `compartment unlock`, which cannot help, and + `verify` and `2fa status` raised FileNotFoundError.""" + missing = str(tmp_path / "nowhere" / "memory.vault") + err = _ends(["--vault", missing, *command], capsys) + assert f"no vault at {missing}" in err and "compartment init" in err + assert "Traceback" not in err and "locked" not in err + + +def test_a_file_named_on_the_command_line_that_is_not_there_is_named( + vault, vault_path, tmp_path, capsys, monkeypatch): + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + vault.lock() + for argv in (["import", str(tmp_path / "missing.jsonl")], + ["pack", "install", str(tmp_path / "missing.mpack")]): + err = _ends(["--vault", vault_path, *argv], capsys) + assert err.startswith("error: ") and "missing." in err + assert len(err.strip().splitlines()) == 1 + + +class _ReaderGone: + """stdout once the program reading it has exited: `| head`.""" + + def __init__(self, fd): + self.fd = fd + + def write(self, text): + raise BrokenPipeError(32, "Broken pipe") + + def flush(self): + raise BrokenPipeError(32, "Broken pipe") + + def fileno(self): + return self.fd + + +def test_output_cut_short_by_its_reader_is_not_a_traceback(tmp_path, capsys, + monkeypatch): + """`compartment recent --all | head` ended in BrokenPipeError twice + over: once from the print, again from the flush at exit.""" + with open(tmp_path / "stdout", "w") as stand_in: + monkeypatch.setattr("sys.stdout", _ReaderGone(stand_in.fileno())) + err = _ends(["embed-daemon", "status"], capsys) + assert err == "" + + +def test_ctrl_c_ends_a_command_without_a_traceback(monkeypatch, capsys): + def interrupted(args): + raise KeyboardInterrupt + monkeypatch.setattr(cli, "cmd_embed_daemon", interrupted) + err = _ends(["embed-daemon", "status"], capsys, code=130) + assert err.strip() == "interrupted" + + +def test_a_vault_found_but_unreadable_names_the_file(vault, vault_path, + capsys, monkeypatch): + if getattr(os, "geteuid", lambda: 1)() == 0: + pytest.skip("root reads any file") + if os.name == "nt": + pytest.skip("chmod cannot take read access away on Windows") + vault.lock() + os.chmod(vault_path, 0) + try: + err = _ends(["--vault", vault_path, "verify"], capsys) + finally: + os.chmod(vault_path, 0o600) + assert vault_path in err and "Traceback" not in err + + +@pytest.mark.parametrize("argv", [ + ["serve", "--vault", "{vp}", "--caller", "cursor", "--assert-offline"], + ["--caller", "cursor", "serve", "--vault", "{vp}", "--assert-offline"], + ["--vault", "{vp}", "--caller", "cursor", "--assert-offline", "serve"], +]) +def test_the_vault_and_caller_may_follow_the_subcommand(tmp_path, monkeypatch, + argv): + """`compartment serve --vault PATH` is the order MCP configs are written + in by hand and by registries, and argparse refused it: "unrecognized + arguments: --vault", and the client showed a server that would not + start.""" + vp = str(tmp_path / "v.vault") + seen = [] + monkeypatch.setattr(cli, "cmd_serve", lambda args: seen.append(args)) + monkeypatch.setattr(cli.offline_guard, "activate", lambda **k: None) + cli.main([a.replace("{vp}", vp) for a in argv]) + (args,) = seen + assert (args.vault, args.caller, args.assert_offline) == (vp, "cursor", + True) + + +def test_a_subcommand_without_them_keeps_the_top_level_values(tmp_path, + monkeypatch): + seen = [] + monkeypatch.setattr(cli, "cmd_serve", lambda args: seen.append(args)) + cli.main(["--vault", str(tmp_path / "v.vault"), "serve"]) + assert seen[0].vault == str(tmp_path / "v.vault") + assert seen[0].caller == "user" and seen[0].assert_offline is False + + +# ------------------------------------------------------- import and export -- +@pytest.fixture() +def open_vault(vault, vault_path, monkeypatch): + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + vault.lock() + return vault_path + + +@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16", "utf-16-le", + "utf-32", "utf-32-be"]) +def test_an_import_saved_by_a_windows_tool_is_read(open_vault, tmp_path, + capsys, encoding): + """PowerShell 5.1 writes UTF-8 with a BOM, or UTF-16 for `>`: the first + ended in "Unexpected UTF-8 BOM", the second in UnicodeDecodeError.""" + f = tmp_path / "memories.jsonl" + f.write_text('{"text": "The staging database runs Postgres 16"}\n', + encoding=encoding) + cli.main(["--vault", open_vault, "import", str(f)]) + assert "imported 1 record" in capsys.readouterr().out + + +@pytest.mark.parametrize("body, said", [ + ('{"text": "fine"}\nnot json\n', "line 2: not JSON"), + ('{"txt": "misspelt"}\n', 'line 1: a record needs a "text" string'), + ("[1, 2]\n", 'line 1: a record needs a "text" string'), +]) +def test_an_import_that_is_not_records_is_refused_by_line(open_vault, + tmp_path, capsys, + body, said): + f = tmp_path / "memories.jsonl" + f.write_text(body, encoding="utf-8") + err = _ends(["--vault", open_vault, "import", str(f)], capsys) + assert said in err and "Traceback" not in err + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +def test_a_plaintext_export_is_readable_by_its_owner_alone(open_vault, + tmp_path, capsys): + """Every memory in the clear, credentials included, was written with the + umask's 0644: any account on the machine could read an export left in + /tmp or in a home other users can enter.""" + out = tmp_path / "export.jsonl" + out.write_text("an older export\n") + os.chmod(out, 0o644) + cli.main(["--vault", open_vault, "export", str(out), "--plaintext"]) + assert os.stat(out).st_mode & 0o777 == 0o600 + assert "an older export" not in out.read_text(encoding="utf-8") + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +def test_an_atomize_listing_is_readable_by_its_owner_alone(open_vault, + tmp_path, capsys): + """`atomize --out` writes the over-limit memories decrypted, behind the + same --plaintext confirmation as export, and with the umask's mode.""" + out = tmp_path / "blobs.jsonl" + cli.main(["--vault", open_vault, "atomize", "--out", str(out), + "--plaintext"]) + assert os.stat(out).st_mode & 0o777 == 0o600 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +@pytest.mark.parametrize("command", [ + ["lock", "--sign", "--identity", "{identity}"], + ["pack", "build", "{source}", "--name", "notes", "--identity", + "{identity}", "--out", "{out}"], +]) +def test_a_new_signing_identity_is_readable_by_its_owner_alone( + open_vault, tmp_path, capsys, command): + """identity.json holds the Ed25519 seed that signs packs and sealed + vaults, and it was written with the umask's 0644: any local account + could have signed packs this user trusts.""" + identity = tmp_path / "keys" / "identity.json" + source = tmp_path / "notes.jsonl" + source.write_text('{"text": "The staging database runs Postgres 16"}\n', + encoding="utf-8") + cli.main(["--vault", open_vault, *[ + a.format(identity=identity, source=source, + out=tmp_path / "notes.mpack") for a in command]]) + assert os.stat(identity).st_mode & 0o777 == 0o600 + assert "generated signing identity" in capsys.readouterr().out + + +def test_serve_ending_with_stdout_closed_by_the_transport_exits_cleanly( + monkeypatch, tmp_path): + """mcp 1.x's stdio transport closes the buffer under sys.stdout when the + session ends; the flush the CLI makes after every command then raised + ValueError, and `compartment serve` ended in a traceback with exit 1 + each time a client hung up.""" + import io + out = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") + monkeypatch.setattr("sys.stdout", out) + monkeypatch.setattr(cli, "cmd_serve", lambda args: out.buffer.close()) + cli.main(["--vault", str(tmp_path / "v.vault"), "serve"]) + + +@pytest.mark.parametrize("command", [["hook", "status"], ["hook", "uninstall"], + ["hook"]]) +def test_a_claude_settings_file_that_is_not_json_is_named(tmp_path, + monkeypatch, capsys, + command): + """`hook status` and `hook uninstall` let claude_hooks' ValueError for a + malformed settings.json out as a traceback; `hook install` already + said it in one line.""" + from compartment import claude_hooks + settings = tmp_path / "settings.json" + settings.write_text('{"hooks": {', encoding="utf-8") + monkeypatch.setattr(claude_hooks, "SETTINGS", settings, raising=False) + err = _ends(command, capsys) + assert str(settings) in err and "Traceback" not in err + + +def test_a_vault_config_that_cannot_be_used_is_named(open_vault, capsys): + """A .config.json that is not JSON raises acl.ConfigError, which + names the file and what to do, and is a ValueError: the CLI, which + caught CryptoError alone, printed it as a traceback.""" + with open(open_vault + ".config.json", "w", encoding="utf-8") as f: + f.write('{"settings": {') + err = _ends(["--vault", open_vault, "search", "anything"], capsys) + assert ".config.json" in err and "Traceback" not in err diff --git a/tests/test_cli_hermes_home.py b/tests/test_cli_hermes_home.py new file mode 100644 index 0000000..b0305f5 --- /dev/null +++ b/tests/test_cli_hermes_home.py @@ -0,0 +1,69 @@ +"""Where `integrate hermes`, `uninstall` and the refresh after an update look +for Hermes Agent: where Hermes itself looks, which on native Windows is +%LOCALAPPDATA%\\hermes and not ~/.hermes.""" +import subprocess +import sys +import types + +import pytest + +from compartment import cli + + +@pytest.fixture() +def windows_hermes(tmp_path, monkeypatch): + """Windows, with Hermes installed by its own installer: no HERMES_HOME + exported, its home under LOCALAPPDATA and its venv in the Windows + layout. Every program it would run is a stub that records the call.""" + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.delenv("HERMES_HOME", raising=False) + local = tmp_path / "AppData" / "Local" + monkeypatch.setenv("LOCALAPPDATA", str(local)) + monkeypatch.setattr(cli.Path, "home", + classmethod(lambda cls: tmp_path / "home")) + venv_py = local / "hermes" / "hermes-agent" / "venv" / "Scripts" / \ + "python.exe" + venv_py.parent.mkdir(parents=True) + venv_py.write_bytes(b"MZ") + ran = [] + + def run(argv, **kwargs): + ran.append([str(a) for a in argv]) + return subprocess.CompletedProcess(argv, 0, "", "") + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(cli.shutil, "which", lambda name, *a, **k: None) + vault = tmp_path / "memory.vault" + vault.write_bytes(b"") + return types.SimpleNamespace(home=local / "hermes", venv_py=venv_py, + ran=ran, vault=str(vault), + wrong=tmp_path / "home" / ".hermes") + + +def test_integrate_hermes_on_windows_writes_where_hermes_reads( + windows_hermes, capsys): + """The plugin went to ~/.hermes/plugins, which Hermes on Windows never + reads, the Hermes venv was looked for as bin/python and never found, + and the install was reported done.""" + cli._integrate_target(types.SimpleNamespace(target="hermes", + vault=windows_hermes.vault)) + plugin = windows_hermes.home / "plugins" / "compartment" + assert (plugin / "plugin.yaml").is_file() + assert (plugin / "__init__.py").is_file() + assert not windows_hermes.wrong.exists() + assert [str(windows_hermes.venv_py), "-c", "import compartment"] in \ + windows_hermes.ran + + +def test_hermes_home_follows_hermes_rules(tmp_path, monkeypatch): + """HERMES_HOME with its variables expanded, as Hermes expands it; else + ~/.hermes, and on Windows %LOCALAPPDATA%\\hermes.""" + monkeypatch.setattr(cli.Path, "home", classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("HERMES_BASE", str(tmp_path / "base")) + monkeypatch.setenv("HERMES_HOME", "$HERMES_BASE/custom-hermes") + assert cli._hermes_home() == tmp_path / "base" / "custom-hermes" + monkeypatch.delenv("HERMES_HOME") + monkeypatch.setattr(sys, "platform", "linux") + assert cli._hermes_home() == tmp_path / ".hermes" + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.delenv("LOCALAPPDATA", raising=False) + assert cli._hermes_home() == tmp_path / "AppData" / "Local" / "hermes" diff --git a/tests/test_cli_home.py b/tests/test_cli_home.py new file mode 100644 index 0000000..c24042a --- /dev/null +++ b/tests/test_cli_home.py @@ -0,0 +1,86 @@ +"""Where the CLI finds the vault: `~` expanded wherever a path comes from, +and no home directory needed when the vault is named.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from compartment import cli +from compartment.vault import Vault + +from conftest import PASS + +SRC = str(Path(__file__).resolve().parents[1] / "src") + + +@pytest.fixture() +def home_dir(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) # expanduser, POSIX + monkeypatch.setenv("USERPROFILE", str(home)) # expanduser, Windows + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + Vault.create(str(home / "v.vault"), PASS, creator="test").lock() + return home + + +@pytest.mark.parametrize("how", ["flag", "flag after", "COMPARTMENT_VAULT", + "ENGRAM_VAULT"]) +def test_a_vault_path_starting_with_a_tilde_is_the_home_one( + home_dir, monkeypatch, capsys, how): + """MCP client configs pass "~/..." in args and env literally, since no + shell runs in between: the CLI looked for a folder named "~" in the + client's working directory and reported no vault.""" + for name in ("COMPARTMENT_VAULT", "ENGRAM_VAULT"): + monkeypatch.delenv(name, raising=False) + argv = ["status"] + if how == "flag": + argv = ["--vault", "~/v.vault", "status"] + elif how == "flag after": + argv = ["status", "--vault", "~/v.vault"] + else: + monkeypatch.setenv(how, "~/v.vault") + cli.main(argv) + status = json.loads(capsys.readouterr().out) + # Compared as paths: JSON escapes a Windows path's backslashes, and + # expanduser there joins with "/" after the profile folder. + assert os.path.normcase(os.path.normpath(status["vault"])) == \ + os.path.normcase(os.path.normpath(str(home_dir / "v.vault"))) + assert status["locked"] is False + + +def _cli_with_no_home(*argv): + """The CLI in a process where Compartment's home cannot be found, as + with HOME unset and no password-database entry for the uid.""" + code = ("import sys, compartment.home as h\n" + "def no_home():\n" + " raise RuntimeError('Could not determine home directory.')\n" + "h.home = no_home\n" + "from compartment import cli\n" + "cli.main(sys.argv[1:])\n") + env = dict(os.environ, COMPARTMENT_EMBED_DAEMON="0", + PYTHONPATH=SRC + os.pathsep + os.environ.get("PYTHONPATH", "")) + for name in ("COMPARTMENT_VAULT", "ENGRAM_VAULT"): + env.pop(name, None) + return subprocess.run([sys.executable, "-c", code, *argv], + capture_output=True, text=True, env=env, timeout=120) + + +def test_a_named_vault_needs_no_home_directory(tmp_path): + """The default vault was a module constant: with no home every command + died at import, even one that named its vault.""" + missing = str(tmp_path / "nowhere.vault") + r = _cli_with_no_home("--vault", missing, "status") + assert "Traceback" not in r.stderr, r.stderr + assert f"no vault at {missing}" in r.stderr + r = _cli_with_no_home("--version") + assert r.returncode == 0 and r.stdout.strip() + + +def test_no_vault_and_no_home_says_what_to_pass(): + r = _cli_with_no_home("status") + assert r.returncode == 1 and "Traceback" not in r.stderr + assert "--vault PATH" in r.stderr and "COMPARTMENT_VAULT" in r.stderr diff --git a/tests/test_cli_hook_stdin.py b/tests/test_cli_hook_stdin.py new file mode 100644 index 0000000..cb76ed7 --- /dev/null +++ b/tests/test_cli_hook_stdin.py @@ -0,0 +1,41 @@ +"""`compartment hook capture` reads Claude Code's hook payload as the UTF-8 +it is written in, whatever the locale would decode stdin as.""" +import io +import json + +import pytest + +from compartment import cli, session +from compartment.vault import Vault + +from conftest import PASS + + +def test_a_memory_note_under_a_non_ascii_home_is_captured(tmp_path, + vault_path, + monkeypatch, + capsys): + """On Windows Python decodes a piped stdin with the ANSI code page, so + the path of a note under C:\\Users\\José\\.claude arrived as + C:\\Users\\José\\.claude and every capture there was dropped.""" + note = (tmp_path / "José" / ".claude" / "projects" / "app" / "memory" + / "deploy.md") + note.parent.mkdir(parents=True) + note.write_text("The deploy key lives in 1Password under 'prod'.\n", + encoding="utf-8") + payload = json.dumps({"hook_event_name": "PostToolUse", + "tool_name": "Write", + "tool_input": {"file_path": str(note)}}, + ensure_ascii=False).encode("utf-8") + # stdin as a Windows pipe presents it: the bytes, decoded as cp1252. + monkeypatch.setattr("sys.stdin", + io.TextIOWrapper(io.BytesIO(payload), + encoding="cp1252")) + v = Vault.create(vault_path, PASS, creator="t") + session.store(vault_path, v._master) + v.lock() + with pytest.raises(SystemExit) as e: + cli.main(["--vault", vault_path, "hook", "capture", "--json"]) + assert e.value.code == 0 + res = json.loads(capsys.readouterr().out) + assert res["stored"] is True, res diff --git a/tests/test_cli_init.py b/tests/test_cli_init.py new file mode 100644 index 0000000..4a7a125 --- /dev/null +++ b/tests/test_cli_init.py @@ -0,0 +1,28 @@ +"""`compartment init` finishes on machines that cannot do everything.""" +from compartment import cli, session +from compartment.crypto import CryptoError +from compartment.vault import Vault + +from conftest import PASS + + +def test_init_finishes_where_no_unlock_credential_can_be_kept( + tmp_path, monkeypatch, capsys): + """Termux, OpenBSD and containers without /dev/shm have no volatile + holder for the per-boot secret, and session.store says so with + CryptoError. init died there half-way: the vault made and seeded, but + not saved after seeding, no agent wired, no app, and exit 1.""" + def no_holder(path, key): + raise CryptoError("No per-boot secret is available on this system.") + monkeypatch.setattr(session, "store", no_holder) + monkeypatch.setattr(cli, "_seed_blobs", lambda: []) + wired = [] + monkeypatch.setattr(cli, "connect_present_agents", + lambda path: wired.append(path) or []) + path = str(tmp_path / "new.vault") + cli.main(["--vault", path, "init", "--passphrase", PASS, "--no-app"]) + out = capsys.readouterr().out + assert "stays locked" in out and "COMPARTMENT_PASSPHRASE" in out + assert wired == [path] # the rest of init ran + assert "Vault ready" in out + Vault.unlock(path, passphrase=PASS) diff --git a/tests/test_cli_install_kind.py b/tests/test_cli_install_kind.py new file mode 100644 index 0000000..98a8111 --- /dev/null +++ b/tests/test_cli_install_kind.py @@ -0,0 +1,84 @@ +"""Which tool installed this copy, so that `update` upgrades it with that +tool and `uninstall` names that tool's command for the last step.""" +import subprocess +import sys +import types + +import pytest + +from compartment import cli + + +@pytest.fixture() +def env_root(tmp_path, monkeypatch): + """sys.prefix an environment of this test's own, and nothing on PATH or + in argv that points into a tool's directory.""" + root = tmp_path / "env" + root.mkdir() + monkeypatch.setattr(sys, "prefix", str(root)) + monkeypatch.setattr(sys, "argv", [str(tmp_path / "bin" / "compartment")]) + monkeypatch.setattr(cli.shutil, "which", lambda name: None) + return root + + +def test_a_uv_tool_install_on_windows_is_upgraded_with_uv(env_root, + monkeypatch): + """uv's launcher on Windows is an .exe of its own in ~/.local/bin, not a + symlink into uv's tools directory, and its paths have backslashes: the + "uv/tools" test never matched there, and `update` ran `python -m pip` + in an environment uv builds without pip.""" + (env_root / "uv-receipt.toml").write_text("[tool]\n") + monkeypatch.setattr(sys, "argv", + [r"C:\Users\me\.local\bin\compartment.exe"]) + assert cli._install_kind() == ("uv", ["uv", "tool", "upgrade", + "compartment"]) + + +def test_a_pipx_install_is_upgraded_and_removed_with_pipx(env_root, + monkeypatch, capsys): + (env_root / "pipx_metadata.json").write_text("{}") + assert cli._install_kind() == ("pipx", ["pipx", "upgrade", "compartment"]) + ran = [] + + def run(cmd, *a, **k): + ran.append(list(cmd)) + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + monkeypatch.setattr(cli.subprocess, "run", run) + from compartment import embed_daemon + monkeypatch.setattr(embed_daemon, "stop", lambda *a, **k: {}) + cli.cmd_update(types.SimpleNamespace(source=True, no_app=True, + vault="unused")) + assert ran[0][:3] == ["pipx", "install", "--force"] + assert ran[0][3].startswith("git+https://github.com/") + + +def test_a_plain_pip_install_is_still_upgraded_with_its_own_pip(env_root): + kind, cmd = cli._install_kind() + assert kind == "pip" and cmd[:3] == [sys.executable, "-m", "pip"] + + +def test_uninstall_names_the_pipx_command_for_a_pipx_install(env_root, + monkeypatch, + capsys): + """`python -m pip uninstall` inside a pipx environment removes the + package and leaves pipx listing a broken install whose launcher points + at nothing.""" + (env_root / "pipx_metadata.json").write_text("{}") + monkeypatch.setattr(cli, "_tray_app", + lambda: (_ for _ in ()).throw(RuntimeError("no app"))) + monkeypatch.setattr(subprocess, "run", lambda argv, **k: + subprocess.CompletedProcess(argv, 1, "", "")) + for name in ("unregister", "is_registered"): + monkeypatch.setattr(cli.clients, name, lambda *a, **k: False) + monkeypatch.setattr(cli.claude_desktop, "unregister", lambda: False) + monkeypatch.setattr(cli.claude_hooks, "is_installed", lambda: False) + monkeypatch.setattr(cli.agent_skill, "SKILL_TARGETS", {}) + monkeypatch.setattr(cli, "_remove_managed_claude_md", lambda: None) + from compartment import menubar + monkeypatch.setattr(menubar, "_is_generated", lambda path: False) + monkeypatch.setenv("OPENCLAW_HOME", str(env_root / "openclaw")) + monkeypatch.setenv("HERMES_HOME", str(env_root / "hermes")) + cli.cmd_uninstall(types.SimpleNamespace(purge=False, + vault=str(env_root / "v.vault"))) + out = capsys.readouterr().out + assert out.rstrip().endswith("pipx uninstall compartment") diff --git a/tests/test_cli_openclaw.py b/tests/test_cli_openclaw.py new file mode 100644 index 0000000..9afe6af --- /dev/null +++ b/tests/test_cli_openclaw.py @@ -0,0 +1,90 @@ +"""`integrate openclaw` and `uninstall` against the config current OpenClaw +reads: servers under `mcp.servers`, the file found by OpenClaw's own rule +for OPENCLAW_STATE_DIR, OPENCLAW_CONFIG_PATH and OPENCLAW_HOME.""" +import json +import subprocess +import types + +import pytest + +from compartment import cli, menubar + +VAULT = "/v/memory.vault" + + +@pytest.fixture() +def openclaw_home(tmp_path, monkeypatch): + for name in ("OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENCLAW_HOME", str(tmp_path)) + d = tmp_path / ".openclaw" + d.mkdir() + return d / "openclaw.json" + + +def _integrate(): + cli._integrate_target(types.SimpleNamespace(target="openclaw", + vault=VAULT)) + + +def test_the_server_goes_under_mcp_servers_and_an_old_entry_moves_there( + openclaw_home, capsys): + """OpenClaw reads `mcp.servers`, and its root schema rejects keys it does + not know: the top-level `mcpServers` integrate wrote was never loaded, + and could keep the whole config from loading.""" + openclaw_home.write_text(json.dumps({ + "gateway": {"port": 18789}, + "mcp": {"servers": {"theirs": {"command": "x"}}}, + "mcpServers": {"compartment": {"command": "old"}}}), + encoding="utf-8") + _integrate() + cfg = json.loads(openclaw_home.read_text(encoding="utf-8")) + assert "mcpServers" not in cfg + servers = cfg["mcp"]["servers"] + assert set(servers) == {"theirs", "compartment"} + assert servers["compartment"]["args"] == ["--vault", VAULT, "--caller", + "openclaw", "serve"] + assert cfg["gateway"] == {"port": 18789} + assert menubar.integration_status(VAULT)["openclaw"] is True + + +def test_the_block_to_paste_has_the_current_shape(openclaw_home, capsys): + openclaw_home.write_text("[]\n", encoding="utf-8") + _integrate() + out = capsys.readouterr().out + block = json.loads(out[out.index("{"):out.rindex("}") + 1]) + assert "compartment" in block["mcp"]["servers"] + + +@pytest.mark.parametrize("var", ["OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"]) +def test_openclaws_own_variables_name_the_config(tmp_path, monkeypatch, var): + """OPENCLAW_HOME was taken for the .openclaw folder itself, one level too + high, and the two variables that move the folder were not read.""" + d = tmp_path / "state" + monkeypatch.setenv(var, str(d if var == "OPENCLAW_STATE_DIR" + else d / "openclaw.json")) + assert menubar._openclaw_config() == d / "openclaw.json" + + +def test_uninstall_takes_it_out_of_both_places(openclaw_home, monkeypatch, + capsys): + openclaw_home.write_text(json.dumps({ + "mcp": {"servers": {"compartment": {}, "theirs": {}}}, + "mcpServers": {"compartment": {}, "also-theirs": {}}}), + encoding="utf-8") + monkeypatch.setattr(cli, "_tray_app", + lambda: (_ for _ in ()).throw(RuntimeError("no app"))) + monkeypatch.setattr(menubar, "_is_generated", lambda path: False) + monkeypatch.setattr(subprocess, "run", lambda argv, **k: + subprocess.CompletedProcess(argv, 1, "", "")) + for name in ("unregister", "is_registered"): + monkeypatch.setattr(cli.clients, name, lambda *a, **k: False) + monkeypatch.setattr(cli.claude_desktop, "unregister", lambda: False) + monkeypatch.setattr(cli.claude_hooks, "is_installed", lambda: False) + monkeypatch.setattr(cli.agent_skill, "SKILL_TARGETS", {}) + monkeypatch.setattr(cli, "_remove_managed_claude_md", lambda: None) + cli.cmd_uninstall(types.SimpleNamespace(purge=False, vault=VAULT)) + cfg = json.loads(openclaw_home.read_text(encoding="utf-8")) + assert cfg == {"mcp": {"servers": {"theirs": {}}}, + "mcpServers": {"also-theirs": {}}} + assert "OpenClaw registration removed" in capsys.readouterr().out diff --git a/tests/test_cli_status_bar.py b/tests/test_cli_status_bar.py new file mode 100644 index 0000000..ecc617e --- /dev/null +++ b/tests/test_cli_status_bar.py @@ -0,0 +1,87 @@ +"""What `compartment init` does to put the status bar app up: nothing +registered for a panel that cannot draw, and no console window behind the +Windows tray app.""" +import subprocess +import sys +import types + +from compartment import cli + + +class _Running: + """A child still running when the installer looks.""" + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired("tray", timeout) + + +def test_a_python_without_tkinter_registers_nothing_to_start_at_login( + monkeypatch, capsys, tmp_path): + """Start at login was registered before tkinter was checked, so a Linux + Python without it got an autostart entry or a systemd user unit that + failed at every login, with nothing on screen to say why.""" + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setenv("DISPLAY", ":0") + monkeypatch.setattr(cli, "_linux_gui_available", lambda: False) + registered = [] + app = types.SimpleNamespace( + set_login=lambda on, vault=None: registered.append(on) or "on", + tk_install_hint=lambda: "sudo zypper install python312-tk") + monkeypatch.setattr(cli, "_tray_app", lambda: app) + monkeypatch.setattr(cli.subprocess, "Popen", lambda *a, **k: + (_ for _ in ()).throw(AssertionError("launched"))) + cli._start_status_bar_app(str(tmp_path / "v.vault")) + out = capsys.readouterr().out + assert registered == [] + assert "sudo zypper install python312-tk" in out + assert "uv tool install compartment" in out + assert "compartment panel --login on" in out + assert "start at login: on" not in out + + +def _windows(monkeypatch, tmp_path, pythonw: bool): + scripts = tmp_path / "venv" / "Scripts" + scripts.mkdir(parents=True) + names = ["python.exe", "compartment.exe"] + (["pythonw.exe"] if pythonw + else []) + for name in names: + (scripts / name).write_bytes(b"MZ") + monkeypatch.setattr(sys, "platform", "win32") + + def parts(vault=None): # systray._autostart_parts, in short + pyw = scripts / "pythonw.exe" + return ((str(pyw), "-m compartment.cli tray") if pyw.is_file() + else (str(scripts / "compartment.exe"), "tray")) + app = types.SimpleNamespace(set_login=lambda on, vault=None: "on", + start_supervised=lambda: False, + _autostart_parts=parts) + monkeypatch.setattr(cli, "_tray_app", lambda: app) + monkeypatch.setattr(cli.shutil, "which", + lambda name, *a, **k: str(scripts / f"{name}.exe")) + monkeypatch.setattr(cli, "_wait_for_status_bar_app", lambda *a, **k: True) + started = [] + monkeypatch.setattr(cli.subprocess, "Popen", lambda argv, **k: + started.append((argv, k)) or _Running()) + return scripts, started + + +def test_windows_starts_the_tray_with_pythonw(monkeypatch, tmp_path, capsys): + """The console script compartment.exe was started with DETACHED_PROCESS, + and the Python it runs could still open a console window of its own + behind a tray app that has no window. pythonw.exe opens none, and it is + what start at sign-in runs.""" + scripts, started = _windows(monkeypatch, tmp_path, pythonw=True) + vault = str(tmp_path / "Zoë Ann" / "v.vault") + cli._start_status_bar_app(vault) + (argv, kwargs), = started + assert argv == [str(scripts / "pythonw.exe"), "-m", "compartment.cli", + "--vault", vault, "tray"] + assert kwargs["stdin"] is subprocess.DEVNULL + + +def test_windows_without_pythonw_still_starts_the_console_script( + monkeypatch, tmp_path, capsys): + scripts, started = _windows(monkeypatch, tmp_path, pythonw=False) + cli._start_status_bar_app(str(tmp_path / "v.vault")) + (argv, _), = started + assert argv[0] == str(scripts / "compartment.exe") diff --git a/tests/test_clients.py b/tests/test_clients.py index 0dbcd2b..79813e9 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -189,10 +189,13 @@ def test_codex_gets_a_table_appended_and_keeps_the_rest(tmp_path): def test_codex_is_not_registered_twice(tmp_path): p = tmp_path / "config.toml" p.write_text('[mcp_servers.compartment]\ncommand = "compartment"\n') + clients.register(clients.CLIENTS["codex"], VAULT, path=p) res = clients.register(clients.CLIENTS["codex"], VAULT, path=p) assert res["written"] is False assert p.read_text().count("[mcp_servers.compartment]") == 1 + entry = tomllib.loads(p.read_text())["mcp_servers"]["compartment"] + assert entry["args"][:2] == ["--vault", VAULT] # refreshed once def test_codex_unregister_takes_only_our_table(tmp_path): @@ -435,7 +438,7 @@ def test_a_client_known_by_a_bare_filename_is_not_found_everywhere( assert clients.CLIENTS["librechat"].present() is True assert clients.CLIENTS["open-webui"].present() is False cli.main(["--vault", VAULT, "integrate", "librechat"]) - assert '"mcpServers"' in capsys.readouterr().out + assert "mcpServers:" in capsys.readouterr().out def test_one_client_that_fails_does_not_stop_integrate_all(tmp_path, @@ -610,10 +613,10 @@ def test_an_openclaw_write_cut_short_leaves_its_config_whole_with_a_backup( with every other server in it, cut short as well.""" home = tmp_path / "openclaw-home" home.mkdir() - monkeypatch.setenv("OPENCLAW_HOME", str(home)) + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(home)) cfg = home / "openclaw.json" - cfg.write_text(json.dumps({"mcpServers": {f"server-{i}": OTHER - for i in range(100)}}, + cfg.write_text(json.dumps({"mcp": {"servers": {f"server-{i}": OTHER + for i in range(100)}}}, indent=2), encoding="utf-8") before = cfg.read_bytes() args = types.SimpleNamespace(target="openclaw", vault=VAULT) @@ -626,7 +629,7 @@ def test_an_openclaw_write_cut_short_leaves_its_config_whole_with_a_backup( assert backup.read_bytes() == before cli._integrate_target(args) - servers = json.loads(cfg.read_text(encoding="utf-8"))["mcpServers"] + servers = json.loads(cfg.read_text(encoding="utf-8"))["mcp"]["servers"] assert "compartment" in servers and servers["server-99"] == OTHER assert backup.read_bytes() == before @@ -639,7 +642,7 @@ def test_an_openclaw_config_that_is_json_but_not_an_object_is_refused( readers leave such a file alone and the user gets the block to paste.""" home = tmp_path / "openclaw-home" home.mkdir() - monkeypatch.setenv("OPENCLAW_HOME", str(home)) + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(home)) cfg = home / "openclaw.json" cfg.write_text(raw, encoding="utf-8") cli._integrate_target(types.SimpleNamespace(target="openclaw", @@ -705,8 +708,8 @@ def test_a_claude_md_or_openclaw_config_kept_in_dotfiles_stays_a_link( write = cli._write_managed_claude_md else: link = tmp_path / "openclaw-home" / name - monkeypatch.setenv("OPENCLAW_HOME", str(link.parent)) - real.write_text(json.dumps({"mcpServers": {"theirs": OTHER}}), + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(link.parent)) + real.write_text(json.dumps({"mcp": {"servers": {"theirs": OTHER}}}), encoding="utf-8") write = lambda: cli._integrate_target( # noqa: E731 types.SimpleNamespace(target="openclaw", vault=VAULT)) @@ -719,8 +722,8 @@ def test_a_claude_md_or_openclaw_config_kept_in_dotfiles_stays_a_link( text = real.read_text(encoding="utf-8") assert ("kept in my dotfiles" in text and cli._CLAUDE_MD_BEGIN in text if name == "CLAUDE.md" - else set(json.loads(text)["mcpServers"]) == {"theirs", - "compartment"}) + else set(json.loads(text)["mcp"]["servers"]) == { + "theirs", "compartment"}) backup = link.with_name(link.name + clients.BACKUP_SUFFIX) assert backup.read_bytes() == before assert os.listdir(dot) == [name] # nothing left beside it diff --git a/tests/test_connect_environment.py b/tests/test_connect_environment.py index 10c0e5f..6abbb72 100644 --- a/tests/test_connect_environment.py +++ b/tests/test_connect_environment.py @@ -202,6 +202,9 @@ def test_without_the_path_fix_it_would_have_written_a_bare_name(monkeypatch, @pytest.fixture() def fake_home(tmp_path, monkeypatch): monkeypatch.setattr(menubar.Path, "home", classmethod(lambda cls: tmp_path)) + # Hermes on native Windows lives under %LOCALAPPDATA%, which must not be + # the runner's real one. + monkeypatch.setenv("LOCALAPPDATA", str(tmp_path / "AppData" / "Local")) monkeypatch.delenv("HERMES_HOME", raising=False) monkeypatch.delenv("OPENCLAW_HOME", raising=False) monkeypatch.setattr(menubar, "integration_status", @@ -226,7 +229,7 @@ def test_hermes_counts_as_connected_once_the_plugin_is_there(fake_home, monkeypatch): from compartment import claude_hooks monkeypatch.setattr(claude_hooks, "is_installed", lambda *a, **k: False) - plug = fake_home / ".hermes" / "plugins" / "compartment" + plug = menubar.hermes_home() / "plugins" / "compartment" # per platform plug.mkdir(parents=True) (plug / "plugin.yaml").write_text("name: compartment\n") st = menubar.integration_status("/v") @@ -240,8 +243,12 @@ def test_openclaw_counts_as_connected_once_it_is_in_the_config(fake_home, d = fake_home / ".openclaw" d.mkdir() (d / "openclaw.json").write_text( - json.dumps({"mcpServers": {"compartment": {}}})) + json.dumps({"mcp": {"servers": {"compartment": {}}}})) assert menubar.integration_status("/v")["openclaw"] is True + # Where earlier releases wrote it: a key current OpenClaw never loads. + (d / "openclaw.json").write_text( + json.dumps({"mcpServers": {"compartment": {}}})) + assert menubar.integration_status("/v")["openclaw"] is False def test_a_broken_config_file_is_not_a_crash(fake_home, monkeypatch): diff --git a/tests/test_embed_batching.py b/tests/test_embed_batching.py index c4cfa52..e403236 100644 --- a/tests/test_embed_batching.py +++ b/tests/test_embed_batching.py @@ -141,3 +141,21 @@ def test_a_vault_pinned_to_another_model_is_still_refused(vault, vault_path): vault.lock() with pytest.raises(Exception, match="does not match the model"): Vault.unlock(vault_path, passphrase=PASS) + + +def test_a_lone_surrogate_embeds_as_the_replacement_character(emb): + """Half of a cut emoji (a JSON "\\ud83d" escape from an MCP client, or a + string sliced in the middle of a pair) made tokenizers 0.16 and later + raise TypeError from every embedding call, so a store or a search with + one in it failed with a traceback. In a Python str every code point in + U+D800..U+DFFF is a lone surrogate, and it now embeds as U+FFFD.""" + cut, whole = "router \ud83d passphrase", "router � passphrase" + assert np.array_equal(emb.embed_query(cut), emb.embed_query(whole)) + assert np.array_equal(emb.embed_passages([cut, "x \udc00"]), + emb.embed_passages([whole, "x �"])) + long = "a sentence about vaults \udfff and agents. " * 400 + assert emb.chunk(long) == emb.chunk(long.replace("\udfff", "�")) + assert np.array_equal(emb.embed_record(long), + emb.embed_record(long.replace("\udfff", "�"))) + # A real pair, written as one code point, is untouched. + assert emb.chunk("a \U0001F600 b") == ["a \U0001F600 b"] diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index 79eaaf5..1c7bfc1 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -6,11 +6,13 @@ away on its own, and that a vault never notices when it is missing. """ import contextlib +import errno import json import os import shutil import signal import socket +import struct import subprocess import sys import tempfile @@ -33,6 +35,45 @@ reason="the daemon needs Unix domain sockets") +def _bind_refused_here() -> str | None: + """Why this machine refuses this process a listening AF_UNIX socket, or + None. An agent's sandbox does (macOS Seatbelt as Claude Code and Codex + apply it to their shell commands refuses bind() with EPERM), and a test + that needs a real daemon cannot run there; what such a sandbox gets + instead, a model of its own at once, is tested below on every machine + by refusing bind() in-process.""" + if not embed_daemon.supported(): + return None + d = tempfile.mkdtemp(prefix="cmpt-") + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind(os.path.join(d, "probe")) + except PermissionError as exc: + return str(exc) + except OSError: + return None + finally: + s.close() + shutil.rmtree(d, ignore_errors=True) + return None + + +_BIND_REFUSED = _bind_refused_here() + +needs_bind = pytest.mark.skipif( + _BIND_REFUSED is not None, + reason=f"this machine refuses AF_UNIX bind() ({_BIND_REFUSED}), as an " + "agent's sandbox does, so no daemon can run here; the fallback " + "such a sandbox gets is tested by test_a_sandbox_*") + + +@pytest.fixture(autouse=True) +def no_refusal_carried_over(monkeypatch): + """embed_daemon remembers a refusal for the life of the process; each + test starts as a fresh process would.""" + monkeypatch.setattr(embed_daemon, "_refused", None) + + @pytest.fixture() def daemon_env(monkeypatch): """A daemon of this test's own, on a short socket path, gone afterwards. @@ -64,6 +105,7 @@ def _wait_until(pred, timeout=15.0): # ------------------------------------------------------------- same vectors -- +@needs_bind def test_the_daemon_answers_with_the_in_process_vectors(daemon_env, local): r = RemoteEmbedder.connect() assert r is not None and r.shared and r.daemon_pid @@ -81,6 +123,7 @@ def test_the_daemon_answers_with_the_in_process_vectors(daemon_env, local): # ------------------------------------------------------------ one of them -- +@needs_bind def test_several_clients_share_one_daemon(daemon_env): clients = [RemoteEmbedder.connect() for _ in range(4)] assert all(c is not None for c in clients) @@ -92,6 +135,7 @@ def test_several_clients_share_one_daemon(daemon_env): assert _wait_until(lambda: embed_daemon.status(daemon_env).get("clients") == 0) +@needs_bind def test_clients_arriving_together_start_exactly_one_daemon(daemon_env): """Six MCP servers start within the same second when a client comes up; the spawn lock makes one of them start the daemon and the rest wait.""" @@ -110,6 +154,7 @@ def go(): assert all(r.embed_query("x").shape == (r.dim,) for r in results) +@needs_bind def test_the_daemon_starts_fast_and_loads_the_model_on_demand(daemon_env): t0 = time.monotonic() r = RemoteEmbedder.connect() @@ -123,6 +168,7 @@ def test_the_daemon_starts_fast_and_loads_the_model_on_demand(daemon_env): # ------------------------------------------------------------- lifecycle -- +@needs_bind def test_the_daemon_exits_when_nobody_is_connected(daemon_env): r = RemoteEmbedder.connect() assert r is not None @@ -131,6 +177,7 @@ def test_the_daemon_exits_when_nobody_is_connected(daemon_env): assert _wait_until(lambda: not daemon_env.exists(), 5) +@needs_bind def test_a_client_arriving_as_an_idle_daemon_exits_starts_the_next_one( daemon_env, monkeypatch): """An idle daemon takes its socket file away first, so that a client @@ -151,6 +198,7 @@ def test_a_client_arriving_as_an_idle_daemon_exits_starts_the_next_one( assert late is not None and late.shared and late.daemon_pid != pid +@needs_bind def test_a_client_survives_the_daemon_being_killed(daemon_env): r = RemoteEmbedder.connect() first = r.daemon_pid @@ -172,6 +220,7 @@ def _returns_within(seconds, fn): return out[0] +@needs_bind def test_a_daemon_that_stops_answering_does_not_hang_its_clients( daemon_env, monkeypatch, local): """A daemon that is alive but not serving (suspended with Ctrl-Z, stuck @@ -196,23 +245,83 @@ def test_a_daemon_that_stops_answering_does_not_hang_its_clients( assert isinstance(fresh, Embedder) -def test_a_text_the_model_rejects_is_an_error_not_a_lost_daemon(daemon_env): - """A text the tokenizer cannot encode, such as one with a lone surrogate - from a cut emoji, comes back from the daemon as an error reply. The - client took that for a daemon that had gone: it dropped the connection, - loaded a model of its own for the rest of the process, ran the text - again there, and the caller saw the tokenizer's bare TypeError.""" +@needs_bind +def test_a_text_with_a_lone_surrogate_embeds_through_the_daemon(daemon_env, + local): + """Half of a cut emoji, which an MCP client may send as a JSON \\ud83d + escape, made tokenizers 0.16 and later raise TypeError, in the daemon + and in-process alike (0.15 let it through). It now embeds as U+FFFD + would, through the same daemon, which stays the one serving.""" r = RemoteEmbedder.connect() r.embed_query("warm up") pid = r.daemon_pid - with pytest.raises(ValueError, match="TypeError"): - r.embed_query("where is the router \ud83d passphrase") + vec = r.embed_query("where is the router \ud83d passphrase") assert r.shared and r.daemon_pid == pid assert embed_daemon.status(daemon_env)["clients"] == 1 + assert np.array_equal( + vec, local.embed_query("where is the router \ufffd passphrase")) + + +def _served(monkeypatch, model=None): + """RemoteEmbedder.connect() wired to a daemon's own connection handler + over a socketpair, which needs no bind(), so this runs everywhere, + sandboxes included. `model` stands in for Embedder in the daemon.""" + if model is not None: + monkeypatch.setattr(embed_daemon, "Embedder", model) + daemon = embed_daemon._Daemon(Path("unused.sock"), idle=60) + + def pair(*a, **k): + mine, theirs = socket.socketpair(socket.AF_UNIX) + threading.Thread(target=daemon._serve, args=(theirs,), + daemon=True).start() + return mine + monkeypatch.setattr(embed_daemon, "connect_or_spawn", pair) + r = RemoteEmbedder.connect(path=Path("unused.sock"), spawn=False) + assert r is not None and r.shared + return r, daemon + + +def test_a_text_the_model_rejects_is_an_error_not_a_lost_daemon(monkeypatch): + """A request the daemon's model fails on comes back as an error reply. + The client took that for a daemon that had gone: it dropped the + connection, loaded a model of its own for the rest of the process, ran + the text again there, and the caller saw the model's bare error.""" + class Refusing: + def __init__(self, name): + self.dim = model_info(name).dim + + def embed_query(self, text): + if "refuse" in text: + raise TypeError("TextInputSequence must be str") + return np.ones(self.dim, np.float32) + + r, daemon = _served(monkeypatch, Refusing) + pid = r.daemon_pid + with pytest.raises(ValueError, match="TypeError"): + r.embed_query("please refuse this one") + assert r.shared and r.daemon_pid == pid + assert daemon._status()["clients"] == 1 assert r.embed_query("an ordinary query afterwards").shape == (r.dim,) assert r.shared + r.close() + + +def test_the_daemon_and_this_process_agree_on_a_lone_surrogate(monkeypatch, + local): + """The daemon gets the text through JSON, where a lone surrogate is a + \\ud83d escape; it must embed what this process's own model embeds.""" + r, _ = _served(monkeypatch) + text = "where is the router \ud83d passphrase" + assert np.array_equal(r.embed_query(text), local.embed_query(text)) + assert np.array_equal(r.embed_passages([text, "plain"]), + local.embed_passages([text, "plain"])) + long = ("sentence about vaults \udc00 and agents. " * 400) + assert r.chunk(long) == local.chunk(long) + assert r.shared + r.close() +@needs_bind def test_a_long_list_of_passages_is_embedded_once_through_the_daemon( monkeypatch): """A reply carries 2 KB a text, so from 32,768 texts on it was longer @@ -258,6 +367,7 @@ def embed_passages(self, texts, batch=64): shutil.rmtree(d, ignore_errors=True) +@needs_bind def test_a_dead_socket_file_is_replaced(daemon_env): daemon_env.parent.mkdir(parents=True, exist_ok=True) s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -268,6 +378,7 @@ def test_a_dead_socket_file_is_replaced(daemon_env): assert r is not None and r.embed_query("x").shape == (r.dim,) +@needs_bind def test_stop_ends_it_and_status_says_so(daemon_env): r = RemoteEmbedder.connect() pid = r.daemon_pid @@ -278,6 +389,7 @@ def test_stop_ends_it_and_status_says_so(daemon_env): assert r.embed_query("after stop").shape == (r.dim,) # starts a new one +@needs_bind def test_status_and_stop_from_the_command_line(daemon_env, capsys): r = RemoteEmbedder.connect() assert r is not None @@ -292,6 +404,7 @@ def test_status_and_stop_from_the_command_line(daemon_env, capsys): # ------------------------------------------------------------ versions -- +@needs_bind def test_an_older_client_leaves_a_newer_daemon_alone(daemon_env, monkeypatch): keep = RemoteEmbedder.connect() # the "newer" daemon pid = keep.daemon_pid @@ -300,6 +413,7 @@ def test_an_older_client_leaves_a_newer_daemon_alone(daemon_env, monkeypatch): assert embed_daemon.status(daemon_env)["pid"] == pid +@needs_bind def test_a_newer_client_replaces_an_older_daemon(daemon_env, monkeypatch): old = RemoteEmbedder.connect() pid = old.daemon_pid @@ -340,6 +454,7 @@ def cleanup(self): """ +@needs_bind def test_a_newer_client_takes_over_from_an_older_daemon_at_once(daemon_env): """Every upgrade meets the daemon the older install left running. The newer client told it to go and started its own as soon as its socket @@ -389,6 +504,7 @@ def _hello(path): s.close() +@needs_bind def test_the_daemon_a_guarded_process_starts_is_guarded(daemon_env, guarded_by_the_flag): """The flag guarded the server and the CLI, and the daemon they started, @@ -432,6 +548,7 @@ def serve(): srv.close() +@needs_bind def test_a_guarded_client_keeps_its_own_model_beside_an_unguarded_daemon( daemon_env, guarded_by_the_flag): """The daemon is shared, so a guarded server could send its texts to a @@ -464,6 +581,7 @@ def test_disabled_means_a_local_encoder(monkeypatch): assert isinstance(get_embedder(), Embedder) +@needs_bind def test_enabled_means_the_shared_one(daemon_env): e = get_embedder() assert isinstance(e, RemoteEmbedder) and e.shared @@ -507,7 +625,36 @@ def test_the_peer_credential_is_this_user(): b.close() +class _PeerCred: + """A connection whose SO_PEERCRED answer is `raw`.""" + + def __init__(self, raw): + self.raw = raw + + def getsockopt(self, level, option, size): + assert size >= len(self.raw) + return self.raw + + +@pytest.mark.parametrize("platform, raw", [ + # Linux, musl and Android: struct ucred { pid_t pid; uid_t uid; gid_t gid; } + ("linux", struct.pack("iII", 4242, 1000, 1001)), + # OpenBSD: struct sockpeercred { uid_t uid; gid_t gid; pid_t pid; } + ("openbsd7", struct.pack("IIi", 1000, 1001, 4242)), +], ids=["linux", "openbsd"]) +def test_the_peer_uid_is_read_from_the_platforms_own_struct(monkeypatch, + platform, raw): + """OpenBSD answers SO_PEERCRED with its fields in another order, and + the daemon read the peer's group id as its user id: every client of a + user whose gid is not their uid was turned away, and the daemon was + useless there.""" + monkeypatch.setattr(socket, "SO_PEERCRED", 17, raising=False) + monkeypatch.setattr(embed_daemon.sys, "platform", platform) + assert embed_daemon._peer_uid(_PeerCred(raw)) == 1000 + + # ------------------------------------------------------------- the vault -- +@needs_bind def test_a_vault_searches_identically_through_the_daemon(daemon_env, tmp_path, monkeypatch): vp = str(tmp_path / "shared.vault") v = Vault.create(vp, PASS, creator="test") @@ -575,3 +722,186 @@ def test_a_home_that_cannot_hold_the_socket_embeds_with_its_own_model( # A model that is not installed is still the error that names the remedy. with pytest.raises(ModelError, match="download-model"): get_embedder("multilingual-e5-small-int8") + + +# ------------------------------------------------------- inside a sandbox -- +class _RefusingSocket(socket.socket): + """A socket the way an agent's sandbox hands them out: it can be made, + but bind() and connect() on a path are refused with EPERM.""" + + def bind(self, address): + if self.family == socket.AF_UNIX: + raise PermissionError(errno.EPERM, os.strerror(errno.EPERM)) + return super().bind(address) + + def connect(self, address): + if self.family == socket.AF_UNIX: + raise PermissionError(errno.EPERM, os.strerror(errno.EPERM)) + return super().connect(address) + + +@pytest.fixture() +def sandboxed(monkeypatch): + """Unix domain sockets refused as a sandbox refuses them, the daemon + switched on, and every attempt to start one recorded.""" + d = tempfile.mkdtemp(prefix="cmpt-") + monkeypatch.setenv("COMPARTMENT_EMBED_DAEMON", "1") + monkeypatch.setenv("COMPARTMENT_EMBED_SOCKET", str(Path(d) / "embed.sock")) + monkeypatch.setattr(socket, "socket", _RefusingSocket) + spawned = [] + + def spawn(path, idle): + spawned.append(path) + raise AssertionError("started a daemon that could never listen") + monkeypatch.setattr(embed_daemon, "_spawn", spawn) + yield Path(d), spawned + shutil.rmtree(d, ignore_errors=True) + + +def test_a_sandbox_that_refuses_binding_gets_a_local_model_at_once(sandboxed): + """Claude Code and Codex run an agent's shell commands under a sandbox + that refuses bind() on a Unix domain socket. Every `compartment search` + an agent ran there started a daemon, waited for it to import numpy and + onnxruntime and fail, and only then used a model of its own: a wasted + process and a quarter of a second or more per command, and again at + every reconnect in a server.""" + d, spawned = sandboxed + for _ in range(3): + t0 = time.monotonic() + assert isinstance(get_embedder(), Embedder) + assert time.monotonic() - t0 < 5 + assert spawned == [] + assert "bind" in embed_daemon._refused + assert not any(p.name.endswith(".p") for p in d.iterdir()) + st = embed_daemon.status() + assert st["running"] is False and "sandbox" in st["reason"] + + +def test_a_sandbox_that_refuses_connecting_starts_no_daemon(sandboxed, + monkeypatch): + """A daemon an unsandboxed server started is listening, and the sandbox + refuses this process the connection: starting a second daemon for the + same socket cannot help, and used to be tried on every call.""" + d, spawned = sandboxed + monkeypatch.setattr(embed_daemon, "_owned_socket", lambda p: True) + assert RemoteEmbedder.connect() is None + assert spawned == [] + assert "connect" in embed_daemon._refused + assert isinstance(get_embedder(), Embedder) + assert spawned == [] + + +def test_a_sandbox_that_refuses_starting_processes_is_asked_once(monkeypatch): + """Some sandboxes refuse to start a process at all; every vault opened + asked again.""" + d = tempfile.mkdtemp(prefix="cmpt-") + monkeypatch.setenv("COMPARTMENT_EMBED_DAEMON", "1") + monkeypatch.setenv("COMPARTMENT_EMBED_SOCKET", str(Path(d) / "embed.sock")) + monkeypatch.setattr(embed_daemon, "_bind_refused", lambda path: False) + tries = [] + + def popen(*a, **k): + tries.append(a) + raise PermissionError(errno.EPERM, os.strerror(errno.EPERM)) + monkeypatch.setattr(embed_daemon.subprocess, "Popen", popen) + try: + for _ in range(3): + assert isinstance(get_embedder(), Embedder) + assert len(tries) == 1 + finally: + shutil.rmtree(d, ignore_errors=True) + + +def test_a_daemon_run_where_it_cannot_listen_says_why(sandboxed, monkeypatch, + capsys): + """`compartment embed-daemon run` in a sandbox exited 1 and said + nothing.""" + d, _ = sandboxed + with pytest.raises(SystemExit) as e: + cli.main(["embed-daemon", "run", "--socket", str(d / "embed.sock"), + "--idle", "1"]) + assert e.value.code == 1 + err = capsys.readouterr().err + assert "cannot listen" in err and os.strerror(errno.EPERM) in err + + +# ------------------------------------------------- the shared temp directory -- +@pytest.mark.parametrize("link", ["symlink", "hardlink"]) +def test_a_link_left_at_the_lock_files_name_is_not_written_through( + tmp_path, link, capsys): + """A home too deep for an AF_UNIX path puts the socket, and its lock and + spawn files, in the shared temp directory under names anyone can work + out from the home path. Another user's symlink or hard link there led + the daemon to truncate what it pointed at, the user's own shell profile + say, and write its pid into it. Linux refuses to follow a stranger's + link in /tmp; the BSDs do not.""" + profile = tmp_path / "profile" + profile.write_text("export PATH=/opt/bin:$PATH\n") + sock = tmp_path / "compartment-1000-abc.sock" + for suffix in (".lock", ".spawn"): + target = Path(str(sock) + suffix) + if link == "symlink": + target.symlink_to(profile) + else: + os.link(profile, target) + code = embed_daemon._Daemon(sock, idle=1).run() + assert profile.read_text() == "export PATH=/opt/bin:$PATH\n" + assert code == 1 and "lock file" in capsys.readouterr().err + assert RemoteEmbedder.connect(path=sock, wait=0.5) is None + assert embed_daemon._locked_pid(sock) is None + assert profile.read_text() == "export PATH=/opt/bin:$PATH\n" + + +def test_a_module_in_the_home_directory_does_not_replace_the_daemons_own( + tmp_path, monkeypatch): + """The daemon starts in the home directory with `python -m`, which puts + that directory first on sys.path: a tempfile.py, random.py, json.py or + compartment.py kept there was imported instead of the real module, the + daemon died at start, and every agent loaded a model of its own. The + command the client builds is run here with --help, from such a home.""" + monkeypatch.setenv("HOME", str(tmp_path)) + (tmp_path / "tempfile.py").write_text( + "raise SystemExit('the home directory shadowed tempfile')\n") + built = {} + real_popen = subprocess.Popen + + def popen(cmd, **kwargs): + built.update(cmd=cmd, **kwargs) + raise OSError("not started in this test") + monkeypatch.setattr(subprocess, "Popen", popen) + with pytest.raises(OSError): + embed_daemon._spawn(tmp_path / "embed.sock", 1) + monkeypatch.setattr(subprocess, "Popen", real_popen) + assert built["cwd"] == str(tmp_path) + cmd = built["cmd"][:built["cmd"].index("--socket")] + ["--help"] + out = subprocess.run(cmd, cwd=built["cwd"], env=built["env"], + capture_output=True, text=True, timeout=120) + assert "shadowed" not in out.stdout + out.stderr + assert out.returncode == 0 and "--socket" in out.stdout + + +@pytest.mark.parametrize("error", [ + PermissionError(13, "Permission denied"), + RuntimeError("Could not determine home directory."), +]) +def test_no_session_directory_means_a_local_model_not_a_crash(monkeypatch, + capsys, error): + """The socket lives in the session directory. Where that cannot be made + (a read-only home) or there is no home at all (HOME unset and no + password-database entry, as for a container's arbitrary uid), the + daemon is given up and this process embeds with its own model; status, + stop and `embed-daemon run` say why instead of raising.""" + from compartment import session + + def no_dir(): + raise error + monkeypatch.setenv("COMPARTMENT_EMBED_DAEMON", "1") + monkeypatch.delenv("COMPARTMENT_EMBED_SOCKET", raising=False) + monkeypatch.setattr(session, "_session_dir", no_dir) + assert RemoteEmbedder.connect() is None + assert isinstance(get_embedder(), Embedder) + st = embed_daemon.status() + assert st["running"] is False and "session directory" in st["reason"] + assert embed_daemon.stop()["stopped"] is False + assert embed_daemon.run() == 1 + assert "--socket" in capsys.readouterr().err diff --git a/tests/test_expiry.py b/tests/test_expiry.py index d09c603..85d29ea 100644 --- a/tests/test_expiry.py +++ b/tests/test_expiry.py @@ -269,7 +269,7 @@ def test_with_the_toggle_on_an_expired_memory_is_never_handed_over(v): it, so the read paths filter even inside the hour before the sweep.""" dead = _store(v, "The old wifi password is hunter2.", expires=_day(-1), _expiry_strict=False) - v._last_expiry_sweep = 9e18 # pretend a sweep just ran + v._last_expiry_sweep = __import__("time").time() # a sweep just ran hits = v.search("wifi password", caller="t")["results"] assert not any(r["id"] == dead["id"] for r in hits) assert not any(r["id"] == dead["id"] @@ -308,6 +308,18 @@ def test_a_read_sweeps_at_most_once_an_hour(v, monkeypatch): assert len(calls) == 1, "swept twice inside the hour" +def test_a_clock_set_back_does_not_hold_the_sweep_off(v, monkeypatch): + """The last sweep ran by a clock that was ahead, or before a snapshot + was restored. Counted as moments ago, no read swept again until the + clock caught up, however far that was.""" + calls = [] + monkeypatch.setattr(Vault, "expire", + lambda self, caller="expiry": calls.append(caller)) + v._last_expiry_sweep = __import__("time").time() + 90 * 86400 + v.search("anything", caller="t") + assert len(calls) == 1 + + def test_a_sweep_that_fails_does_not_take_the_search_down(v, monkeypatch): def boom(self, caller="expiry"): raise RuntimeError("disk on fire") diff --git a/tests/test_integrations_claude_config_dir.py b/tests/test_integrations_claude_config_dir.py new file mode 100644 index 0000000..d627a97 --- /dev/null +++ b/tests/test_integrations_claude_config_dir.py @@ -0,0 +1,76 @@ +"""CLAUDE_CONFIG_DIR moves everything Claude Code keeps in ~/.claude: +settings.json, the per-project memory under projects/, skills/. People set +it to run a second account, or to keep the directory in a dotfiles tree +under a name that is not .claude.""" +import pytest + +from compartment import agent_skill, claude_hooks, claude_memory + + +@pytest.fixture() +def config_dir(monkeypatch, tmp_path): + """CLAUDE_CONFIG_DIR set, as it is in the environment Claude Code and + `compartment` are started with. SETTINGS and DEFAULT_ROOT are worked out + when used; a value an earlier test's monkeypatch left assigned on its + way out would pin them, so none is left in place for this one.""" + d = tmp_path / "dotfiles" / "claude-work" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(d)) + for module, name in ((claude_memory, "DEFAULT_ROOT"), + (claude_hooks, "SETTINGS")): + if name in vars(module): + monkeypatch.delattr(module, name) + return d + + +def test_the_hook_goes_into_the_settings_claude_code_reads(config_dir): + """The hook was installed into ~/.claude/settings.json whatever + CLAUDE_CONFIG_DIR said, a file that Claude Code then never read.""" + assert claude_hooks.SETTINGS == config_dir / "settings.json" + claude_hooks.install(compartment_bin="compartment") + assert claude_hooks.is_installed() is True + assert (config_dir / "settings.json").is_file() + + +def test_the_import_finds_the_memories_where_claude_code_keeps_them( + config_dir): + mem = config_dir / "projects" / "-home-ana-site" / "memory" + mem.mkdir(parents=True) + (mem / "likes-tea.md").write_text("---\nname: likes-tea\n---\nAna likes " + "tea.\n", encoding="utf-8") + (mem / claude_memory.INDEX_NAME).write_text("- likes-tea\n", + encoding="utf-8") + assert claude_memory.DEFAULT_ROOT == config_dir / "projects" + assert claude_memory.discover() == [mem / "likes-tea.md"] + + +def test_the_hook_captures_memory_written_under_the_config_dir(config_dir): + """is_memory_file wanted a `.claude` directory in the path, so with the + config directory named anything else every memory write was passed + over.""" + note = config_dir / "projects" / "-home-ana-site" / "memory" / "a.md" + assert claude_hooks.is_memory_file(note) is True + assert claude_hooks.is_memory_file( + config_dir / "projects" / "-home-ana-site" / "notes.md") is False + assert claude_hooks.is_memory_file( + config_dir.parent / "elsewhere" / "memory" / "a.md") is False + + +def test_the_skill_goes_where_claude_code_loads_skills(config_dir): + assert agent_skill.skill_path("claude") == ( + config_dir / "skills" / "compartmentalize" / "SKILL.md") + + +@pytest.mark.parametrize("value", ["", " ", "relative/dir"]) +def test_an_empty_or_relative_config_dir_is_ignored(value, monkeypatch, + tmp_path): + monkeypatch.setattr(claude_memory.Path, "home", + classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", value) + assert claude_memory.claude_config_dir() == tmp_path / ".claude" + + +def test_a_config_dir_under_home_is_expanded(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", "~/claude-alt") + assert claude_memory.claude_config_dir() == tmp_path / "claude-alt" diff --git a/tests/test_integrations_claude_desktop.py b/tests/test_integrations_claude_desktop.py new file mode 100644 index 0000000..9645358 --- /dev/null +++ b/tests/test_integrations_claude_desktop.py @@ -0,0 +1,114 @@ +"""Claude Desktop on Windows is an MSIX package now, from the Store or from +the direct download, and Windows keeps a packaged app's %APPDATA% writes in +the package's own LocalCache. The app reads that private copy first, so a +config written to the real %APPDATA%\\Claude is one it never sees.""" +import json + +import pytest + +from compartment import claude_desktop + +DIRECT, STORE = claude_desktop.MSIX_FAMILIES +ARGS = ["--vault", r"C:\Users\ana\.compartment\memory.vault", "--caller", + "claude-desktop", "serve"] + + +@pytest.fixture() +def windows(monkeypatch, tmp_path): + """A Windows profile under tmp_path: %APPDATA% and %LOCALAPPDATA%.""" + monkeypatch.setattr(claude_desktop, "_windows", lambda: True, + raising=False) + monkeypatch.setattr(claude_desktop.sys, "platform", "win32") + roaming = tmp_path / "AppData" / "Roaming" + local = tmp_path / "AppData" / "Local" + roaming.mkdir(parents=True) + local.mkdir(parents=True) + monkeypatch.setenv("APPDATA", str(roaming)) + monkeypatch.setenv("LOCALAPPDATA", str(local)) + + class Profile: + real = roaming / "Claude" / claude_desktop.FILENAME + + @staticmethod + def package(family): + (local / "Packages" / family).mkdir(parents=True, exist_ok=True) + return (local / "Packages" / family / "LocalCache" / "Roaming" + / "Claude" / claude_desktop.FILENAME) + + @staticmethod + def write(p, servers): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps({"mcpServers": servers}), encoding="utf-8") + + return Profile + + +def test_an_msix_install_is_written_where_the_app_reads( + windows, real_claude_desktop_config_path): + """The package had saved its settings to its private copy, so the app + no longer read %APPDATA%\\Claude, and that is where the entry went: + `integrate claude` reported Claude Desktop connected, and it never saw + the server.""" + private = windows.package(DIRECT) + windows.write(private, {"theirs": {"command": "npx"}}) + windows.write(windows.real, {"stale": {"command": "old"}}) + stale = windows.real.read_text(encoding="utf-8") + + p = real_claude_desktop_config_path() + assert p == private + claude_desktop.register(r"C:\x\compartment.exe", ARGS, path=p) + + servers = json.loads(private.read_text(encoding="utf-8"))["mcpServers"] + assert set(servers) == {"theirs", "compartment"} + assert windows.real.read_text(encoding="utf-8") == stale + + +def test_the_store_build_is_found_too(windows, + real_claude_desktop_config_path): + private = windows.package(STORE) + windows.write(private, {}) + assert real_claude_desktop_config_path() == private + + +def test_a_package_that_has_not_saved_yet_reads_the_real_file( + windows, real_claude_desktop_config_path): + """Until the package saves its own copy, Windows shows it the real file; + a private file holding only our server would hide every server in + it.""" + windows.package(DIRECT) + windows.write(windows.real, {"theirs": {"command": "npx"}}) + assert real_claude_desktop_config_path() == windows.real + + +def test_a_package_with_no_config_anywhere_gets_its_private_one( + windows, real_claude_desktop_config_path): + private = windows.package(DIRECT) + assert real_claude_desktop_config_path() == private + + +def test_of_two_packages_the_one_saved_last_is_written( + windows, real_claude_desktop_config_path): + import os + old, new = windows.package(DIRECT), windows.package(STORE) + windows.write(old, {}) + windows.write(new, {}) + os.utime(old, (1_000_000, 1_000_000)) + assert real_claude_desktop_config_path() == new + + +def test_without_a_package_the_regular_install_uses_appdata( + windows, real_claude_desktop_config_path): + assert real_claude_desktop_config_path() == windows.real + + +@pytest.mark.parametrize("installed", ["direct", "store", "squirrel"]) +def test_an_install_that_was_never_opened_is_present(windows, installed, + tmp_path): + """A package that has never run has no %APPDATA%\\Claude, so it was + reported as not installed and `integrate claude` skipped it.""" + assert claude_desktop.present() is False + if installed == "squirrel": + (tmp_path / "AppData" / "Local" / "AnthropicClaude").mkdir() + else: + windows.package(DIRECT if installed == "direct" else STORE) + assert claude_desktop.present() is True diff --git a/tests/test_integrations_claude_memory.py b/tests/test_integrations_claude_memory.py new file mode 100644 index 0000000..713abf4 --- /dev/null +++ b/tests/test_integrations_claude_memory.py @@ -0,0 +1,77 @@ +"""Claude Code's auto memory, wherever Claude Code keeps it.""" +import json + +import pytest + +from compartment import claude_hooks, claude_memory + + +@pytest.fixture() +def claude_home(tmp_path, monkeypatch): + root = tmp_path / ".claude" / "projects" + root.mkdir(parents=True) + monkeypatch.setattr(claude_memory, "DEFAULT_ROOT", root) + return tmp_path / ".claude" + + +def _settings(claude_home, **values): + (claude_home / "settings.json").write_text(json.dumps(values), + encoding="utf-8") + + +def _note(d, name, body="Ana prefers tea."): + d.mkdir(parents=True, exist_ok=True) + p = d / name + p.write_text(f"---\nname: {p.stem}\ndescription: x\nmetadata:\n type: " + f"user\n---\n{body}\n", encoding="utf-8") + return p + + +def test_memory_in_the_auto_memory_directory_is_found_and_captured( + claude_home, tmp_path): + """autoMemoryDirectory (Claude Code 2.1.74) moves auto memory out of + projects//memory to any directory. Notes written there were + invisible to `import-claude` and to the capture hook.""" + custom = tmp_path / "notes" / "claude" + _settings(claude_home, autoMemoryDirectory=str(custom)) + moved = _note(custom, "tea.md") + (custom / claude_memory.INDEX_NAME).write_text("- tea\n", + encoding="utf-8") + usual = _note(claude_home / "projects" / "p" / "memory", "old.md") + + assert claude_memory.auto_memory_dir() == custom + assert claude_memory.discover() == [usual, moved] + assert claude_hooks.is_memory_file(moved) is True + assert claude_hooks.is_memory_file(custom / claude_memory.INDEX_NAME) is ( + False) + assert claude_hooks.is_memory_file(tmp_path / "notes" / "x.md") is False + + +def test_an_auto_memory_directory_under_home_is_expanded(claude_home, + tmp_path, + monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + _settings(claude_home, autoMemoryDirectory="~/mem") + assert claude_memory.auto_memory_dir() == tmp_path / "mem" + + +@pytest.mark.parametrize("value", [None, "", "relative/mem", 3]) +def test_no_or_unusable_auto_memory_directory_changes_nothing(claude_home, + value): + _settings(claude_home, **({} if value is None + else {"autoMemoryDirectory": value})) + assert claude_memory.auto_memory_dir() is None + + +def test_a_note_saved_with_a_byte_order_mark_keeps_its_frontmatter( + tmp_path): + """The mark hid the opening ---, so the frontmatter was not recognised + and the whole block was stored as the fact, tagged with nothing.""" + p = tmp_path / "memory" / "tea.md" + p.parent.mkdir() + p.write_bytes(b"\xef\xbb\xbf---\nname: tea\ndescription: Ana likes tea" + b"\nmetadata:\n type: user\n---\nGreen, no sugar.\n") + rec = claude_memory.parse(p) + assert rec["name"] == "tea" and "user" in rec["tags"] + assert "---" not in rec["text"] and "" not in rec["text"] diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py new file mode 100644 index 0000000..a866c2e --- /dev/null +++ b/tests/test_integrations_clients.py @@ -0,0 +1,348 @@ +"""Where each client reads its MCP servers today. + +Every case puts a fake home directory under tmp_path and lays out only what +the client itself would have created, then checks that `integrate` finds +the client and writes the file that client reads.""" +import json +import os +import sys + +import pytest + +from compartment import clients + +VAULT = "/v/memory.vault" + + +@pytest.fixture() +def home(tmp_path, monkeypatch): + h = tmp_path / "home" + h.mkdir() + monkeypatch.setattr(clients.Path, "home", classmethod(lambda cls: h)) + monkeypatch.setenv("APPDATA", str(h / "AppData" / "Roaming")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(h / ".config")) + for var in ("CLINE_DIR", "CLINE_DATA_DIR", "CLINE_MCP_SETTINGS_PATH", + "CODEX_HOME", "GEMINI_CLI_HOME", "QWEN_HOME", "COPILOT_HOME", + "OPENCODE_CONFIG_DIR", "CRUSH_GLOBAL_CONFIG", "KIRO_HOME", + "GOOSE_PATH_ROOT", "CONTINUE_GLOBAL_DIR"): + monkeypatch.delenv(var, raising=False) + return h + + +def _editor_storage(editor, ext): + """An extension's globalStorage directory in `editor`, as the editor + creates it the first time the extension runs.""" + d = clients._vscode_user(editor) / "globalStorage" / ext + d.mkdir(parents=True) + return d + + +def _servers(p, root="mcpServers"): + return clients._plain(clients._jsonc_parse( + p.read_text(encoding="utf-8")))[root] + + +# ------------------------------------------------------------------ Cline + +def test_cline_4_is_written_in_its_shared_settings_file(home): + """Cline 4 reads ~/.cline/data/settings/cline_mcp_settings.json and + imports the old globalStorage file once, so a server written there + afterwards was never seen: `integrate cline` said it was wired.""" + (home / ".cline").mkdir() + _editor_storage("Code", "saoudrizwan.claude-dev") + cline = clients.CLIENTS["cline"] + + assert cline.present() is True + want = home / ".cline" / "data" / "settings" / "cline_mcp_settings.json" + assert cline.config_path() == want + assert clients.register(cline, VAULT)["written"] is True + assert "compartment" in _servers(want) + + +def test_cline_honours_its_directory_variables(home, monkeypatch, tmp_path): + cline = clients.CLIENTS["cline"] + monkeypatch.setenv("CLINE_DIR", str(tmp_path / "c")) + assert cline.config_path() == (tmp_path / "c" / "data" / "settings" + / "cline_mcp_settings.json") + monkeypatch.setenv("CLINE_DATA_DIR", str(tmp_path / "d")) + assert cline.config_path() == (tmp_path / "d" / "settings" + / "cline_mcp_settings.json") + monkeypatch.setenv("CLINE_MCP_SETTINGS_PATH", str(tmp_path / "m.json")) + assert cline.config_path() == tmp_path / "m.json" + + +def test_an_older_cline_in_cursor_is_found_and_written_there(home): + """Only VS Code's own storage was looked at, so Cline installed in + Cursor (or Windsurf, VSCodium, Insiders) was reported as not installed + and wiring it wrote into a VS Code that was not there.""" + storage = _editor_storage("Cursor", "saoudrizwan.claude-dev") + cline = clients.CLIENTS["cline"] + assert cline.present() is True + assert cline.config_path() == (storage / "settings" + / "cline_mcp_settings.json") + + +def test_roo_code_in_windsurf_is_found(home): + storage = _editor_storage("Windsurf", "rooveterinaryinc.roo-cline") + roo = clients.CLIENTS["roo"] + assert roo.present() is True + assert roo.config_path() == storage / "settings" / "mcp_settings.json" + + +@pytest.mark.skipif(sys.platform == "darwin" or os.name == "nt", + reason="Flatpak is Linux") +def test_an_extension_in_flatpak_vs_code_is_found(home): + storage = (home / ".var" / "app" / "com.visualstudio.code" / "config" + / "Code" / "User" / "globalStorage" + / "rooveterinaryinc.roo-cline") + storage.mkdir(parents=True) + assert clients.CLIENTS["roo"].config_path().parent.parent == storage + + +# ------------------------------------------------------------------- Kilo + +def test_kilo_code_7_gets_an_opencode_style_entry_in_its_global_config(home): + """Kilo Code 7 reads MCP servers from ~/.config/kilo/kilo.jsonc under + "mcp"; the old mcp_settings.json in VS Code's storage is read only for + migration, and never by the Kilo CLI.""" + kilo_dir = home / ".config" / "kilo" + kilo_dir.mkdir(parents=True) + kilo = clients.CLIENTS["kilo"] + assert kilo.present() is True + assert kilo.config_path() == kilo_dir / "kilo.jsonc" + + clients.register(kilo, VAULT) + entry = _servers(kilo_dir / "kilo.jsonc", "mcp")["compartment"] + assert entry["type"] == "local" and entry["enabled"] is True + assert entry["command"][1:3] == ["--vault", VAULT] + + +def test_kilo_code_writes_the_config_file_it_already_has(home): + kilo_dir = home / ".config" / "kilo" + kilo_dir.mkdir(parents=True) + (kilo_dir / "opencode.json").write_text( + '{\n // mine\n "model": "x",\n}\n', encoding="utf-8") + kilo = clients.CLIENTS["kilo"] + assert kilo.config_path() == kilo_dir / "opencode.json" + assert clients.register(kilo, VAULT)["written"] is True + text = (kilo_dir / "opencode.json").read_text(encoding="utf-8") + assert "// mine" in text and '"compartment"' in text + + +def test_kilo_code_installed_in_an_editor_but_never_run_is_present(home): + _editor_storage("VSCodium", "kilocode.kilo-code") + assert clients.CLIENTS["kilo"].present() is True + + +# ------------------------------------------ homes moved by an environment + +@pytest.mark.parametrize("key, var, expect", [ + ("codex", "CODEX_HOME", "{d}/config.toml"), + ("gemini", "GEMINI_CLI_HOME", "{d}/.gemini/settings.json"), + ("qwen", "QWEN_HOME", "{d}/settings.json"), + ("copilot-cli", "COPILOT_HOME", "{d}/mcp-config.json"), + ("opencode", "OPENCODE_CONFIG_DIR", "{d}/opencode.json"), +]) +def test_a_home_moved_by_its_variable_is_followed(key, var, expect, home, + monkeypatch, tmp_path): + """Each of these moves its whole configuration directory with one + variable; the default was written whatever it said, a file the client + never read, and `--list` called a client with only the moved directory + not installed.""" + d = tmp_path / "moved" + d.mkdir() + monkeypatch.setenv(var, str(d)) + c = clients.CLIENTS[key] + assert str(c.config_path()) == expect.format(d=d).replace("/", os.sep) + if key == "gemini": + (d / ".gemini").mkdir() + assert c.present() is True + assert clients.register(c, VAULT)["written"] is True + assert clients.is_registered(c) is True + + +def test_an_opencode_jsonc_is_written_rather_than_split_in_two(home): + d = home / ".config" / "opencode" + d.mkdir(parents=True) + (d / "opencode.jsonc").write_text('{\n // theme\n "theme": "x"\n}\n', + encoding="utf-8") + oc = clients.CLIENTS["opencode"] + assert oc.config_path() == d / "opencode.jsonc" + assert clients.register(oc, VAULT)["written"] is True + assert not (d / "opencode.json").exists() + assert "// theme" in (d / "opencode.jsonc").read_text(encoding="utf-8") + + +def test_gemini_settings_with_comments_are_written(home): + """Gemini CLI and Qwen Code strip comments before parsing settings.""" + p = home / ".gemini" / "settings.json" + p.parent.mkdir() + p.write_text('{\n // dark mode\n "theme": "Dracula"\n}\n', + encoding="utf-8") + assert clients.register(clients.CLIENTS["gemini"], VAULT)["written"] + assert "// dark mode" in p.read_text(encoding="utf-8") + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS rule") +def test_zed_on_macos_ignores_xdg_config_home(home, monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + assert clients.CLIENTS["zed"].config_path() == ( + home / ".config" / "zed" / "settings.json") + + +def test_lm_studio_is_found_where_its_home_pointer_says(home, tmp_path): + """LM Studio records its home in ~/.lmstudio-home-pointer, which is how + a home moved in its settings (to another disk, say) is found.""" + moved = tmp_path / "Volumes" / "Big" / "lmstudio" + moved.mkdir(parents=True) + (home / ".lmstudio-home-pointer").write_text(f"{moved}\n", + encoding="utf-8") + lm = clients.CLIENTS["lmstudio"] + assert lm.config_path() == moved / "mcp.json" + assert lm.present() is True + + +def test_lm_studio_without_a_pointer_uses_its_default_home(home): + assert clients.CLIENTS["lmstudio"].config_path() == ( + home / ".lmstudio" / "mcp.json") + (home / ".cache" / "lm-studio").mkdir(parents=True) + assert clients.CLIENTS["lmstudio"].config_path() == ( + home / ".cache" / "lm-studio" / "mcp.json") + + +# --------------------------------------------------------------- Continue + +def test_continue_is_wired_with_a_file_of_its_own(home): + """Continue was print-only, since its config.yaml is YAML with comments. + It also loads every JSON file in ~/.continue/mcpServers in the Claude + Desktop shape, so it gets a file of its own and nothing of the user's + is edited.""" + cont = home / ".continue" + cont.mkdir() + (cont / "config.yaml").write_text("# mine\nname: x\n", encoding="utf-8") + c = clients.CLIENTS["continue"] + assert c.writes is True and c.present() is True + + res = clients.register(c, VAULT) + assert res["written"] is True + p = cont / "mcpServers" / "compartment.json" + assert json.loads(p.read_text(encoding="utf-8"))["mcpServers"][ + "compartment"]["args"][:2] == ["--vault", VAULT] + assert (cont / "config.yaml").read_text(encoding="utf-8") == ( + "# mine\nname: x\n") + assert clients.unregister(c) is True + + +def test_continue_follows_its_global_dir_variable(home, monkeypatch, + tmp_path): + monkeypatch.setenv("CONTINUE_GLOBAL_DIR", str(tmp_path / "cont")) + assert clients.CLIENTS["continue"].config_path() == ( + tmp_path / "cont" / "mcpServers" / "compartment.json") + assert clients.CLIENTS["continue"].present() is False + + +# ------------------------------------------------------------------ Goose + +def test_the_goose_block_is_an_extension_goose_reads(home): + """Goose was handed `{"mcpServers": ...}` JSON, a shape Goose's + config.yaml does not have: its servers are extensions, with `cmd`, not + `command`, and a `type`.""" + goose = clients.CLIENTS["goose"] + block = clients.snippet(goose, "/Users/ana/My Vault/mémoire.vault") + lines = block.splitlines() + assert lines[:3] == ["extensions:", " compartment:", " enabled: true"] + assert " type: stdio" in lines and " name: compartment" in lines + cmd = next(x for x in lines if x.startswith(" cmd: ")) + args = next(x for x in lines if x.startswith(" args: ")) + # Double-quoted YAML scalars are JSON strings. + assert json.loads(cmd.split(": ", 1)[1]) == clients.executable() + assert json.loads(args.split(": ", 1)[1])[:2] == [ + "--vault", "/Users/ana/My Vault/mémoire.vault"] + assert "mcpServers" not in block + + +class _WindowsOs: + """`os` as clients.py sees it on Windows, without telling pathlib.""" + name = "nt" + + def __getattr__(self, attr): + return getattr(os, attr) + + +def test_goose_on_windows_is_under_appdata_block(home, monkeypatch): + """The XDG path was printed on Windows too, a file Goose never reads: + there its config is %APPDATA%\\Block\\goose\\config\\config.yaml.""" + monkeypatch.setattr(clients, "os", _WindowsOs()) + p = clients.CLIENTS["goose"].config_path() + assert p == (home / "AppData" / "Roaming" / "Block" / "goose" / "config" + / "config.yaml") + + +def test_goose_path_root_moves_its_config(home, monkeypatch, tmp_path): + monkeypatch.delenv("GOOSE_PATH_ROOT", raising=False) + default = clients.CLIENTS["goose"].config_path() # per-OS default + monkeypatch.setenv("GOOSE_PATH_ROOT", str(tmp_path / "g")) + assert clients.CLIENTS["goose"].config_path() == ( + tmp_path / "g" / "config" / "config.yaml") + monkeypatch.setenv("GOOSE_PATH_ROOT", "relative") # ignored by Goose + assert clients.CLIENTS["goose"].config_path() == default + + +# ---------------------------------------------------------- newer clients + +@pytest.mark.parametrize("key, rel, root, typed", [ + ("kiro", ".kiro/settings/mcp.json", "mcpServers", False), + ("amazon-q", ".aws/amazonq/mcp.json", "mcpServers", False), + ("junie", ".junie/mcp/mcp.json", "mcpServers", False), + ("factory", ".factory/mcp.json", "mcpServers", False), + ("crush", ".config/crush/crush.json", "mcp", True), +]) +def test_newer_clients_are_known_and_wired(key, rel, root, typed, home): + c = clients.CLIENTS[key] + p = home.joinpath(*rel.split("/")) + assert c.config_path() == p + assert c.present() is False + p.parent.mkdir(parents=True) + p.write_text(json.dumps({root: {"theirs": {"command": "x"}}}), + encoding="utf-8") + assert c.present() is True + assert clients.register(c, VAULT)["written"] is True + servers = json.loads(p.read_text(encoding="utf-8"))[root] + assert servers["theirs"] == {"command": "x"} + assert servers["compartment"]["args"][:2] == ["--vault", VAULT] + assert ("type" in servers["compartment"]) is typed + + +def test_kiro_and_crush_follow_their_variables(home, monkeypatch, tmp_path): + monkeypatch.setenv("KIRO_HOME", str(tmp_path / "k")) + monkeypatch.setenv("CRUSH_GLOBAL_CONFIG", str(tmp_path / "c")) + assert clients.CLIENTS["kiro"].config_path() == ( + tmp_path / "k" / "settings" / "mcp.json") + assert clients.CLIENTS["crush"].config_path() == ( + tmp_path / "c" / "crush.json") + + +def test_junie_is_present_before_its_first_server(home): + (home / ".junie").mkdir() + assert clients.CLIENTS["junie"].present() is True + assert clients.resolve("droid").key == "factory" + assert clients.resolve("amazonq").key == "amazon-q" + + +def test_flathub_vs_code_on_linux_is_found_and_written(home, monkeypatch): + """The Flathub VS Code keeps its User directory in its sandbox, + ~/.var/app/com.visualstudio.code/config/Code/User; only the native + path was looked at, so it was not found and never written.""" + monkeypatch.setattr(clients.sys, "platform", "linux") + if os.name == "nt": + pytest.skip("POSIX layout") + user = (home / ".var" / "app" / "com.visualstudio.code" / "config" + / "Code" / "User") + user.mkdir(parents=True) + vscode = clients.CLIENTS["vscode"] + assert vscode.config_path() == user / "mcp.json" + assert vscode.present() is True + # A native install, where there is one, is still the one written. + (home / ".config" / "Code" / "User").mkdir(parents=True) + assert vscode.config_path() == (home / ".config" / "Code" / "User" + / "mcp.json") diff --git a/tests/test_integrations_codex.py b/tests/test_integrations_codex.py new file mode 100644 index 0000000..9077a68 --- /dev/null +++ b/tests/test_integrations_codex.py @@ -0,0 +1,134 @@ +"""Codex reads one TOML file, and refuses to start at all when it cannot +read it. Whatever `integrate codex` appends has to leave a document Codex +can read, with every other setting in it exactly as it was.""" +import tomllib + +import pytest + +from compartment import clients + +CODEX = clients.CLIENTS["codex"] + + +@pytest.mark.parametrize("vault", [ + "/Users/ana/Vaults \U0001F512/memory.vault", # an emoji in a folder + "/home/\U00020BB7\u5409/memory.vault", # CJK Extension B + "C:\\Users\\Jos\u00e9\\.compartment\\memory.vault", + '/tmp/a "quoted"\tname\x7f/memory.vault', +]) +def test_the_codex_table_reads_back_whatever_the_vault_path_holds(vault, + tmp_path): + """The values were escaped with json.dumps, which writes a character + outside the Basic Multilingual Plane as a UTF-16 surrogate pair; TOML + allows only scalar values in an escape, so a vault in a folder with an + emoji in its name left config.toml unreadable and Codex would not + start.""" + p = tmp_path / "config.toml" + p.write_text('model = "o3"\n', encoding="utf-8") + + res = clients.register(CODEX, vault, path=p) + + assert res["written"] is True, res["reason"] + doc = tomllib.loads(p.read_text(encoding="utf-8")) + assert doc["model"] == "o3" + assert doc["mcp_servers"]["compartment"]["args"][1] == vault + assert tomllib.loads(clients.snippet(CODEX, vault))["mcp_servers"][ + "compartment"]["args"][1] == vault + + +@pytest.mark.parametrize("existing", [ + 'mcp_servers = { other = { command = "x" } }\n', + 'mcp_servers = []\n', + '[[mcp_servers]]\nname = "x"\ncommand = "y"\n', +]) +def test_a_config_a_table_cannot_be_added_to_is_left_alone(existing, + tmp_path): + """An inline `mcp_servers = { ... }` cannot be extended by a + [mcp_servers.compartment] header, so the append turned a working config + into one Codex could not read, every server in it gone with it.""" + p = tmp_path / "config.toml" + p.write_text(existing, encoding="utf-8") + + res = clients.register(CODEX, "/v/memory.vault", path=p) + + assert res["written"] is False + assert "refusing to touch it" in res["reason"] + assert "[mcp_servers.compartment]" in res["snippet"] + assert p.read_text(encoding="utf-8") == existing + assert not p.with_name(p.name + clients.BACKUP_SUFFIX).exists() + + +def test_dotted_keys_for_another_server_still_take_the_table(tmp_path): + p = tmp_path / "config.toml" + p.write_text('mcp_servers.other.command = "x"\n', encoding="utf-8") + res = clients.register(CODEX, "/v/memory.vault", path=p) + assert res["written"] is True, res["reason"] + servers = tomllib.loads(p.read_text(encoding="utf-8"))["mcp_servers"] + assert servers["other"] == {"command": "x"} and "compartment" in servers + + +def _registered(p): + return tomllib.loads(p.read_text(encoding="utf-8"))["mcp_servers"][ + "compartment"] + + +def test_registering_codex_again_refreshes_a_stale_entry(tmp_path, + monkeypatch): + """Re-registering said "already registered; left as it is" whatever the + entry held, so an entry naming a bare `compartment` that Codex could + not find, a script an upgrade had moved, or another vault was never + put right by running `integrate codex` again, which is what every JSON + client does.""" + p = tmp_path / "config.toml" + p.write_text( + '# my settings\nmodel = "o3"\n\n' + '[mcp_servers.compartment]\ncommand = "compartment"\n' + 'args = ["--vault", "/old.vault", "--caller", "codex", "serve"]\n' + 'startup_timeout_sec = 30\n\n' + '[mcp_servers.compartment.env]\nCOMPARTMENT_EMBED_DAEMON = "0"\n\n' + "# Bob's server\n[mcp_servers.bob]\ncommand = \"bob\"\n", + encoding="utf-8") + monkeypatch.setattr(clients, "executable", lambda: "/opt/bin/compartment") + + res = clients.register(CODEX, "/new.vault", path=p) + + assert res["written"] is True, res["reason"] + text = p.read_text(encoding="utf-8") + doc = tomllib.loads(text) + ours = doc["mcp_servers"]["compartment"] + assert ours["command"] == "/opt/bin/compartment" + assert ours["args"][:2] == ["--vault", "/new.vault"] + # What the user added to our table stays with it. + assert ours["startup_timeout_sec"] == 30 + assert ours["env"] == {"COMPARTMENT_EMBED_DAEMON": "0"} + assert doc["mcp_servers"]["bob"] == {"command": "bob"} + assert doc["model"] == "o3" + assert text.startswith("# my settings\n") and "# Bob's server" in text + assert text.count("[mcp_servers.compartment]") == 1 + backup = p.with_name(p.name + clients.BACKUP_SUFFIX) + assert "/old.vault" in backup.read_text(encoding="utf-8") + + # Refreshing again changes nothing and does not grow the file. + again = clients.register(CODEX, "/new.vault", path=p) + assert again["written"] is False + assert again["reason"] == "already registered; left as it is" + assert p.read_text(encoding="utf-8") == text + monkeypatch.setattr(clients, "executable", lambda: "/usr/bin/compartment") + clients.register(CODEX, "/new.vault", path=p) + clients.register(CODEX, "/other.vault", path=p) + after = p.read_text(encoding="utf-8") + assert "\n\n\n" not in after + assert _registered(p)["args"][1] == "/other.vault" + + +def test_a_stale_codex_entry_that_cannot_be_taken_out_is_left_for_the_user( + tmp_path): + p = tmp_path / "config.toml" + text = ('[mcp_servers]\ncompartment = { command = "compartment", ' + 'args = ["serve"] }\nother = { command = "x" }\n') + p.write_text(text, encoding="utf-8") + res = clients.register(CODEX, "/v/memory.vault", path=p) + assert res["written"] is False + assert "left as it is" in res["reason"] + assert "[mcp_servers.compartment]" in res["snippet"] + assert p.read_text(encoding="utf-8") == text diff --git a/tests/test_integrations_command.py b/tests/test_integrations_command.py new file mode 100644 index 0000000..887285b --- /dev/null +++ b/tests/test_integrations_command.py @@ -0,0 +1,150 @@ +"""The command written into every client's config has to start the server +from a process that has none of the user's shell: a GUI app launched from +the Dock or the Start menu, or the panel's own child process. + +These are about where that command is found, for the ways Compartment is +actually installed: `pip install --user` (what the install scripts run), a +venv, and a package manager that keeps a link on PATH to a versioned tree. +""" +import sys +import sysconfig + +import pytest + +from compartment import clients + + +def _fake_schemes(monkeypatch, tmp_path, installed_in: str): + """Three install schemes whose directories are under tmp_path, and this + package imported from the library directory of `installed_in`.""" + layout = {} + for scheme in ("venv", "posix_prefix", "posix_user", "osx_framework_user", + "nt", "nt_user"): + lib = tmp_path / scheme / "site-packages" + layout[scheme] = {"purelib": str(lib), "platlib": str(lib), + "scripts": str(tmp_path / scheme / "bin")} + + def get_paths(scheme=None, vars=None, expand=True): + if scheme not in layout: + raise KeyError(scheme) + return dict(layout[scheme]) + + lib = tmp_path / installed_in / "site-packages" + lib.mkdir(parents=True) + monkeypatch.setattr(sysconfig, "get_paths", get_paths) + monkeypatch.setattr(clients, "_package_root", lambda: lib.resolve(), + raising=False) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "interpreter")) + monkeypatch.setattr(sys, "executable", + str(tmp_path / "interpreter" / "bin" / "python3")) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: None) + return {k: tmp_path / k / "bin" for k in layout} + + +def _script(directory, name="compartment"): + directory.mkdir(parents=True, exist_ok=True) + s = directory / name + s.write_text("#!/bin/sh\n", encoding="utf-8") + return s + + +@pytest.mark.parametrize("platform", ["macos", "windows", "linux"]) +def test_a_user_install_names_the_script_pip_wrote(platform, tmp_path, + monkeypatch): + """`pip install --user` (every install script) puts the script in the + user scheme's bin or Scripts directory, beside no interpreter and on no + PATH a Dock-launched app or the panel's child has. Only the interpreter's + own directories and PATH were looked in, so the bare name was written, + and every GUI client failed with "spawn compartment ENOENT".""" + user = {"macos": "osx_framework_user", "windows": "nt_user", + "linux": "posix_user"}[platform] + monkeypatch.setattr(sysconfig, "get_default_scheme", + lambda: "nt" if platform == "windows" + else "posix_prefix") + monkeypatch.setattr(sysconfig, "get_preferred_scheme", + lambda key: user if key == "user" + else ("nt" if platform == "windows" + else "posix_prefix")) + bins = _fake_schemes(monkeypatch, tmp_path, installed_in=user) + name = "compartment.exe" if platform == "windows" else "compartment" + script = _script(bins[user], name) + # A stale copy in a scheme this package is not installed in is not it. + _script(bins["posix_prefix" if platform != "windows" else "nt"], name) + + assert clients.executable() == str(script) + + +def test_a_venv_install_is_still_named_by_its_venv_script(tmp_path, + monkeypatch): + monkeypatch.setattr(sysconfig, "get_default_scheme", lambda: "venv") + monkeypatch.setattr(sysconfig, "get_preferred_scheme", + lambda key: "posix_user" if key == "user" else "venv") + bins = _fake_schemes(monkeypatch, tmp_path, installed_in="venv") + script = _script(bins["venv"]) + _script(bins["posix_user"]) # an older --user copy, not this one + assert clients.executable() == str(script) + + +def _symlink_or_skip(link, target): + link.parent.mkdir(parents=True, exist_ok=True) + try: + link.symlink_to(target) + except (OSError, NotImplementedError) as exc: + pytest.skip(f"no symlinks here ({exc})") + + +def test_a_package_managers_link_is_written_instead_of_its_versioned_tree( + tmp_path, monkeypatch): + """Homebrew runs a Python formula from Cellar///libexec, + so the script beside the interpreter has the version in its path and + is gone after the next `brew upgrade` and cleanup: every client config + written before then pointed at a file that no longer existed. The link + Homebrew keeps in its bin directory is repointed by every upgrade.""" + prefix = tmp_path / "Cellar" / "compartment" / "4.12.0" / "libexec" + script = _script(prefix / "bin") + link = tmp_path / "homebrew" / "bin" / "compartment" + _symlink_or_skip(link, script) + monkeypatch.setattr(clients, "_package_root", + lambda: tmp_path / "not-a-scheme", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(clients, "_link_dirs", lambda: [], raising=False) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: str(link)) + + assert clients.executable() == str(link) + + +def test_a_link_to_some_other_install_is_not_taken_for_this_one( + tmp_path, monkeypatch): + """A `compartment` on PATH that is another copy (an old --user install + while this one runs from a venv) is not the one to name.""" + prefix = tmp_path / "venv" + script = _script(prefix / "bin") + other = _script(tmp_path / "old-install" / "bin") + link = tmp_path / "local" / "bin" / "compartment" + _symlink_or_skip(link, other) + monkeypatch.setattr(clients, "_package_root", + lambda: tmp_path / "not-a-scheme", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(clients, "_link_dirs", lambda: [link.parent], + raising=False) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: str(link)) + + assert clients.executable() == str(script) + + +def test_a_link_in_a_package_managers_bin_off_path_is_found(tmp_path, + monkeypatch): + """The panel's child process has the login shell's PATH at best, which + may not hold ~/.local/bin; pipx's and uv's link there is found anyway.""" + prefix = tmp_path / "pipx" / "venvs" / "compartment" + script = _script(prefix / "bin") + link = tmp_path / "home" / ".local" / "bin" / "compartment" + _symlink_or_skip(link, script) + monkeypatch.setattr(clients, "_package_root", + lambda: tmp_path / "not-a-scheme", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(clients.Path, "home", + classmethod(lambda cls: tmp_path / "home")) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: None) + + assert clients.executable() == str(link) diff --git a/tests/test_integrations_encodings.py b/tests/test_integrations_encodings.py new file mode 100644 index 0000000..ea9929c --- /dev/null +++ b/tests/test_integrations_encodings.py @@ -0,0 +1,142 @@ +"""Config files as Windows editors save them. + +Notepad before Windows 10 1903 and PowerShell 5's `Out-File -Encoding utf8` +put a UTF-8 byte order mark at the front of the file; PowerShell 5's `>` and +plain `Out-File` write UTF-16. The Claude Desktop and Cursor guides send +Windows users to exactly those tools, so these are real files on real +machines, and each of them has another server in it that must survive. +""" +import codecs +import json +import tomllib + +import pytest + +from compartment import claude_desktop, claude_hooks, clients + +VAULT = r"C:\Users\Ana\.compartment\memory.vault" +THEIRS = {"mcpServers": {"theirs": {"command": "npx", "args": ["-y", "x"]}}, + "globalShortcut": "Ctrl+Space"} + +ENCODINGS = { + "utf-8 with a mark (Notepad, Out-File -Encoding utf8)": + lambda t: codecs.BOM_UTF8 + t.encode("utf-8"), + "utf-16 (PowerShell 5's > and Out-File)": + lambda t: t.encode("utf-16"), # BOM + little-endian + "utf-16 big-endian with a mark": + lambda t: codecs.BOM_UTF16_BE + t.encode("utf-16-be"), + "utf-8 with CRLF line endings": + lambda t: t.replace("\n", "\r\n").encode("utf-8"), +} + + +def _write(path, text, how): + path.parent.mkdir(parents=True, exist_ok=True) + raw = ENCODINGS[how](text) + path.write_bytes(raw) + return raw + + +def _is_plain_utf8(raw: bytes) -> bool: + return not raw.startswith((codecs.BOM_UTF8, codecs.BOM_UTF16_LE, + codecs.BOM_UTF16_BE)) and b"\r\r" not in raw + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_a_client_config_saved_by_a_windows_editor_is_registered(how, + tmp_path): + """The mark is not whitespace to json.loads, so a marked file was + refused as "not valid JSON", and a UTF-16 one did not decode at all: + the user was handed a block to paste into the file just refused.""" + p = tmp_path / ".cursor" / "mcp.json" + before = _write(p, json.dumps(THEIRS, indent=2) + "\n", how) + + res = clients.register(clients.CLIENTS["cursor"], VAULT, path=p) + + assert res["written"] is True, res["reason"] + raw = p.read_bytes() + assert _is_plain_utf8(raw) + data = json.loads(raw.decode("utf-8")) + assert data["mcpServers"]["theirs"] == THEIRS["mcpServers"]["theirs"] + assert data["globalShortcut"] == "Ctrl+Space" + assert data["mcpServers"]["compartment"]["args"][1] == VAULT + assert open(res["backup"], "rb").read() == before # byte-exact + assert clients.is_registered(clients.CLIENTS["cursor"], path=p) + assert clients.unregister(clients.CLIENTS["cursor"], path=p) is True + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_claude_desktops_config_saved_by_a_windows_editor_is_registered( + how, tmp_path): + """Claude Desktop's own guide has Windows users open the file in + Notepad. Its reader refused a marked file, and a UTF-16 one escaped + as a bare UnicodeDecodeError.""" + p = tmp_path / "Claude" / claude_desktop.FILENAME + _write(p, json.dumps(THEIRS, indent=2) + "\n", how) + + claude_desktop.register(r"C:\x\compartment.exe", ["serve"], path=p) + + raw = p.read_bytes() + assert _is_plain_utf8(raw) + data = json.loads(raw.decode("utf-8")) + assert set(data["mcpServers"]) == {"theirs", "compartment"} + assert data["mcpServers"]["compartment"]["command"] == ( + r"C:\x\compartment.exe") + assert claude_desktop.is_registered(path=p) is True + assert claude_desktop.unregister(path=p) is True + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_claude_code_settings_saved_by_a_windows_editor_get_the_hook( + how, tmp_path): + p = tmp_path / ".claude" / "settings.json" + _write(p, json.dumps({"permissions": {"allow": ["Bash"]}}) + "\n", how) + + claude_hooks.install(compartment_bin=r"C:\x\compartment.exe", + settings=p) + + raw = p.read_bytes() + assert _is_plain_utf8(raw) + data = json.loads(raw.decode("utf-8")) + assert data["permissions"] == {"allow": ["Bash"]} + assert claude_hooks.is_installed(settings=p) is True + assert claude_hooks.uninstall(settings=p) is True + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_a_codex_config_saved_by_a_windows_editor_is_registered(how, + tmp_path): + """Codex's TOML parser skips a leading mark; tomllib does not, so a + config Codex reads happily was refused as "not valid TOML".""" + p = tmp_path / ".codex" / "config.toml" + _write(p, '# mine\nmodel = "o3"\n\n[mcp_servers.other]\ncommand = "x"\n', + how) + codex = clients.CLIENTS["codex"] + + res = clients.register(codex, VAULT, path=p) + + assert res["written"] is True, res["reason"] + raw = p.read_bytes() + assert _is_plain_utf8(raw) + doc = tomllib.loads(raw.decode("utf-8")) + assert doc["model"] == "o3" and doc["mcp_servers"]["other"] == { + "command": "x"} + assert doc["mcp_servers"]["compartment"]["args"][1] == VAULT + text = raw.decode("utf-8") + assert text.replace("\r\n", "\n").startswith("# mine\n") + # The file keeps the line endings it had, on every system. + assert ("\r\n" in text) == ("CRLF" in how) + assert clients.is_registered(codex, path=p) is True + assert clients.unregister(codex, path=p) is True + assert "compartment" not in tomllib.loads( + p.read_bytes().decode("utf-8"))["mcp_servers"] + + +def test_a_file_that_is_not_text_in_any_encoding_is_still_refused(tmp_path): + p = tmp_path / ".cursor" / "mcp.json" + p.parent.mkdir() + p.write_bytes(b"\x00\xff\xfe\x80 not text") + before = p.read_bytes() + res = clients.register(clients.CLIENTS["cursor"], VAULT, path=p) + assert res["written"] is False and "refusing to touch it" in res["reason"] + assert p.read_bytes() == before diff --git a/tests/test_integrations_homeless.py b/tests/test_integrations_homeless.py new file mode 100644 index 0000000..0736b2a --- /dev/null +++ b/tests/test_integrations_homeless.py @@ -0,0 +1,81 @@ +"""A container run as an arbitrary uid has no HOME and no passwd entry, so +Path.home() raises there. Importing a module must not ask for it: every +`compartment` command imports these, whether or not it touches Claude Code +or a skills directory.""" +import os +import subprocess +import sys + +import pytest + +MODULES = ["clients", "claude_desktop", "claude_hooks", "claude_memory", + "agent_skill", "wire"] + +PRELUDE = """ +import pathlib, sys +try: + import pwd +except ImportError: # Windows has no passwd database + pass +else: + def _no_entry(uid): + raise KeyError(f"getpwuid(): uid not found: {uid}") + pwd.getpwuid = _no_entry +try: + pathlib.Path.home() +except RuntimeError: + pass +else: + sys.exit("the home directory could still be found") +""" + + +def _homeless_env(): + env = {k: v for k, v in os.environ.items() + if k not in ("HOME", "USERPROFILE", "HOMEDRIVE", "HOMEPATH", + "CLAUDE_CONFIG_DIR", "HERMES_HOME", "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH")} + src = os.path.join(os.path.dirname(__file__), os.pardir, "src") + env["PYTHONPATH"] = os.pathsep.join( + [os.path.abspath(src), env.get("PYTHONPATH", "")]) + return env + + +@pytest.mark.parametrize("module", MODULES) +def test_a_module_imports_where_there_is_no_home_directory(module): + """claude_hooks, claude_memory and agent_skill each called Path.home() + at import, so in such a container every command died before it ran.""" + code = PRELUDE + f"import compartment.{module}\n" + ran = subprocess.run([sys.executable, "-c", code], env=_homeless_env(), + capture_output=True, text=True, timeout=60) + assert ran.returncode == 0, ran.stderr[-2000:] + + +def test_the_import_root_has_a_value_to_show_with_no_home(): + """`import-claude --help` shows DEFAULT_ROOT; with no home it names a + path that holds nothing, and discovery finds nothing in it.""" + code = PRELUDE + ( + "from compartment import claude_memory\n" + "print(claude_memory.DEFAULT_ROOT)\n" + "assert claude_memory.discover() == []\n") + ran = subprocess.run([sys.executable, "-c", code], env=_homeless_env(), + capture_output=True, text=True, timeout=60) + assert ran.returncode == 0, ran.stderr[-2000:] + assert ".claude" in ran.stdout + + +def test_the_env_overrides_still_win_when_asked_for(monkeypatch, tmp_path): + from compartment import agent_skill, claude_hooks, claude_memory + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "c")) + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "h")) + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(tmp_path / "o")) + for module, name in ((claude_memory, "DEFAULT_ROOT"), + (claude_hooks, "SETTINGS")): + if name in vars(module): + monkeypatch.delattr(module, name) + assert claude_hooks.SETTINGS == tmp_path / "c" / "settings.json" + assert claude_hooks.settings_path() == tmp_path / "c" / "settings.json" + assert claude_memory.DEFAULT_ROOT == tmp_path / "c" / "projects" + assert agent_skill.agent_home("claude") == tmp_path / "c" + assert agent_skill.agent_home("hermes") == tmp_path / "h" + assert agent_skill.agent_home("openclaw") == tmp_path / "o" diff --git a/tests/test_integrations_hooks.py b/tests/test_integrations_hooks.py new file mode 100644 index 0000000..01e6a60 --- /dev/null +++ b/tests/test_integrations_hooks.py @@ -0,0 +1,118 @@ +"""The capture hook's command, as the shell Claude Code runs it in reads it. + +Claude Code runs hook commands through bash on every platform: Git Bash on +Windows since 2.1.47. To bash an unquoted backslash is an escape character, +and $ and ` expand inside double quotes.""" +import os +import subprocess +import sys + +import pytest + +from compartment import claude_hooks + +WIN_EXE = r"C:\Users\ana\AppData\Roaming\Python\Python312\Scripts\compartment.exe" +WIN_VAULT = r"C:\Users\ana\.compartment\memory.vault" + + +def _bash_words(cmd): + """The words bash makes of `cmd`, as bash itself splits them.""" + ran = subprocess.run(["bash", "-c", f'set -- {cmd}; for w in "$@"; do ' + 'printf "%s\\0" "$w"; done'], + capture_output=True, text=True, timeout=30) + assert ran.returncode == 0, ran.stderr + return ran.stdout.split("\0")[:-1] + + +needs_bash = pytest.mark.skipif(sys.platform == "win32" or not any( + os.access(os.path.join(d, "bash"), os.X_OK) + for d in os.environ.get("PATH", "").split(os.pathsep)), + reason="needs a bash to split the command") + + +@needs_bash +def test_a_windows_executable_path_reaches_git_bash_whole(monkeypatch): + """Quoted only when it held a space, the usual Windows path reached + Git Bash with its backslashes taken as escapes, + C:UsersanaAppDataRoaming...compartment.exe, so the hook failed with + "command not found" after every edit and captured nothing.""" + monkeypatch.setattr(claude_hooks, "_windows", lambda: True, + raising=False) + cmd = claude_hooks.hook_command(WIN_EXE, WIN_VAULT) + words = _bash_words(cmd) + assert words[0].replace("/", "\\") == WIN_EXE + assert words[1:3] == ["--vault", WIN_VAULT.replace("\\", "/")] + assert words[3:] == ["hook", "capture"] + assert claude_hooks.is_ours(cmd) + + +@needs_bash +@pytest.mark.parametrize("exe, vault", [ + ("/Applications/My Tools/compartment", "/v/m.vault"), + ("/home/ana/$HOME/bin/compartment", "/home/ana/a `b` $c \"d\"/m.vault"), + ("/opt/R&D (x86)/compartment", "/v/it's mine/m.vault"), +]) +def test_a_path_with_shell_characters_in_it_is_one_word(exe, vault): + cmd = claude_hooks.hook_command(exe, vault) + assert _bash_words(cmd) == [exe, "--vault", vault, "hook", "capture"] + assert claude_hooks.is_ours(cmd) + + +def test_a_windows_command_has_no_backslash_for_any_shell_to_eat(monkeypatch): + monkeypatch.setattr(claude_hooks, "_windows", lambda: True, + raising=False) + cmd = claude_hooks.hook_command(WIN_EXE, WIN_VAULT) + assert "\\" not in cmd + # A plain path stays bare, which PowerShell also runs as a command. + assert cmd.startswith("C:/Users/ana/AppData/Roaming/Python/Python312/" + "Scripts/compartment.exe --vault ") + spaced = claude_hooks.hook_command( + r"C:\Program Files (x86)\Compartment\compartment.exe", WIN_VAULT) + assert spaced.startswith('"C:/Program Files (x86)/Compartment/' + 'compartment.exe" --vault ') + + +def test_the_forward_slash_vault_opens_the_same_session(monkeypatch): + """The session store keys a vault by its normalised absolute path, so + the forward-slash spelling the hook now writes on Windows is the same + vault, unlocked by the same session, as the backslash one.""" + import ntpath + from compartment import session + with monkeypatch.context() as m: + m.setattr(session.os, "path", ntpath) + a = session._canon(WIN_VAULT) + b = session._canon(WIN_VAULT.replace("\\", "/")) + assert a == b + + +@pytest.mark.parametrize("settings", [ + '{"hooks": []}', + '{"hooks": {"PostToolUse": {"a": 1}}}', + '{"hooks": {"PostToolUse": ["x"]}}', + '{"hooks": {"PostToolUse": [{"matcher": "Edit", "hooks": "x"}]}}', +]) +def test_settings_whose_hooks_are_the_wrong_shape_are_refused_untouched( + settings, tmp_path): + """A list where the hooks object belongs, or a string where a matcher + group does, was walked as if it were right and died with + AttributeError, which `integrate claude`, `hook status` and the + uninstaller do not catch: a traceback instead of the file's name and + what is wrong with it. An event that was an object was replaced.""" + p = tmp_path / "settings.json" + p.write_text(settings, encoding="utf-8") + for call in (lambda: claude_hooks.install("compartment", settings=p), + lambda: claude_hooks.uninstall(settings=p), + lambda: claude_hooks.is_installed(settings=p)): + with pytest.raises(ValueError, match="refusing to touch it"): + call() + assert p.read_text(encoding="utf-8") == settings + + +def test_settings_whose_hooks_are_null_get_the_hook(tmp_path): + """`"hooks": null` is no hooks at all; install died with TypeError on + it, which nothing catches.""" + p = tmp_path / "settings.json" + p.write_text('{"hooks": null, "model": "opus"}', encoding="utf-8") + claude_hooks.install("compartment", settings=p) + assert claude_hooks.is_installed(settings=p) is True + assert claude_hooks.uninstall(settings=p) is True diff --git a/tests/test_integrations_jsonc.py b/tests/test_integrations_jsonc.py new file mode 100644 index 0000000..ed31816 --- /dev/null +++ b/tests/test_integrations_jsonc.py @@ -0,0 +1,179 @@ +"""Clients that read their config as JSON with comments. + +Zed writes settings.json on first run with a block of // comments above the +opening brace, so a strict reader refused every Zed user's file. VS Code's +mcp.json is JSON with comments too. Re-serialising would delete what the +user wrote, so the edit is textual and proven by reading it back: every +comment, every trailing comma, every other byte stays where it was. +""" +import json + +import pytest + +from compartment import cli, clients + +VAULT = "/v/memory.vault" + +#: Zed's own first-run settings.json, with a server and a comment the user +#: added since. +ZED_FIRST_RUN = """\ +// Zed settings +// +// For information on how to configure Zed, see the Zed +// documentation: https://zed.dev/docs/configuring-zed +// +// To see all of Zed's default settings without changing your +// custom settings, run `zed: open default settings` from the +// command palette (cmd-shift-p / ctrl-shift-p) +{ + "ui_font_size": 16, + "buffer_font_size": 15, + "theme": { + "mode": "system", + "light": "One Light", + "dark": "One Dark" // the one I use + }, + /* servers */ + "context_servers": { + "github": { "command": "gh-mcp", "args": [] }, // keep this one + }, +} +""" + +COMMENTS = ["// Zed settings", "// the one I use", "/* servers */", + "// keep this one", "(cmd-shift-p / ctrl-shift-p)"] + + +def _lenient(text): + return clients._plain(clients._jsonc_parse(text)) + + +def test_a_zed_settings_file_with_comments_is_written_keeping_them(tmp_path): + """The strict reader refused the file Zed itself writes, so `integrate + zed` printed a block on every machine instead of connecting Zed.""" + p = tmp_path / "settings.json" + p.write_text(ZED_FIRST_RUN, encoding="utf-8") + zed = clients.CLIENTS["zed"] + + res = clients.register(zed, VAULT, path=p) + + assert res["written"] is True, res["reason"] + text = p.read_text(encoding="utf-8") + for comment in COMMENTS: + assert comment in text + data = _lenient(text) + assert data["context_servers"]["github"] == {"command": "gh-mcp", + "args": []} + assert data["context_servers"]["compartment"]["args"][:2] == [ + "--vault", VAULT] + assert data["theme"]["dark"] == "One Dark" + assert open(res["backup"], encoding="utf-8").read() == ZED_FIRST_RUN + assert clients.is_registered(zed, path=p) is True + + # Again: refreshed in place, not duplicated, comments still there. + clients.register(zed, "/other.vault", path=p) + again = p.read_text(encoding="utf-8") + assert again.count('"compartment"') == 1 + assert _lenient(again)["context_servers"]["compartment"]["args"][1] == ( + "/other.vault") + for comment in COMMENTS: + assert comment in again + + # And out again: only ours goes. + assert clients.unregister(zed, path=p) is True + out = p.read_text(encoding="utf-8") + assert _lenient(out) == _lenient(ZED_FIRST_RUN) + for comment in COMMENTS: + assert comment in out + assert clients.is_registered(zed, path=p) is False + + +@pytest.mark.parametrize("text", [ + "// mine\n{}\n", + "{\n // nothing here yet\n}\n", + '{\n\t"theme": "dark", // tabs\n}\n', + '{"theme": "dark", /* one line */ "vim_mode": true}', + '{\n "context_servers": {}, // none yet\n "vim_mode": true\n}\n', + '{\n "context_servers": {\n // none yet\n }\n}\n', + '{\n "a": [1, 2,], // trailing in an array\n "b": "//not a comment"\n}', +]) +def test_every_shape_of_jsonc_reads_back_with_our_entry_and_out_again( + text, tmp_path): + p = tmp_path / "settings.json" + p.write_text(text, encoding="utf-8") + zed = clients.CLIENTS["zed"] + before = _lenient(text) + + assert clients.register(zed, VAULT, path=p)["written"] is True + after = _lenient(p.read_text(encoding="utf-8")) + assert after["context_servers"]["compartment"]["args"][1] == VAULT + del after["context_servers"]["compartment"] + if "context_servers" not in before: + assert after.pop("context_servers") == {} + assert after == before + + assert clients.unregister(zed, path=p) is True + restored = _lenient(p.read_text(encoding="utf-8")) + if "context_servers" not in before: + assert restored.pop("context_servers") == {} + assert restored == before + + +@pytest.mark.parametrize("text, why", [ + ('// c\n{"theme": "a", "theme": "b"}', "duplicate key"), + ("// c\n[1, 2]", "not a JSON object"), + ('// c\n{"context_servers": []}', "not an object"), + ("{'single': 'quotes'}", "expected a quoted key"), + ('{"a": 1 /* never closed', "unterminated comment"), +]) +def test_jsonc_this_cannot_edit_safely_is_left_alone(text, why, tmp_path): + p = tmp_path / "settings.json" + p.write_text(text, encoding="utf-8") + res = clients.register(clients.CLIENTS["zed"], VAULT, path=p) + assert res["written"] is False + assert why in res["reason"] and "refusing to touch it" in res["reason"] + assert "compartment" in res["snippet"] + assert p.read_text(encoding="utf-8") == text + + +def test_a_vscode_mcp_json_with_comments_is_written(tmp_path): + p = tmp_path / "mcp.json" + text = ('{\n\t// my servers\n\t"servers": {\n\t\t"x": {"type": "stdio", ' + '"command": "x"},\n\t},\n\t"inputs": [],\n}\n') + p.write_text(text, encoding="utf-8") + res = clients.register(clients.CLIENTS["vscode"], VAULT, path=p) + assert res["written"] is True, res["reason"] + out = p.read_text(encoding="utf-8") + assert "// my servers" in out + entry = _lenient(out)["servers"]["compartment"] + assert entry["type"] == "stdio" and entry["args"][1] == VAULT + + +def test_a_strict_json_client_still_refuses_comments(tmp_path): + """Cursor reads strict JSON: a file with comments in it is not one + Cursor can read either, so it is not ours to rewrite.""" + p = tmp_path / "mcp.json" + text = '{"mcpServers": {} // a comment\n}' + p.write_text(text, encoding="utf-8") + res = clients.register(clients.CLIENTS["cursor"], VAULT, path=p) + assert res["written"] is False + assert p.read_text(encoding="utf-8") == text + + +def test_removing_a_zed_entry_that_cannot_be_taken_out_says_so( + tmp_path, monkeypatch, capsys): + """A registration this cannot take out cleanly is reported, with the + entry to remove, rather than claimed never to have been there.""" + zed = clients.CLIENTS["zed"] + p = tmp_path / "settings.json" + p.write_text('// mine\n{"context_servers": {"compartment": ' + '{"command": "x"}}}\n', encoding="utf-8") + assert clients.is_registered(zed, path=p) is True + monkeypatch.setattr(clients, "_jsonc_delete", + lambda *a: (_ for _ in ()).throw(ValueError("no"))) + monkeypatch.setattr(zed.__class__, "config_path", lambda self: p) + cli._integrate_client(zed, type("A", (), {"remove": True, + "vault": VAULT})()) + out = capsys.readouterr().out + assert "left as it is" in out and "was not in" not in out + assert json.loads(p.read_text(encoding="utf-8").split("\n", 1)[1]) diff --git a/tests/test_integrations_skills.py b/tests/test_integrations_skills.py new file mode 100644 index 0000000..5aa3b3e --- /dev/null +++ b/tests/test_integrations_skills.py @@ -0,0 +1,59 @@ +"""Where each agent loads the /compartmentalize skill from.""" +import pytest + +from compartment import agent_skill + + +@pytest.fixture() +def no_openclaw_env(monkeypatch): + for var in ("OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH", "OPENCLAW_HOME"): + monkeypatch.delenv(var, raising=False) + + +def test_openclaw_home_stands_in_for_the_home_directory(no_openclaw_env, + monkeypatch, + tmp_path): + """OPENCLAW_HOME replaces the home directory and OpenClaw keeps + .openclaw under it (src/infra/config-dir.ts). It was taken for the + .openclaw directory itself, so the skill landed one level above the + skills/ OpenClaw loads.""" + monkeypatch.setenv("OPENCLAW_HOME", str(tmp_path / "h")) + assert agent_skill.skill_path("openclaw") == ( + tmp_path / "h" / ".openclaw" / "skills" / "compartmentalize" + / "SKILL.md") + + +def test_openclaw_state_dir_and_config_path_move_it(no_openclaw_env, + monkeypatch, tmp_path): + monkeypatch.setenv("OPENCLAW_HOME", str(tmp_path / "h")) + monkeypatch.setenv("OPENCLAW_CONFIG_PATH", + str(tmp_path / "cfg" / "openclaw.json")) + assert agent_skill.agent_home("openclaw") == tmp_path / "cfg" + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(tmp_path / "state")) + assert agent_skill.agent_home("openclaw") == tmp_path / "state" + + +def test_openclaw_without_any_variable_is_in_the_home_directory( + no_openclaw_env, monkeypatch, tmp_path): + monkeypatch.setitem(agent_skill.SKILL_TARGETS, "openclaw", + (agent_skill._openclaw_dir, tmp_path / ".openclaw")) + assert agent_skill.agent_home("openclaw") == tmp_path / ".openclaw" + r = agent_skill.install("openclaw") + assert r["action"] == "written" + assert agent_skill.is_current("openclaw") + assert agent_skill.remove("openclaw") is True + + +def test_a_relative_claude_config_dir_does_not_scatter_the_skill( + monkeypatch, tmp_path): + """The hook and the import ignore a CLAUDE_CONFIG_DIR that is not + absolute, as Claude Code's own VS Code extension does; the skill must + not land relative to whatever directory the command ran in.""" + monkeypatch.setitem(agent_skill.SKILL_TARGETS, "claude", + (agent_skill._claude_dir, tmp_path / ".claude")) + monkeypatch.setattr(agent_skill.Path, "home", + classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", "relative/claude") + assert agent_skill.agent_home("claude") == tmp_path / ".claude" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "abs")) + assert agent_skill.agent_home("claude") == tmp_path / "abs" diff --git a/tests/test_linux_panel.py b/tests/test_linux_panel.py index b1c5edc..a51cde8 100644 --- a/tests/test_linux_panel.py +++ b/tests/test_linux_panel.py @@ -242,3 +242,57 @@ def test_it_never_signals_itself(monkeypatch, tmp_path): lambda pid, sig: killed.append(pid)) systray.quit_running(vault) assert killed == [], "an uninstall that kills its own process finishes nothing" + + +# ------------------------------------------------------- app sandboxes + +@pytest.mark.parametrize("var,value,word", [ + ("FLATPAK_ID", "com.visualstudio.code", "Flatpak"), + ("SNAP_NAME", "code", "Snap"), +]) +def test_start_at_login_from_inside_an_app_sandbox_says_it_cannot( + as_linux, monkeypatch, var, value, word): + """Inside a Flatpak the XDG directories are the app's own, and inside a + Snap so is HOME: the entries landed where no desktop reads them, naming + a Python the desktop cannot run, and start at login reported "on".""" + monkeypatch.setattr(systray, "_is_linux", lambda: True) + monkeypatch.setattr(systray, "_FLATPAK_INFO", str(as_linux / "none")) + monkeypatch.setenv(var, value) + if var == "SNAP_NAME": # strict confinement moves HOME + monkeypatch.setenv("SNAP_USER_DATA", str(as_linux / "snap" / "1")) + monkeypatch.setenv("HOME", str(as_linux / "snap" / "1")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(as_linux / "cfg")) + out = systray.set_login(True, "/home/x/memory.vault") + assert out.startswith("error") and word in out and value in out + assert "outside it" in out + assert not systray.autostart_entry_path().exists() + assert not systray.desktop_entry_path().exists() + + +def test_outside_a_sandbox_nothing_changes(as_linux, monkeypatch): + monkeypatch.setattr(systray, "_is_linux", lambda: True) + monkeypatch.setattr(systray, "_app_sandbox", lambda: None) + monkeypatch.setattr(systray, "systemd_available", lambda: False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(as_linux / "cfg")) + assert systray.set_login(True, "/home/x/memory.vault") == "on" + + +def test_a_flatpak_is_recognised_by_its_info_file(as_linux, monkeypatch): + for var in ("FLATPAK_ID", "SNAP_NAME"): + monkeypatch.delenv(var, raising=False) + info = as_linux / ".flatpak-info" + monkeypatch.setattr(systray, "_FLATPAK_INFO", str(info)) + assert systray._app_sandbox() is None + info.write_text("[Application]\nname=org.example.Editor\n") + assert "Flatpak" in systray._app_sandbox() + + +def test_a_classic_snap_terminal_is_not_a_sandbox(as_linux, monkeypatch): + """VS Code's snap is classic: it sets SNAP_NAME in every terminal it + opens and confines nothing, so start at login works from there.""" + monkeypatch.delenv("FLATPAK_ID", raising=False) + monkeypatch.setattr(systray, "_FLATPAK_INFO", str(as_linux / "none")) + monkeypatch.setenv("SNAP_NAME", "code") + monkeypatch.setenv("SNAP_USER_DATA", str(as_linux / "snap" / "code" / "1")) + monkeypatch.setenv("HOME", str(as_linux / "home")) + assert systray._app_sandbox() is None diff --git a/tests/test_macos_bundle.py b/tests/test_macos_bundle.py index 6dafb14..1aa880c 100644 --- a/tests/test_macos_bundle.py +++ b/tests/test_macos_bundle.py @@ -200,3 +200,356 @@ def test_a_system_python_is_never_embedded(): """The whole prefix gets copied in, so /usr would mean copying /usr.""" src = SPEC.read_text(encoding="utf-8") assert '("/usr", "/System", "/Library")' in src + + +# ------------------------------------- every binary runs where the app claims + +def _macho(arch="arm64", minos=(13, 0), legacy=False) -> bytes: + """A minimal 64-bit Mach-O header with the one load command that names + its minimum macOS: LC_BUILD_VERSION, or the older LC_VERSION_MIN_MACOSX.""" + import struct + cpu = {"arm64": 0x0100000C, "x86_64": 0x01000007}[arch] + v = (minos[0] << 16) | (minos[1] << 8) + cmd = (struct.pack(" bytes: + import struct + cpus = {"arm64": 0x0100000C, "x86_64": 0x01000007} + entries, body = b"", b"" + for arch, data in slices: + entries += struct.pack(">IIIII", cpus[arch], 0, 4096 + len(body), + len(data), 12) + body += data.ljust(4096, b"\0") + head = struct.pack(">II", 0xCAFEBABE, len(slices)) + entries + return head.ljust(4096, b"\0") + body + + +def test_the_mach_o_reader_finds_each_slice_and_its_minimum(tmp_path): + b = _builder() + thin = tmp_path / "thin.so" + thin.write_bytes(_macho("arm64", (14, 0))) + assert b._macho_slices(thin) == {"arm64": (14, 0)} + old = tmp_path / "old.so" + old.write_bytes(_macho("x86_64", (10, 9), legacy=True)) + assert b._macho_slices(old) == {"x86_64": (10, 9)} + fat = tmp_path / "fat.so" + fat.write_bytes(_fat(("x86_64", _macho("x86_64", (10, 13))), + ("arm64", _macho("arm64", (11, 0))))) + assert b._macho_slices(fat) == {"x86_64": (10, 13), "arm64": (11, 0)} + # Not Mach-O: text, and a Java class file, which shares the fat magic. + (tmp_path / "a.py").write_text("print('hi')\n") + (tmp_path / "A.class").write_bytes(b"\xca\xfe\xba\xbe\x00\x00\x00\x34") + assert b._macho_slices(tmp_path / "a.py") == {} + assert b._macho_slices(tmp_path / "A.class") == {} + + +def _fake_app(tmp_path, files: dict) -> pathlib.Path: + app = tmp_path / "Compartment.app" + for rel, data in files.items(): + p = app / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(data) + return app + + +def test_a_library_built_for_a_newer_macos_than_the_app_is_refused(tmp_path): + """onnxruntime 1.24 and later ship macOS wheels tagged macosx_14_0 only, + and pip on a current build machine takes them, so the bundle declaring + 13.0 carried an embedding model that macOS 13 cannot load.""" + lib = ("Contents/Resources/runtime/lib/python3.12/site-packages/" + "onnxruntime/capi/onnxruntime_pybind11_state.so") + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("arm64", (13, 0)), + lib: _macho("arm64", (14, 0)), + "Contents/Resources/runtime/lib/python3.12/os.py": b"# stdlib\n"}) + b = _builder() + with pytest.raises(SystemExit) as e: + b._verify_deployment_target(app, "arm64", "13.0") + assert "onnxruntime_pybind11_state.so needs macOS 14.0" in str(e.value) + b._verify_deployment_target(app, "arm64", "14.0") # must not raise + + +def test_an_intel_build_refuses_a_library_without_an_intel_slice(tmp_path): + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("x86_64", (13, 0)), + "Contents/Resources/runtime/lib/x.so": _macho("arm64", (11, 0)), + "Contents/Resources/runtime/lib/u.so": _fat( + ("x86_64", _macho("x86_64", (10, 13))), + ("arm64", _macho("arm64", (11, 0))))}) + b = _builder() + with pytest.raises(SystemExit) as e: + b._verify_deployment_target(app, "x86_64", "13.0") + assert "x.so (arm64)" in str(e.value) + assert "u.so" not in str(e.value), "a universal2 library is fine" + + +def test_the_build_runs_the_deployment_check_before_signing(): + src = SPEC.read_text(encoding="utf-8") + body = src.split("def build_app")[1] + assert body.index("_verify_deployment_target(app, arch, min_macos)") \ + < body.index('_run("codesign"') + + +def test_wheels_are_chosen_for_the_declared_minimum_macos(tmp_path, + monkeypatch): + """pip on its own picks wheels for the macOS it runs on. The build has to + ask for the macOS the app declares, binary only, and install nothing + else.""" + import subprocess + b = _builder() + b._EMBEDDED_VER[0] = "3.12" + calls = [] + + def fake_run(*args, **kw): + argv = [str(a) for a in args] + calls.append(argv) + if "wheel" in argv: + house = pathlib.Path(argv[argv.index("--wheel-dir") + 1]) + (house / "compartment-4.13.0-py3-none-any.whl").write_bytes(b"") + return subprocess.CompletedProcess(argv, 0) + + def fake_subprocess_run(argv, **kw): + calls.append([str(a) for a in argv]) + return subprocess.CompletedProcess(argv, 0, "", "") + + monkeypatch.setattr(b, "_run", fake_run) + monkeypatch.setattr(b.subprocess, "run", fake_subprocess_run) + b._install_packages(pathlib.Path("/rt/bin/python3.12"), "/src", + "arm64", "13.0") + download = next(c for c in calls if "download" in c) + assert download[download.index("--platform") + 1] == "macosx_13_0_arm64" + assert "--only-binary=:all:" in download + install = next(c for c in calls if "install" in c) + assert "--no-index" in install, "only the wheels chosen above" + assert not any("install" in c and "--no-index" not in c for c in calls) + + +def test_no_wheels_for_the_minimum_says_what_to_embed_instead(tmp_path, + monkeypatch): + import subprocess + b = _builder() + b._EMBEDDED_VER[0] = "3.14" + + def fake_run(*args, **kw): + argv = [str(a) for a in args] + if "wheel" in argv: + house = pathlib.Path(argv[argv.index("--wheel-dir") + 1]) + (house / "compartment-4.13.0-py3-none-any.whl").write_bytes(b"") + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(b, "_run", fake_run) + monkeypatch.setattr(b.subprocess, "run", lambda argv, **kw: ( + subprocess.CompletedProcess(argv, 1, "", "ERROR: No matching " + "distribution found for onnxruntime"))) + with pytest.raises(SystemExit) as e: + b._install_packages(pathlib.Path("/rt/bin/python3.14"), "/src", + "arm64", "13.0") + msg = str(e.value) + assert "Python 3.14 on macOS 13.0" in msg and "onnxruntime" in msg + assert "--min-macos 14.0" in msg and "3.12" in msg + + +def test_the_installer_accepts_only_the_macos_the_app_declares(tmp_path, + monkeypatch): + import plistlib + b = _builder() + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("arm64", (14, 0))}) + with open(app / "Contents" / "Info.plist", "wb") as fh: + plistlib.dump({"LSMinimumSystemVersion": "14.0"}, fh) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + monkeypatch.setattr(b, "_run", lambda *a, **k: None) + b.build_pkg(app) + dist = (tmp_path / "build" / "distribution.xml").read_text() + assert '' in dist + + +def test_a_minimum_below_what_the_login_item_needs_is_refused(): + with pytest.raises(SystemExit, match="SMAppService"): + _builder().build_app(python="/nonexistent", min_macos="12.0") + + +# ---------------------------- the CLI inside the bundle runs on another Mac + +def _pip_written(runtime: pathlib.Path, name: str, long_form=False) -> None: + """A console script as pip writes it: the absolute path of the + interpreter that installed it, plain or in the exec form pip uses for a + path with a space in it.""" + py = runtime / "bin" / "python3.12" + head = (f"#!/bin/sh\n'''exec' \"{py}\" \"$0\" \"$@\"\n' '''\n" + if long_form else f"#!{py}\n") + (runtime / "bin" / name).write_text( + head + "# -*- coding: utf-8 -*-\nimport sys\n" + "print('ran', sys.argv[1:])\n", encoding="utf-8") + (runtime / "bin" / name).chmod(0o755) + + +@pytest.mark.skipif(sys.platform == "win32", reason="the bundle is POSIX") +def test_console_scripts_still_run_after_the_bundle_moves(tmp_path): + """pip names the build directory's interpreter in every script it writes, + and `compartment` inside the app is what the panel writes into each + agent's config. Installed anywhere else - which is everywhere but the + build machine - it failed with "bad interpreter".""" + import shutil + import subprocess + if not shutil.which("realpath"): + pytest.skip("no realpath here") + build = tmp_path / "build dir" / "runtime" + (build / "bin").mkdir(parents=True) + (build / "bin" / "python3.12").symlink_to(sys.executable) + _pip_written(build, "compartment") + _pip_written(build, "cmpt", long_form=True) + relocatable = "#!/bin/sh\n'''exec' \"$(dirname -- \"$0\")/python3.12\" \"$0\"\n' '''\nprint(1)\n" + (build / "bin" / "pip").write_text(relocatable, encoding="utf-8") + assert sorted(_builder()._relocate_scripts(build)) == ["cmpt", + "compartment"] + assert (build / "bin" / "pip").read_text() == relocatable, \ + "a script that is relocatable already is left alone" + installed = tmp_path / "Applications" / "runtime" + installed.parent.mkdir() + shutil.move(str(build), str(installed)) + for name in ("compartment", "cmpt"): + script = installed / "bin" / name + assert str(build) not in script.read_text() + r = subprocess.run([str(script), "--version"], capture_output=True, + text=True, timeout=60) + assert r.returncode == 0, r.stderr + assert r.stdout.strip() == "ran ['--version']" + + +def test_a_script_naming_the_build_directory_fails_the_build(tmp_path): + app = tmp_path / "Compartment.app" + runtime = app / "Contents" / "Resources" / "runtime" + (runtime / "bin").mkdir(parents=True) + _pip_written(runtime, "compartment") + with pytest.raises(SystemExit, match="name the build directory"): + _builder()._verify_console_scripts(app) + + +def test_the_build_makes_its_scripts_relocatable_and_checks_them(): + src = SPEC.read_text(encoding="utf-8") + body = src.split("def build_app")[1] + assert body.index("_relocate_scripts(runtime)") < body.index("compileall") + assert body.index("_verify_console_scripts(app)") \ + < body.index('_run("codesign"') + + +# ------------------------------- an Intel build is an installer of its own + +def test_release_names_keep_apple_silicon_where_it_was_and_mark_intel(): + b = _builder() + assert b.artifact_name("4.13.0", "arm64", "pkg") == "Compartment-4.13.0.pkg" + assert b.artifact_name("4.13.0", "arm64", "dmg") == "Compartment-4.13.0.dmg" + assert b.artifact_name("4.13.0", "x86_64", "pkg") == \ + "Compartment-4.13.0-intel.pkg" + + +@pytest.mark.parametrize("arch,suffix", [("arm64", ""), + ("x86_64", "-intel")]) +def test_the_installer_runs_only_on_the_macs_the_app_can_start_on( + tmp_path, monkeypatch, arch, suffix): + """It said arm64,x86_64 whatever was inside, so an Intel Mac installed + the Apple silicon app, which could never launch.""" + import plistlib + b = _builder() + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho(arch, (13, 0))}) + with open(app / "Contents" / "Info.plist", "wb") as fh: + plistlib.dump({"LSMinimumSystemVersion": "13.0"}, fh) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + ran = [] + monkeypatch.setattr(b, "_run", lambda *a, **k: ran.append(a)) + out = b.build_pkg(app) + dist = (tmp_path / "build" / "distribution.xml").read_text() + assert f'hostArchitectures="{arch}"' in dist + version = b._version() + assert out.name == f"Compartment-{version}{suffix}.pkg" + dmg = b.build_dmg(app) + assert dmg.name == f"Compartment-{version}{suffix}.dmg" + readme = (tmp_path / "build" / "dmg" / "READ ME FIRST.txt").read_text() + assert f"Compartment-{version}{suffix}.pkg" in readme + + +# ----------------------------------------------------- a build CI can run + +def test_the_icon_needs_no_pillow_in_the_python_running_the_build( + tmp_path, monkeypatch): + """A runner given only uv's standalone Python has no Pillow, and the + build died at the icon after every other step had passed.""" + monkeypatch.setitem(sys.modules, "PIL", None) # import PIL fails + b = _builder() + out = b.build_icon(tmp_path / "Resources" / "Compartment.icns") + shipped = ROOT / "src" / "compartment" / "data" / "app.icns" + assert out.read_bytes() == shipped.read_bytes() + + +def _main(b, monkeypatch, tmp_path, argv): + monkeypatch.setattr(b.sys, "platform", "darwin") + monkeypatch.setattr(b.sys, "argv", ["build_macos_app.py", *argv]) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + return b.main() + + +def test_a_failed_step_exits_non_zero_naming_the_step(tmp_path, monkeypatch, + capsys): + import subprocess + b = _builder() + + def fail(*a, **k): + raise subprocess.CalledProcessError(2, ["pkgbuild", "--root", "x"]) + + monkeypatch.setattr(b, "build_app", fail) + assert _main(b, monkeypatch, tmp_path, ["--pkg"]) == 1 + err = capsys.readouterr().err + assert "error: `pkgbuild --root x` exited 2" in err + assert "Traceback" not in err + + +def test_the_build_lists_what_it_made(tmp_path, monkeypatch, capsys): + b = _builder() + app = tmp_path / "build" / "Compartment.app" + monkeypatch.setattr(b, "build_app", lambda *a, **k: app) + monkeypatch.setattr(b, "build_pkg", lambda a: a.with_name("C-1.pkg")) + monkeypatch.setattr(b, "build_dmg", lambda a: a.with_name("C-1.dmg")) + assert _main(b, monkeypatch, tmp_path, ["--pkg", "--dmg"]) == 0 + listed = [ln.split(": ", 1)[1] for ln in capsys.readouterr().out + .splitlines() if ln.startswith("artifact: ")] + assert listed == [str(app), str(app.with_name("C-1.dmg")), + str(app.with_name("C-1.pkg"))] + + +def test_a_busy_disk_image_is_tried_again(tmp_path, monkeypatch): + import subprocess + b = _builder() + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("arm64", (13, 0))}) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + monkeypatch.setattr(b.time, "sleep", lambda s: None) + tries = [] + + def run(*args, **kw): + if args[0] == "hdiutil": + tries.append(args) + if len(tries) < 3: + raise subprocess.CalledProcessError(1, list(args)) + + monkeypatch.setattr(b, "_run", run) + b.build_dmg(app) + assert len(tries) == 3 + + +def test_the_default_minimum_is_what_the_bundled_onnxruntime_needs(): + """onnxruntime 1.23.2's macosx_13_0 wheels hold libraries that record a + minimum of macOS 13.4 (arm64 and x86_64), and 1.23.2 is the newest + release a macOS 13 build can have. A bundle declaring 13.0 installed on + 13.0 to 13.3 and could not load its model there.""" + b = _builder() + assert b._version_tuple(b.MIN_MACOS) >= (13, 4) + # pip's tags for macOS 11 and later are major versions only, so 13.4 + # still selects the macosx_13_0 wheels it has to. + major, _minor = b._version_tuple(b.MIN_MACOS) + assert major == 13 diff --git a/tests/test_menu_bar_persistence.py b/tests/test_menu_bar_persistence.py index 82d1075..875a6f7 100644 --- a/tests/test_menu_bar_persistence.py +++ b/tests/test_menu_bar_persistence.py @@ -874,6 +874,29 @@ def test_a_machine_with_no_user_manager_still_starts_at_login(as_linux, assert not systray.systemd_unit_path().exists() +def test_a_desktop_that_never_starts_the_target_keeps_its_autostart_entry( + as_linux, monkeypatch): + """Xfce, MATE, Cinnamon and LXQt never activate graphical-session.target, + which the unit is WantedBy, so an enabled unit there never starts at + login - and the autostart entry, which they all read, was removed as + the redundant second starter. The panel came up at install time and + never again.""" + monkeypatch.setattr(systray, "systemd_available", lambda: True) + base = _fake_systemctl() + + def xfce(*argv): + if argv[:2] == ("is-active", "graphical-session.target"): + return 3, "inactive" + return base(*argv) + + monkeypatch.setattr(systray, "_systemctl", xfce) + assert systray.set_login(True) == "on" + assert systray.autostart_entry_path().is_file() + assert not systray.systemd_unit_path().exists() + assert not any(c[0] == "enable" for c in base.calls), \ + "nothing enabled that will never start" + + def test_a_unit_that_will_not_run_is_undone_rather_than_left_half_done( as_linux, monkeypatch): """Enabled but not active proves nothing. Rather than leave a unit that @@ -1030,6 +1053,33 @@ def fake_run(argv, *a, **k): assert not [c for c in ran if c[0] == "taskkill"] +def test_windows_counts_only_the_tray_app_of_this_session(monkeypatch): + """Fast user switching: a second person signed in on the same PC runs a + tray app of their own, and from an elevated prompt its command line is + readable. `init` took it as this user's app coming up and started none; + an update counted it as the copy to wait for.""" + _as_windows(monkeypatch) + exe = WINDOWS_EXE + mine = f'"{exe}" --vault "C:\\Users\\ana\\.compartment\\memory.vault" tray' + theirs = f'"{exe}" --vault "C:\\Users\\bo\\.compartment\\memory.vault" tray' + table = ('"ProcessId","ParentProcessId","CommandLine","SessionId"\r\n' + + f'"4100","900","{mine.replace(chr(34), chr(34) * 2)}","1"\r\n' + + f'"6100","950","{theirs.replace(chr(34), chr(34) * 2)}","2"\r\n') + asked = [] + + def fake_run(argv, *a, **k): + asked.append(argv[-1]) + return subprocess.CompletedProcess(argv, 0, table, "") + monkeypatch.setattr(systray.subprocess, "run", fake_run) + monkeypatch.setattr(systray, "_windows_session", lambda: "1", + raising=False) + assert systray.running_pids() == [4100] + assert "SessionId" in asked[-1], "the session has to be asked for" + # Where the session cannot be told, nothing is left out. + monkeypatch.setattr(systray, "_windows_session", lambda: None) + assert systray.running_pids() == [4100, 6100] + + def test_the_task_restarts_a_panel_that_failed(monkeypatch): _as_windows(monkeypatch) xml = systray.scheduled_task_xml("C:\\v\\memory.vault") diff --git a/tests/test_menubar.py b/tests/test_menubar.py index bd52143..798f701 100644 --- a/tests/test_menubar.py +++ b/tests/test_menubar.py @@ -580,9 +580,13 @@ def test_the_dashboard_button_starts_dash_once_and_reuses_it(monkeypatch, "time.sleep(30)\n") monkeypatch.setattr(menubar, "_cli_argv", lambda: [sys.executable, "-c", script]) - opened = [] - monkeypatch.setattr(menubar.webbrowser, "open", - lambda u: opened.append(u) or True) + # Recorded at open_in_browser, not webbrowser.open: whether a browser + # exists is that function's business (none on a headless Linux runner), + # and the reuse path calls it on a thread, so wait for the call. + import threading + opened, called = [], threading.Event() + monkeypatch.setattr(menubar, "open_in_browser", + lambda u: (opened.append(u), called.set(), True)[-1]) menubar._DASH.update(proc=None, url=None) vault = str(tmp_path / "x.vault") url = menubar.open_dashboard(vault) @@ -591,6 +595,7 @@ def test_the_dashboard_button_starts_dash_once_and_reuses_it(monkeypatch, assert proc is not None and proc.poll() is None assert opened == [] # dash opened the browser itself assert menubar.open_dashboard(vault) == url + assert called.wait(5) assert menubar._DASH["proc"] is proc and opened == [url] menubar.stop_dashboard() assert proc.poll() is not None and menubar._DASH["proc"] is None diff --git a/tests/test_offline_guard_paths.py b/tests/test_offline_guard_paths.py index 02af643..e188300 100644 --- a/tests/test_offline_guard_paths.py +++ b/tests/test_offline_guard_paths.py @@ -97,6 +97,51 @@ def test_local_ipc_still_works(guarded, tmp_path, monkeypatch): server.close() +#: Families with a far end on another machine, by the name each platform +#: knows them by. Each one runs where its platform has it. +_FAR_FAMILIES = ["AF_PACKET", "AF_BLUETOOTH", "AF_VSOCK", "AF_HYPERV", + "AF_CAN", "AF_TIPC", "AF_RDS", "AF_QIPCRTR", "AF_APPLETALK", + "AF_IPX", "AF_DECnet", "AF_SNA", "AF_IRDA", "AF_X25", + "AF_AX25", "AF_UNSPEC"] + + +@pytest.mark.parametrize("name", _FAR_FAMILIES) +def test_a_socket_that_can_reach_another_machine_is_refused_whatever_its_family( + guarded, name): + """Only AF_INET and AF_INET6 were refused, and "every other local family" + let through - but AF_PACKET writes raw frames, AF_BLUETOOTH reaches a + paired device with no privilege at all, and AF_VSOCK/AF_HYPERV reach the + host of a VM (WSL2's included). The refusal comes before the operating + system is asked, so where it would refuse the family anyway the answer + must still be the guard's.""" + family = getattr(socket, name, None) + if family is None: + pytest.skip(f"{name} does not exist here") + for make in (socket.socket, _socket.socket): + with pytest.raises(offline_guard.OfflineViolation): + make(family, socket.SOCK_DGRAM).close() + + +@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="no AF_UNIX here") +def test_a_socketpair_and_the_kernel_interfaces_are_still_local(guarded): + """What stays open: the AF_UNIX pair asyncio builds its self-pipe from on + POSIX, and the kernel's own sockets, which have no far end.""" + a, b = socket.socketpair() + with a, b: + a.sendall(b"x") + assert b.recv(1) == b"x" + for name in ("AF_NETLINK", "AF_ROUTE", "AF_SYSTEM"): + family = getattr(socket, name, None) + if family is None: + continue + try: + socket.socket(family, socket.SOCK_RAW).close() + except offline_guard.OfflineViolation: + raise + except OSError: + pass # the platform's own answer, not the guard's + + def test_literal_and_localhost_lookups_still_resolve(guarded): """Resolving something already local asks no one anything, so it stays allowed: blocking it would break loopback code paths for no gain.""" diff --git a/tests/test_platform_boot.py b/tests/test_platform_boot.py new file mode 100644 index 0000000..a9520d0 --- /dev/null +++ b/tests/test_platform_boot.py @@ -0,0 +1,126 @@ +"""Reading the boot instant on macOS and the BSDs: in-process, without a +fork on every vault open, and from every sysctl dialect.""" +import ctypes +import ctypes.util +import platform +import subprocess + +import pytest + +from compartment import platforms + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + + +class _RefusedLibc: + """libc as Seatbelt leaves it: sysctlbyname is there and answers EPERM.""" + + def sysctlbyname(self, *a): + return -1 + + +def _record_subprocesses(monkeypatch, stdout="", fail=False): + calls = [] + + def run(cmd, *a, **kw): + calls.append(cmd) + if fail: + raise subprocess.CalledProcessError(1, cmd) + return subprocess.CompletedProcess(cmd, 0, stdout, "") + monkeypatch.setattr(platforms.subprocess, "run", run) + return calls + + +def test_a_refused_kern_boottime_on_macos_does_not_fork_sysctl(monkeypatch): + """Under Seatbelt (Claude Code, Codex) the in-process sysctl is refused, + and the sysctl binary makes the same call and is refused as well. It was + forked anyway on every vault open from the agent's shell.""" + monkeypatch.setattr(platforms.platform, "system", lambda: "Darwin") + monkeypatch.setattr(ctypes, "CDLL", lambda *a, **kw: _RefusedLibc()) + calls = _record_subprocesses(monkeypatch, fail=True) + assert platforms._sysctl_boot_estimate() is None + assert calls == [] + + +@pytest.mark.parametrize("system", ["FreeBSD", "NetBSD", "DragonFly", "Darwin"]) +def test_the_in_process_read_runs_no_helper_to_find_libc(monkeypatch, system): + """ctypes.util.find_library("c") runs /sbin/ldconfig -r on FreeBSD, + OpenBSD and DragonFly and the C compiler on NetBSD, in a subprocess, on + every call: the in-process read forked anyway.""" + looked_up = [] + monkeypatch.setattr(ctypes.util, "find_library", + lambda name: looked_up.append(name) or None) + monkeypatch.setattr(platforms.platform, "system", lambda: system) + _record_subprocesses(monkeypatch, fail=True) + platforms._sysctl_boot_estimate() + assert looked_up == [] + + +@pytest.mark.skipif(platform.system() != "Linux", + reason="compares against /proc/stat") +def test_the_linux_clock_estimate_matches_the_kernel_btime(): + """The CLOCK_BOOTTIME fallback (Termux, a sandbox hiding /proc/stat) must + name the boot instant the kernel reports, or a process that can read + /proc/stat and one that cannot would pin different tokens.""" + try: + with open("/proc/stat") as f: + btime = next(float(line.split()[1]) for line in f + if line.startswith("btime ")) + except (OSError, StopIteration): + pytest.skip("/proc/stat btime not readable here") + clock = platforms._clock_boot_estimate() + assert clock is not None and abs(clock[0] - btime) < 2 + + +@pytest.mark.parametrize("stdout", [ + "1700000000\n", # OpenBSD -n + "{ sec = 1700000000, usec = 123456 } Tue Nov 14 22:13:20 2023\n", # FreeBSD +]) +def test_every_sysctl_dialect_of_kern_boottime_is_read(monkeypatch, stdout): + """OpenBSD has no sysctlbyname, so the binary is its only kernel record, + and `sysctl -n kern.boottime` prints the bare seconds there + (sbin/sysctl/sysctl.c). Only FreeBSD's "sec = N" was understood, so + OpenBSD always fell through to the clock.""" + monkeypatch.setattr(platforms.platform, "system", lambda: "OpenBSD") + + class NoSysctlbyname: + pass + monkeypatch.setattr(ctypes, "CDLL", lambda *a, **kw: NoSysctlbyname()) + _record_subprocesses(monkeypatch, stdout=stdout) + est = platforms._sysctl_boot_estimate() + assert est is not None and est[0] == 1_700_000_000.0 + + +# --- Windows: the tick count is 64 bits wide --------------------------------- + +class _CtypesFunction: + """A foreign function as ctypes models one: its result is read as the + restype says, and a function nobody described returns a C int.""" + + def __init__(self, value): + self.value = value + self.restype = ctypes.c_int + self.argtypes = None + + def __call__(self): + return self.restype(self.value).value + + +def test_windows_uptime_past_24_days_is_not_cut_to_32_bits(tmp_path, + monkeypatch): + """Bare ctypes reads GetTickCount64's 64-bit result as a 32-bit int. + Past 2**31 ms (24.9 days, routine for a Windows machine that sleeps or + uses Fast Startup) the uptime went negative, the boot instant jumped by + 49.7 days, the pin took it for a reboot, and the stored unlock + credential was deleted - and again every 49.7 days.""" + import types + thirty_days_ms = 30 * 86_400 * 1000 + kernel32 = types.SimpleNamespace( + GetTickCount64=_CtypesFunction(thirty_days_ms)) + monkeypatch.setattr(ctypes, "windll", + types.SimpleNamespace(kernel32=kernel32), raising=False) + monkeypatch.setattr(platforms.time, "time", lambda: 1_800_000_000.0) + assert platforms._windows_boot_token() == str(1_800_000_000 - 30 * 86_400) diff --git a/tests/test_platform_docker.py b/tests/test_platform_docker.py new file mode 100644 index 0000000..8733240 --- /dev/null +++ b/tests/test_platform_docker.py @@ -0,0 +1,55 @@ +"""The Dockerfile's contract, read from the file: Docker is not available to +the suite, so what the image promises is pinned here instead. + +The behaviour behind the run line - a uid with no home, HOME=/, the +passphrase in the environment - is exercised for real in +test_platform_session_store.py.""" +import pathlib +import re + +DOCKERFILE = (pathlib.Path(__file__).resolve().parents[1] / "Dockerfile" + ).read_text(encoding="utf-8") + + +def _env(): + """Every ENV key=value the final stage sets.""" + final = DOCKERFILE.split("\nFROM ")[-1] + pairs = {} + for block in re.findall(r"^ENV ((?:.*\\\n)*.*)$", final, re.M): + for key, value in re.findall(r"(\w+)=(\S+)", block): + pairs[key] = value + return pairs + + +def test_the_container_is_held_offline_at_runtime(): + """The header promised no outbound connection and nothing enforced it.""" + assert _env().get("COMPARTMENT_ASSERT_OFFLINE") == "1" + assert "--network none" in DOCKERFILE + + +def test_one_server_per_container_runs_no_shared_embedding_daemon(): + """embed_daemon.enabled() names containers as wanting it off; the daemon + also needs a writable session directory a --user run does not have.""" + assert _env().get("COMPARTMENT_EMBED_DAEMON") == "0" + + +def test_the_run_line_can_read_a_vault_owned_by_the_host_user(): + """The vault is 0600 and owned by the host user, so the image's uid + 10001 could not open a bind-mounted one on a Linux host.""" + assert '--user "$(id -u):$(id -g)"' in DOCKERFILE + + +def test_the_wheel_is_built_from_the_package_alone(): + """`COPY . /src` brought a checkout's stale build/ directory, which + setuptools folds into the wheel, into the image's build.""" + build = DOCKERFILE.split("\nFROM ")[1] + assert not re.search(r"^COPY \. ", build, re.M) + assert re.search(r"^COPY src /src/src$", build, re.M) + + +def test_a_supported_python_and_both_architectures(): + bases = re.findall(r"^FROM python:(\d+\.\d+)-slim", DOCKERFILE, re.M) + assert bases and all((3, 11) <= tuple(map(int, b.split("."))) <= (3, 14) + for b in bases) + assert len(set(bases)) == 1 + assert "linux/amd64,linux/arm64" in DOCKERFILE diff --git a/tests/test_platform_filelock.py b/tests/test_platform_filelock.py new file mode 100644 index 0000000..7bf1bf3 --- /dev/null +++ b/tests/test_platform_filelock.py @@ -0,0 +1,45 @@ +"""FileLock on filesystems that cannot lock: NFS without its lock service or +mounted nolock, some SMB and FUSE mounts. That is not "busy", and waiting +does not change it.""" +import errno +import os +import time + +import pytest + +from compartment import platforms +from compartment.crypto import CryptoError + +pytestmark = pytest.mark.skipif(os.name == "nt", reason="the fcntl branch") + + +@pytest.mark.parametrize("code", [errno.ENOLCK, errno.EOPNOTSUPP, errno.EINVAL]) +def test_a_filesystem_without_locks_is_named_at_once_not_called_busy( + tmp_path, monkeypatch, code): + """Before: every write waited out the timeout and then reported "Vault + is busy: another process holds the write lock", on a vault nobody else + had open.""" + def no_locks(fh, op): + raise OSError(code, os.strerror(code)) + monkeypatch.setattr(platforms.fcntl, "flock", no_locks) + started = time.monotonic() + with pytest.raises(CryptoError) as err: + with platforms.FileLock(str(tmp_path / "v.vault.flock"), timeout=3): + pass + assert time.monotonic() - started < 1.5 + assert "busy" not in str(err.value) + assert "does not support file locks" in str(err.value) + assert "local disk" in str(err.value) + + +@pytest.mark.parametrize("code", [errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES]) +def test_a_lock_someone_holds_is_still_waited_for_and_called_busy( + tmp_path, monkeypatch, code): + """BSD flock answers EWOULDBLOCK, and Linux's NFS emulation can answer + EAGAIN or EACCES: all three are another holder.""" + def held(fh, op): + raise OSError(code, os.strerror(code)) + monkeypatch.setattr(platforms.fcntl, "flock", held) + with pytest.raises(CryptoError, match="busy"): + with platforms.FileLock(str(tmp_path / "v.vault.flock"), timeout=0.2): + pass diff --git a/tests/test_platform_installers.py b/tests/test_platform_installers.py new file mode 100644 index 0000000..57b9b56 --- /dev/null +++ b/tests/test_platform_installers.py @@ -0,0 +1,523 @@ +"""install/install-linux.sh and install/install-macos.sh, run for real by +bash against stand-ins for python, pip, venv, pipx and uv. + +Every external program the scripts start is a stub on a PATH that holds +nothing else: each one logs how it was called, and the python stub runs the +scripts' own probes with the platform facts a test gives it (its version, +glibc or musl, CPU, macOS version, PEP 668, an onnxruntime already +installed). So these tests are about the scripts' decisions, on any POSIX +machine, and nothing is installed or downloaded. +""" +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +LINUX = ROOT / "install" / "install-linux.sh" +MACOS = ROOT / "install" / "install-macos.sh" +BASH = shutil.which("bash") + +needs_bash = pytest.mark.skipif(os.name == "nt" or BASH is None, + reason="the shell installers need bash") + +_PYTHON_STUB = r'''#!{python} +import json, os, shutil, stat, sys +me = os.path.abspath(sys.argv[0]) +name = os.path.basename(me) +args = sys.argv[1:] +facts = json.loads(os.environ.get("STUB_FACTS", "{{}}")) +f = dict(facts.get("*", {{}})) +f.update(facts.get(name, {{}})) +in_venv = os.path.isfile(os.path.join(os.path.dirname(os.path.dirname(me)), + "pyvenv.cfg")) +kind = "plain" if in_venv else f.get("kind", "plain") + +def log(*what): + with open(os.environ["STUB_LOG"], "a") as fh: + fh.write(json.dumps([me, *what]) + "\n") + +def console_script(where): + os.makedirs(where, exist_ok=True) + target = os.path.join(where, "compartment") + shutil.copy(os.environ["STUB_COMPARTMENT"], target) + os.chmod(target, 0o755) + +if args[:1] == ["-c"]: + import importlib.util, platform, sysconfig + if "import compartment.cli" in args[1]: + sys.exit(1) + sys.version_info = tuple(f.get("version", [3, 12])) + (0, "final", 0) + platform.system = lambda: f.get("system", "Linux") + platform.machine = lambda: f.get("machine", "x86_64") + libc = f.get("libc", "glibc 2.39") + def confstr(n): + if libc == "musl": + raise OSError(22, "Invalid argument") + return libc + os.confstr = confstr + if f.get("android"): + sys.getandroidapilevel = lambda: 34 + real_find_spec = importlib.util.find_spec + def find_spec(n, *a, **k): + if n == "onnxruntime": + return object() if f.get("ort") else None + if n == "pip": + return None if f.get("no_pip") else object() + return real_find_spec(n, *a, **k) + importlib.util.find_spec = find_spec + stdlib = os.path.join(os.environ["STUB_ROOT"], "stdlib-" + kind) + os.makedirs(stdlib, exist_ok=True) + if kind == "managed": + open(os.path.join(stdlib, "EXTERNALLY-MANAGED"), "w").close() + real_get_path = sysconfig.get_path + def get_path(n, scheme=None, *a, **k): + if n == "stdlib": + return stdlib + if n == "scripts": + return os.environ["STUB_USER_BIN" if scheme == "posix_user" + else "STUB_VENV_BIN"] + return real_get_path(n, *a, **k) + sysconfig.get_path = get_path + if kind == "venv": + sys.prefix, sys.base_prefix = "/venv", "/base" + else: + sys.prefix = sys.base_prefix + exec(compile(args[1], "", "exec"), {{"__name__": "__main__"}}) + sys.exit(0) +if args[:2] == ["-m", "pip"]: + log("pip", *args[2:]) + if kind == "managed" and "--user" in args: + print("error: externally-managed-environment", file=sys.stderr) + sys.exit(1) + console_script(os.path.dirname(me) if in_venv + else os.environ["STUB_USER_BIN"] if "--user" in args + else os.environ["STUB_VENV_BIN"]) + sys.exit(0) +if args[:2] == ["-m", "venv"]: + log("venv", *args[2:]) + if f.get("no_venv"): + print("ensurepip is not available", file=sys.stderr) + sys.exit(1) + d = args[-1] + os.makedirs(os.path.join(d, "bin"), exist_ok=True) + open(os.path.join(d, "pyvenv.cfg"), "w").close() + shutil.copy(me, os.path.join(d, "bin", "python")) + sys.exit(0) +log("unexpected", *args) +sys.exit(3) +''' + +_TOOL_STUB = r'''#!{python} +import json, os, shutil, sys +args = sys.argv[1:] +tool = os.path.basename(sys.argv[0]) +with open(os.environ["STUB_LOG"], "a") as fh: + fh.write(json.dumps([tool, *args]) + "\n") +if tool == "compartment": + sys.exit(0) +if tool == "sw_vers": + print(os.environ.get("STUB_MACOS", "15.1")) + sys.exit(0) +bindir = os.environ["STUB_TOOL_BIN"] +if args[:1] in (["environment"], ["tool"]) and "install" not in args: + print(bindir) + sys.exit(0) +os.makedirs(bindir, exist_ok=True) +shutil.copy(os.environ["STUB_COMPARTMENT"], os.path.join(bindir, "compartment")) +os.chmod(os.path.join(bindir, "compartment"), 0o755) +''' + + +class Run: + def __init__(self, proc, log): + self.rc = proc.returncode + self.out = proc.stdout + proc.stderr + self.calls = [json.loads(line) for line in log.read_text().splitlines()] + + def of(self, what): + """The argument lists of every call to `what` (pip, venv, pipx, + uv, compartment).""" + found = [] + for call in self.calls: + head = call[0] if call[0] in ("pipx", "uv", "compartment", "sw_vers") else call[1] + if head == what: + found.append(call[1:] if call[0] in ("pipx", "uv", "compartment", "sw_vers") else call[2:]) + return found + + def interpreter_of(self, what): + return [c[0] for c in self.calls if len(c) > 1 and c[1] == what] + + def init_args(self): + """How `init` was run, or how the script told the user to run it + when there was no terminal to ask for a passphrase on (no pty in + this sandbox, say). None when it was neither.""" + inits = [c for c in self.of("compartment") if c[:1] == ["init"]] + if inits: + return inits[0] + m = re.search(r"Create the vault with:\n\s+compartment (init[^\n]*)", + self.out) + return m.group(1).split() if m else None + + +@pytest.fixture() +def installer(tmp_path): + """Run an installer with the stand-ins. `pythons` names the python + stubs to put on PATH; `facts` is per-stub (or "*") platform facts; + `tools` adds pipx and/or uv.""" + if os.name == "nt" or BASH is None: + pytest.skip("the shell installers need bash") + + def run(script, pythons=("python3",), facts=None, tools=(), env=None, + tty=True, stdin_script=False, cwd=None): + root = tmp_path / "stubs" + bin_ = root / "bin" + bin_.mkdir(parents=True, exist_ok=True) + home = tmp_path / "home" + home.mkdir(exist_ok=True) + log = root / "log" + log.write_text("") + py = _PYTHON_STUB.format(python=sys.executable) + tool = _TOOL_STUB.format(python=sys.executable) + comp = root / "compartment-stub" + comp.write_text(tool) + comp.chmod(0o755) + for name in pythons: + (bin_ / name).write_text(py) + (bin_ / name).chmod(0o755) + for name in (*tools, "sw_vers"): + (bin_ / name).write_text(tool) + (bin_ / name).chmod(0o755) + for name in ("sed", "grep", "dirname", "mkdir", "ln", "cat", "uname"): + real = shutil.which(name, path="/usr/bin:/bin") + if real: + (bin_ / name).symlink_to(real) + e = {"PATH": str(bin_), "HOME": str(home), + "STUB_LOG": str(log), "STUB_ROOT": str(root), + "STUB_FACTS": json.dumps(facts or {}), + "STUB_COMPARTMENT": str(comp), + "STUB_USER_BIN": str(home / ".local" / "bin"), + "STUB_VENV_BIN": str(root / "active-venv" / "bin"), + "STUB_TOOL_BIN": str(home / ".local" / "bin"), + # The macOS script also tries Homebrew's and python.org's + # absolute paths; on a real Mac those are this machine's own + # interpreters, which a test must never install into. + "CMPT_EXTRA_PYTHONS": ""} + e.update(env or {}) + master = slave = None + if stdin_script: + argv, stdin = [BASH, "-s"], open(script, "rb") + else: + argv, stdin = [BASH, str(script)], subprocess.DEVNULL + if tty: + try: + import pty # POSIX only, like these tests + master, slave = pty.openpty() + stdin = slave + except OSError: + pass # no ptys here: the no-terminal branch + try: + # A session of its own, so /dev/tty is never the terminal of + # whoever runs the suite. + proc = subprocess.run(argv, env=e, stdin=stdin, cwd=cwd or tmp_path, + capture_output=True, text=True, timeout=120, + start_new_session=True) + finally: + if stdin_script: + stdin.close() + for fd in (master, slave): + if fd is not None: + os.close(fd) + return Run(proc, log) + return run + + +# --- Linux: PEP 668 ----------------------------------------------------------- + +def test_an_externally_managed_python_is_installed_with_pipx(installer): + """Debian 12+, Ubuntu 23.04+, Arch, Crostini: `pip install --user` fails + with externally-managed-environment, and under `set -e` the script + ended there with nothing installed.""" + r = installer(LINUX, facts={"*": {"kind": "managed"}}, tools=("pipx",)) + assert r.rc == 0, r.out + assert not [a for a in r.of("pip") if "--user" in a] + install = [a for a in r.of("pipx") if a[:1] == ["install"]] + assert install and str(ROOT) in install[0] + assert r.init_args()[0] == "init" + + +def test_uv_is_used_where_there_is_no_pipx(installer): + r = installer(LINUX, facts={"*": {"kind": "managed"}}, tools=("uv",)) + assert r.rc == 0, r.out + assert [a for a in r.of("uv") if a[:2] == ["tool", "install"]] + + +def test_with_neither_a_private_environment_is_made_and_linked(installer, + tmp_path): + r = installer(LINUX, facts={"*": {"kind": "managed"}}) + assert r.rc == 0, r.out + venv = tmp_path / "home" / ".local" / "share" / "compartment" / "venv" + assert r.of("venv") == [[str(venv)]] + assert r.interpreter_of("pip") == [str(venv / "bin" / "python")] + link = tmp_path / "home" / ".local" / "bin" / "compartment" + assert link.is_symlink() and link.resolve() == (venv / "bin" / "compartment").resolve() + assert r.init_args()[0] == "init" + + +def test_a_python_without_the_venv_module_says_which_package_to_add(installer): + """Debian splits ensurepip into python3-venv, and Crostini has none.""" + r = installer(LINUX, facts={"*": {"kind": "managed", "no_venv": True}}) + assert r.rc == 1 + assert "python3-venv" in r.out and "pipx" in r.out + + +def test_a_python_that_allows_user_installs_keeps_the_old_route(installer): + r = installer(LINUX) + assert r.rc == 0, r.out + assert r.of("pip") == [["install", "--user", "--upgrade", str(ROOT)]] + + +def test_a_python_without_pip_gets_a_private_environment(installer, tmp_path): + """Minimal containers and some distributions ship python3 without pip: + `python3 -m pip` died with "No module named pip".""" + r = installer(LINUX, facts={"*": {"no_pip": True}}) + assert r.rc == 0, r.out + venv = tmp_path / "home" / ".local" / "share" / "compartment" / "venv" + assert r.of("venv") == [[str(venv)]] + assert r.interpreter_of("pip") == [str(venv / "bin" / "python")] + + +def test_an_activated_virtualenv_is_installed_into_without_user(installer): + """pip refuses --user inside a virtualenv.""" + r = installer(LINUX, facts={"*": {"kind": "venv"}}) + assert r.rc == 0, r.out + assert r.of("pip") == [["install", "--upgrade", str(ROOT)]] + + +# --- Linux: which Python ------------------------------------------------------ + +def test_a_too_old_python3_is_passed_over_for_a_versioned_one(installer): + """Ubuntu 22.04's python3 is 3.10; python3.11 is a separate package.""" + r = installer(LINUX, pythons=("python3", "python3.11"), + facts={"python3": {"version": [3, 10]}, + "python3.11": {"version": [3, 11]}}) + assert r.rc == 0, r.out + assert r.interpreter_of("pip")[0].endswith("python3.11") + + +def test_a_python_newer_than_the_wheels_is_passed_over_too(installer): + """Python 3.15 is out before onnxruntime builds for it; the supported + range is 3.11 to 3.14 (tools/wheel_coverage.py).""" + r = installer(LINUX, pythons=("python3", "python3.14"), + facts={"python3": {"version": [3, 15]}, + "python3.14": {"version": [3, 14]}}) + assert r.rc == 0, r.out + assert r.interpreter_of("pip")[0].endswith("python3.14") + + +def test_no_python_in_range_is_explained(installer): + r = installer(LINUX, facts={"*": {"version": [3, 9]}}) + assert r.rc == 1 + assert "3.11 to 3.14" in r.out and "python3 is Python 3.9" in r.out + assert not r.of("pip") + + +# --- Linux: platforms onnxruntime has no wheel for ---------------------------- + +@pytest.mark.parametrize("facts, says", [ + ({"libc": "musl"}, "apk add python3 py3-onnxruntime"), + ({"android": True}, "pkg install python onnxruntime"), + ({"system": "FreeBSD", "version": [3, 12]}, "pkg install py312-onnxruntime"), + ({"machine": "armv7l"}, "64-bit OS"), + ({"libc": "glibc 2.17"}, "glibc 2.28 or newer"), + ({"libc": "glibc 2.27"}, "glibc 2.28 or newer"), +]) +def test_a_platform_without_onnxruntime_wheels_is_told_what_to_install( + installer, facts, says): + """pip used to fail there with a bare "No matching distribution found + for onnxruntime", after a long resolve.""" + r = installer(LINUX, facts={"*": facts}) + assert r.rc == 1 + assert says in r.out + assert not r.of("pip") and not r.of("venv") + + +def test_onnxruntime_from_the_os_is_used_through_system_site_packages( + installer, tmp_path): + """Alpine's py3-onnxruntime, Termux's onnxruntime, FreeBSD's port: pip + must see that copy, or it goes looking for a wheel that does not + exist.""" + r = installer(LINUX, facts={"*": {"libc": "musl", "ort": True, + "kind": "managed"}}, tools=("pipx",)) + assert r.rc == 0, r.out + venv = tmp_path / "home" / ".local" / "share" / "compartment" / "venv" + assert r.of("venv") == [["--system-site-packages", str(venv)]] + assert not r.of("pipx") + + +def test_termux_creates_the_vault_locked_because_it_has_no_shared_memory( + installer): + """Termux has no shm_open, so `init` would create the vault and then + fail storing the session credential.""" + r = installer(LINUX, facts={"*": {"android": True, "ort": True}}, + env={"TERMUX_VERSION": "0.119"}) + assert r.rc == 0, r.out + assert r.init_args() == ["init", "--no-session"] + assert "COMPARTMENT_PASSPHRASE" in r.out + + +# --- Linux: what gets installed, and re-running ------------------------------ + +def test_piped_into_bash_it_installs_compartment_not_the_project_next_door( + installer, tmp_path): + """`curl ... | bash` has no script file: $0 is "bash", so `dirname $0` + was the current directory, and a pyproject.toml one level up - any + other Python project the user was standing in - was installed instead + of Compartment.""" + other = tmp_path / "someone-elses-project" + (other / "docs").mkdir(parents=True) + (other / "pyproject.toml").write_text('[project]\nname = "other"\n') + vault = tmp_path / "home" / "existing.vault" + vault.parent.mkdir(parents=True, exist_ok=True) + vault.write_text("") + r = installer(LINUX, stdin_script=True, cwd=other / "docs", + env={"COMPARTMENT_VAULT": str(vault)}) + assert r.rc == 0, r.out + assert r.of("pip") == [["install", "--user", "--upgrade", "compartment"]] + + +def test_running_it_again_leaves_the_vault_alone_and_succeeds(installer, + tmp_path): + """`init` refuses to overwrite a vault, so a second run used to end in + an error after upgrading.""" + (tmp_path / "home" / ".compartment").mkdir(parents=True) + (tmp_path / "home" / ".compartment" / "memory.vault").write_text("") + r = installer(LINUX) + assert r.rc == 0, r.out + assert r.init_args() is None + assert "kept as it is" in r.out + + +def test_a_legacy_engram_vault_counts_as_existing(installer, tmp_path): + (tmp_path / "home" / ".engram").mkdir(parents=True) + (tmp_path / "home" / ".engram" / "memory.vault").write_text("") + r = installer(LINUX) + assert r.rc == 0, r.out + assert r.init_args() is None + + +# --- macOS -------------------------------------------------------------------- + +def test_homebrew_python_is_installed_with_pipx_on_macos(installer): + r = installer(MACOS, facts={"*": {"kind": "managed", "machine": "arm64"}}, + tools=("pipx",)) + assert r.rc == 0, r.out + assert not [a for a in r.of("pip") if "--user" in a] + assert [a for a in r.of("pipx") if a[:1] == ["install"]] + + +@pytest.mark.parametrize("machine, macos", [("x86_64", "15.1"), + ("arm64", "13.6")]) +def test_python_3_14_is_passed_over_where_onnxruntime_stops_at_3_13( + installer, machine, macos): + """onnxruntime has no Intel Mac build after 1.23.2 and no macOS 13 build + after it either; 1.23.2 covers Python 3.11-3.13. With Homebrew's + python3 at 3.14, pip failed on onnxruntime.""" + r = installer(MACOS, pythons=("python3", "python3.13"), + facts={"*": {"machine": machine}, + "python3": {"version": [3, 14]}, + "python3.13": {"version": [3, 13]}}, + env={"STUB_MACOS": macos}) + assert r.rc == 0, r.out + assert r.interpreter_of("pip")[0].endswith("python3.13") + + +def test_only_python_3_14_on_an_intel_mac_names_the_python_to_install( + installer): + r = installer(MACOS, facts={"*": {"machine": "x86_64", + "version": [3, 14]}}) + assert r.rc == 1 + assert "brew install python@3.13" in r.out + assert "x86_64 Python" in r.out + assert not r.of("pip") + + +def test_a_mac_older_than_macos_13_is_told_it_is_unsupported(installer): + r = installer(MACOS, facts={"*": {"machine": "arm64"}}, + env={"STUB_MACOS": "12.7"}) + assert r.rc == 1 + assert "macOS 13 or newer" in r.out + assert not r.of("pip") + + +def test_apple_silicon_on_a_current_macos_takes_python_3_14(installer): + r = installer(MACOS, facts={"*": {"machine": "arm64", "version": [3, 14]}}, + env={"STUB_MACOS": "15.1"}) + assert r.rc == 0, r.out + assert r.init_args() == ["init"] + + +@needs_bash +def test_both_scripts_parse(): + for script in (LINUX, MACOS): + subprocess.run([BASH, "-n", str(script)], check=True) + + +# --- Windows: the interpreter probe ------------------------------------------ +# +# PowerShell is not here to run the script itself, so its probe - the part +# that decides which Python is used - runs under this Python with the +# interpreter facts it reads stood in for. + +WINDOWS = ROOT / "install" / "install-windows.ps1" + + +def _windows_probe(version, plat, monkeypatch, capsys): + text = WINDOWS.read_text(encoding="utf-8") + m = re.search(r"\$probe = @'\r?\n(.*?)\r?\n'@", text, re.S) + assert m, "install-windows.ps1 has no interpreter probe" + import sysconfig + monkeypatch.setattr(sys, "version_info", tuple(version) + (0, "final", 0)) + monkeypatch.setattr(sysconfig, "get_platform", lambda: plat) + try: + exec(compile(m.group(1), "", "exec"), {"__name__": "__main__"}) + code = 0 + except SystemExit as exc: + code = exc.code or 0 + return code, capsys.readouterr().out.strip() + + +@pytest.mark.parametrize("version, plat, usable", [ + ((3, 13), "win-amd64", True), + ((3, 11), "win-amd64", True), + ((3, 14), "win-arm64", True), + ((3, 11), "win-arm64", False), # no pyyaml wheel for 3.11 on ARM64 + ((3, 12), "win32", False), # no 32-bit onnxruntime at all + ((3, 15), "win-amd64", False), # newer than the wheels + ((3, 10), "win-amd64", False), +]) +def test_the_windows_installer_picks_a_python_inside_the_wheel_matrix( + version, plat, usable, monkeypatch, capsys): + """The old script took whatever `py` started: the default Python, which + may be 32-bit, 3.10, 3.15, or 3.11 on ARM64, where pip then failed on a + dependency with no wheel.""" + code, out = _windows_probe(version, plat, monkeypatch, capsys) + assert (code == 0) is usable, out + if not usable: + assert out + + +@needs_bash +def test_the_macos_script_tries_the_well_known_paths_unless_told_otherwise(): + """The absolute interpreter paths are on by default for users and + replaceable, so the suite's stand-in Mac never reaches a real one: on a + GitHub macOS runner it found the real Python 3.13, ran a real install + and then waited for a passphrase until the job timed out.""" + script = MACOS.read_text(encoding="utf-8") + assert "${CMPT_EXTRA_PYTHONS-/opt/homebrew/bin/python3.13" in script + assert "$EXTRA_PYTHONS" in script diff --git a/tests/test_platform_session_store.py b/tests/test_platform_session_store.py new file mode 100644 index 0000000..7f8c870 --- /dev/null +++ b/tests/test_platform_session_store.py @@ -0,0 +1,147 @@ +"""Where the session store cannot be written: a container run as an arbitrary +uid (HOME is `/`), a read-only root, and an agent's shell under macOS +Seatbelt, which may read the user's session directory but not write it. + +Reading a credential, or finding there is none, must not need a write.""" +import errno +import os + +import pytest + +from compartment import session +from compartment.vault import Vault + +from conftest import PASS + + +def _unmakeable_store(tmp_path, monkeypatch): + """A session directory nobody can create: its parent is a regular file. + mkdir refuses that for every user, root included, on every platform.""" + blocker = tmp_path / "home-is-not-writable" + blocker.write_text("") + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(blocker / "session")) + return blocker / "session" + + +def test_finding_no_credential_needs_no_session_directory(tmp_path, monkeypatch): + store = _unmakeable_store(tmp_path, monkeypatch) + vault = str(tmp_path / "v.vault") + assert session.get(vault) is None + assert session.clear(vault) is False # `compartment lock` + assert not store.exists() + + +def test_the_passphrase_variable_opens_a_vault_where_home_is_not_writable( + tmp_path, monkeypatch): + """`docker run --user 1000 -e COMPARTMENT_PASSPHRASE ...`: HOME is `/`, so + the session store would be /.compartment/session. Every open used to die + with PermissionError making it, before the variable was ever read.""" + vault = str(tmp_path / "v.vault") + Vault.create(vault, PASS, creator="test").lock() + _unmakeable_store(tmp_path, monkeypatch) + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + assert Vault.find_credential(vault) == (PASS, None) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX modes") +def test_a_store_that_refuses_chmod_still_yields_its_credential( + tmp_path, monkeypatch): + """The agent-sandbox case. The user unlocked in a terminal; the agent's + `compartment search` runs under Seatbelt, which lets it read the 0700 + session directory but answers chmod on it with EPERM (seen in Claude + Code's own sandbox). The credential is there and readable, so the + lookup must find it rather than die on a mode it did not need to set.""" + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + vault = str(tmp_path / "v.vault") + key = b"k" * 32 + session.store(vault, key) + real_chmod = os.chmod + + def seatbelt(path, mode, *a, **kw): + if os.fspath(path).startswith(str(tmp_path / "sess")): + raise PermissionError(errno.EPERM, "Operation not permitted", + os.fspath(path)) + return real_chmod(path, mode, *a, **kw) + monkeypatch.setattr(session.os, "chmod", seatbelt) + assert session.get(vault) == key + assert session.credential_path(vault).is_file() + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX modes") +def test_a_store_with_a_wide_mode_is_still_narrowed(tmp_path, monkeypatch): + """Skipping a chmod that is not needed must not skip one that is.""" + d = tmp_path / "sess" + d.mkdir(mode=0o755) + os.chmod(d, 0o755) + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(d)) + session.store(str(tmp_path / "v.vault"), b"k" * 32) + assert (d.stat().st_mode & 0o777) == 0o700 + + +# --- no usable holder: the refusal has to say what to do instead ---------- + +@pytest.mark.skipif(session.IS_WINDOWS, reason="POSIX holder") +@pytest.mark.parametrize("code", [errno.ENOENT, errno.EACCES, errno.EROFS, + errno.ENOSPC]) +def test_a_holder_that_cannot_be_made_names_the_way_round_it( + tmp_path, monkeypatch, code): + """glibc keeps shm objects in /dev/shm and fails ENOENT where it is not + mounted (chroots, some CI and build containers); a container's can be + read-only or full, and a sandbox can refuse it. The bare "[Errno 2] No + such file or directory" left nobody knowing what to do.""" + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + + def refuse(name, flags, mode=None): + if flags & os.O_CREAT: + raise OSError(code, os.strerror(code), name) + raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), name) + monkeypatch.setattr(session, "_shm_open", refuse) + with pytest.raises(session.CryptoError) as err: + session.store(str(tmp_path / "v.vault"), b"k" * 32) + msg = str(err.value) + assert "/dev/shm" in msg + assert "COMPARTMENT_PASSPHRASE" in msg and "--once" in msg + + +@pytest.mark.skipif(session.IS_WINDOWS, reason="POSIX holder") +def test_openbsd_does_not_keep_the_secret_in_a_file_on_disk( + tmp_path, monkeypatch): + """OpenBSD's libc implements shm_open as /tmp/.shm, an ordinary + file on disk (lib/libc/gen/shm_open.c), so the secret would travel with + any copy of the disk the session file is on. Storing must refuse there, + the way it does where shm_open is missing, and say what to use.""" + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + monkeypatch.setattr(session, "_libc_cache", None) + monkeypatch.setattr(session.sys, "platform", "openbsd7") + with pytest.raises(session.CryptoError) as err: + session.store(str(tmp_path / "v.vault"), b"k" * 32) + assert "OpenBSD" in str(err.value) + assert "COMPARTMENT_PASSPHRASE" in str(err.value) + assert not session.credential_path(str(tmp_path / "v.vault")).exists() + + +# --- Windows: who the user is must not hang on one environment variable --- + +def test_a_windows_process_started_without_username_is_still_the_same_user( + monkeypatch): + """The holder's name and the wrap key's context both fold in the user. + A process started without USERNAME - an MCP client that replaces the + environment with its config's `env` block, a service wrapper - called + itself "user", found no secret under that holder, and get() deleted the + credential every other process of the user was opening.""" + monkeypatch.delattr(session.os, "getuid", raising=False) + monkeypatch.setattr(session, "_windows_account_name", lambda: "Zoë", + raising=False) + monkeypatch.setenv("USERNAME", "Zoë") + with_variable = session._uid() + monkeypatch.delenv("USERNAME") + assert session._uid() == with_variable == "Zoë" + + +def test_the_username_variable_still_wins_so_existing_holders_keep_their_name( + monkeypatch): + monkeypatch.delattr(session.os, "getuid", raising=False) + monkeypatch.setattr(session, "_windows_account_name", lambda: "other", + raising=False) + monkeypatch.setenv("USERNAME", "Zoë") + assert session._uid() == "Zoë" diff --git a/tests/test_platforms.py b/tests/test_platforms.py index 857ce8b..cb960e1 100644 --- a/tests/test_platforms.py +++ b/tests/test_platforms.py @@ -130,6 +130,69 @@ def test_a_clock_step_does_not_kill_the_session_credential(tmp_path, monkeypatch assert session._file_for(vault).is_file() +def _no_kernel_boot_record(monkeypatch, system): + """The sandbox case: kern.boottime refused (Seatbelt under Claude Code and + Codex), /proc/stat hidden (Android's SELinux under Termux).""" + monkeypatch.setattr(platforms.platform, "system", lambda: system) + monkeypatch.setattr(platforms, "_sysctl_boot_estimate", lambda: None) + real_open = open + + def no_proc(path, *a, **kw): + if str(path).startswith("/proc/"): + raise PermissionError(13, "Permission denied", str(path)) + return real_open(path, *a, **kw) + monkeypatch.setattr("builtins.open", no_proc) + + +@pytest.mark.parametrize("system", ["Darwin", "Linux", "FreeBSD"]) +def test_a_sandbox_that_hides_the_boot_record_still_gets_a_boot_token( + tmp_path, monkeypatch, system): + """Before the clock fallback every vault open from an agent's sandboxed + shell raised "Cannot identify this boot session", so `compartment store` + run by the agent itself could never use the unlocked vault.""" + _tmp_session_dir(monkeypatch, tmp_path) + _no_kernel_boot_record(monkeypatch, system) + clock = "CLOCK_MONOTONIC" if system == "Darwin" else "CLOCK_BOOTTIME" + monkeypatch.setattr(platforms.time, clock, 777, raising=False) + # raising=False: Windows' time module has no clock_gettime at all. + monkeypatch.setattr(platforms.time, "clock_gettime", + lambda c: 500.0 if c == 777 else 1.0, raising=False) + monkeypatch.setattr(platforms.time, "time", lambda: 10_500.0) + assert platforms.boot_time() == platforms.boot_time() == "10000" + + +def test_the_clock_fallback_agrees_with_the_pinned_kernel_reading( + tmp_path, monkeypatch): + """An unsandboxed process pins the kernel's value; a sandboxed one that + can only read the clock a second later must land on the same token, or + the credential the first one stored is unreadable to the second.""" + _tmp_session_dir(monkeypatch, tmp_path) + monkeypatch.setattr(platforms.platform, "system", lambda: "Darwin") + monkeypatch.setattr(platforms, "_sysctl_boot_estimate", + lambda: (10_000.0, 500.0)) + assert platforms.boot_time() == "10000" + monkeypatch.setattr(platforms, "_sysctl_boot_estimate", lambda: None) + monkeypatch.setattr(platforms.time, "CLOCK_MONOTONIC", 1, raising=False) + monkeypatch.setattr(platforms.time, "clock_gettime", lambda c: 501.0, + raising=False) + monkeypatch.setattr(platforms.time, "time", lambda: 10_501.7) + assert platforms.boot_time() == "10000" + + +@pytest.mark.skipif(platform.system() not in ("Darwin", "Linux"), + reason="needs a POSIX clock that counts from boot") +def test_the_clock_estimate_is_close_to_the_kernel_record(): + """The fallback is only safe if it reads the same boot instant the kernel + reports; a clock that skipped sleep would drift from it by hours.""" + clock = platforms._clock_boot_estimate() + assert clock is not None + kernel = (platforms._sysctl_boot_estimate() if platform.system() == "Darwin" + else None) + if kernel is None: + pytest.skip("kernel boot record not readable here") + assert abs(clock[0] - kernel[0]) < 120 + + def test_filelock_is_exclusive(tmp_path): lockpath = str(tmp_path / "x.flock") with platforms.FileLock(lockpath, timeout=0.3): @@ -357,3 +420,44 @@ def _linux(monkeypatch, *paths): monkeypatch.setattr(platforms, "_LINUX_MACHINE_ID_PATHS", tuple(str(p) for p in paths)) yield + + +def test_the_windows_lock_always_takes_byte_zero(monkeypatch, tmp_path): + """msvcrt.locking locks from the handle's position, and an "a+" handle + starts at end of file: with anything in the lock file, two processes + locked different bytes and both held the "exclusive" lock.""" + import importlib + import types + calls = [] + fake = types.SimpleNamespace( + LK_NBLCK=2, LK_UNLCK=0, + locking=lambda fd, mode, n: calls.append((mode, os.lseek(fd, 0, os.SEEK_CUR)))) + monkeypatch.setitem(sys.modules, "msvcrt", fake) + monkeypatch.setattr(os, "name", "nt") + try: + win = importlib.reload(platforms) + lockfile = tmp_path / "v.flock" + lockfile.write_text("left behind by something else") + with win.FileLock(str(lockfile), timeout=0.2): + pass + finally: + monkeypatch.setattr(os, "name", "posix" if sys.platform != "win32" else "nt") + monkeypatch.delitem(sys.modules, "msvcrt", raising=False) + importlib.reload(platforms) + assert calls and all(pos == 0 for _, pos in calls) + + +def test_a_session_dir_from_the_environment_needs_no_home_folder( + monkeypatch, tmp_path): + """A container run as a uid with no passwd entry and no HOME: Path.home() + raises, and setting COMPARTMENT_SESSION_DIR - the documented way to put + the session somewhere writable - still went through it.""" + from compartment import home as home_mod, session + + def no_home(cls=None): + raise RuntimeError("Could not determine home directory.") + monkeypatch.setattr(home_mod.Path, "home", classmethod(no_home)) + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "s")) + assert home_mod.session_dir() == tmp_path / "s" + assert platforms._posix_marker().parent == tmp_path / "s" + assert session.credential_path(str(tmp_path / "v.vault")).parent == tmp_path / "s" diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py new file mode 100644 index 0000000..d2b57d7 --- /dev/null +++ b/tests/test_plugin_manifests.py @@ -0,0 +1,545 @@ +"""Every store listing in the repository parses, says what its host needs, +and launches the same server. + +Compartment is listed in a dozen places from one repository: the root is +at once a Claude Code plugin, a Codex plugin (through the Agent Plugins +manifest), an OpenClaw bundle, a Gemini CLI extension and a Hermes +package, and packaging/ holds the manifests other catalogues want. None of +those hosts runs in CI, so a manifest that drifts from its host's format, +or from the server it describes, fails on a user's machine and nowhere +else. These tests check each one against the parts of its host's current +format that matter (docs/PLUGINS.md names the source of each), and check +them against each other: one command, the same subcommand, a caller each +host can be told apart by, and no claim about the server that the server +does not make. + +Offline, like the rest of the suite: nothing here fetches a schema. +""" +import json +import pathlib +import re + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def _json(rel: str) -> dict: + return json.loads((ROOT / rel).read_text(encoding="utf-8")) + + +# ------------------------------------------------------------ MCP Registry -- +#: The newest released server.json schema in modelcontextprotocol/registry +#: (docs/reference/server-json/CHANGELOG.md), still current in October 2026. +REGISTRY_SCHEMA = ("https://static.modelcontextprotocol.io/schemas/" + "2025-12-11/server.schema.json") + + +def test_server_json_targets_the_current_registry_schema(): + assert _json("server.json")["$schema"] == REGISTRY_SCHEMA + + +def test_server_json_name_is_the_one_pypi_proves_ownership_of(): + """The registry checks a PyPI package by finding `mcp-name: ` in + its README, so the two must be the same string or every publish fails.""" + name = _json("server.json")["name"] + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert f"mcp-name: {name}" in readme + + +def test_server_json_meets_the_registry_limits(): + d = _json("server.json") + assert 1 <= len(d["description"]) <= 100 # schema maxLength + assert d["repository"]["url"] == "https://github.com/MaxFreedomPollard/Compartment" + pkg = d["packages"][0] + assert pkg["registryType"] == "pypi" + assert pkg["registryBaseUrl"] == "https://pypi.org" # the only one allowed + assert pkg["identifier"] == "compartment" + assert pkg["transport"] == {"type": "stdio"} + assert [a["value"] for a in pkg["packageArguments"]][-1] == "serve" + + +def test_server_json_gives_registry_clients_a_website_and_an_icon(): + """Clients that render the registry (VS Code's MCP gallery, GitHub's MCP + registry, directory sites) show a blank tile and no link without these.""" + d = _json("server.json") + assert d["websiteUrl"].startswith("https://") + icon = d["icons"][0] + assert icon["src"].startswith("https://") and len(icon["src"]) <= 255 + assert icon["mimeType"] == "image/png" + assert all(re.fullmatch(r"\d+x\d+|any", s) for s in icon["sizes"]) + # the icon it names is the one in this repository, at the size it says + local = ROOT / icon["src"].split("/main/", 1)[1] + head = local.read_bytes()[:24] + w, h = int.from_bytes(head[16:20], "big"), int.from_bytes(head[20:24], "big") + assert icon["sizes"] == [f"{w}x{h}"] + + +def test_listing_pages_never_state_a_client_count_that_is_wrong(): + """`integrate --list` grows; a page that says "28 clients" is wrong the + day it does. Pages in this set either say no number or the right one.""" + from compartment import clients + for rel in ("registry.html", "llms-install.md", "docs/PLUGINS.md"): + p = ROOT / rel + if not p.exists(): + continue + for n in re.findall(r"\b(\d+) (?:MCP )?clients\b", p.read_text(encoding="utf-8")): + assert int(n) == len(clients.CLIENTS), (rel, n) + + +# ------------------------------------------- Agent Plugins (Hermes, Codex) -- +#: Agent Plugins 1.0.0 is still the published release (1.1.0 is a working +#: draft that Codex, Hermes and OpenClaw do not accept yet), and both of its +#: schemas are closed: a host must reject the plugin on any field outside +#: these, so this is the whole vocabulary. +AGENT_PLUGIN_FIELDS = {"$schema", "name", "version", "description", "author", + "homepage", "repository", "license", "keywords", + "extensions"} +AGENT_PLUGIN_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json" +AGENT_MCP_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json" + + +def test_the_agent_plugins_manifest_stays_inside_the_closed_schema(): + d = _json("plugin.json") + assert set(d) <= AGENT_PLUGIN_FIELDS, set(d) - AGENT_PLUGIN_FIELDS + assert d["$schema"] == AGENT_PLUGIN_SCHEMA + assert re.fullmatch(r"(?!.*(?:--|\.\.))[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", + d["name"]) and len(d["name"]) <= 64 + assert set(d["author"]) <= {"name", "email", "url"} + assert all(isinstance(v, dict) for v in d.get("extensions", {}).values()) + + +def test_the_agent_plugins_mcp_json_stays_inside_the_closed_schema(): + d = _json("mcp.json") + assert set(d) == {"$schema", "mcpServers"} + assert d["$schema"] == AGENT_MCP_SCHEMA + for server in d["mcpServers"].values(): + assert set(server) <= {"type", "command", "args", "env", "cwd"} + assert server["type"] == "stdio" + # one executable token, never a path or a shell line: Codex, Hermes + # and OpenClaw all refuse anything else for this format + assert re.fullmatch(r"[A-Za-z0-9._-]+", server["command"]) + cwd = server.get("cwd", "./") + assert re.match(r"(?:\./|\$\{PLUGIN_ROOT\}(?:/|$)|\$\{PLUGIN_DATA\}(?:/|$))", cwd) + + +def test_codex_gets_a_listing_of_its_own_from_the_agent_plugins_manifest(): + """Codex installs from .claude-plugin/marketplace.json but loads the root + plugin.json first when it declares the Agent Plugins schema, and without + a com.openai extension it lists the plugin as "compartment", category + Other, with no prompt or logo. The extension is where Codex looks.""" + ui = _json("plugin.json")["extensions"]["com.openai"]["interface"] + for field in ("displayName", "shortDescription", "longDescription", + "developerName", "category", "capabilities", "defaultPrompt"): + assert ui[field], field + assert 1 <= len(ui["defaultPrompt"]) <= 3 + assert all(len(p) <= 128 for p in ui["defaultPrompt"]) + for field in ("websiteURL", "privacyPolicyURL"): + assert ui[field].startswith("https://") + assert ui["logo"].startswith("./") and (ROOT / ui["logo"]).is_file() + + +# ------------------------------------------------------ one-click pages ----- +def _deeplinks(rel: str) -> list[str]: + """The meta refresh and the fallback anchor, which must be one link.""" + import html + s = (ROOT / rel).read_text(encoding="utf-8") + refresh = re.search(r'http-equiv="refresh" content="0; url=([^"]+)"', s).group(1) + anchors = re.findall(r' str: + return args[args.index("--caller") + 1] + + +def test_the_lm_studio_button_registers_what_integrate_registers(): + """The caller names the client in the audit log and is the key an ACL + entry is written against; `integrate lmstudio` writes `lmstudio`, so a + vault configured for that caller must see the same name from the + button. The manual block on the page is the same entry.""" + import base64 + from urllib.parse import parse_qs, urlsplit + from compartment import clients + links = _deeplinks("add/lmstudio.html") + assert len(set(links)) == 1, links + q = parse_qs(urlsplit(links[0]).query) + assert q["name"] == ["compartment"] + entry = json.loads(base64.b64decode(q["config"][0])) + assert entry["command"] == "compartment" and entry["args"][-1] == "serve" + assert _caller(entry["args"]) == clients.CLIENTS["lmstudio"].key + page = (ROOT / "add/lmstudio.html").read_text(encoding="utf-8") + manual = json.loads(re.search(r"
(.*?)
", page, re.S).group(1)) + assert manual["mcpServers"]["compartment"] == entry + + +def test_the_goose_button_uses_a_command_goose_accepts(): + """goose refuses a deeplink whose command is not one of its launchers + (cu, docker, jbang, npx, uvx, goose), so the bare `compartment` the + other manifests use would be rejected; uvx runs the PyPI package.""" + from urllib.parse import parse_qs, urlsplit + from compartment import clients + links = _deeplinks("add/goose.html") + assert len(set(links)) == 1, links + q = parse_qs(urlsplit(links[0]).query) + assert q["cmd"] == ["uvx"] and q["name"] == ["Compartment"] + assert q["arg"][0] == "compartment" and q["arg"][-1] == "serve" + assert _caller(q["arg"]) == clients.CLIENTS["goose"].key + page = (ROOT / "add/goose.html").read_text(encoding="utf-8") + assert f"
uvx {' '.join(q['arg'])}
" in page + + +# --------------------------------------------- Claude Desktop (MCP Bundle) -- +def _registered_tools() -> list[str]: + import asyncio + from compartment import server + return [t.name for t in asyncio.run(server.mcp.list_tools())] + + +def _build_mcpb(): + import importlib.util + spec = importlib.util.spec_from_file_location( + "build_mcpb", ROOT / "tools" / "build_mcpb.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_the_bundle_is_a_uv_bundle_claude_desktop_can_run(): + """manifest_version 0.4 is the MCPB release that added the uv server + type, the only type that can carry compiled dependencies (onnxruntime, + PyNaCl, argon2-cffi, numpy) to every platform from one bundle.""" + m = _json("packaging/mcpb/manifest.json") + assert m["manifest_version"] == "0.4" + assert m["name"] == "compartment" and m["author"]["name"] + server = m["server"] + assert server["type"] == "uv" + assert (ROOT / "packaging/mcpb" / server["entry_point"]).is_file() + cfg = server["mcp_config"] + assert cfg["command"] == "uv" + assert cfg["args"][:3] == ["run", "--directory", "${__dirname}"] + assert cfg["args"][-1] == server["entry_point"] + + +def test_the_bundle_lists_exactly_the_tools_the_server_registers(): + """Claude Desktop shows this list before the server ever runs.""" + m = _json("packaging/mcpb/manifest.json") + assert [t["name"] for t in m["tools"]] == _registered_tools() + assert m["tools_generated"] is False + + +def test_the_bundle_starts_the_server_as_claude_desktop(): + """The same caller `integrate claude` writes into Claude Desktop's own + config, so the two routes are one client to the audit log and ACL.""" + entry = (ROOT / "packaging/mcpb/server/main.py").read_text(encoding="utf-8") + assert 'main(["--caller", "claude-desktop", "serve"])' in entry + assert '"--caller", "claude-desktop", "serve"' in ( + ROOT / "src/compartment/cli.py").read_text(encoding="utf-8") + + +def test_the_bundle_pins_an_interpreter_its_dependencies_have_wheels_for(): + """uv otherwise picks the newest Python it can get, and onnxruntime + trails Python releases: a bundle without a pin, installed the week a + new Python ships, cannot build its environment.""" + pinned = (ROOT / "packaging/mcpb/.python-version").read_text().strip() + text = (ROOT / "packaging/mcpb/pyproject.toml").read_text(encoding="utf-8") + lo, hi = re.search(r'requires-python = ">=(\d+\.\d+),<(\d+\.\d+)"', text).groups() + as_tuple = lambda v: tuple(int(x) for x in v.split(".")) # noqa: E731 + assert as_tuple(lo) <= as_tuple(pinned) < as_tuple(hi) + + +def test_the_bundle_build_is_complete_and_reproducible(tmp_path): + import zipfile + mod = _build_mcpb() + a = mod.build(tmp_path / "a", with_lock=False) + b = mod.build(tmp_path / "b", with_lock=False) + assert a.name == f"compartment-{mod.package_version()}.mcpb" + assert a.read_bytes() == b.read_bytes() + with zipfile.ZipFile(a) as z: + names = set(z.namelist()) + manifest = json.loads(z.read("manifest.json")) + assert names == set(mod.CONTENTS) + assert manifest == _json("packaging/mcpb/manifest.json") + assert {manifest["icon"], manifest["server"]["entry_point"]} <= names + + +def test_the_bundle_lock_failure_says_what_to_do(tmp_path, monkeypatch): + """The lock pins the release, so it only resolves once the release is on + PyPI; a build run too early must say so, not end in a traceback.""" + import subprocess + import pytest + mod = _build_mcpb() + monkeypatch.setattr(mod.shutil, "which", lambda name: None) + with pytest.raises(SystemExit, match="--no-lock"): + mod.build(tmp_path, with_lock=True) + monkeypatch.setattr(mod.shutil, "which", lambda name: "/usr/bin/uv") + monkeypatch.setattr(mod.subprocess, "run", + lambda *a, **k: subprocess.CompletedProcess(a, 1)) + with pytest.raises(SystemExit, match="after the PyPI publish"): + mod.build(tmp_path, with_lock=True) + assert not list(tmp_path.glob("*.mcpb*")) + + +def test_the_bundle_build_refuses_a_bundle_for_another_release(): + import pytest + mod = _build_mcpb() + with pytest.raises(SystemExit, match="not the package's"): + mod.check("0.0.1") + + +# ---------------------------------------------------- Docker MCP Catalog ---- +def _yaml(rel: str): + import pytest + yaml = pytest.importorskip("yaml") + return yaml.safe_load((ROOT / rel).read_text(encoding="utf-8")) + + +def _dockerfile_entrypoint() -> list[str]: + text = (ROOT / "Dockerfile").read_text(encoding="utf-8") + return json.loads(re.search(r"^ENTRYPOINT (\[.*\])$", text, re.M).group(1)) + + +def test_the_docker_catalog_entry_builds_from_this_repository(): + """docker/mcp-registry builds mcp/compartment from the Dockerfile at a + pinned commit; the entry is useless if either is wrong.""" + d = _yaml("packaging/docker-mcp/server.yaml") + assert d["name"] == "compartment" and d["type"] == "server" + assert d["image"] == "mcp/compartment" + assert d["source"]["project"] == "https://github.com/MaxFreedomPollard/Compartment" + assert re.fullmatch(r"[0-9a-f]{40}", d["source"]["commit"]) + assert d["about"]["title"] and d["about"]["icon"].startswith("https://") + assert d["meta"]["category"] + + +def test_the_docker_catalog_runs_the_container_offline_on_the_users_vault(): + """No network at all, enforced by Docker; the user's vault folder mounted + where the image's entrypoint looks for it; the passphrase from Docker + Desktop's secret store, as the Dockerfile documents.""" + d = _yaml("packaging/docker-mcp/server.yaml") + assert d["run"]["disableNetwork"] is True + entry = _dockerfile_entrypoint() + vault = entry[entry.index("--vault") + 1] + (volume,) = d["run"]["volumes"] + param, target = re.fullmatch(r"\{\{compartment\.(\w+)\}\}:(/\S+)", volume).groups() + assert pathlib.PurePosixPath(vault).parent == pathlib.PurePosixPath(target) + assert param in d["config"]["parameters"]["properties"] + assert param in d["config"]["parameters"]["required"] + assert [s["env"] for s in d["config"]["secrets"]] == ["COMPARTMENT_PASSPHRASE"] + assert all(s["name"].startswith("compartment.") for s in d["config"]["secrets"]) + assert entry[-1] == "serve" + assert not re.search(r"^EXPOSE\b", (ROOT / "Dockerfile").read_text( + encoding="utf-8"), re.M) + + +# -------------------------------------------------------------- Continue ---- +def test_the_continue_block_is_a_block_continue_loads(): + from compartment import clients + d = _yaml("packaging/continue/compartment.yaml") + assert d["schema"] == "v1" and d["name"] and d["version"] + (server,) = d["mcpServers"] + assert server["type"] == "stdio" and server["command"] == "compartment" + assert server["args"][-1] == "serve" + assert _caller(server["args"]) == clients.CLIENTS["continue"].key + + +# ------------------------------------------------- across every listing ----- +def _listings() -> dict[str, tuple[str, list[str]]]: + """host -> (command, compartment arguments) for every place this + repository tells a host how to start the server.""" + import base64 + from urllib.parse import parse_qs, urlsplit + out = {} + out["claude-code"] = tuple(_json(".claude-plugin/plugin.json")["mcpServers"] + ["compartment"][k] for k in ("command", "args")) + out["agent-plugins"] = tuple(_json("mcp.json")["mcpServers"]["compartment"][k] + for k in ("command", "args")) + out["gemini"] = tuple(_json("gemini-extension.json")["mcpServers"] + ["compartment"][k] for k in ("command", "args")) + q = parse_qs(urlsplit(_deeplinks("add/lmstudio.html")[0]).query) + lm = json.loads(base64.b64decode(q["config"][0])) + out["lmstudio"] = (lm["command"], lm["args"]) + q = parse_qs(urlsplit(_deeplinks("add/goose.html")[0]).query) + out["goose"] = (q["arg"][0], q["arg"][1:]) # uvx + entry = _dockerfile_entrypoint() + out["docker"] = (entry[0], entry[1:]) + main_py = (ROOT / "packaging/mcpb/server/main.py").read_text(encoding="utf-8") + out["claude-desktop"] = ("compartment", json.loads( + re.search(r"main\((\[.*?\])\)", main_py).group(1))) + try: + cont = _yaml("packaging/continue/compartment.yaml")["mcpServers"][0] + out["continue"] = (cont["command"], cont["args"]) + except Exception: # noqa: BLE001 + pass # no PyYAML here + return out + + +def test_every_listing_starts_the_same_server(): + """One executable, global options only before the subcommand, and the + subcommand last: `compartment [--vault V] [--caller C] serve`.""" + for host, (command, args) in _listings().items(): + assert command == "compartment", host + assert args[-1] == "serve", host + opts = args[:-1] + assert len(opts) % 2 == 0, (host, args) + assert set(opts[::2]) <= {"--vault", "--caller"}, (host, args) + + +def test_every_listing_names_its_host_as_integrate_does(): + """The caller is the audit log's name for a client and the key ACL + entries are written against. Where `compartment integrate` wires the + same host, a listing must use the caller integrate writes, or one app + shows up as two clients; and no two hosts may share a caller.""" + from compartment import clients + listings = _listings() + callers = {host: _caller(args) for host, (_, args) in listings.items()} + assert len(set(callers.values())) == len(callers), callers + for host, caller in callers.items(): + if host in clients.CLIENTS: + assert caller == clients.CLIENTS[host].key, (host, caller) + assert callers["claude-code"] == "claude-code" # integrate claude + assert callers["claude-desktop"] == "claude-desktop" + + +def test_no_listing_claims_a_tool_count_the_server_does_not_have(): + count = len(_registered_tools()) + for rel in (".claude-plugin/plugin.json", ".claude-plugin/marketplace.json", + "plugin.json", "gemini-extension.json", "server.json", + "packaging/mcpb/manifest.json", "packaging/docker-mcp/server.yaml", + "packaging/continue/compartment.yaml", "llms-install.md", + "docs/PLUGINS.md", "registry.html"): + p = ROOT / rel + if not p.exists(): + continue + for n in re.findall(r"\b(\d+) (?:memory |MCP )?tools\b", + p.read_text(encoding="utf-8")): + assert int(n) == count, (rel, n, count) + + +# ------------------------------------------------------------- Oh My Pi ----- +def test_the_omp_extension_declares_what_each_tool_does(): + """omp's ToolDefinition treats a missing `approval` as "exec" and a + missing `loadMode` on an extension tool as "discoverable" (removed from + the top-level tool list, found by search). Recall must not be gated + like running code, and a tool the model is told to call before + answering must be in front of it.""" + src = (ROOT / "integrations/omp/extension.ts").read_text(encoding="utf-8") + tools = {} + for block in src.split("pi.registerTool({")[1:]: + name = re.search(r'name:\s*"([^"]+)"', block).group(1) + head = block.split("async execute")[0] + tools[name] = dict(re.findall(r'\b(approval|loadMode):\s*"([^"]+)"', head)) + assert tools == { + "memory_search": {"approval": "read", "loadMode": "essential"}, + "memory_store": {"approval": "write", "loadMode": "essential"}, + } + + +# ---------------------------------------------------- Cline (llms-install) -- +def test_llms_install_gets_an_agent_past_the_common_install_failures(): + """Cline's marketplace installs by handing this file to the agent, which + then meets what users meet: pip refusing on an OS-managed Python + (PEP 668), a Windows shell, a GUI client that cannot see the PATH.""" + from compartment import clients + text = (ROOT / "llms-install.md").read_text(encoding="utf-8") + assert "externally-managed-environment" in text + assert "pipx install compartment" in text and "uv tool install compartment" in text + # --break-system-packages appears only as something not to do + for line in text.splitlines(): + if "--break-system-packages" in line: + assert "do not" in line.lower(), line + assert "py -m pip install compartment" in text + assert "where compartment" in text and "which compartment" in text + for client in re.findall(r"compartment integrate ([a-z0-9][a-z0-9-]*)", text): + assert client in clients.CLIENTS, client + + +# ---------------------------------- the repository root is a plugin root ---- +#: What each host that installs the repository root loads from it without +#: being told to. skills/ and mcp.json are shared on purpose; everything +#: else here would be read by several hosts in several incompatible ways +#: (a Claude hooks/hooks.json is not a Gemini one; Gemini's commands/ are +#: TOML, Claude's Markdown; OpenClaw imports a root settings.json into its +#: own settings). Adding one is a decision for every host at once, which is +#: why this test names them. +HOST_AUTOLOADED = ("commands", "agents", "hooks", "policies", "output-styles", + "monitors", "themes", ".mcp.json", ".lsp.json", + "settings.json", ".codex-plugin", ".cursor-plugin", + "openclaw.plugin.json") + + +def test_the_root_holds_nothing_a_host_would_load_by_surprise(): + present = [n for n in HOST_AUTOLOADED if (ROOT / n).exists()] + assert not present, ( + f"{present} at the repository root would be auto-loaded by Claude " + "Code, Gemini CLI, Codex, OpenClaw or Hermes, each in its own format; " + "see HOST_AUTOLOADED and docs/PLUGINS.md before adding it") + + +def test_every_shared_skill_meets_the_agent_skills_specification(): + """All five hosts load skills/*/SKILL.md, and Hermes and Codex skip a + skill that fails the specification instead of loading it loosely.""" + allowed = {"name", "description", "license", "compatibility", "metadata", + "allowed-tools"} + skills = sorted((ROOT / "skills").glob("*/SKILL.md")) + assert skills + for path in skills: + text = path.read_text(encoding="utf-8") + assert text.startswith("---\n") + front = text.split("---\n", 2)[1] + keys = set(re.findall(r"^([A-Za-z][\w-]*):", front, re.M)) + assert keys <= allowed, (path, keys - allowed) + name = re.search(r"^name:\s*(\S+)", front, re.M).group(1) + assert name == path.parent.name + assert re.fullmatch(r"[a-z0-9]+(-[a-z0-9]+)*", name) and len(name) <= 64 + desc = re.search(r"^description:\s*(.+)$", front, re.M).group(1) + assert 1 <= len(desc) <= 1024 + # metadata is a map of strings: quoted, never a YAML list or number + for value in re.findall(r"^ [\w-]+:\s*(.*)$", front, re.M): + assert value.startswith('"') and value.endswith('"'), (path, value) + + +# --------------------------------------------------- Claude Code, Gemini ---- +def test_the_claude_code_marketplace_serves_the_plugin_at_the_root(): + market = _json(".claude-plugin/marketplace.json") + plugin = _json(".claude-plugin/plugin.json") + assert market["owner"]["name"] + (entry,) = market["plugins"] + assert entry["name"] == plugin["name"] == "compartment" + assert entry["source"] == "./" + assert re.fullmatch(r"[a-z][a-z0-9]*(-[a-z0-9]+)*", plugin["name"]) + assert re.fullmatch(r"\d+\.\d+\.\d+", plugin["version"]) + server = plugin["mcpServers"]["compartment"] + assert set(server) <= {"type", "command", "args", "env"} + + +def test_the_gemini_extension_is_what_the_gallery_indexes(): + """The gallery crawler wants gemini-extension.json at the repository + root with a name matching the install directory, a version and a + description; extensions may not set `trust` on a server.""" + d = _json("gemini-extension.json") + assert d["name"] == "compartment" + assert re.fullmatch(r"\d+\.\d+\.\d+", d["version"]) and d["description"] + assert (ROOT / d["contextFileName"]).is_file() + for server in d["mcpServers"].values(): + assert "trust" not in server + + +def test_gemini_md_is_the_handshake_instructions_verbatim(): + """Gemini CLI loads GEMINI.md into every session as the extension's + context, on top of the instructions the server sends in its handshake; + two versions of the rules would tell the model two things.""" + from compartment import server + flat = lambda s: re.sub(r"\s+", " ", s).strip() # noqa: E731 + body = (ROOT / "GEMINI.md").read_text(encoding="utf-8").split("\n", 1)[1] + assert flat(body) == flat(server.COMPARTMENT_INSTRUCTIONS) + + +def test_the_website_does_not_publish_the_plugin_files_as_pages(): + """GitHub Pages builds the repository root with Jekyll, which renders + any file with YAML front matter (every SKILL.md) as a page and copies + the rest; the manifests are for hosts, not for the site.""" + import pytest + yaml = pytest.importorskip("yaml") + excluded = set(yaml.safe_load((ROOT / "_config.yml").read_text( + encoding="utf-8"))["exclude"]) + for name in ("skills", "plugin.json", "mcp.json", "gemini-extension.json", + "GEMINI.md", "server.json", "glama.json", "packaging", "tools"): + assert name in excluded, name diff --git a/tests/test_server.py b/tests/test_server.py index bb1a84b..9fc569a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -522,6 +522,40 @@ def test_a_locked_vault_tells_the_model_what_to_run(vault_path, monkeypatch): assert "compartment unlock" in str(e.value), str(e.value) +def test_a_vault_that_is_not_there_is_not_called_locked(tmp_path, + monkeypatch): + """A client configured before `compartment init` ran, or with a --vault + that points nowhere: every tool said the vault was locked and to run + `compartment unlock`, which cannot make a vault that is not there.""" + missing = str(tmp_path / "nowhere.vault") + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", "CorrectHorse") + for key, value in (("vault", None), ("path", missing), + ("caller", "agent")): + monkeypatch.setitem(server._state, key, value) + with pytest.raises(Exception) as e: + _through_the_sdk(monkeypatch, "memory_search", {"query": "anything"}) + assert f"No vault at {missing}" in str(e.value), str(e.value) + assert "compartment init" in str(e.value) + status = json.loads(server.memory_status.__wrapped__()) + assert status["exists"] is False and "compartment init" in \ + status["message"] + + +def test_a_failure_that_is_not_the_vaults_own_still_reaches_the_model( + serving, monkeypatch): + """mcp 2.x passes the model the text of a ToolError only. A full disk, a + file this user cannot read, the embedding process refusing a text, all + arrived as "Error executing tool memory_store" and nothing else.""" + def full(*a, **k): + raise OSError(28, "No space left on device") + monkeypatch.setattr(serving, "store", full) + with pytest.raises(Exception) as e: + _through_the_sdk(monkeypatch, "memory_store", + {"text": "The staging database runs Postgres 16", + "source": "from chat"}) + assert "No space left on device" in str(e.value), str(e.value) + + @pytest.fixture() def own_state(monkeypatch): """A copy of server._state for this test, so that main() leaves nothing @@ -533,8 +567,8 @@ def _serve(monkeypatch, vault_path, session, passphrase="CorrectHorse"): """server.main() with `session` in place of the stdio transport, its two background loops not started, and the passphrase in the environment as the only credential (no keychain lookup; None for no credential at - all). main() handles SIGTERM for the whole process, so the handler it - found is put back.""" + all). main() handles SIGTERM and SIGHUP for the whole process, so the + handlers it found are put back.""" from compartment import vault as vault_module monkeypatch.setattr(vault_module, "keychain_get", lambda path: None) monkeypatch.delenv("ENGRAM_PASSPHRASE", raising=False) @@ -545,11 +579,41 @@ def _serve(monkeypatch, vault_path, session, passphrase="CorrectHorse"): monkeypatch.setattr(server, "_autolock_loop", lambda: None) monkeypatch.setattr(server, "_retag_loop", lambda: None) monkeypatch.setattr(server.mcp, "run", lambda *a, **k: session()) - before = signal.getsignal(signal.SIGTERM) + handled = [getattr(signal, n) for n in ("SIGTERM", "SIGHUP") + if hasattr(signal, n)] + before = {sig: signal.getsignal(sig) for sig in handled} try: server.main(["--vault", vault_path, "--caller", "agent"]) finally: - signal.signal(signal.SIGTERM, before) + for sig, handler in before.items(): + signal.signal(sig, handler) + + +def test_a_vault_path_with_a_tilde_is_the_home_one(vault, vault_path, + own_state, monkeypatch, + tmp_path): + """Run as `python -m compartment.server --vault ~/...` from a client's + config, where no shell expands the ~.""" + vault.lock() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + rel = os.path.relpath(vault_path, tmp_path) + seen = [] + monkeypatch.setattr(server, "_autolock_loop", lambda: None) + monkeypatch.setattr(server, "_retag_loop", lambda: None) + monkeypatch.setattr(server.mcp, "run", lambda *a, **k: seen.append( + (server._state["path"], server._state["vault"]))) + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", "CorrectHorse") + handled = [getattr(signal, n) for n in ("SIGTERM", "SIGHUP") + if hasattr(signal, n)] + before = {sig: signal.getsignal(sig) for sig in handled} + try: + server.main(["--vault", os.path.join("~", rel)]) + finally: + for sig, handler in before.items(): + signal.signal(sig, handler) + (path, opened), = seen + assert path == vault_path and opened is not None def _audited_on_disk(vault_path): @@ -597,6 +661,39 @@ def session(): assert ended == [(os.getpid(), signal.SIGTERM, signal.SIG_DFL, "search")] +def test_a_server_whose_vault_is_missing_still_starts(tmp_path, own_state, + monkeypatch): + """With the passphrase in the environment, a vault path that pointed + nowhere raised FileNotFoundError out of main(): the client showed a + server that failed to start, the reason in a log nobody opens.""" + ran = [] + _serve(monkeypatch, str(tmp_path / "nowhere.vault"), + lambda: ran.append(True)) + assert ran == [True] and server._state["vault"] is None + + +@pytest.mark.skipif(not hasattr(signal, "SIGHUP"), + reason="Windows has no SIGHUP") +def test_a_server_whose_terminal_closes_saves_them_too( + vault, vault_path, own_state, monkeypatch): + """A client running in a terminal starts its servers in its process + group, and closing the terminal sends them all SIGHUP, whose default + action ended the server with the session's audited reads unsaved.""" + vault.lock() + ended = [] + monkeypatch.setattr(os, "kill", lambda pid, sig: ended.append( + (pid, sig, signal.getsignal(sig), _audited_on_disk(vault_path)[-1]))) + + def session(): + _call(server.memory_search, "which Postgres version does staging run") + if not callable(signal.getsignal(signal.SIGHUP)): + pytest.fail("compartment serve leaves SIGHUP to end it on the " + "spot, with nothing saved") + signal.raise_signal(signal.SIGHUP) + _serve(monkeypatch, vault_path, session) + assert ended == [(os.getpid(), signal.SIGHUP, signal.SIG_DFL, "search")] + + def _set(vault_path, **settings): from compartment.acl import VaultConfig cfg = VaultConfig.load(vault_path) diff --git a/tests/test_server_embed_runtime.py b/tests/test_server_embed_runtime.py new file mode 100644 index 0000000..c75d5eb --- /dev/null +++ b/tests/test_server_embed_runtime.py @@ -0,0 +1,118 @@ +"""What the embedder says when its runtime cannot be loaded: the remedy, in +a ModelError, which the CLI prints as one line and the MCP server passes to +the model, instead of an ImportError neither of them shows usefully.""" +import sys + +import pytest + +from compartment import embed +from compartment.crypto import CryptoError + + +@pytest.mark.parametrize("platform, installer", [ + ("win-amd64", "vc_redist.x64.exe"), + ("win-arm64", "vc_redist.arm64.exe"), + ("win32", "vc_redist.x86.exe"), +]) +def test_the_runtime_named_is_the_one_this_python_needs(monkeypatch, platform, + installer): + """Windows on ARM with an ARM64 Python was sent to the x64 installer, + which installs a runtime that Python cannot load.""" + monkeypatch.setattr(embed.os, "name", "nt") + monkeypatch.setattr(embed.sysconfig, "get_platform", lambda: platform) + err = embed._missing_runtime_error(ImportError( + "DLL load failed while importing onnxruntime_pybind11_state: The " + "specified module could not be found.")) + assert installer in str(err) + + +@pytest.mark.parametrize("module", ["onnxruntime", "tokenizers"]) +def test_a_runtime_that_is_not_installed_says_how_to_install_it(monkeypatch, + module): + """onnxruntime has no wheel for musl, 32-bit ARM, the BSDs or Android, + and an install there can lack it: the CLI printed a traceback and the + MCP server told the model "Error executing tool" and nothing else.""" + monkeypatch.setitem(sys.modules, module, None) # import now fails + with pytest.raises(embed.ModelError) as e: + embed.Embedder() + assert isinstance(e.value, CryptoError) # what the CLI and server report + assert f"-m pip install {module}" in str(e.value) + assert sys.executable in str(e.value) + + +def test_any_other_missing_module_is_left_as_it_is(monkeypatch): + err = ModuleNotFoundError("No module named 'flatbuffers'", + name="flatbuffers") + assert embed._runtime_not_installed(err) is None + + +# ------------------------------------------------- CPUs in a container -- +def _cgroup(tmp_path, own, files): + root = tmp_path / "sys" / "fs" / "cgroup" + for rel, text in files.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + proc = tmp_path / "proc_self_cgroup" + proc.write_text(own) + return str(root), str(proc) + + +def test_a_cgroup_v2_quota_is_read_from_the_tightest_level(tmp_path): + root, own = _cgroup(tmp_path, "0::/user.slice/agent.scope\n", { + "user.slice/cpu.max": "150000 100000\n", + "user.slice/agent.scope/cpu.max": "max 100000\n", + }) + assert embed._cgroup_cpu_limit(root, own) == 1.5 + + +def test_a_container_sees_its_own_quota_at_the_root(tmp_path): + root, own = _cgroup(tmp_path, "0::/\n", {"cpu.max": "200000 100000\n"}) + assert embed._cgroup_cpu_limit(root, own) == 2.0 + + +def test_a_cgroup_v1_quota_and_none(tmp_path): + root, own = _cgroup(tmp_path, "4:cpu,cpuacct:/docker/abc\n", { + "cpu,cpuacct/cpu.cfs_quota_us": "50000\n", + "cpu,cpuacct/cpu.cfs_period_us": "100000\n", + }) + assert embed._cgroup_cpu_limit(root, own) == 0.5 + root, own = _cgroup(tmp_path / "unlimited", "0::/\n", + {"cpu.max": "max 100000\n"}) + assert embed._cgroup_cpu_limit(root, own) is None + + +@pytest.mark.parametrize("limit, affinity, threads", [ + (2.0, 64, 2), # docker run --cpus 2 on a 64-core host + (0.5, 64, 1), # never fewer than one + (None, 64, 0), # nothing limits it: onnxruntime's own choice + (48.0, 64, 0), # above half: its own choice already fits + (None, 4, 4), # taskset -c 0-3 +]) +def test_the_inference_pool_fits_the_cpus_this_process_may_use( + monkeypatch, limit, affinity, threads): + """onnxruntime sizes its pool from the host's cores and does not see a + container's quota: 32 threads sharing two CPUs' worth of time, every + search throttled until the next period.""" + monkeypatch.setattr(embed.os, "cpu_count", lambda: 64) + monkeypatch.setattr(embed.os, "process_cpu_count", lambda: affinity, + raising=False) + monkeypatch.setattr(embed.sys, "platform", "linux") + monkeypatch.setattr(embed, "_cgroup_cpu_limit", lambda: limit) + assert embed.intra_op_threads() == threads + + +def test_the_embedder_runs_with_that_many_threads(monkeypatch): + monkeypatch.setattr(embed, "intra_op_threads", lambda: 2) + e = embed.Embedder() + assert e.sess.get_session_options().intra_op_num_threads == 2 + assert e.embed_query("router passphrase").shape == (e.dim,) + + +def test_a_model_folder_named_in_the_environment_needs_no_home(tmp_path, + monkeypatch): + def no_home(): + raise RuntimeError("Could not determine home directory.") + monkeypatch.setattr(embed, "home", no_home) + monkeypatch.setenv("COMPARTMENT_MODEL_DIR", str(tmp_path / "models")) + assert embed.user_model_dir() == tmp_path / "models" diff --git a/tests/test_server_stdio.py b/tests/test_server_stdio.py new file mode 100644 index 0000000..226101e --- /dev/null +++ b/tests/test_server_stdio.py @@ -0,0 +1,58 @@ +"""`compartment serve` driven over real pipes, as a client drives it: the +handshake, a tool list, then the client hanging up. The rest of the suite +calls the server in-process and cannot see what only a client sees: a +stray line on stdout, a traceback on the way out, an exit status that +tells the client the server crashed.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +from compartment.vault import Vault + +from conftest import PASS + +SRC = str(Path(__file__).resolve().parents[1] / "src") + + +def test_a_session_over_stdio_ends_cleanly_when_the_client_hangs_up( + tmp_path): + vault = str(tmp_path / "v.vault") + Vault.create(vault, PASS, creator="test").lock() + env = dict(os.environ, COMPARTMENT_PASSPHRASE=PASS, + COMPARTMENT_EMBED_DAEMON="0", + PYTHONPATH=SRC + os.pathsep + os.environ.get("PYTHONPATH", "")) + requests = [ + {"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0"}}}, + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, + ] + proc = subprocess.Popen( + [sys.executable, "-m", "compartment.cli", "--vault", vault, "serve"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + try: + for r in requests: + proc.stdin.write((json.dumps(r) + "\n").encode()) + proc.stdin.flush() + # The client hangs up once it has its answers, as a client does: + # closing stdin at once would race the replies. + replies = [json.loads(proc.stdout.readline()) for _ in range(2)] + rest, err = proc.communicate(timeout=60) # closes stdin first + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + err = err.decode("utf-8", "replace") + assert proc.returncode == 0, err + assert "Traceback" not in err, err + assert rest == b"" # nothing but frames + assert [r.get("id") for r in replies] == [1, 2] + init, listed = replies + assert init["result"]["serverInfo"]["name"] == "compartment" + assert "memory_search" in init["result"]["instructions"] + assert {t["name"] for t in listed["result"]["tools"]} >= { + "memory_search", "memory_store", "memory_status"} diff --git a/tests/test_server_tools.py b/tests/test_server_tools.py new file mode 100644 index 0000000..8dd6f82 --- /dev/null +++ b/tests/test_server_tools.py @@ -0,0 +1,96 @@ +"""What `tools/list` tells a client about each memory tool: the hints it +uses to decide how to present and gate a tool, and input schemas that the +strictest clients' validators take. + +Listed through the SDK the server runs on (mcp 1.x or 2.x), without an +event loop, as test_server.py calls tools: listing awaits nothing.""" +import pytest + +from compartment import server + + +def _listed(): + call = server.mcp.list_tools() + try: + call.send(None) + except StopIteration as done: + return done.value + finally: + call.close() + raise AssertionError("list_tools waited for an event loop") + + +def _dump(tool) -> dict: + return tool.model_dump(by_alias=True, exclude_none=True) + + +def test_every_tool_says_it_stays_on_this_machine(): + """openWorldHint absent means true in the spec: "may interact with an + open world of external entities". Every tool left it out, so a client + that weighs the hint treated a fully offline vault as an internet + tool.""" + tools = [_dump(t) for t in _listed()] + assert len(tools) == 14 + for t in tools: + hints = t["annotations"] + assert hints["openWorldHint"] is False, t["name"] + assert hints["title"] == t["title"], t["name"] + assert isinstance(hints["readOnlyHint"], bool), t["name"] + + +def test_the_hints_match_what_each_tool_does(): + hints = {t["name"]: t["annotations"] for t in map(_dump, _listed())} + for name in ("memory_search", "memory_get", "memory_recent", + "memory_relations", "memory_list_namespaces", + "memory_status", "memory_selftest"): + assert hints[name]["readOnlyHint"] is True, name + for name in ("memory_forget", "memory_unlink"): + assert hints[name]["destructiveHint"] is True, name + for name in ("memory_link", "memory_unlink", "memory_forget", + "memory_lock", "memory_unlock"): + assert hints[name].get("idempotentHint") is True, name + for name in ("memory_store", "memory_store_many"): + assert not hints[name].get("idempotentHint"), name + + +def _typed(schema: dict, where: str) -> None: + """Every schema node carries a type, directly or in each branch of an + anyOf: Gemini's and older Codex's converters dropped or defaulted an + untyped node, and none of them resolve $ref.""" + assert "$ref" not in schema and "$defs" not in schema, where + if "anyOf" in schema: + assert schema["anyOf"], where + for i, branch in enumerate(schema["anyOf"]): + _typed(branch, f"{where}.anyOf[{i}]") + return + assert "type" in schema, where + if schema["type"] == "array": + _typed(schema["items"], f"{where}.items") + if schema["type"] == "object": + for key, sub in schema.get("properties", {}).items(): + _typed(sub, f"{where}.{key}") + + +def test_every_input_schema_is_one_strict_clients_accept(): + """An object at the top, with no union there (OpenAI and Gemini refuse + one), every property typed, and every required name defined.""" + for t in map(_dump, _listed()): + schema = t["inputSchema"] + assert schema["type"] == "object", t["name"] + for key in ("anyOf", "oneOf", "allOf", "not", "$ref"): + assert key not in schema, (t["name"], key) + _typed(schema, t["name"]) + assert set(schema.get("required", [])) <= set(schema["properties"]) + + +def test_mcp_1x_does_not_log_every_request(capsys): + """mcp 1.x's FastMCP logs at INFO by default, "Processing request of type + CallToolRequest" on stderr for every request, which Cursor shows as an + [error] line each time. mcp 2.x logs nothing per request.""" + import logging + if not server.MCPServer.__module__.startswith("mcp.server.fastmcp"): + pytest.skip("mcp 2.x logs nothing per request") + assert server.mcp.settings.log_level == "WARNING" + logging.getLogger("mcp.server.lowlevel.server").info( + "Processing request of type ListToolsRequest") + assert "Processing request" not in capsys.readouterr().err diff --git a/tests/test_status_bar_install.py b/tests/test_status_bar_install.py index efebba7..2af3ec0 100644 --- a/tests/test_status_bar_install.py +++ b/tests/test_status_bar_install.py @@ -187,6 +187,35 @@ def test_windows_autostart_avoids_a_console_window_and_keeps_the_vault(): assert "python.exe" not in cmd.lower() or "pythonw.exe" in cmd.lower() +def test_windows_autostart_prefers_pythonw_over_the_console_script( + tmp_path, monkeypatch): + """compartment.exe, the console script pip and uv write, is a console + program. Run at sign-in it opened a console window that stayed for the + whole session - that process is the tray app - and closing the window + quit the app. pythonw.exe beside the interpreter opens none.""" + from compartment import systray + scripts = tmp_path / "venv" / "Scripts" + scripts.mkdir(parents=True) + for name in ("python.exe", "pythonw.exe", "compartment.exe"): + (scripts / name).write_bytes(b"MZ") + monkeypatch.setattr(systray.sys, "executable", str(scripts / "python.exe")) + monkeypatch.setattr(systray.shutil, "which", + lambda name, *a, **k: str(scripts / "compartment.exe")) + exe, args = systray._autostart_parts("C:/Users/Zoë Ann/v.vault") + assert exe == str(scripts / "pythonw.exe") + assert args == '-m compartment.cli --vault "C:/Users/Zoë Ann/v.vault" tray' + assert systray._autostart_command("C:/v.vault").startswith( + f'"{scripts / "pythonw.exe"}" -m compartment.cli') + # The scheduled task runs the same thing. + xml = systray.scheduled_task_xml("C:/v.vault") + assert f"{scripts / 'pythonw.exe'}" in xml + # With no pythonw.exe (an embeddable or unusual build) the console + # script is still better than nothing. + (scripts / "pythonw.exe").unlink() + assert systray._autostart_parts("C:/v.vault")[0] == \ + str(scripts / "compartment.exe") + + # --- opting out of the GUI at install time --------------------------------- def test_a_scripted_install_never_waits_for_a_keypress(monkeypatch, capsys): diff --git a/tests/test_storage_bench_download.py b/tests/test_storage_bench_download.py new file mode 100644 index 0000000..1b6bd4f --- /dev/null +++ b/tests/test_storage_bench_download.py @@ -0,0 +1,54 @@ +"""The benchmark dataset download, cut short. + +The LongMemEval files run to gigabytes. A download interrupted part way +used to leave a truncated file where the dataset belongs, and every run +after trusted it because it existed. +""" +import io +import urllib.request + +import pytest + +from compartment import longmemeval as L + + +class _Response(io.BytesIO): + """A response body that drops after `cut` bytes, as a connection does.""" + + def __init__(self, body: bytes, cut: int | None = None): + super().__init__(body) + self._cut = cut + + def read(self, n=-1): + if self._cut is not None and self.tell() >= self._cut: + raise ConnectionResetError(54, "Connection reset by peer") + if self._cut is not None and n > 0: + n = min(n, self._cut - self.tell()) + return super().read(n) + + +@pytest.fixture() +def bench_home(tmp_path, monkeypatch): + monkeypatch.setattr(L, "data_dir", lambda: tmp_path / "longmemeval") + return tmp_path / "longmemeval" + + +def test_a_download_cut_short_leaves_nothing_that_looks_like_the_dataset( + bench_home, monkeypatch): + body = b'[{"question_id": "q1", "haystack_sessions": []}]' * 1000 + monkeypatch.setattr(urllib.request, "urlopen", + lambda url: _Response(body, cut=len(body) // 3)) + with pytest.raises(ConnectionResetError): + L.download("s") + assert not L.dataset_path("s").exists() + assert list(bench_home.iterdir()) == [], "and no partial file beside it" + with pytest.raises(L.CryptoError, match="download-longmemeval"): + L.run("s") # says to fetch it, not "bad JSON" + + +def test_a_whole_download_lands_where_run_looks(bench_home, monkeypatch): + body = b'[{"question_id": "q1", "haystack_sessions": []}]' + monkeypatch.setattr(urllib.request, "urlopen", lambda url: _Response(body)) + dest = L.download("s") + assert dest == L.dataset_path("s") and dest.read_bytes() == body + assert [p.name for p in bench_home.iterdir()] == [dest.name] diff --git a/tests/test_storage_credentials.py b/tests/test_storage_credentials.py new file mode 100644 index 0000000..0e5d834 --- /dev/null +++ b/tests/test_storage_credentials.py @@ -0,0 +1,53 @@ +"""Which credential opens the vault when the session store itself fails. + +The boot-session store needs shared memory, a readable session directory +and an identifiable boot. Containers, sandboxes and Android lack one or +another, and the error for that tells the user to set COMPARTMENT_PASSPHRASE; +the lookup raised before it ever reached the variable. +""" +import pytest + +from compartment import session, vault as vault_mod +from compartment.crypto import CryptoError +from compartment.vault import Vault + +from conftest import PASS + +BOOT_ERROR = ("Cannot identify this boot session (no boot clock), so a " + "stored credential cannot be checked. Use COMPARTMENT_PASSPHRASE " + "or `compartment unlock --once`.") + + +@pytest.fixture() +def broken_session_store(monkeypatch): + def get(path): + raise CryptoError(BOOT_ERROR) + monkeypatch.setattr(session, "get", get) + monkeypatch.setattr(vault_mod, "keychain_get", lambda path: None) + monkeypatch.delenv("COMPARTMENT_PASSPHRASE", raising=False) + monkeypatch.delenv("ENGRAM_PASSPHRASE", raising=False) + + +@pytest.mark.parametrize("var", ["COMPARTMENT_PASSPHRASE", + "ENGRAM_PASSPHRASE"]) +def test_a_passphrase_in_the_environment_is_used_when_the_session_store_fails( + broken_session_store, vault_path, monkeypatch, var): + Vault.create(vault_path, PASS, creator="t").lock() + monkeypatch.setenv(var, PASS) + assert Vault.find_credential(vault_path) == (PASS, None) + pw, key = Vault.resolve_credential(vault_path) + assert Vault.unlock(vault_path, passphrase=pw, raw_key=key).db.count() == 0 + + +def test_with_nothing_else_the_session_error_still_surfaces( + broken_session_store, vault_path): + """Not reported as "locked": the status panel must not say a vault is + locked when the truth is that its credential store cannot be read.""" + with pytest.raises(CryptoError, match="Cannot identify this boot"): + Vault.find_credential(vault_path) + + +def test_a_keychain_credential_is_used_when_the_session_store_fails( + broken_session_store, vault_path, monkeypatch): + monkeypatch.setattr(vault_mod, "keychain_get", lambda path: b"k" * 32) + assert Vault.find_credential(vault_path) == (None, b"k" * 32) diff --git a/tests/test_storage_dates.py b/tests/test_storage_dates.py new file mode 100644 index 0000000..3aeeecb --- /dev/null +++ b/tests/test_storage_dates.py @@ -0,0 +1,141 @@ +"""Stored instants that arrive from outside the vault. + +`created` and `affirmed` reach the vault from files on the restore paths. A +value stored as it came - an ISO string, a NaN, an infinity, milliseconds - +used to break every later search, because every search reads every live +record's time for the recency prior. +""" +import datetime +import json +import time + +import pytest + +from compartment.vault import Vault, instant, local_stamp + +from conftest import PASS + + +def _line(**fields) -> str: + return json.dumps(fields) + "\n" + + +def test_instant_reads_what_a_person_would_mean(): + t = datetime.datetime(2024, 5, 1, 9, 30, + tzinfo=datetime.timezone.utc).timestamp() + assert instant(t) == t + assert instant(str(t)) == t + assert instant("2024-05-01T09:30:00Z") == t + assert instant("2024-05-01T09:30:00+00:00") == t + assert instant(t * 1000) == t, "JavaScript's Date.now() is milliseconds" + local_midnight = datetime.datetime(2024, 5, 1).timestamp() + assert instant("2024-05-01") == local_midnight + for nonsense in (None, "", "yesterday", True, float("nan"), + float("inf"), -float("inf"), 1e300, [1]): + assert instant(nonsense) is None, nonsense + + +def test_an_infinite_date_is_shown_as_no_date_instead_of_raising(): + assert local_stamp(float("inf")) == "" + assert local_stamp(1e300) == "" + + +@pytest.mark.parametrize("created", [ + "2024-05-01T09:30:00Z", float("inf"), float("nan"), "last tuesday", + 1714555800000]) +def test_an_imported_date_never_breaks_search(vault, created): + """An ISO string went into the REAL column as text and the next search + failed converting it; infinity was stored and failed every reader of + the record; NaN failed the import half way.""" + vault.store("Tea is kept in the left cupboard.", caller="t") + before = time.time() + n = vault.import_jsonl( + json.dumps({"text": "Coffee beans are kept in the freezer.", + "created": created}, allow_nan=True) + "\n", + caller="t") + assert n == 1 + hits = vault.search("where are the coffee beans kept", caller="t") + top = hits["results"][0] + assert top["text"].startswith("Coffee beans are kept in the freezer.") + assert isinstance(top["created"], float) + assert top["created_local"], "the date reads as a date" + want = instant(created) + if want is None: + assert top["created"] >= before, "an unreadable date becomes now" + else: + assert top["created"] == want + assert vault.recent("t")["results"] + + +def test_an_unreadable_affirmed_time_is_dropped(vault): + vault.import_jsonl(_line(text="Prefers tabs over spaces.", + kind="opinion", affirmed="recently"), + caller="t") + row = vault.db.conn.execute("SELECT affirmed FROM records").fetchone() + assert row["affirmed"] is None + assert vault.search("tabs or spaces", caller="t")["results"] + + +def test_a_vault_an_older_build_broke_is_repaired_when_it_opens(vault, + vault_path): + """A vault that took such a line before this build failed every search + for good, and the record could not be found to remove it.""" + good = vault.store("Tea is kept in the left cupboard.", caller="t")["id"] + bad = vault.store("Coffee beans are kept in the freezer.", + caller="t")["id"] + vault.db.conn.execute( + "UPDATE records SET created = ?, affirmed = ? WHERE id = ?", + ("2024-05-01T09:30:00Z", "soon", bad)) + vault.db._invalidate_recency() + with pytest.raises(ValueError): # what an older build did + vault.search("coffee", caller="t") + vault.save() + + again = Vault.unlock(vault_path, passphrase=PASS) + hits = again.search("where are the coffee beans kept", caller="t") + assert hits["results"][0]["id"] == bad + assert hits["results"][0]["created"] == instant("2024-05-01T09:30:00Z") + row = again.db.get_row(bad) + assert row["affirmed"] is None + assert again.db.get_row(good)["created"] == vault.db.get_row(good)[ + "created"], "a record with a good date is untouched" + ops = [r["op"] for r in again.db.conn.execute("SELECT op FROM audit")] + assert "repair" in ops + # and the repair was compacted in: the next open has nothing to do + third = Vault.unlock(vault_path, passphrase=PASS) + assert [r["op"] for r in third.db.conn.execute( + "SELECT op FROM audit")].count("repair") == 1 + + +def test_a_discovery_date_in_milliseconds_is_the_day_it_names(vault): + """A JavaScript client sends Date.now(). The store raised "year 56759 is + out of range" and the memory was lost.""" + from compartment.vault import discovery_date + ms = datetime.datetime(2024, 5, 1, 12).timestamp() * 1000 + assert discovery_date(ms) == "2024-05-01" + out = vault.store("The office moved to the fourth floor.", caller="t", + discovered=ms) + assert out["discovered"] == "2024-05-01" + + +def test_a_discovery_number_that_names_no_day_is_kept_as_written(): + """The field's rule for anything unparseable: keep what the caller + meant, never raise, never invent a date that reads as verified.""" + from compartment.vault import discovery_date + assert discovery_date(1e300) == "1e+300" + assert discovery_date(float("nan")) == "nan" + + +@pytest.mark.parametrize("expires", ["9999y", float("nan"), + float("inf")]) +def test_an_expiry_nothing_can_date_is_refused_with_a_reason(vault, expires): + """These escaped as ValueError and OverflowError from date arithmetic, + which no surface reports as a refusal with a fix.""" + from compartment.crypto import CryptoError + from compartment.vault import expiry_date + with pytest.raises(CryptoError, match="expires="): + expiry_date(expires) + with pytest.raises(CryptoError, match="expires="): + vault.store("The trial licence lapses eventually.", caller="t", + expires=expires) + assert vault.db.count() == 0 diff --git a/tests/test_storage_durability.py b/tests/test_storage_durability.py new file mode 100644 index 0000000..8fc7df4 --- /dev/null +++ b/tests/test_storage_durability.py @@ -0,0 +1,356 @@ +"""What reaches the disk, and what a failed write leaves behind. + +The journal's promise is that an acknowledged write survives and an +unacknowledged one costs nothing. These tests hold the second half to the +disk running out part way through, which used to leave a torn tail that no +open could get past while the disk stayed full. +""" +import errno +import os + +import pytest + +from compartment import vaultfile +from compartment.vault import Vault + +from conftest import PASS + + +class _With: + """A module as one other module sees it, with some functions replaced.""" + + def __init__(self, module, **replaced): + self._module = module + self.__dict__.update(replaced) + + def __getattr__(self, name): + return getattr(self._module, name) + + +class _Budget: + """A disk with `room` bytes left: writes past it land in part and then + fail with ENOSPC, the way a real full disk answers.""" + + def __init__(self, room: int): + self.room = room + + def take(self, data) -> int: + n = min(len(data), self.room) + self.room -= n + return n + + +def _disk_fills_during_appends(monkeypatch, budget: _Budget): + """Both ways an append can reach the file - os.write on a descriptor and + a Python file object - run out of room together.""" + real_write = os.write + real_open = open + + def write(fd, data): + n = budget.take(data) + if n == 0 and len(data): + raise OSError(errno.ENOSPC, "No space left on device") + real_write(fd, bytes(data[:n])) + if n < len(data): + raise OSError(errno.ENOSPC, "No space left on device") + return n + + class _File: + def __init__(self, f): + self._f = f + + def write(self, data): + n = budget.take(data) + self._f.write(data[:n]) + if n < len(data): + self._f.flush() + raise OSError(errno.ENOSPC, "No space left on device") + return n + + def __getattr__(self, name): + return getattr(self._f, name) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self._f.close() + + def opener(path, mode="r", *args, **kwargs): + f = real_open(path, mode, *args, **kwargs) + return _File(f) if "a" in mode else f + + monkeypatch.setattr(vaultfile, "os", _With(os, write=write)) + monkeypatch.setattr(vaultfile, "open", opener, raising=False) + + +def _disk_stays_full(monkeypatch): + """No save can be written. Patched where the vault calls it, never as + `_atomic_replace`: modules imported for the first time during the test + bind that name for good, and would carry the full disk into the next.""" + def full(*args, **kwargs): + raise OSError(errno.ENOSPC, "No space left on device") + monkeypatch.setattr(vaultfile, "write_vault_file", full) + + +def test_an_append_the_disk_refuses_leaves_no_torn_tail(vault_path, + monkeypatch): + """The partial entry stayed on the end of the file. Nothing had been + acknowledged, but every open after it had to compact the tail away, and + on a disk that was still full that compaction failed and the open with + it: every agent's server, `compartment status` and the panel lost the + vault, reads included, until someone freed space.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("The first memory reached the disk whole.", caller="t") + size = os.path.getsize(vault_path) + + with monkeypatch.context() as m: + _disk_fills_during_appends(m, _Budget(room=40)) + with pytest.raises(OSError, match="No space"): + v.store("The second memory ran out of disk part way.", caller="t") + + assert os.path.getsize(vault_path) == size, \ + "the refused bytes are taken back off the end" + loaded = vaultfile.read_vault_file(vault_path) + assert not loaded.truncated_tail + + _disk_stays_full(monkeypatch) + again = Vault.unlock(vault_path, passphrase=PASS) + texts = [r["text"] for r in again.peek_recent("t")["results"]] + assert len(texts) == 1 + assert texts[0].startswith("The first memory reached the disk whole.") + + +def test_an_append_whose_sync_fails_is_not_left_to_replay(vault_path, + monkeypatch): + """Every byte written, then the sync refused (EIO, a full quota on a + network share). The caller was told the store failed, so the entry must + not come back on the next open as a memory nobody stored.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory that was acknowledged.", caller="t") + size = os.path.getsize(vault_path) + calls = [] + real_sync = vaultfile._sync + + def sync(fd): + calls.append(fd) + if len(calls) == 1: + raise OSError(errno.EIO, "Input/output error") + return real_sync(fd) + + with monkeypatch.context() as m: + m.setattr(vaultfile, "_sync", sync) + with pytest.raises(OSError, match="Input/output"): + v.store("A memory whose sync failed.", caller="t") + assert os.path.getsize(vault_path) == size + assert Vault.unlock(vault_path, passphrase=PASS).status()[ + "organic_records"] == 1 + + +def test_an_append_never_creates_a_vault_that_is_not_there(vault_path): + """An append to a missing file used to create it, holding journal + frames and no header: a file no build can open. The writer's own + staleness check refuses first; this is the floor under it.""" + v = Vault.create(vault_path, PASS, creator="t") + os.unlink(vault_path) + with pytest.raises(FileNotFoundError): + vaultfile.append_journal_entry(vault_path, v.header, 0, + {"op": "noop"}, v._master) + assert not os.path.exists(vault_path) + + +class _Fcntl: + """fcntl as the vault file sees it: records every call, and refuses + them all when `refuse` is set, as SMB and some USB media do.""" + + def __init__(self, refuse: bool = False): + self.calls, self.refuse = [], refuse + + def fcntl(self, fd, op, *args): + self.calls.append(op) + if self.refuse: + raise OSError(errno.ENOTSUP, "Operation not supported") + return 0 + + +def _record_fsyncs(monkeypatch) -> list: + fsyncs = [] + + def fsync(fd): + fsyncs.append(fd) + return os.fsync(fd) + monkeypatch.setattr(vaultfile, "os", _With(os, fsync=fsync)) + return fsyncs + + +def test_appends_and_compactions_reach_the_disk_on_macos(vault_path, + monkeypatch): + """fsync on macOS hands data to the drive's own cache and no further. + An acknowledged store could be lost to a power cut, and a journal entry + whose length landed before its body read back as tampering, so the + vault refused to open. F_FULLFSYNC is the call Apple documents for it.""" + v = Vault.create(vault_path, PASS, creator="t") + fake = _Fcntl() + monkeypatch.setattr(vaultfile, "fcntl", fake) + monkeypatch.setattr(vaultfile, "_FULLFSYNC", 51) # macOS's value + fsyncs = _record_fsyncs(monkeypatch) + v.store("A memory that has to survive a power cut.", caller="t") + assert fake.calls == [51], "the append is flushed past the drive cache" + v.save() + assert fake.calls[1] == 51, "and so is the compaction's copy" + assert not fsyncs, "F_FULLFSYNC does fsync's work and more" + + +def test_a_disk_that_refuses_a_full_sync_still_gets_fsync(vault_path, + monkeypatch): + """A vault on an SMB share or a USB stick, where the filesystem does not + implement F_FULLFSYNC: a store must not fail for want of it.""" + v = Vault.create(vault_path, PASS, creator="t") + monkeypatch.setattr(vaultfile, "fcntl", _Fcntl(refuse=True)) + monkeypatch.setattr(vaultfile, "_FULLFSYNC", 51) + fsyncs = _record_fsyncs(monkeypatch) + v.store("A memory on a share without F_FULLFSYNC.", caller="t") + assert len(fsyncs) == 1 + assert Vault.unlock(vault_path, passphrase=PASS).status()[ + "organic_records"] == 1 + + +def test_elsewhere_an_append_is_an_ordinary_fsync(vault_path, monkeypatch): + v = Vault.create(vault_path, PASS, creator="t") + monkeypatch.setattr(vaultfile, "_FULLFSYNC", None) + fsyncs = _record_fsyncs(monkeypatch) + v.store("A memory on Linux or Windows.", caller="t") + assert len(fsyncs) == 1 + + +def _record_save(monkeypatch, name: str = "posix") -> list: + """What a save does to the disk, in order: syncs (with what they + synced), the rename, and every folder opened.""" + order = [] + real_sync, real_open = vaultfile._sync, os.open + + def sync(fd): + order.append(("sync", None)) + return real_sync(fd) + + def replace(src, dst): + order.append(("replace", None)) + return os.replace(src, dst) + + def opener(path, flags, *args): + if os.path.isdir(path): + order.append(("open folder", path)) + return real_open(path, flags, *args) + + monkeypatch.setattr(vaultfile, "_sync", sync) + monkeypatch.setattr(vaultfile, "os", _With(os, name=name, replace=replace, + open=opener)) + return order + + +@pytest.mark.skipif(os.name == "nt", reason="a folder cannot be opened " + "for syncing on Windows; the test below covers it") +def test_the_rename_of_a_compaction_is_synced_through_its_folder( + vault_path, monkeypatch): + """Syncing the file makes its bytes durable, not the name pointing at + them. After a power cut the old vault could come back, and every store + acknowledged since the compaction - each appended to the NEW file - + went with it.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory that the compaction folds in.", caller="t") + order = _record_save(monkeypatch) + v.save() + kinds = [k for k, _ in order] + assert kinds == ["sync", "replace", "open folder", "sync"] + assert order[2][1] == os.path.dirname(os.path.abspath(vault_path)) + + +def test_on_windows_no_folder_is_opened(vault_path, monkeypatch): + """A directory cannot be opened for syncing on Windows, and NTFS + journals the rename itself.""" + v = Vault.create(vault_path, PASS, creator="t") + order = _record_save(monkeypatch, name="nt") + v.save() + assert [k for k, _ in order] == ["sync", "replace"] + + +def test_a_folder_that_cannot_be_synced_does_not_fail_the_save( + vault_path, monkeypatch): + """Some FUSE and network mounts refuse to sync a directory. The save + has already succeeded by then and must say so.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory on a mount that cannot sync folders.", caller="t") + real_sync, real_replace = vaultfile._sync, os.replace + renamed = [] + + def sync(fd): + if renamed: # the folder's sync, after the rename + raise OSError(errno.EINVAL, "Invalid argument") + return real_sync(fd) + + def replace(src, dst): + renamed.append(dst) + return real_replace(src, dst) + + monkeypatch.setattr(vaultfile, "_sync", sync) + monkeypatch.setattr(vaultfile, "os", _With(os, replace=replace)) + v.save() + assert Vault.unlock(vault_path, passphrase=PASS).status()[ + "organic_records"] == 1 + + +def test_a_store_whose_journal_did_not_land_cannot_be_saved_later( + vault_path, monkeypatch): + """The store changes RAM before it journals. When the journal never + landed - here the write lock stayed busy past its timeout, as behind a + long import in another process - the record stayed in RAM: a retry of + the same store was answered "duplicate" from it and journalled nothing, + and the memory the caller believed stored was gone at the next reopen, + or a later save wrote the one the caller was told had failed.""" + from compartment import vault as vault_mod + from compartment.crypto import CryptoError + from compartment.vault import VaultStaleError + + v = Vault.create(vault_path, PASS, creator="t") + real_lock = vault_mod.FileLock + calls = [] + + def busy_once(path, timeout=10.0): + calls.append(path) + if len(calls) == 1: + raise CryptoError("Vault is busy: another process holds the " + "write lock (waited 10s)") + return real_lock(path, timeout=timeout) + + monkeypatch.setattr(vault_mod, "FileLock", busy_once) + text = "The parking spot is level 3, bay 41." + with pytest.raises(CryptoError, match="busy"): + v.store(text, caller="t") + assert v.is_stale(), "the handle no longer matches the file" + with pytest.raises(VaultStaleError, match="did not reach"): + v.store(text, caller="t") + with pytest.raises(VaultStaleError): + v.save() + + again = Vault.unlock(vault_path, passphrase=PASS) # what the server does + assert again.db.count() == 0 + out = again.store(text, caller="t") + assert not out["duplicate"] + assert Vault.unlock(vault_path, passphrase=PASS).db.count() == 1 + + +def test_a_refused_append_that_wrote_nothing_is_not_saved_later( + vault_path, monkeypatch): + """A disk already full refuses the first byte: the file is untouched, + so nothing about it says this handle is behind - but its RAM holds the + record, and the next save wrote it.""" + from compartment.vault import VaultStaleError + + v = Vault.create(vault_path, PASS, creator="t") + with monkeypatch.context() as m: + _disk_fills_during_appends(m, _Budget(room=0)) + with pytest.raises(OSError, match="No space"): + v.store("A memory refused at the first byte.", caller="t") + with pytest.raises(VaultStaleError): + v.save() + assert Vault.unlock(vault_path, passphrase=PASS).db.count() == 0 diff --git a/tests/test_storage_sqlite_builds.py b/tests/test_storage_sqlite_builds.py new file mode 100644 index 0000000..de71742 --- /dev/null +++ b/tests/test_storage_sqlite_builds.py @@ -0,0 +1,106 @@ +"""Compartment on a Python whose SQLite was built without FTS5. + +Python compiled against an older enterprise distribution's SQLite (CentOS +and RHEL 7, Amazon Linux 2) and some minimal builds have no FTS5. There is no +degraded mode to offer - the full-text table is part of the vault image - so +the requirement is a refusal that names the interpreter and the fix, on the +first command anyone runs, which is `compartment init`. +""" +import sqlite3 + +import pytest + +from compartment import store as store_mod +from compartment.crypto import CryptoError +from compartment.store import Store, StoreError +from compartment.vault import Vault + +from conftest import PASS + + +class _NoFts5: + """A connection to a SQLite with no fts5 module: everything else works, + and anything that needs the module fails the way that SQLite fails.""" + + def __init__(self, conn): + object.__setattr__(self, "_conn", conn) + + @staticmethod + def _needs_fts5(sql: str) -> bool: + low = sql.lower() + return "fts5" in low or "from fts" in low or "into fts" in low + + def execute(self, sql, *args): + if self._needs_fts5(sql): + raise sqlite3.OperationalError("no such module: fts5") + return self._conn.execute(sql, *args) + + def executescript(self, sql): + if self._needs_fts5(sql): + raise sqlite3.OperationalError("no such module: fts5") + return self._conn.executescript(sql) + + def __getattr__(self, name): + return getattr(self._conn, name) + + def __setattr__(self, name, value): + setattr(self._conn, name, value) + + +def _store_connects_through(monkeypatch, proxy) -> None: + """The store's sqlite3, handing out connections wrapped in `proxy`.""" + real = sqlite3.connect + + class _Module: + Row = sqlite3.Row + OperationalError = sqlite3.OperationalError + sqlite_version = sqlite3.sqlite_version + + @staticmethod + def connect(*args, **kwargs): + return proxy(real(*args, **kwargs)) + + monkeypatch.setattr(store_mod, "sqlite3", _Module) + + +@pytest.fixture() +def sqlite_without_fts5(monkeypatch): + _store_connects_through(monkeypatch, _NoFts5) + + +def test_a_new_vault_is_refused_with_the_fix_not_a_bare_sqlite_error( + sqlite_without_fts5): + """The friendly refusal sat after the schema was created, and creating + the schema is where such a SQLite first fails, so a new vault died on + "no such module: fts5" and the message was only ever reached when an + existing vault was opened.""" + with pytest.raises(StoreError) as caught: + Store() + msg = str(caught.value) + assert "FTS5" in msg and sqlite3.sqlite_version in msg + assert "python.org" in msg, "it says where a working Python comes from" + + +def test_init_reports_it_as_a_vault_error(sqlite_without_fts5, vault_path): + """What `compartment init` and the panel's Create button catch.""" + with pytest.raises(CryptoError, match="FTS5"): + Vault.create(vault_path, PASS, creator="t") + + +def test_an_existing_vault_is_refused_the_same_way(monkeypatch): + image = Store().serialize() + _store_connects_through(monkeypatch, _NoFts5) + with pytest.raises(StoreError, match="FTS5"): + Store(image) + + +def test_other_schema_errors_are_not_called_a_missing_fts5(monkeypatch): + """Only the missing module earns that message; anything else is still + the error it was.""" + class _Broken(_NoFts5): + def executescript(self, sql): + raise sqlite3.OperationalError("database is locked") + + _store_connects_through(monkeypatch, _Broken) + with pytest.raises(sqlite3.OperationalError, match="locked"): + Store() diff --git a/tests/test_storage_staleness.py b/tests/test_storage_staleness.py new file mode 100644 index 0000000..3352d49 --- /dev/null +++ b/tests/test_storage_staleness.py @@ -0,0 +1,102 @@ +"""Telling that another process rewrote the vault, on any filesystem. + +A handle knows the vault changed under it by comparing what it read with +what is on disk now. Inode, size and modification time miss a rewrite on +the disks people carry vaults on: FAT and exFAT keep times to two seconds or +ten milliseconds and derive inode numbers from where a file landed, which a +later save reuses; a compaction often leaves the same size, since the +payload is whole SQLite pages; and a clock stepped back stamps a newer file +with an older time anywhere. A missed rewrite is a lost update: this +handle's next save writes its stale image over the other process's work. +""" +import os + +import pytest + +from compartment import vault as vault_mod, vaultfile +from compartment.vault import Vault, VaultStaleError + +from conftest import PASS + + +class _With: + def __init__(self, module, **replaced): + self._module = module + self.__dict__.update(replaced) + + def __getattr__(self, name): + return getattr(self._module, name) + + +def _fat_like_stat(monkeypatch): + """Every stat of the vault reports one inode number and one time, as a + FAT stick can for two writes landing in the same slot within its clock + tick. The size stays true.""" + def fake(result): + fields = list(result) + fields[1] = 4242 # st_ino + fields[8] = 1_700_000_000 # st_mtime + st = os.stat_result(fields, {"st_mtime_ns": 1_700_000_000 * 10**9}) + return st + + def stat(path, *args, **kwargs): + return fake(os.stat(path, *args, **kwargs)) + + def fstat(fd): + return fake(os.fstat(fd)) + + for module in (vaultfile, vault_mod): + monkeypatch.setattr(module, "os", _With(os, stat=stat, fstat=fstat)) + + +def test_a_rewrite_the_stat_cannot_see_is_still_seen(vault_path, monkeypatch): + first = Vault.create(vault_path, PASS, creator="t") + rid = first.store("The build server is called ostrich.", caller="t", + tags=["aaaa"])["id"] + first.lock() + _fat_like_stat(monkeypatch) + + writer = Vault.unlock(vault_path, passphrase=PASS) + stale = Vault.unlock(vault_path, passphrase=PASS) + size = os.path.getsize(vault_path) + writer.db.set_tags(rid, ["bbbb"]) # same length: same pages + writer.save() + assert os.path.getsize(vault_path) == size, \ + "the premise: a rewrite the same size as what it replaced" + + assert stale.is_stale(), "the other handle's save is noticed" + with pytest.raises(VaultStaleError): + stale.save() + assert Vault.unlock(vault_path, passphrase=PASS).get( + rid, caller="t")["tags"] == ["bbbb"], "and its change survives" + + +def test_an_untouched_vault_is_not_called_stale(vault_path): + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory nobody else touches.", caller="t") + assert not v.is_stale() + again = Vault.unlock(vault_path, passphrase=PASS) + assert not again.is_stale() + again.store("Another, from the same handle.", caller="t") + assert not again.is_stale(), "its own writes are not someone else's" + + +def test_the_identity_is_the_header_and_the_payload_nonce(vault_path): + """What changes on every save and on nothing else: an append keeps it, + a save replaces it.""" + v = Vault.create(vault_path, PASS, creator="t") + before = vaultfile.disk_state(vault_path)[3] + v.store("Appended to the journal only.", caller="t") + assert vaultfile.disk_state(vault_path)[3] == before + v.save() + assert vaultfile.disk_state(vault_path)[3] != before + assert vaultfile.read_vault_file(vault_path).disk_state == \ + vaultfile.disk_state(vault_path) + + +def test_a_file_that_is_not_a_vault_still_has_a_state(tmp_path): + p = tmp_path / "junk.vault" + p.write_bytes(b"NUC") + assert vaultfile.disk_state(str(p))[1] == 3 + p.write_bytes(b"NUCV\x00\x01\xff\xff\xff\xff" + b"x" * 100) + assert vaultfile.disk_state(str(p))[1] == 110, "a garbage length is capped" diff --git a/tests/test_storage_text.py b/tests/test_storage_text.py new file mode 100644 index 0000000..9223fcd --- /dev/null +++ b/tests/test_storage_text.py @@ -0,0 +1,78 @@ +"""Text the encoder and SQLite cannot take as it arrives. + +A JSON string may carry "\\ud83d" with no partner, and Python's decoder hands +it on as a lone surrogate. JavaScript clients that cut a string to length cut +emoji in half; Windows returns unpaired UTF-16 from file names and the +clipboard. Such a string has no UTF-8 encoding, so every layer below the +vault refuses it. +""" +import json + +import pytest + +from compartment.vault import Vault, clean_text + +HALF_EMOJI = json.loads('"cut here \\ud83d"') # what a JSON client sends +PAIR_APART = chr(0xD83D) + chr(0xDE00) # a pair, as two points + + +def test_clean_text_replaces_only_what_cannot_be_encoded(): + assert clean_text("plain, ünïcode, 🦄 and עברית") == \ + "plain, ünïcode, 🦄 and עברית" + assert clean_text(HALF_EMOJI) == "cut here \ufffd" + assert clean_text(PAIR_APART) == "\N{GRINNING FACE}", "a split pair is joined, not lost" + assert clean_text(["a\udc80b", "ok"]) == ["a\ufffdb", "ok"] + assert clean_text(None) is None and clean_text(3) == 3 + + +def test_a_memory_with_half_an_emoji_is_stored_and_found(vault): + """It failed with "TextInputSequence must be str" from the tokenizer, + and the memory was not stored.""" + text = "The deploy channel is #ops-alerts " + HALF_EMOJI + out = vault.store(text, caller="t", source="chat \udc81", + tags=["ops\ud800"]) + got = vault.get(out["id"], caller="t") + assert "\ufffd" in got["text"] and got["source"] == "chat \ufffd" + assert got["tags"] == ["ops\ufffd"] + hits = vault.search("deploy channel " + HALF_EMOJI, caller="t") + assert [h["id"] for h in hits["results"]][:1] == [out["id"]] + + +def test_the_memory_survives_a_reopen(vault, vault_path): + out = vault.store("Half an emoji at the end " + HALF_EMOJI, caller="t") + again = Vault.unlock(vault_path, passphrase="CorrectHorse") + assert again.get(out["id"], caller="t")["text"].startswith( + "Half an emoji at the end cut here \ufffd") + + +def test_the_graph_takes_and_finds_the_same_text(vault): + vault.link("Ana " + HALF_EMOJI, "works on", "the \udcffrelease", caller="t") + rels = vault.relations("t", entity="Ana " + HALF_EMOJI)["relations"] + assert [(r["subject"], r["object"]) for r in rels] == [ + ("Ana cut here \ufffd", "the \ufffdrelease")] + + +def test_an_import_line_with_half_an_emoji_is_imported(vault): + """The import computes its vectors before store() sees the text, so the + tokenizer refused it there and the whole import failed.""" + line = '{"text": "Imported with half an emoji \\ud83d at the end."}\n' + assert vault.import_jsonl(line, caller="t") == 1 + hit = vault.search("imported half an emoji \ud83d", caller="t") + assert hit["results"][0]["text"].startswith( + "Imported with half an emoji � at the end.") + + +def test_a_pack_record_with_half_an_emoji_reads_clean(): + """Pack records are JSON lines too; a recomputed vector or the insert + refused the lone surrogate the escape decodes to.""" + import numpy as np + + from compartment import packs + ident = packs.new_identity("tester") + blob = packs.build_pack( + name="t", version="1.0.0", description="", identity=ident, + records=[{"id": "r1", "text": "Pack fact with half an emoji \ud83d"}], + vectors=np.zeros((1, 4), dtype=np.float32), + model={"name": "m", "sha256": "0", "dim": 4}) + _, records, _ = packs.read_pack(blob, trusted_keys=[ident["pub_hex"]]) + assert records[0]["text"] == "Pack fact with half an emoji �" diff --git a/tests/test_storage_windows_files.py b/tests/test_storage_windows_files.py new file mode 100644 index 0000000..2bfa2c9 --- /dev/null +++ b/tests/test_storage_windows_files.py @@ -0,0 +1,92 @@ +"""Files a Windows editor or shell wrote. + +Notepad's "UTF-8 with BOM", Windows PowerShell 5.1's Set-Content and +Out-File (-Encoding UTF8) put a byte order mark in front of a file, and +that PowerShell's `>` redirection writes UTF-16. Each used to fail on its +first line: an access-rules file the user had just edited stopped every +unlock, and an export they had passed through PowerShell would not import. +""" +import codecs +import json + +import pytest + +from compartment import curate +from compartment.acl import AclError, ConfigError, VaultConfig +from compartment.vault import Vault + +from conftest import PASS + +CONFIG = {"callers": {"*": {"default_namespace": "main", + "grants": {"*": "rw"}}, + "reader": {"default_namespace": "main", + "grants": {"*": "ro"}}}, + "settings": {"auto_lock_minutes": 12}} + + +@pytest.mark.parametrize("encoding,bom", [ + ("utf-8", codecs.BOM_UTF8), ("utf-16-le", codecs.BOM_UTF16_LE), + ("utf-16-be", codecs.BOM_UTF16_BE), ("utf-8", b"")]) +def test_a_config_saved_by_a_windows_editor_opens(vault, vault_path, + encoding, bom): + vault.lock() + with open(VaultConfig.path_for(vault_path), "wb") as f: + f.write(bom + json.dumps(CONFIG, indent=2).encode(encoding)) + cfg = VaultConfig.load(vault_path) + assert cfg.settings["auto_lock_minutes"] == 12 + assert cfg.grant_for("reader", "main") == "ro" + v = Vault.unlock(vault_path, passphrase=PASS) + with pytest.raises(AclError): + v.store("A reader may not write here.", caller="reader") + + +@pytest.mark.parametrize("content,why", [ + (b'{"callers": {', "not valid JSON"), + (b'["not", "an", "object"]', "must be a JSON object"), + (b'{"callers": [], "settings": {}}', "must be a JSON object"), +]) +def test_a_config_that_cannot_be_used_is_refused_with_what_to_do( + vault_path, content, why): + """Refused, not guessed at: it holds the access rules. The message + names the file and says what moving it aside would grant.""" + with open(VaultConfig.path_for(vault_path), "wb") as f: + f.write(content) + with pytest.raises(ConfigError) as caught: + VaultConfig.load(vault_path) + msg = str(caught.value) + assert why in msg and VaultConfig.path_for(vault_path) in msg + assert "move it aside" in msg + assert isinstance(caught.value, ValueError), \ + "still what a caller catching the old json error catches" + from compartment.crypto import CryptoError + assert not isinstance(caught.value, CryptoError), \ + "callers read a CryptoError from an unlock as 'locked'; this is not" + + +def test_an_export_passed_through_powershell_imports(vault): + vault.store("The router admin page is at 192.168.1.1.", caller="t") + exported = vault.export_jsonl() + vault.forget(vault.db.conn.execute("SELECT id FROM records") + .fetchone()["id"], caller="t") + assert vault.import_jsonl("" + exported, caller="t") == 1 + + +def test_a_curation_plan_with_a_byte_order_mark_parses(): + plan = curate.parse_plan('{"id": "abc", "pieces": [{"text": "x"}]}\n') + assert plan[0]["id"] == "abc" + + +def test_an_export_from_another_tool_with_line_separators_in_text_imports( + vault): + """JSONL written with ensure_ascii=False leaves U+2028, U+2029 and U+0085 + unescaped inside strings. splitlines() cut those lines in two and the + whole import failed on the first memory that contained one; CRLF line + ends are the Windows shape of the same file.""" + texts = ["Line one
line two of the same memory.", + "A paragraph
and the next.", + "Next line\u0085character inside."] + jsonl = "\r\n".join(json.dumps({"text": t}, ensure_ascii=False) + for t in texts) + "\r\n" + assert vault.import_jsonl(jsonl, caller="t") == 3 + stored = sorted(r["text"] for r in vault.recent("t")["results"]) + assert [t.split(" [")[0] for t in stored] == sorted(texts) diff --git a/tests/test_twofa.py b/tests/test_twofa.py index d785af5..7ffa1d9 100644 --- a/tests/test_twofa.py +++ b/tests/test_twofa.py @@ -102,3 +102,32 @@ def test_legacy_recovery_slot_still_opens(vault_path): v.lock() v2 = Vault.unlock(vault_path, passphrase=" ".join(words)) assert v2.status()["records"] == 0 + + +def test_a_new_keyfile_is_created_private_and_exclusive(vault, vault_path, + tmp_path, monkeypatch): + """`2fa enable` wrote the keyfile and chmod'ed it afterwards, so the + second factor sat at the umask's mode (0644 on most systems) for that + moment, and a file planted at the path in between was written through. + It is now created 0600 with O_EXCL in one call.""" + import os + from compartment import cli + vault.lock() + keyfile = tmp_path / "factor.key" + seen = [] + real_open = os.open + + def spy(path, flags, mode=0o777, *a, **k): + if str(path) == str(keyfile): + seen.append((flags, mode)) + return real_open(path, flags, mode, *a, **k) + monkeypatch.setattr(os, "open", spy) + monkeypatch.delenv("COMPARTMENT_PASSPHRASE", raising=False) + cli.main(["--vault", vault_path, "2fa", "enable", "--passphrase", PASS, + "--keyfile", str(keyfile)]) + assert seen, "the keyfile was not created through os.open" + flags, mode = seen[0] + assert flags & os.O_EXCL and flags & os.O_CREAT and mode == 0o600 + assert len(keyfile.read_bytes()) == crypto.KEYFILE_LEN + if os.name != "nt": + assert (keyfile.stat().st_mode & 0o777) == 0o600 diff --git a/tests/test_uninstall.py b/tests/test_uninstall.py index be27180..ae2c9f0 100644 --- a/tests/test_uninstall.py +++ b/tests/test_uninstall.py @@ -33,7 +33,9 @@ def machine(tmp_path, monkeypatch): "openclaw": ("OPENCLAW_HOME", home / ".openclaw"), }) monkeypatch.setenv("HERMES_HOME", str(home / ".hermes")) - monkeypatch.setenv("OPENCLAW_HOME", str(home / ".openclaw")) + # OPENCLAW_HOME stands in for the home directory, as OpenClaw reads it: + # its config is .openclaw/openclaw.json under it. + monkeypatch.setenv("OPENCLAW_HOME", str(home)) monkeypatch.setattr(cli, "_tray_app", lambda: (_ for _ in ()).throw(RuntimeError("no app"))) monkeypatch.setattr(menubar, "_is_generated", lambda path: False) @@ -118,7 +120,7 @@ def test_uninstall_takes_back_every_registration_integrate_wrote(machine, assert json.loads(w.cursor.config_path().read_text(encoding="utf-8")) == { "mcpServers": {"theirs": {"command": "x"}}} assert json.loads(w.openclaw.read_text(encoding="utf-8")) == { - "gateway": {"port": 18789}, "mcpServers": {}} + "gateway": {"port": 18789}, "mcp": {"servers": {}}} assert w.md.read_text(encoding="utf-8") == "# My notes\nkeep me\n" assert not w.plugin.exists() for said in ("Claude Desktop registration removed", diff --git a/tests/test_vault_ops.py b/tests/test_vault_ops.py index 9ab2179..97acc6a 100644 --- a/tests/test_vault_ops.py +++ b/tests/test_vault_ops.py @@ -520,7 +520,10 @@ def nap(seconds): m.setattr(vaultfile, "time", _With(time, sleep=nap)) with pytest.raises(PermissionError): vault.save() - assert len(tries) == 20, "the Windows retries all ran" + # The retries ran for the whole patience budget, one try per nap plus + # the last, before giving up. + assert sum(naps) >= vaultfile._REPLACE_PATIENCE_S, "the Windows retries all ran" + assert len(tries) == len(naps) + 1 assert not os.path.exists(vault_path + ".tmp") assert Vault.unlock(vault_path, passphrase=PASS).db.count() == 1 @@ -574,19 +577,21 @@ def test_a_save_still_syncs_its_copy_before_the_rename(vault, vault_path, synced before it is renamed over the vault, and the vault stays 0600.""" vault.store("A memory saved the ordinary way.", caller="test") order = [] + real_sync = vaultfile._sync - def fsync(fd): - order.append("fsync") - return os.fsync(fd) + def sync(fd): + order.append("sync") + return real_sync(fd) def replace(src, dst): order.append("replace") return os.replace(src, dst) with monkeypatch.context() as m: - m.setattr(vaultfile, "os", _With(os, fsync=fsync, replace=replace)) + m.setattr(vaultfile, "_sync", sync) + m.setattr(vaultfile, "os", _With(os, replace=replace)) vault.save() - assert order == ["fsync", "replace"] + assert order[:2] == ["sync", "replace"] assert not os.path.exists(vault_path + ".tmp") if os.name != "nt": assert os.stat(vault_path).st_mode & 0o777 == 0o600 @@ -711,3 +716,28 @@ def stuck(argv, **kwargs): assert vault_module.keychain_clear("/x/a.vault") is False with pytest.raises(vault_module.CryptoError, match="did not answer"): vault_module.keychain_store("/x/a.vault", b"\x00" * 32) + + +def test_a_nan_similarity_is_never_a_duplicate(vault, monkeypatch): + """A vector from a broken model run scores NaN against everything, and + NaN compares false both ways: `score < thr` read it as a match, so every + later store came back as a "duplicate" of whatever was nearest.""" + first = vault.store("The staging database is Postgres 16", caller="test") + monkeypatch.setattr(vault.index, "search", + lambda vec, k: [(next(iter(vault._id_by_ikey)), float("nan"))]) + second = vault.store("Deploys go out on Tuesdays", caller="test") + assert not second["duplicate"] and second["id"] != first["id"] + + +def test_creating_a_vault_does_not_load_the_model(tmp_path, monkeypatch): + """Vault.create needs only the model's pinned hash and dimension; it used + to build a whole embedder for them, loading the model or starting the + shared embedding process before anything needed a vector.""" + from compartment import vault as vault_mod + + def boom(*a, **k): + raise AssertionError("create built an embedder") + monkeypatch.setattr(vault_mod, "get_embedder", boom) + v = Vault.create(str(tmp_path / "new.vault"), "pw-not-a-secret") + assert v.header.model["dim"] == 384 and len(v.header.model["sha256"]) == 64 + assert v._embedder is None diff --git a/tests/test_version_lockstep.py b/tests/test_version_lockstep.py index 1f3a8e4..f3a71e9 100644 --- a/tests/test_version_lockstep.py +++ b/tests/test_version_lockstep.py @@ -75,6 +75,140 @@ def test_plugin_json_matches_the_package(): assert d["version"] == _semver(_version()) +def test_claude_code_plugin_json_matches_the_package(): + """Claude Code decides whether an installed plugin is out of date by this + field, and a plugin that declares none is pinned to whatever commit it + was installed from, so it has to exist and has to move with every + release. Codex and OpenClaw read the same file when they install from + this marketplace.""" + d = json.loads((ROOT / ".claude-plugin" / "plugin.json").read_text( + encoding="utf-8")) + assert d["version"] == _semver(_version()) + + +def test_marketplace_entries_that_carry_a_version_match_the_package(): + """The entry may repeat the plugin's version; Claude Code shows the + entry's metadata before plugin.json's, so a stale copy there would be + the number users see. None is set today; this stops one from rotting.""" + d = json.loads((ROOT / ".claude-plugin" / "marketplace.json").read_text( + encoding="utf-8")) + for entry in d["plugins"]: + if "version" in entry: + assert entry["version"] == _semver(_version()), entry["name"] + + +def test_the_claude_desktop_bundle_installs_this_release(): + """The .mcpb attached to a release must install that release: its + manifest is the version Claude Desktop shows and compares on update, and + the pin is what uv installs.""" + d = json.loads((ROOT / "packaging" / "mcpb" / "manifest.json").read_text( + encoding="utf-8")) + assert d["version"] == _semver(_version()) + text = (ROOT / "packaging" / "mcpb" / "pyproject.toml").read_text( + encoding="utf-8") + assert re.search(r'^version\s*=\s*"([^"]+)"', text, re.M).group(1) == _version() + assert re.search(r'"compartment==([^"]+)"', text).group(1) == _version() + + +def test_the_continue_block_carries_this_release(): + """Continue's hub shows a block's version; it is published from here.""" + assert _yaml_version(ROOT / "packaging" / "continue" / "compartment.yaml") \ + == _version() + + +# ----------------------------------------------------------- coverage ------ +#: Every file above, and nothing else, may carry the package's version. +GATED = { + "pyproject.toml", + "server.json", + "plugin.json", + "gemini-extension.json", + ".claude-plugin/plugin.json", + ".claude-plugin/marketplace.json", + "integrations/hermes/compartment/plugin.yaml", + "src/compartment/data/hermes-plugin/plugin.yaml", + "packaging/mcpb/manifest.json", + "packaging/mcpb/pyproject.toml", + "packaging/continue/compartment.yaml", + "integrations/hermes/upstream/plugin.yaml", +} + +#: Files whose version field is their own, not the package's. +OWN_VERSION = { + # The tombstone for the pre-rename PyPI name. It only depends on + # compartment; republishing it under every Compartment version would + # upload an identical empty package each release. + "packaging/engram-memory-vault/pyproject.toml", + # The skill's own revision, which Hermes shows; it changes when the + # skill's text does, not when the package does. + "skills/compartmentalize/SKILL.md", + "src/compartment/data/agent-skill/SKILL.md", +} + +#: Known drift in a file this test does not own the fix for. Strict, so the +#: day it is fixed the marker has to go and the file joins GATED. +KNOWN_STALE: set[str] = set() + +_SKIP_DIRS = {".git", "tests", "models", "build", "dist", ".venv", "venv", + "node_modules", "__pycache__"} + + +def _carries_a_version(p: pathlib.Path) -> bool: + """A top-level version field, in whatever syntax the file is written in, + or a pin on the compartment package itself.""" + text = p.read_text(encoding="utf-8", errors="replace") + if p.suffix == ".json": + try: + d = json.loads(text) + except ValueError: + return False + if not isinstance(d, dict): + return False + return "version" in d or any( + isinstance(e, dict) and "version" in e + for e in d.get("plugins", []) or []) + if p.name == "SKILL.md": + head = text.split("---", 2)[1] if text.startswith("---") else "" + return bool(re.search(r"^\s*version:", head, re.M)) + if p.suffix in (".yaml", ".yml", ".toml"): + return bool(re.search(r"^version\s*[:=]", text, re.M) + or re.search(r"\bcompartment\s*==\s*[0-9]", text)) + return False + + +def _manifests_with_a_version() -> set[str]: + found = set() + for p in ROOT.rglob("*"): + rel = p.relative_to(ROOT) + if any(part in _SKIP_DIRS or part.endswith(".egg-info") + for part in rel.parts[:-1]): + continue + if not p.is_file(): + continue + if p.suffix in (".json", ".yaml", ".yml", ".toml") or p.name == "SKILL.md": + if _carries_a_version(p): + found.add(rel.as_posix()) + return found + + +def test_every_file_that_carries_a_version_is_gated(): + """A new manifest nobody added to this file is exactly how plugin.yaml + drifted. Anything in the tree with a version field must be checked + above, or say here why its version is its own.""" + unaccounted = _manifests_with_a_version() - GATED - OWN_VERSION - KNOWN_STALE + assert not unaccounted, ( + f"{sorted(unaccounted)} carry a version this test does not check: " + "gate it against the package version above, or add it to " + "OWN_VERSION with the reason it differs") + + +@pytest.mark.parametrize("rel", sorted(KNOWN_STALE)) +@pytest.mark.xfail(strict=True, reason="stale copy in a file this test does " + "not own; see KNOWN_STALE") +def test_known_stale_copies_match_the_package(rel): + assert _yaml_version(ROOT / rel) == _version() + + @pytest.mark.parametrize("path", PLUGIN_YAMLS, ids=lambda p: p.parent.name) def test_hermes_plugin_yaml_matches_the_package(path): """`hermes plugins list` prints this in its Version column, so a stale @@ -87,3 +221,14 @@ def test_the_shipped_plugin_is_the_one_in_integrations(): something no one has read.""" a, b = (p.read_text(encoding="utf-8") for p in PLUGIN_YAMLS) assert a == b + + +def test_the_upstream_hermes_plugin_yaml_matches_the_package(): + """The copy prepared for a PR to hermes-agent sat at 1.7.0 and + compartment>=2.3 while the package shipped 4.12.0, because nothing read + it. It names this release and the same engine floor as the shipped one.""" + up = ROOT / "integrations" / "hermes" / "upstream" / "plugin.yaml" + assert _yaml_version(up) == _version() + shipped = PLUGIN_YAMLS[0].read_text(encoding="utf-8") + floor = re.search(r'"compartment>=([0-9.]+)"', shipped).group(1) + assert f'"compartment>={floor}"' in up.read_text(encoding="utf-8") diff --git a/tests/test_vindex.py b/tests/test_vindex.py index 4336b1e..32bd3df 100644 --- a/tests/test_vindex.py +++ b/tests/test_vindex.py @@ -167,3 +167,76 @@ def test_an_add_writes_one_row_instead_of_copying_the_index(): moves += 1 before = built._mat assert moves <= 2, f"the rows moved on {moves} of 1,000 adds" + + +def test_a_mistyped_precision_setting_still_builds_an_index(capsys): + """index_precision is hand-edited JSON, and usearch raised KeyError for + anything it does not spell. Above the brute-force limit the index is + built on every open, so one typo meant the vault would not open.""" + import pytest + pytest.importorskip("usearch") + from compartment.vindex import build_index + rng = np.random.default_rng(3) + mat = rng.standard_normal((50, 16)).astype(np.float32) + for setting in ("int-8", 8, None, "b1", ["f32"]): + idx = build_index(16, list(range(1, 51)), mat, precision=setting, + force="hnsw") + assert idx.kind == "hnsw/usearch (f32)" and len(idx) == 50 + assert len(idx.search(mat[7], 3)) == 3 # built, and it searches + assert "index_precision" in capsys.readouterr().err + for spelled, kind in (("INT8", "i8"), (" i8", "i8"), ("float16", "f16")): + assert build_index(16, list(range(1, 51)), mat, precision=spelled, + force="hnsw").kind == f"hnsw/usearch ({kind})" + + +def test_a_large_vault_is_not_given_an_hnsw_index_usearch_would_build_serially( + monkeypatch): + """Without SIMD - which is what usearch reports inside macOS Seatbelt, + where it cannot ask the kernel what the CPU supports - the graph is + built at a few hundred vectors a second, on every open: minutes for a + vault past the limit, against milliseconds per exact search.""" + import pytest + pytest.importorskip("usearch") + from usearch import index as usearch_index + + from compartment import vindex + + built = [] + + class _Serial(usearch_index.Index): + hardware_acceleration = "serial" + + def add(self, *args, **kwargs): + built.append(args) + return super().add(*args, **kwargs) + + monkeypatch.setattr(usearch_index, "Index", _Serial) + monkeypatch.setattr(vindex, "BRUTE_FORCE_LIMIT", 10) + rng = np.random.default_rng(5) + mat = rng.standard_normal((40, 16)).astype(np.float32) + idx = vindex.build_index(16, list(range(1, 41)), mat) + assert isinstance(idx, BruteForceIndex) + assert not built, "no graph was built only to be thrown away" + assert idx.search(mat[3], 1)[0][0] == 4 + + forced = vindex.build_index(16, list(range(1, 41)), mat, force="hnsw") + assert forced.kind.startswith("hnsw"), "an explicit force is honoured" + + +def test_with_simd_a_large_vault_still_gets_hnsw(monkeypatch): + import pytest + pytest.importorskip("usearch") + from usearch import index as usearch_index + + from compartment import vindex + + class _Simd(usearch_index.Index): + hardware_acceleration = "neon" + + monkeypatch.setattr(usearch_index, "Index", _Simd) + monkeypatch.setattr(vindex, "BRUTE_FORCE_LIMIT", 10) + mat = np.random.default_rng(6).standard_normal((40, 16)).astype(np.float32) + idx = vindex.build_index(16, list(range(1, 41)), mat) + assert idx.kind.startswith("hnsw") + assert idx.search(mat[3], 1)[0][0] == 4 + diff --git a/tools/build_macos_app.py b/tools/build_macos_app.py index 7a9284d..e398bb0 100644 --- a/tools/build_macos_app.py +++ b/tools/build_macos_app.py @@ -5,6 +5,17 @@ python tools/build_macos_app.py --pkg # …and a .pkg with an optional # "start at login" component +From CI, on a GitHub macOS runner (arm64, or macos-15-intel for x86_64), +with uv's standalone Python and nothing else installed: + + uv python install 3.12 + PY="$(uv python find --managed-python 3.12)" + "$PY" tools/build_macos_app.py --python "$PY" --pkg --dmg + +It exits non-zero on any failure, with one line saying which step, and +writes build/Compartment-.pkg and .dmg (-intel before the +extension for an x86_64 build), listing them as `artifact: `. + Why the app has to embed its own Python --------------------------------------- A status bar item belongs to whatever process created it, and macOS decides @@ -48,18 +59,33 @@ import os import pathlib import plistlib +import re import shutil +import struct import subprocess import sys import sysconfig import tempfile +import time ROOT = pathlib.Path(__file__).resolve().parents[1] _EMBEDDED_VER = [""] # X.Y of the interpreter actually embedded BUILD = ROOT / "build" APP_NAME = "Compartment" BUNDLE_ID = "io.github.maxfreedompollard.compartment" -MIN_MACOS = "13.0" # also LSMinimumSystemVersion, below +# The oldest macOS the app is built for, and the default of --min-macos. It +# is LSMinimumSystemVersion, the launcher's -mmacosx-version-min, the .pkg's +# allowed-os-versions, and - the part that is easy to miss - the platform +# every wheel in the bundle is chosen for (_install_packages). 13 is also the +# floor: SMAppService, which registers the login item, is macOS 13 and up. +# +# 13.4, not 13.0. onnxruntime 1.23.2 is the last release with macOS 13 (and +# Intel) wheels, and its wheels are tagged macosx_13_0 while the libraries +# inside record a minimum of 13.4, on arm64 and x86_64 alike (found by +# _verify_deployment_target on a real build). Declaring 13.0 let macOS 13.0 +# to 13.3 install an app that could not load its embedding model. 13.4 is a +# free update every macOS 13 Mac can take. +MIN_MACOS = "13.4" # Shown under the app's name in System Settings > General > Login Items. DESCRIPTION = ("Compartment keeps your AI agents' memory encrypted on this Mac. " "The menu bar item shows what it has remembered and lets you " @@ -78,8 +104,25 @@ def _run(*args, **kw) -> subprocess.CompletedProcess: def build_icon(dest: pathlib.Path) -> pathlib.Path: + """The app's icon: the .icns the package already ships, which + tools/make_icon.py draws and the pip install's login bundle uses too. + + Drawing it again here needed Pillow in whatever Python ran this script + and iconutil, and a CI runner given only uv's standalone Python has no + Pillow: the build died at the icon after every other step had passed. + The copy is the same picture, and only a checkout missing it is drawn. + """ + shipped = ROOT / "src" / "compartment" / "data" / "app.icns" + if shipped.is_file(): + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(shipped, dest) + return dest sys.path.insert(0, str(ROOT / "tools")) - from make_icon import build as build_icns + try: + from make_icon import build as build_icns + except ImportError as exc: + raise SystemExit(f"error: {shipped} is missing, and drawing it needs " + f"Pillow ({exc}): pip install pillow") from exc return build_icns(dest) @@ -120,7 +163,11 @@ def ok(exe: str) -> bool: if ok(sys.executable): return sys.executable - uv_root = pathlib.Path.home() / ".local" / "share" / "uv" / "python" + # Where uv keeps its Pythons, which UV_PYTHON_INSTALL_DIR moves (CI + # caches often do). + uv_root = pathlib.Path(os.environ.get("UV_PYTHON_INSTALL_DIR") + or pathlib.Path.home() / ".local" / "share" / "uv" + / "python") for d in sorted(uv_root.glob("cpython-3.*-macos-*"), reverse=True): for exe in sorted(d.glob("bin/python3.*"), reverse=True): if exe.is_file() and ok(str(exe)): @@ -147,6 +194,189 @@ def _python_version(exe: str) -> str: return out.stdout.strip() +def _python_arch(exe: str) -> str: + """The CPU the embedded interpreter is built for: arm64 or x86_64. + + Asked of that interpreter, not of this one or of the machine. It decides + which wheels go into the bundle, which architecture the launcher is + compiled for, and which Macs the installer will agree to run on, and all + three have to be the interpreter's or the app cannot start.""" + out = subprocess.run( + [exe, "-c", "import platform;print(platform.machine())"], + capture_output=True, text=True, check=True) + return out.stdout.strip() + + +def _version_tuple(text: str) -> tuple[int, int]: + parts = (str(text).split(".") + ["0"])[:2] + return int(parts[0]), int(parts[1]) + + +def _install_packages(rt_python: pathlib.Path, spec: str, arch: str, + min_macos: str) -> None: + """Install Compartment and its dependencies into the embedded runtime, + each as the wheel built for `min_macos` on `arch`. + + pip on its own picks the wheel for the macOS it is running on, and the + build machine is the newest macOS there is. From onnxruntime 1.24 on, + the macOS wheels are arm64 only and tagged macosx_14_0, and numpy ships + a macosx_14_0 build beside its macosx_11_0 one; pip prefers the newer + tag whenever the machine allows it. So an app built on a current runner + carried libraries that need macOS 14 inside a bundle declaring 13.0: + Installer accepted it on macOS 13, and the app could not load its own + embedding model there. + + So the wheels are chosen for the declared minimum instead. `pip + download --platform` resolves against that platform (and every older + one, so universal2 and macosx_11_0 wheels qualify), binary only, into a + wheelhouse; the install then takes only from the wheelhouse. On arm64 + with a minimum of 13.0 that resolves onnxruntime 1.23.2, the last + release with a macosx_13_0 build, and on Intel the last with any macOS + x86_64 build at all. _verify_deployment_target checks the result. + """ + major, minor = _version_tuple(min_macos) + platform_tag = f"macosx_{major}_{minor}_{arch}" + with tempfile.TemporaryDirectory(prefix="compartment-wheels-") as td: + house = pathlib.Path(td) + # Compartment itself, as a wheel, so the binary-only resolve below + # can take it: it is pure Python, built from this checkout (or + # fetched, for a --spec naming a release). + _run(rt_python, "-m", "pip", "wheel", "--quiet", "--no-deps", + "--wheel-dir", house, spec) + wheels = sorted(house.glob("compartment-*.whl")) + if not wheels: + raise SystemExit(f"error: building a wheel from {spec} produced " + "no compartment wheel") + wanted = [str(wheels[-1]), "pyobjc-framework-Cocoa>=10.0"] + got = subprocess.run( + [str(rt_python), "-m", "pip", "download", "--quiet", + "--only-binary=:all:", "--platform", platform_tag, + "--dest", str(house), *wanted], + capture_output=True, text=True) + if got.returncode != 0: + raise SystemExit( + f"error: no complete set of wheels for Python " + f"{_EMBEDDED_VER[0]} on macOS {min_macos} ({arch})\n" + f"{(got.stderr or got.stdout).strip()[-1500:]}\n" + " onnxruntime 1.23.2 is the last release with macOS 13 and " + "Intel wheels, and it\n goes up to Python 3.13: embed 3.12 " + "or 3.13 (uv python install 3.12), or, on\n Apple silicon " + "only, build for macOS 14 with --min-macos 14.0") + print(f"wheels chosen for {platform_tag}: " + + ", ".join(sorted(p.name.split("-")[0] + " " + + p.name.split("-")[1] + for p in house.glob("*.whl")))) + _run(rt_python, "-m", "pip", "install", "--quiet", "--no-index", + "--find-links", house, *wanted) + + +# Mach-O, read directly: what the loader reads, with no tool to install and +# a check that a test can feed bytes to on any OS. +_FAT_MAGICS = (b"\xca\xfe\xba\xbe", b"\xca\xfe\xba\xbf") +_THIN_MAGICS = (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe") # 64, 32 bit LE +_CPU_NAMES = {0x01000007: "x86_64", 0x0100000C: "arm64", 7: "i386", + 12: "arm"} +_LC_VERSION_MIN_MACOSX = 0x24 +_LC_BUILD_VERSION = 0x32 +_PLATFORM_MACOS = 1 + + +def _macho_thin(f, start: int) -> tuple[str, tuple[int, int] | None] | None: + """(architecture, minimum macOS) of the Mach-O image at `start`.""" + f.seek(start) + head = f.read(28) + if len(head) < 28 or head[:4] not in _THIN_MAGICS: + return None + cputype, _sub, _ftype, ncmds, sizeofcmds, _flags = struct.unpack( + " len(cmds): + break + cmd, size, a, b = struct.unpack_from("> 16, (b >> 8) & 0xFF) + elif cmd == _LC_VERSION_MIN_MACOSX: + minos = (a >> 16, (a >> 8) & 0xFF) + if size < 8: + break + pos += size + return _CPU_NAMES.get(cputype, hex(cputype)), minos + + +def _macho_slices(path: pathlib.Path) -> dict[str, tuple[int, int] | None]: + """{architecture: minimum macOS} for each slice of a Mach-O file, or {} + for anything that is not one. The minimum is None for an image that + records none.""" + try: + with open(path, "rb") as f: + magic = f.read(4) + if magic in _THIN_MAGICS: + one = _macho_thin(f, 0) + return dict([one]) if one else {} + if magic not in _FAT_MAGICS: + return {} + wide = magic == _FAT_MAGICS[1] + (count,) = struct.unpack(">I", f.read(4)) + # A Java class file starts with the same four bytes, followed by + # its version, 45 and up. No universal binary has that many + # slices. + if not 0 < count <= 16: + return {} + entry = ">IIQQII" if wide else ">IIIII" + offsets = [struct.unpack(entry, f.read(struct.calcsize(entry)))[2] + for _ in range(count)] + out = {} + for off in offsets: + one = _macho_thin(f, off) + if one is None: + return {} + out[one[0]] = one[1] + return out + except (OSError, struct.error): + return {} + + +def _verify_deployment_target(app: pathlib.Path, arch: str, + min_macos: str) -> None: + """Every binary in the bundle runs on `arch` under `min_macos`. + + The check that would have caught the macOS 14 onnxruntime in a bundle + declaring 13.0, and an arm64-only library in an Intel build. A Mach-O + built for a newer macOS than the bundle declares fails to load on the + older one with nothing on screen - the menu bar app has no window to + report it in - so this is fatal, as the signature check is.""" + floor = _version_tuple(min_macos) + too_new, wrong_arch, count = [], [], 0 + for p in sorted(app.rglob("*")): + if p.is_symlink() or not p.is_file(): + continue + slices = _macho_slices(p) + if not slices: + continue + count += 1 + rel = p.relative_to(app) + if arch not in slices: + wrong_arch.append(f"{rel} ({', '.join(sorted(slices))})") + elif slices[arch] is not None and slices[arch] > floor: + need = slices[arch] + too_new.append(f"{rel} needs macOS {need[0]}.{need[1]}") + problems = [] + if wrong_arch: + problems.append(f"built without {arch}, so it cannot load in this " + "app:\n " + "\n ".join(wrong_arch[:20])) + if too_new: + problems.append(f"built for a newer macOS than the {min_macos} the " + "app declares:\n " + "\n ".join(too_new[:20])) + if problems: + raise SystemExit("error: " + "\nerror: ".join(problems)) + print(f"{count} binaries, every one {arch} and none needing more than " + f"macOS {min_macos}") + + def _strip(lib: pathlib.Path) -> None: """Drop what a shipped app has no use for. @@ -174,6 +404,99 @@ def _strip(lib: pathlib.Path) -> None: print(f"stripped {freed / 1024 / 1024:.0f} MB of test and build files") +#: What a console script in the bundle starts with: the interpreter beside +#: the script, wherever the bundle is, the form uv gives its own Pythons' +#: scripts. Valid sh and valid Python at once: sh runs the exec line, and +#: Python reads lines two and three as one string and moves on. -B because +#: the bundle is signed, and bytecode written into it breaks the seal. +#: realpath is in macOS 13 and up, which is MIN_MACOS's floor. +_RELOCATABLE_SHEBANG = ( + "#!/bin/sh\n" + "'''exec' \"$(dirname -- \"$(realpath -- \"$0\")\")/{python}\" -B " + "\"$0\" \"$@\"\n" + "' '''\n") + + +def _relocate_scripts(runtime: pathlib.Path) -> list[str]: + """Rewrite the console scripts pip put in runtime/bin so they run from + wherever the bundle ends up. + + pip writes the absolute path of the interpreter that installed them into + each script's first line, and that interpreter is the copy inside the + BUILD directory. So `compartment` inside the app, which is what the + panel's Connect buttons write into every agent's config (clients. + executable picks the console script beside the running interpreter) and + what the capture hook runs, named a Python on the build machine: on + anybody else's Mac the agent's memory server failed with "bad + interpreter" on its first start. The long-path form pip uses for a + folder with a space in it names the same absolute path, inside an exec + line, and is rewritten the same way.""" + bin_dir = runtime / "bin" + here = {str(runtime), str(runtime.resolve())} + done = [] + for script in sorted(bin_dir.iterdir()): + if script.is_symlink() or not script.is_file(): + continue + try: + lines = script.read_text(encoding="utf-8").split("\n") + except (UnicodeDecodeError, OSError): + continue # a binary + first = lines[0] + if (first.startswith("#!") and len(lines) > 2 + and first.strip() == "#!/bin/sh" + and lines[1].startswith("'''exec' ") + and lines[2] == "' '''"): + named, rest = lines[1], lines[3:] + elif first.startswith("#!"): + named, rest = first, lines[1:] + else: + continue + if not any(h in named for h in here): + continue # not ours, or relocatable already + # The interpreter's own name, python3.12, from wherever it sat: the + # path itself may hold spaces, so it is not split on them. + found = re.search(r"/bin/(python[0-9.]*)", named) + python = found.group(1) if found else "python3" + script.write_text(_RELOCATABLE_SHEBANG.format(python=python) + + "\n".join(rest), encoding="utf-8") + done.append(script.name) + return done + + +def _verify_console_scripts(app: pathlib.Path) -> None: + """No script in the bundle names the build machine's paths, and the CLI + an agent is given really runs, from a clean environment.""" + runtime = app / "Contents" / "Resources" / "runtime" + built_at = {str(app), str(app.resolve())} + strays = [] + for script in sorted((runtime / "bin").iterdir()): + if script.is_symlink() or not script.is_file(): + continue + try: + head = script.read_text(encoding="utf-8")[:1024] + except (UnicodeDecodeError, OSError): + continue + if any(p in head for p in built_at): + strays.append(script.name) + if strays: + raise SystemExit( + "error: these scripts name the build directory and would fail " + "on any other Mac:\n " + "\n ".join(strays)) + cli = runtime / "bin" / "compartment" + if not cli.is_file(): + raise SystemExit("error: the bundle has no compartment command for " + "agents to run") + with tempfile.TemporaryDirectory(prefix="compartment-buildprobe-") as tmp: + out = subprocess.run([str(cli), "--version"], capture_output=True, + text=True, timeout=120, + env={"PATH": "/usr/bin:/bin", "HOME": tmp}) + if out.returncode != 0: + raise SystemExit("error: the bundle's compartment command does not " + "run\n" + (out.stderr or out.stdout).strip()[:2000]) + print(f"the compartment command agents are given runs: " + f"{out.stdout.strip()}") + + def _unseal_hazards(contents: pathlib.Path) -> None: """Delete files that make the bundle unsignable. @@ -284,12 +607,21 @@ def _verify_self_contained(app: pathlib.Path) -> None: def build_app(spec: str | None = None, out: pathlib.Path | None = None, - python: str | None = None) -> pathlib.Path: + python: str | None = None, + min_macos: str = MIN_MACOS) -> pathlib.Path: """spec: what to pip-install (default: this checkout, so the build always matches the source it came from).""" version = _version() + if _version_tuple(min_macos) < (13, 0): + raise SystemExit("error: --min-macos cannot go below 13.0: the login " + "item is registered with SMAppService, which is " + "macOS 13 and up") base_python = python or find_standalone_python() - print(f"embedding {base_python}") + arch = _python_arch(base_python) + if arch not in ("arm64", "x86_64"): + raise SystemExit(f"error: {base_python} is built for {arch!r}; the " + "app is built for arm64 or x86_64") + print(f"embedding {base_python} ({arch}), for macOS {min_macos} and up") app = (out or BUILD / f"{APP_NAME}.app") if app.exists(): shutil.rmtree(app) @@ -322,9 +654,12 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # bundle, and installing into it is the entire point. (runtime / "lib" / f"python{ver}" / "EXTERNALLY-MANAGED").unlink( missing_ok=True) - _run(rt_python, "-m", "pip", "install", "--quiet", spec or str(ROOT)) - _run(rt_python, "-m", "pip", "install", "--quiet", - "pyobjc-framework-Cocoa>=10.0") + if subprocess.run([str(rt_python), "-m", "pip", "--version"], + capture_output=True).returncode != 0: + _run(rt_python, "-m", "ensurepip", "--default-pip") # a pip-less build + _install_packages(rt_python, spec or str(ROOT), arch, min_macos) + fixed = _relocate_scripts(runtime) + print(f"console scripts made relocatable: {', '.join(fixed) or 'none'}") _strip(runtime / "lib") @@ -353,7 +688,7 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # an image and a window - but the menu bar never gives it a slot, and # the icon simply never appears. See tools/launcher.c. launcher = macos / APP_NAME - _run("cc", "-O2", f"-mmacosx-version-min={MIN_MACOS}", + _run("cc", "-O2", "-arch", arch, f"-mmacosx-version-min={min_macos}", "-o", str(launcher), str(pathlib.Path(__file__).with_name("launcher.c")), f"-I{runtime / 'include' / f'python{ver}'}", f"-L{runtime / 'lib'}", f"-lpython{ver}", @@ -375,7 +710,7 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, "CFBundleVersion": version, "CFBundleInfoDictionaryVersion": "6.0", "LSUIElement": True, # menu bar only, no dock icon - "LSMinimumSystemVersion": MIN_MACOS, + "LSMinimumSystemVersion": min_macos, "NSHumanReadableCopyright": "https://github.com/" "MaxFreedomPollard/Compartment", "NSHumanReadableDescription": DESCRIPTION, @@ -391,6 +726,8 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # Checked BEFORE signing: running the interpreter is what writes bytecode # into the bundle, and doing that afterwards is what broke the seal. _verify_self_contained(app) + _verify_console_scripts(app) + _verify_deployment_target(app, arch, min_macos) _run("codesign", "--force", "--deep", "--sign", "-", "--identifier", BUNDLE_ID, app) @@ -464,9 +801,42 @@ def _login_plist(app: pathlib.Path) -> dict: } +def _app_arch(app: pathlib.Path) -> str: + """The architecture of the built app, read from its executable.""" + slices = _macho_slices(app / "Contents" / "MacOS" / APP_NAME) + if not slices: + raise SystemExit(f"error: {app} has no executable to read") + return ",".join(sorted(slices)) + + +def artifact_name(version: str, arch: str, ext: str) -> str: + """build/: Compartment-.pkg for Apple silicon, the build + the README and every release so far have linked, and + Compartment--intel.pkg for an Intel build. The two cannot share + a name: a release carries both, and whichever was uploaded second would + replace the first.""" + suffix = "" if arch == "arm64" else "-intel" + return f"{APP_NAME}-{version}{suffix}.{ext}" + + +def _app_min_macos(app: pathlib.Path) -> str: + """The minimum macOS the built app declares. The installers take it from + the app rather than from a constant of their own, so a .pkg can never + accept a macOS the app inside it was not built for.""" + with open(app / "Contents" / "Info.plist", "rb") as fh: + return str(plistlib.load(fh).get("LSMinimumSystemVersion") + or MIN_MACOS) + + def build_pkg(app: pathlib.Path) -> pathlib.Path: """A .pkg whose second, optional component starts it at login.""" version = _version() + min_macos = _app_min_macos(app) + # The installer agrees to run only on the Macs this build can start on. + # It used to say arm64,x86_64 whatever was inside, so an Intel Mac + # installed the Apple silicon app - which could never launch, and being + # a menu bar app said nothing about it. + arch = _app_arch(app) staging = BUILD / "pkgroot" login_root = BUILD / "pkgroot-login" for d in (staging, login_root): @@ -539,8 +909,8 @@ def _scripts(name: str, body: str) -> pathlib.Path: dist.write_text(f""" Compartment - - + + @@ -559,7 +929,7 @@ def _scripts(name: str, body: str) -> pathlib.Path: """, encoding="utf-8") - out = BUILD / f"{APP_NAME}-{version}.pkg" + out = BUILD / artifact_name(version, arch, "pkg") _run("productbuild", "--distribution", dist, "--package-path", BUILD, out) print(f"built {out}") return out @@ -567,6 +937,7 @@ def _scripts(name: str, body: str) -> pathlib.Path: def build_dmg(app: pathlib.Path) -> pathlib.Path: version = _version() + arch = _app_arch(app) stage = BUILD / "dmg" if stage.exists(): shutil.rmtree(stage) @@ -581,7 +952,8 @@ def build_dmg(app: pathlib.Path) -> pathlib.Path: (stage / "READ ME FIRST.txt").write_text( "Compartment " + version + "\n" "=================\n\n" - "Easiest install: use Compartment-" + version + ".pkg from the release page\n" + "Easiest install: use " + artifact_name(version, arch, "pkg") + + " from the release page\n" "instead of this disk image. It sets everything up in one click and\n" "skips the warning below entirely.\n\n" "If you would rather drag the app across:\n\n" @@ -596,10 +968,22 @@ def build_dmg(app: pathlib.Path) -> pathlib.Path: "just open Compartment.app again: it will show its panel in a window.\n\n" "Source, and every other way to install:\n" "https://github.com/MaxFreedomPollard/Compartment\n", encoding="utf-8") - out = BUILD / f"{APP_NAME}-{version}.dmg" + out = BUILD / artifact_name(version, arch, "dmg") out.unlink(missing_ok=True) - _run("hdiutil", "create", "-volname", f"{APP_NAME} {version}", - "-srcfolder", stage, "-ov", "-format", "UDZO", out) + # hdiutil on a CI runner now and then fails with "Resource busy" while + # something (Spotlight, the previous image) still has the volume; the + # same command a few seconds later succeeds. Three tries, then the + # error stands. + for attempt in range(3): + try: + _run("hdiutil", "create", "-volname", f"{APP_NAME} {version}", + "-srcfolder", stage, "-ov", "-format", "UDZO", out) + break + except subprocess.CalledProcessError: + if attempt == 2: + raise + print("hdiutil failed; trying again in 5 s") + time.sleep(5) print(f"built {out}") return out @@ -611,15 +995,29 @@ def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--spec", help="what to pip install (default: this repo)") ap.add_argument("--python", help="standalone interpreter to embed") + ap.add_argument("--min-macos", default=MIN_MACOS, + help=f"oldest macOS the app runs on (default {MIN_MACOS}); " + "every wheel is chosen for it") ap.add_argument("--dmg", action="store_true") ap.add_argument("--pkg", action="store_true") args = ap.parse_args() BUILD.mkdir(exist_ok=True) - app = build_app(args.spec, python=args.python) - if args.dmg: - build_dmg(app) - if args.pkg: - build_pkg(app) + try: + artifacts = [build_app(args.spec, python=args.python, + min_macos=args.min_macos)] + if args.dmg: + artifacts.append(build_dmg(artifacts[0])) + if args.pkg: + artifacts.append(build_pkg(artifacts[0])) + except subprocess.CalledProcessError as exc: + # A step that failed says which, in one line, instead of a + # traceback through _run that a CI log buries the command in. + cmd = " ".join(str(a) for a in exc.cmd) + print(f"error: `{cmd[:300]}` exited {exc.returncode}", + file=sys.stderr) + return 1 + for path in artifacts: + print(f"artifact: {path}") return 0 diff --git a/tools/build_mcpb.py b/tools/build_mcpb.py new file mode 100644 index 0000000..091c6dc --- /dev/null +++ b/tools/build_mcpb.py @@ -0,0 +1,154 @@ +"""Build dist/compartment-.mcpb, the Claude Desktop extension. + +An MCP Bundle is a zip with manifest.json at its root (the format lives in +github.com/modelcontextprotocol/mcpb, formerly anthropics/dxt). This one is +server type "uv", manifest_version 0.4: it carries a pyproject.toml that +pins the release, a three-line entry point and the icon, and Claude Desktop +has uv install compartment and its compiled dependencies for the user's own +platform when the extension is installed. Bundling the dependencies instead +("python" type) cannot work for Compartment: onnxruntime, PyNaCl, +argon2-cffi, numpy and tokenizers are compiled, so a bundle built on one +platform is dead on the others, and the MCPB README says as much. + + python tools/build_mcpb.py # what the release runs; needs uv + python tools/build_mcpb.py --no-lock # no uv.lock (development, tests) + +With uv on the PATH the bundle also carries a uv.lock, resolved here, with +hashes, for every platform: Claude Desktop then installs exactly what the +release was built against instead of resolving again on each user's +machine, and a launch whose environment is already in place has nothing to +look up. Resolving is the one step that uses the network, and it happens at +build time, on the release runner. The built bundle and the server in it +make no connection. + +Standard library only, and deterministic: the same inputs give the same +bytes, so a rebuilt bundle can be compared with the one attached to a +release. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SRC = ROOT / "packaging" / "mcpb" + +#: Archive path -> source file. The manifest's entry point and icon must be +#: in here, and nothing that is not declared ships. +CONTENTS = { + "manifest.json": SRC / "manifest.json", + "pyproject.toml": SRC / "pyproject.toml", + ".python-version": SRC / ".python-version", + "server/main.py": SRC / "server" / "main.py", + "icon.png": ROOT / "docs" / "images" / "logo-400.png", + "LICENSE": ROOT / "LICENSE", +} + +#: 1980-01-01, the earliest time a zip can record: every entry gets it, so +#: the archive does not change with the checkout's file times. +_EPOCH = (1980, 1, 1, 0, 0, 0) + + +def package_version() -> str: + text = (ROOT / "src" / "compartment" / "__init__.py").read_text(encoding="utf-8") + m = re.search(r'^__version__\s*=\s*"([^"]+)"', text, re.M) + if not m: + raise SystemExit("cannot find __version__ in src/compartment/__init__.py") + return m.group(1) + + +def semver(v: str) -> str: + parts = v.split(".") + return ".".join(parts + ["0"] * (3 - len(parts))) + + +def check(version: str) -> dict: + """Refuse to build a bundle that would install the wrong release or + that Claude Desktop would reject. Returns the manifest.""" + manifest = json.loads((SRC / "manifest.json").read_text(encoding="utf-8")) + problems = [] + for field in ("manifest_version", "name", "version", "description", + "author", "server"): + if field not in manifest: + problems.append(f"manifest.json has no {field}") + if manifest.get("version") != semver(version): + problems.append(f"manifest.json version {manifest.get('version')!r} is " + f"not the package's {semver(version)!r}") + pin = re.search(r'"compartment==([^"]+)"', + (SRC / "pyproject.toml").read_text(encoding="utf-8")) + if not pin or pin.group(1) != version: + problems.append(f"pyproject.toml must pin compartment=={version}") + server = manifest.get("server", {}) + if server.get("type") != "uv": + problems.append("server.type must be uv: the dependencies are compiled") + for declared in (server.get("entry_point"), manifest.get("icon")): + if declared and declared not in CONTENTS: + problems.append(f"manifest.json names {declared}, which the bundle " + "does not contain") + for name, source in CONTENTS.items(): + if not source.is_file(): + problems.append(f"{name}: {source} is missing") + if problems: + raise SystemExit("cannot build the bundle:\n " + "\n ".join(problems)) + return manifest + + +def lock(staging: pathlib.Path) -> pathlib.Path: + """Resolve uv.lock for the pinned release. Network: PyPI, build time only.""" + uv = shutil.which("uv") + if not uv: + raise SystemExit("uv is not on PATH, so the bundle cannot carry a lock " + "file. Install it (`pip install uv`) and run again, or " + "pass --no-lock for a development build.") + for name in ("pyproject.toml", ".python-version"): + shutil.copy2(SRC / name, staging / name) + if subprocess.run([uv, "lock", "--directory", str(staging)]).returncode: + # Almost always the release is not on PyPI yet: the lock pins the + # version this bundle installs, so it can only be resolved after + # publish.yml has uploaded it. uv has printed its own reason above. + raise SystemExit(f"uv could not lock compartment=={package_version()}. " + "If that version is not on PyPI yet, build the bundle " + "after the PyPI publish; --no-lock skips the lock.") + return staging / "uv.lock" + + +def build(out_dir: pathlib.Path, with_lock: bool = True) -> pathlib.Path: + version = package_version() + check(version) + files = dict(CONTENTS) + with tempfile.TemporaryDirectory() as tmp: + if with_lock: + files["uv.lock"] = lock(pathlib.Path(tmp)) + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / f"compartment-{version}.mcpb" + partial = out.with_suffix(".mcpb.partial") + with zipfile.ZipFile(partial, "w", zipfile.ZIP_DEFLATED) as z: + for name in sorted(files): + info = zipfile.ZipInfo(name, date_time=_EPOCH) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + z.writestr(info, files[name].read_bytes()) + partial.replace(out) + return out + + +def main(argv: list[str] | None = None) -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--out", default=str(ROOT / "dist"), + help="directory to write the bundle into (default dist/)") + ap.add_argument("--no-lock", action="store_true", + help="leave out uv.lock (no uv, no network); not for release") + args = ap.parse_args(argv) + out = build(pathlib.Path(args.out), with_lock=not args.no_lock) + print(f"built {out} ({out.stat().st_size / 1024:.0f} KB)") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_report.py b/tools/ci_report.py new file mode 100644 index 0000000..4464a86 --- /dev/null +++ b/tools/ci_report.py @@ -0,0 +1,98 @@ +"""Run pytest and, when it fails or stalls, surface why where it can be read. + + python tools/ci_report.py [pytest args...] + +A red CI job's log is only readable by someone signed in to GitHub, and +pytest's verdict (which test, which assertion, which import failed during +collection) sits at the end of thousands of lines. This runs pytest with the +given arguments, streams its output unchanged, and under GitHub Actions also +emits error annotations - they appear on the run's summary page, inline on a +pull request, and in the check-run annotations API: + +* when pytest fails: the short test summary and the last lines; +* when pytest falls silent for STALL_S seconds (a hung test): the last lines + as they stand, the unfinished line included - with `-v` that line names + the test that is running, and with `-o faulthandler_timeout=N` the stacks + of every thread are already in it. + +A job that a hung test makes run into its timeout used to end with no +verdict at all; the stall annotation is written while the job is alive. +Outside GitHub Actions it is plain pytest. +""" +from __future__ import annotations + +import os +import subprocess +import sys +import threading +import time + +TAIL_LINES = 60 +STALL_S = int(os.environ.get("CI_REPORT_STALL_S", "150")) +MAX_STALL_REPORTS = 4 +ON_ACTIONS = os.environ.get("GITHUB_ACTIONS") == "true" + + +def _escape(text: str) -> str: + # Workflow-command data escaping, so a multi-line message stays one + # annotation. + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def _annotate(title: str, text: str) -> None: + sys.stdout.write(f"::error title={title}::{_escape(text)}\n") + sys.stdout.flush() + + +def main(argv: list[str]) -> int: + proc = subprocess.Popen([sys.executable, "-m", "pytest", "-rfE", *argv], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + assert proc.stdout is not None + buf: list[str] = [] # complete lines so far + partial = [""] # the line being written now + last = [time.monotonic()] + lock = threading.Lock() + + def reader() -> None: + fd = proc.stdout.fileno() + while True: + chunk = os.read(fd, 4096) + if not chunk: + return + text = chunk.decode("utf-8", "replace") + sys.stdout.write(text) + sys.stdout.flush() + with lock: + last[0] = time.monotonic() + pieces = (partial[0] + text).split("\n") + partial[0] = pieces.pop() + buf.extend(pieces) + del buf[:-2000] + + t = threading.Thread(target=reader, daemon=True) + t.start() + reports = 0 + while proc.poll() is None: + time.sleep(5) + with lock: + idle = time.monotonic() - last[0] + tail = buf[-TAIL_LINES:] + ([partial[0]] if partial[0] else []) + if ON_ACTIONS and idle >= STALL_S and reports < MAX_STALL_REPORTS: + reports += 1 + _annotate(f"pytest silent for {int(idle)}s (still running)", + "\n".join(tail)) + with lock: + last[0] = time.monotonic() # next report after another stall + t.join(timeout=10) + rc = proc.returncode + lines = buf + ([partial[0]] if partial[0] else []) + if rc != 0 and ON_ACTIONS: + summary = [ln for ln in lines if ln.startswith(("FAILED ", "ERROR "))] + if summary: + _annotate("pytest failures", "\n".join(summary[:50])) + _annotate(f"pytest exited {rc} (last lines)", "\n".join(lines[-TAIL_LINES:])) + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/mcp_smoke.py b/tools/mcp_smoke.py new file mode 100644 index 0000000..2a3ac7f --- /dev/null +++ b/tools/mcp_smoke.py @@ -0,0 +1,152 @@ +"""Talk to `compartment serve` the way an MCP client does, and fail loudly. + + python tools/mcp_smoke.py [--command compartment] [--vault PATH] + +Starts the server over stdio, sends `initialize`, the `initialized` +notification and `tools/list`, then calls `memory_status`, and checks every +answer. CI runs it against the INSTALLED package on macOS, Windows and Linux: +the unit suite drives the server in-process, which cannot see the failures a +client sees first - an entry point that does not start, a stray line on +stdout that corrupts the JSON-RPC stream, a handshake that never answers. + +Standard library only, so it runs before or without the package's own +dependencies on the interpreter that launches it. +""" +from __future__ import annotations + +import argparse +import json +import os +import queue +import shutil +import subprocess +import sys +import threading + +#: Tools every build must expose. memory_unlock is off by default, so it is +#: not required here. +REQUIRED_TOOLS = { + "memory_search", "memory_store", "memory_store_many", "memory_get", + "memory_recent", "memory_forget", "memory_link", "memory_relations", + "memory_unlink", "memory_list_namespaces", "memory_status", "memory_lock", + "memory_selftest", +} +TIMEOUT_S = 120.0 # the first call may load the embedding model + + +def _reader(stream, q: queue.Queue) -> None: + for line in iter(stream.readline, b""): + q.put(line) + q.put(None) + + +def _expect(q: queue.Queue, want_id: int, stderr_tail: list[str]) -> dict: + """The response with this id. Every stdout line must be JSON-RPC: a line + that is not is exactly the stream corruption clients choke on.""" + while True: + try: + line = q.get(timeout=TIMEOUT_S) + except queue.Empty: + raise SystemExit(f"no answer to request {want_id} within " + f"{TIMEOUT_S:.0f}s\nstderr:\n" + "".join(stderr_tail)) + if line is None: + raise SystemExit(f"the server exited before answering {want_id}\n" + "stderr:\n" + "".join(stderr_tail)) + text = line.decode("utf-8", "replace").strip() + if not text: + continue + try: + msg = json.loads(text) + except ValueError: + raise SystemExit(f"non-JSON line on the server's stdout: {text!r}") + if msg.get("jsonrpc") != "2.0": + raise SystemExit(f"not a JSON-RPC 2.0 message: {text!r}") + if msg.get("id") == want_id: + if "error" in msg: + raise SystemExit(f"request {want_id} failed: {msg['error']}") + return msg["result"] + # notifications (logging, progress) are fine; keep reading + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--command", default="compartment", + help="the executable to start (default: compartment on PATH)") + ap.add_argument("--vault", help="pass --vault to the server") + args = ap.parse_args() + + exe = shutil.which(args.command) or args.command + cmd = [exe] + if args.vault: + cmd += ["--vault", args.vault] + cmd += ["serve"] + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=os.environ.copy()) + out_q: queue.Queue = queue.Queue() + err_tail: list[str] = [] + + def _drain_err(): + for line in iter(proc.stderr.readline, b""): + err_tail.append(line.decode("utf-8", "replace")) + del err_tail[:-40] + threading.Thread(target=_reader, args=(proc.stdout, out_q), daemon=True).start() + threading.Thread(target=_drain_err, daemon=True).start() + + def send(obj: dict) -> None: + proc.stdin.write((json.dumps(obj) + "\n").encode("utf-8")) + proc.stdin.flush() + + try: + send({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "compartment-smoke", "version": "1"}}}) + init = _expect(out_q, 1, err_tail) + name = init.get("serverInfo", {}).get("name", "") + print(f"initialize: server {name!r} {init.get('serverInfo', {}).get('version', '')}, " + f"protocol {init.get('protocolVersion')}") + if not init.get("instructions"): + raise SystemExit("the server sent no instructions in initialize") + send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + + send({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + tools = {t["name"]: t for t in _expect(out_q, 2, err_tail)["tools"]} + missing = REQUIRED_TOOLS - set(tools) + if missing: + raise SystemExit(f"tools missing from tools/list: {sorted(missing)}") + for t in tools.values(): + schema = t.get("inputSchema") or {} + if schema.get("type") != "object": + raise SystemExit(f"{t['name']}: inputSchema is not an object schema") + print(f"tools/list: {len(tools)} tools") + + send({"jsonrpc": "2.0", "id": 3, "method": "tools/call", + "params": {"name": "memory_status", "arguments": {}}}) + res = _expect(out_q, 3, err_tail) + if res.get("isError"): + raise SystemExit(f"memory_status reported an error: {res}") + print("tools/call memory_status: ok") + finally: + try: + proc.stdin.close() + except OSError: + pass + try: + proc.wait(timeout=20) + except subprocess.TimeoutExpired: + proc.kill() + raise SystemExit("the server did not exit after its stdin closed") + # A client that hangs up is the normal end of a session, so it has to be + # a clean one: exit 0 and no traceback on stderr. A regression that + # crashed on shutdown would otherwise pass every request above. + stderr = "".join(err_tail) + if proc.returncode != 0: + raise SystemExit(f"the server exited {proc.returncode} after its stdin " + f"closed\nstderr:\n{stderr}") + if "Traceback" in stderr: + raise SystemExit(f"the server printed a traceback\nstderr:\n{stderr}") + print("MCP smoke test passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/wheel_coverage.py b/tools/wheel_coverage.py new file mode 100644 index 0000000..cd9e4aa --- /dev/null +++ b/tools/wheel_coverage.py @@ -0,0 +1,104 @@ +"""Fail when a plain install would have to compile something on a supported platform. + + python tools/wheel_coverage.py # needs `uv` on PATH + +pip installs the newest release of every dependency, direct or not, whether +or not that release has a wheel for the machine, and then tries to build it +from source. A user without a compiler (or a Rust toolchain, for +cryptography) gets a wall of build errors from a package they never asked +for. That is how `pip install compartment` broke on Windows on ARM and on +Intel Macs in 2026: cryptography and argon2-cffi-bindings stopped shipping +wheels there and nothing here noticed. + +For every platform and Python Compartment supports, this resolves the +dependencies twice with uv - as pip would, and with wheels only - and fails +if the two disagree, naming the package. pyproject.toml's platform caps are +the fix it points at. +""" +from __future__ import annotations + +import os +import pathlib +import subprocess +import sys +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +#: (uv --python-platform, Pythons, macOS deployment target or None). Kept in +#: step with README's "Where it runs": onnxruntime decides most of it. +SUPPORTED = [ + ("x86_64-pc-windows-msvc", ["3.11", "3.12", "3.13", "3.14"], None), + # 3.11 on Windows on ARM has no pyyaml wheel (via tokenizers -> + # huggingface-hub), so ARM64 Windows starts at 3.12. + ("aarch64-pc-windows-msvc", ["3.12", "3.13", "3.14"], None), + ("x86_64-manylinux_2_28", ["3.11", "3.12", "3.13", "3.14"], None), + ("aarch64-manylinux_2_28", ["3.11", "3.12", "3.13", "3.14"], None), + # onnxruntime's arm64 wheels need macOS 14 from 1.24 on, and 1.24 is the + # first with 3.14 wheels. + ("aarch64-apple-darwin", ["3.11", "3.12", "3.13", "3.14"], "14.0"), + # Intel Macs stop at onnxruntime 1.23.2, which stops at Python 3.13. + ("x86_64-apple-darwin", ["3.11", "3.12", "3.13"], "13.0"), +] + + +def _compile(platform: str, python: str, target: str | None, + binary_only: bool, extra: str | None = None) -> tuple[int, str]: + env = dict(os.environ) + if target: + env["MACOSX_DEPLOYMENT_TARGET"] = target + with tempfile.TemporaryDirectory() as td: + out = pathlib.Path(td) / "req.txt" + cmd = ["uv", "pip", "compile", str(ROOT / "pyproject.toml"), + *(["--extra", extra] if extra else []), + "--python-platform", platform, + "--python-version", python, "--no-header", "--no-annotate", + "-q", "-o", str(out)] + if binary_only: + cmd += ["--only-binary", ":all:"] + p = subprocess.run(cmd, capture_output=True, text=True, env=env) + text = out.read_text() if out.exists() else "" + return p.returncode, text if p.returncode == 0 else p.stderr + + +def _check(platform: str, py: str, target: str | None, + extra: str | None) -> str | None: + """None when a plain install needs no compiler; otherwise why.""" + rc_any, any_ = _compile(platform, py, target, False, extra) + rc_bin, bin_ = _compile(platform, py, target, True, extra) + if rc_any != 0: + return f"does not resolve at all\n{any_}" + if rc_bin != 0: + return f"no wheel-only solution exists\n{bin_}" + if any_ != bin_: + a, b = set(any_.splitlines()), set(bin_.splitlines()) + return "pip would build from source:\n" + "\n".join( + [f" {ln} (newest; no wheel here)" for ln in sorted(a - b)] + + [f" {ln} (newest with a wheel)" for ln in sorted(b - a)]) + return None + + +def main() -> int: + bad = 0 + for platform, pythons, target in SUPPORTED: + for py in pythons: + label = f"{platform} / Python {py}" + why = _check(platform, py, target, None) + if why: + print(f"FAIL {label}: {why}") + bad += 1 + continue + # The optional [hnsw] extra is reported, not enforced: it is an + # opt-in for vaults past 20k records, and its own dependency + # (numkong) has shipped partial wheel sets that fix themselves + # within days. + why = _check(platform, py, target, "hnsw") + print(f"ok {label}" + (f" (note: [hnsw] {why})" if why else "")) + if bad: + print(f"\n{bad} platform(s) would compile a dependency on install. " + "Cap the package for that platform in pyproject.toml, with the " + "reason, or drop the platform from SUPPORTED and the README.") + return 1 if bad else 0 + +if __name__ == "__main__": + sys.exit(main())