From c557f1a671bca0106ed7d40b2eb1abbbf831cda2 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:24:19 -0700 Subject: [PATCH 001/135] The boot session is read from a clock that counts from boot when the kernel's record is out of reach Claude Code and Codex run an agent's shell commands under macOS Seatbelt, which refuses kern.boottime both to the sysctl binary and to sysctlbyname, and Android's SELinux hides /proc/stat from Termux. Every vault open there raised "Cannot identify this boot session", so `compartment store` run by the agent itself could never use the vault the user had unlocked. The estimate now falls back to CLOCK_MONOTONIC on macOS (mach_continuous_time, which counts through sleep) and CLOCK_BOOTTIME elsewhere, and the result is pinned per boot like every other reading, so a sandboxed process and an unsandboxed one agree on the token. macOS and the BSDs read kern.boottime in-process first instead of forking sysctl on every vault open, and FreeBSD, NetBSD and DragonFly get the kernel record at all. Co-Authored-By: Claude Opus 5.5 --- src/compartment/platforms.py | 78 ++++++++++++++++++++++++++++++------ tests/test_platforms.py | 60 +++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index e35aa69..7471f97 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -133,18 +133,10 @@ def _posix_boot_estimate() -> tuple[float, float] | None: `kern.boottime` reading, so a step moves both ends equally and cancels. """ system = platform.system() - if system == "Darwin": - try: - out = subprocess.run(["sysctl", "-n", "kern.boottime"], - capture_output=True, text=True, check=True, - timeout=_SUBPROCESS_TIMEOUT).stdout - import re - m = re.search(r"sec = (\d+)", out) - if m: - sec = float(m.group(1)) - return sec, time.time() - sec - except (OSError, subprocess.SubprocessError, ValueError): - pass + if system == "Darwin" or system.endswith("BSD") or system == "DragonFly": + est = _sysctl_boot_estimate() + if est is not None: + return est elif system == "Linux": try: btime = None @@ -159,6 +151,68 @@ def _posix_boot_estimate() -> tuple[float, float] | None: return btime, up except (OSError, ValueError): pass + # The kernel's own boot record could not be read. That is the normal + # state inside an agent's sandbox - Claude Code and Codex run shell + # commands under macOS Seatbelt, which refuses kern.boottime - and on + # Android, where SELinux hides /proc/stat from apps such as Termux. A + # clock that counts from boot answers the same question in-process. + return _clock_boot_estimate() + + +def _sysctl_boot_estimate() -> tuple[float, float] | None: + """kern.boottime on macOS and the BSDs: in-process first, so a vault open + does not fork, then through the sysctl binary for a libc without + sysctlbyname (OpenBSD).""" + import struct + try: + import ctypes + import ctypes.util + libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + buf = ctypes.create_string_buffer(32) + size = ctypes.c_size_t(len(buf)) + if libc.sysctlbyname(b"kern.boottime", buf, ctypes.byref(size), None, 0) == 0: + # struct timeval: a 64-bit time_t on every 64-bit system, and a + # 32-bit one where the whole struct is 8 bytes. + fmt = "=q" if size.value >= 16 else "=i" + sec = float(struct.unpack_from(fmt, buf.raw)[0]) + if sec > 0: + return sec, time.time() - sec + except (ImportError, OSError, AttributeError, ValueError, struct.error): + pass + try: + out = subprocess.run(["sysctl", "-n", "kern.boottime"], + capture_output=True, text=True, check=True, + timeout=_SUBPROCESS_TIMEOUT).stdout + import re + m = re.search(r"sec = (\d+)", out) + if m: + sec = float(m.group(1)) + return sec, time.time() - sec + except (OSError, subprocess.SubprocessError, ValueError): + pass + return None + + +def _clock_boot_estimate() -> tuple[float, float] | None: + """(boot instant, uptime) from a clock that starts at boot, or None. + + Only clocks that keep counting through sleep qualify, or every resume + would read as a reboot: CLOCK_BOOTTIME on Linux, Android and OpenBSD, and + CLOCK_MONOTONIC on macOS, where it is mach_continuous_time and includes + sleep. The result feeds _pinned_boot_token like any other estimate, so + the first value seen this boot is the one every process agrees on.""" + names = ("CLOCK_MONOTONIC",) if platform.system() == "Darwin" else ( + "CLOCK_BOOTTIME",) + for name in names: + clock = getattr(time, name, None) + if clock is None: + continue + try: + up = time.clock_gettime(clock) + except OSError: + continue + if up > 0: + return time.time() - up, up return None diff --git a/tests/test_platforms.py b/tests/test_platforms.py index 857ce8b..8f634a2 100644 --- a/tests/test_platforms.py +++ b/tests/test_platforms.py @@ -130,6 +130,66 @@ def test_a_clock_step_does_not_kill_the_session_credential(tmp_path, monkeypatch assert session._file_for(vault).is_file() +def _no_kernel_boot_record(monkeypatch, system): + """The sandbox case: kern.boottime refused (Seatbelt under Claude Code and + Codex), /proc/stat hidden (Android's SELinux under Termux).""" + monkeypatch.setattr(platforms.platform, "system", lambda: system) + monkeypatch.setattr(platforms, "_sysctl_boot_estimate", lambda: None) + real_open = open + + def no_proc(path, *a, **kw): + if str(path).startswith("/proc/"): + raise PermissionError(13, "Permission denied", str(path)) + return real_open(path, *a, **kw) + monkeypatch.setattr("builtins.open", no_proc) + + +@pytest.mark.parametrize("system", ["Darwin", "Linux", "FreeBSD"]) +def test_a_sandbox_that_hides_the_boot_record_still_gets_a_boot_token( + tmp_path, monkeypatch, system): + """Before the clock fallback every vault open from an agent's sandboxed + shell raised "Cannot identify this boot session", so `compartment store` + run by the agent itself could never use the unlocked vault.""" + _tmp_session_dir(monkeypatch, tmp_path) + _no_kernel_boot_record(monkeypatch, system) + clock = "CLOCK_MONOTONIC" if system == "Darwin" else "CLOCK_BOOTTIME" + monkeypatch.setattr(platforms.time, clock, 777, raising=False) + monkeypatch.setattr(platforms.time, "clock_gettime", + lambda c: 500.0 if c == 777 else 1.0) + monkeypatch.setattr(platforms.time, "time", lambda: 10_500.0) + assert platforms.boot_time() == platforms.boot_time() == "10000" + + +def test_the_clock_fallback_agrees_with_the_pinned_kernel_reading( + tmp_path, monkeypatch): + """An unsandboxed process pins the kernel's value; a sandboxed one that + can only read the clock a second later must land on the same token, or + the credential the first one stored is unreadable to the second.""" + _tmp_session_dir(monkeypatch, tmp_path) + monkeypatch.setattr(platforms.platform, "system", lambda: "Darwin") + monkeypatch.setattr(platforms, "_sysctl_boot_estimate", + lambda: (10_000.0, 500.0)) + assert platforms.boot_time() == "10000" + monkeypatch.setattr(platforms, "_sysctl_boot_estimate", lambda: None) + monkeypatch.setattr(platforms.time, "clock_gettime", lambda c: 501.0) + monkeypatch.setattr(platforms.time, "time", lambda: 10_501.7) + assert platforms.boot_time() == "10000" + + +@pytest.mark.skipif(platform.system() not in ("Darwin", "Linux"), + reason="needs a POSIX clock that counts from boot") +def test_the_clock_estimate_is_close_to_the_kernel_record(): + """The fallback is only safe if it reads the same boot instant the kernel + reports; a clock that skipped sleep would drift from it by hours.""" + clock = platforms._clock_boot_estimate() + assert clock is not None + kernel = (platforms._sysctl_boot_estimate() if platform.system() == "Darwin" + else None) + if kernel is None: + pytest.skip("kernel boot record not readable here") + assert abs(clock[0] - kernel[0]) < 120 + + def test_filelock_is_exclusive(tmp_path): lockpath = str(tmp_path / "x.flock") with platforms.FileLock(lockpath, timeout=0.3): From 1f3855be77537d5ab322b1542421d85de7598def Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:34:56 -0700 Subject: [PATCH 002/135] The command written into a client's config is the console script pip actually installed, or the package manager's link to it, instead of a bare name a GUI app cannot find executable() looked for the script beside the interpreter, then on PATH, then gave up and wrote the bare name `compartment`. `pip install --user`, which is what the install scripts for macOS, Windows and Linux all run, puts the script in the user scheme's directory (~/Library/Python/3.X/bin, ~/.local/bin, %APPDATA%\Python\PythonXY\Scripts): beside no interpreter, and on no PATH that a Dock- or Start-menu-launched client, or the panel's own child process, has. So the Connect button and `integrate` from a fresh terminal wrote `"command": "compartment"`, and Claude Desktop, Cursor and the rest failed to start the server with "spawn compartment ENOENT"; the capture hook named the same bare word. The scripts directory of the install scheme whose library directory holds the running package is now asked first: it is where pip wrote the script for this copy, venv, system or --user alike. A stale copy in another scheme is not taken for it. A script inside a package manager's versioned tree, such as Homebrew's Cellar/compartment//libexec/bin, stops existing after the next upgrade and cleanup, so configs written before it pointed at nothing. When a link on PATH, or in a package manager's bin directory (Homebrew, pipx's and uv's ~/.local/bin, Nix profiles), resolves to the very script found, the link is written instead, since every upgrade repoints it. A link to some other install is not used. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 128 +++++++++++++++++++++++- tests/test_integrations_command.py | 150 +++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 tests/test_integrations_command.py diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 4cd2b3e..8261aaf 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -291,16 +291,134 @@ def executable() -> str: `Compartment`, macOS filesystems are case-insensitive by default, and a config pointed at it would start the menu bar app every time a client tried to open a memory server. + + `pip install --user`, which is what the install scripts for all three + platforms run, puts the script in the user scheme's directory + (~/Library/Python/3.X/bin, ~/.local/bin, or on Windows + %APPDATA%\\Python\\PythonXY\\Scripts), beside no interpreter and on no + PATH a GUI app or the panel's child process has. Looking only beside the interpreter and then on PATH + found nothing there and wrote the bare name: every client then failed + to start the server with "spawn compartment ENOENT". So the scripts + directory of the scheme this package was actually installed under is + asked first; it is the one pip wrote the script into, by definition. + + A script found inside a package manager's versioned tree (Homebrew's + Cellar/compartment//libexec/bin) is dead after the next + upgrade and cleanup, while the link the package manager keeps on PATH + for it (/opt/homebrew/bin/compartment, and pipx's and uv's + ~/.local/bin links) is repointed by every upgrade. So when such a link + resolves to the very script found, the link is what is written. """ - prefix = Path(sys.prefix) - for c in (prefix / "bin" / NAME, prefix / "Scripts" / f"{NAME}.exe", - Path(sys.executable).parent / NAME): + found = _installed_script() + if found is None: + which = shutil.which(NAME) + return which or NAME + return str(_stable_link(found)) + + +def _script_in(directory: Path) -> Path | None: + """The compartment console script in `directory`, if there is one and it + is not an app bundle's launcher.""" + for c in (directory / NAME, directory / f"{NAME}.exe"): try: if c.is_file() and c.parent.name != "MacOS": - return str(c) + return c except OSError: pass - return shutil.which(NAME) or NAME + return None + + +def _package_root() -> Path: + """The site-packages directory this copy of compartment was imported + from. An editable install's is the source tree, which no scheme + names.""" + return Path(__file__).resolve().parent.parent + + +def _scheme_scripts() -> list[Path]: + """The scripts directory of each install scheme whose library directory + holds this package: where pip put the console script for this copy. + + The schemes are expanded against the running interpreter's own prefix, + the venv's inside a venv, and the user scheme is the one pip uses for + `--user` on this platform (osx_framework_user for python.org's macOS + builds, nt_user on Windows, posix_user elsewhere).""" + import sysconfig + try: + here = _package_root() + except OSError: + return [] + names = [] + for get in (sysconfig.get_default_scheme, + lambda: sysconfig.get_preferred_scheme("prefix"), + lambda: sysconfig.get_preferred_scheme("user")): + try: + names.append(get()) + except (KeyError, ValueError, AttributeError): + pass + out = [] + for scheme in dict.fromkeys(names): + try: + paths = sysconfig.get_paths(scheme, vars={ + "base": sys.prefix, "platbase": sys.exec_prefix}) + libs = {Path(paths["purelib"]).resolve(), + Path(paths["platlib"]).resolve()} + except (KeyError, ValueError, OSError): + continue + if here in libs: + out.append(Path(paths["scripts"])) + return out + + +def _installed_script() -> Path | None: + """The console script that belongs to the running copy of compartment, + or None when there is none to be found (a source checkout run with + `python -m`).""" + prefix = Path(sys.prefix) + for d in (*_scheme_scripts(), prefix / "bin", prefix / "Scripts"): + hit = _script_in(d) + if hit is not None: + return hit + beside = Path(sys.executable).parent / NAME + try: + if beside.is_file() and beside.parent.name != "MacOS": + return beside + except OSError: + pass + return None + + +def _link_dirs() -> list[Path]: + """Where package managers keep the links they repoint on upgrade.""" + h = Path.home() + dirs = [h / ".local" / "bin", Path("/opt/homebrew/bin"), + Path("/usr/local/bin"), Path("/home/linuxbrew/.linuxbrew/bin"), + h / ".linuxbrew" / "bin", h / ".nix-profile" / "bin", + Path("/run/current-system/sw/bin"), h / "bin"] + user = os.environ.get("USER") + if user: + dirs.append(Path("/etc/profiles/per-user") / user / "bin") + return dirs + + +def _stable_link(script: Path) -> Path: + """A link on PATH, or in a package manager's bin directory, that + resolves to `script` itself; `script` when there is none.""" + try: + real = os.path.realpath(script) + except (OSError, ValueError): + return script + which = shutil.which(NAME) + candidates = ([Path(which)] if which else []) + [d / NAME + for d in _link_dirs()] + for c in candidates: + try: + if (c.is_absolute() and c != script and c.is_symlink() + and os.path.realpath(c) == real): + return c + except (OSError, ValueError): + continue + return script def server_args(vault: str, caller: str) -> list[str]: diff --git a/tests/test_integrations_command.py b/tests/test_integrations_command.py new file mode 100644 index 0000000..887285b --- /dev/null +++ b/tests/test_integrations_command.py @@ -0,0 +1,150 @@ +"""The command written into every client's config has to start the server +from a process that has none of the user's shell: a GUI app launched from +the Dock or the Start menu, or the panel's own child process. + +These are about where that command is found, for the ways Compartment is +actually installed: `pip install --user` (what the install scripts run), a +venv, and a package manager that keeps a link on PATH to a versioned tree. +""" +import sys +import sysconfig + +import pytest + +from compartment import clients + + +def _fake_schemes(monkeypatch, tmp_path, installed_in: str): + """Three install schemes whose directories are under tmp_path, and this + package imported from the library directory of `installed_in`.""" + layout = {} + for scheme in ("venv", "posix_prefix", "posix_user", "osx_framework_user", + "nt", "nt_user"): + lib = tmp_path / scheme / "site-packages" + layout[scheme] = {"purelib": str(lib), "platlib": str(lib), + "scripts": str(tmp_path / scheme / "bin")} + + def get_paths(scheme=None, vars=None, expand=True): + if scheme not in layout: + raise KeyError(scheme) + return dict(layout[scheme]) + + lib = tmp_path / installed_in / "site-packages" + lib.mkdir(parents=True) + monkeypatch.setattr(sysconfig, "get_paths", get_paths) + monkeypatch.setattr(clients, "_package_root", lambda: lib.resolve(), + raising=False) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "interpreter")) + monkeypatch.setattr(sys, "executable", + str(tmp_path / "interpreter" / "bin" / "python3")) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: None) + return {k: tmp_path / k / "bin" for k in layout} + + +def _script(directory, name="compartment"): + directory.mkdir(parents=True, exist_ok=True) + s = directory / name + s.write_text("#!/bin/sh\n", encoding="utf-8") + return s + + +@pytest.mark.parametrize("platform", ["macos", "windows", "linux"]) +def test_a_user_install_names_the_script_pip_wrote(platform, tmp_path, + monkeypatch): + """`pip install --user` (every install script) puts the script in the + user scheme's bin or Scripts directory, beside no interpreter and on no + PATH a Dock-launched app or the panel's child has. Only the interpreter's + own directories and PATH were looked in, so the bare name was written, + and every GUI client failed with "spawn compartment ENOENT".""" + user = {"macos": "osx_framework_user", "windows": "nt_user", + "linux": "posix_user"}[platform] + monkeypatch.setattr(sysconfig, "get_default_scheme", + lambda: "nt" if platform == "windows" + else "posix_prefix") + monkeypatch.setattr(sysconfig, "get_preferred_scheme", + lambda key: user if key == "user" + else ("nt" if platform == "windows" + else "posix_prefix")) + bins = _fake_schemes(monkeypatch, tmp_path, installed_in=user) + name = "compartment.exe" if platform == "windows" else "compartment" + script = _script(bins[user], name) + # A stale copy in a scheme this package is not installed in is not it. + _script(bins["posix_prefix" if platform != "windows" else "nt"], name) + + assert clients.executable() == str(script) + + +def test_a_venv_install_is_still_named_by_its_venv_script(tmp_path, + monkeypatch): + monkeypatch.setattr(sysconfig, "get_default_scheme", lambda: "venv") + monkeypatch.setattr(sysconfig, "get_preferred_scheme", + lambda key: "posix_user" if key == "user" else "venv") + bins = _fake_schemes(monkeypatch, tmp_path, installed_in="venv") + script = _script(bins["venv"]) + _script(bins["posix_user"]) # an older --user copy, not this one + assert clients.executable() == str(script) + + +def _symlink_or_skip(link, target): + link.parent.mkdir(parents=True, exist_ok=True) + try: + link.symlink_to(target) + except (OSError, NotImplementedError) as exc: + pytest.skip(f"no symlinks here ({exc})") + + +def test_a_package_managers_link_is_written_instead_of_its_versioned_tree( + tmp_path, monkeypatch): + """Homebrew runs a Python formula from Cellar///libexec, + so the script beside the interpreter has the version in its path and + is gone after the next `brew upgrade` and cleanup: every client config + written before then pointed at a file that no longer existed. The link + Homebrew keeps in its bin directory is repointed by every upgrade.""" + prefix = tmp_path / "Cellar" / "compartment" / "4.12.0" / "libexec" + script = _script(prefix / "bin") + link = tmp_path / "homebrew" / "bin" / "compartment" + _symlink_or_skip(link, script) + monkeypatch.setattr(clients, "_package_root", + lambda: tmp_path / "not-a-scheme", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(clients, "_link_dirs", lambda: [], raising=False) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: str(link)) + + assert clients.executable() == str(link) + + +def test_a_link_to_some_other_install_is_not_taken_for_this_one( + tmp_path, monkeypatch): + """A `compartment` on PATH that is another copy (an old --user install + while this one runs from a venv) is not the one to name.""" + prefix = tmp_path / "venv" + script = _script(prefix / "bin") + other = _script(tmp_path / "old-install" / "bin") + link = tmp_path / "local" / "bin" / "compartment" + _symlink_or_skip(link, other) + monkeypatch.setattr(clients, "_package_root", + lambda: tmp_path / "not-a-scheme", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(clients, "_link_dirs", lambda: [link.parent], + raising=False) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: str(link)) + + assert clients.executable() == str(script) + + +def test_a_link_in_a_package_managers_bin_off_path_is_found(tmp_path, + monkeypatch): + """The panel's child process has the login shell's PATH at best, which + may not hold ~/.local/bin; pipx's and uv's link there is found anyway.""" + prefix = tmp_path / "pipx" / "venvs" / "compartment" + script = _script(prefix / "bin") + link = tmp_path / "home" / ".local" / "bin" / "compartment" + _symlink_or_skip(link, script) + monkeypatch.setattr(clients, "_package_root", + lambda: tmp_path / "not-a-scheme", raising=False) + monkeypatch.setattr(sys, "prefix", str(prefix)) + monkeypatch.setattr(clients.Path, "home", + classmethod(lambda cls: tmp_path / "home")) + monkeypatch.setattr(clients.shutil, "which", lambda *a, **k: None) + + assert clients.executable() == str(link) From 35e48dc19f4defa393a97af7f4b69cf9edde2ae6 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:36:35 -0700 Subject: [PATCH 003/135] A store the disk refuses part way takes its bytes back off the journal, so the vault keeps opening on a full disk A journal append that hit ENOSPC or EDQUOT mid write, or whose fsync failed, left its partial entry on the end of the vault. Nothing had been acknowledged, but a torn tail is the one thing an open cannot get past without compacting, and on a disk that was still full the compaction failed and the open with it: every agent's MCP server reopening after the failed store, `compartment status` and the panel lost the vault, reads included, until space was freed. A sync that failed after every byte was written left a complete entry instead, and the memory the caller had been told was not stored came back on the next open. The append now writes through one descriptor and, on any failure, truncates the file back to where it began (which succeeds on the full disk that caused it) before re-raising. It no longer creates a missing vault either: an append to a deleted file used to recreate it as journal frames with no header, which nothing can open. Co-Authored-By: Claude Opus 5.5 --- FORMAT.md | 4 +- src/compartment/vault.py | 7 +- src/compartment/vaultfile.py | 47 +++++++-- tests/test_storage_durability.py | 158 +++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 12 deletions(-) create mode 100644 tests/test_storage_durability.py diff --git a/FORMAT.md b/FORMAT.md index f03e678..092dcd2 100644 --- a/FORMAT.md +++ b/FORMAT.md @@ -99,7 +99,9 @@ u32 section_count, then per section: Each entry: `AEAD_seal(master_key, canonical_json(entry), aad="compartment-journal:"+vault_id+":"+u64_be(seq))` where `seq` starts at 0 after each compaction and increments per entry. Entries are fsync'd on -append (acknowledged). Ops: +append (acknowledged). An append the disk refuses part way (ENOSPC, EDQUOT, +a failed sync) is truncated back to where it began, so an unacknowledged +write leaves no bytes behind. Ops: ```json {"op":"store","audit":AuditRow,"record":{"id","ns","text","vec":base64_f32, diff --git a/src/compartment/vault.py b/src/compartment/vault.py index e04790c..87d5935 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -310,10 +310,11 @@ def is_stale(self) -> bool: A file that has been DELETED counts as changed. It used to count as unchanged, because the existence test was ANDed into the answer, and - the consequence was silent and permanent: _journal appends with - open(path, "ab"), which recreates the file with journal frames and no + the consequence was silent and permanent: _journal appended with + open(path, "ab"), which recreated the file with journal frames and no header, so every later write reported success into something no - version of this program can ever open again. + version of this program can ever open again. (The append no longer + creates a missing file either; this check is what refuses first.) """ if self._disk_state is None: return False diff --git a/src/compartment/vaultfile.py b/src/compartment/vaultfile.py index a1f4347..f26fd4b 100644 --- a/src/compartment/vaultfile.py +++ b/src/compartment/vaultfile.py @@ -393,14 +393,45 @@ def append_journal_entries(path: str, header: VaultHeader, seq: int, crypto.canonical_json(entry), aad=_journal_aad(header.vault_id, seq + i), ) for i, entry in enumerate(entries)] - with open(path, "ab") as f: - for ct in cts: - length = struct.pack(">I", len(ct)) - f.write(length) - f.write(struct.pack(">I", zlib.crc32(length) & 0xFFFFFFFF)) - f.write(ct) - f.flush() - os.fsync(f.fileno()) + frames = bytearray() + for ct in cts: + length = struct.pack(">I", len(ct)) + frames += length + frames += struct.pack(">I", zlib.crc32(length) & 0xFFFFFFFF) + frames += ct + # No O_CREAT: an append is only ever made to a vault that exists, and + # creating one here would write journal frames into a file with no + # header, which nothing can open again. O_BINARY, where it exists, so + # Windows never translates a newline byte inside the ciphertext. + fd = os.open(path, os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0)) + try: + # The caller holds the vault's write lock, so nothing else appends + # between this and the write below: the file ends here. + start = os.fstat(fd).st_size + try: + view = memoryview(frames) + while view: + view = view[os.write(fd, view):] + os.fsync(fd) + except BaseException: + # A write the disk refused part way (ENOSPC, EDQUOT, an I/O + # error, a sync that failed) is a write nobody acknowledged, and + # its bytes are taken back off the end. Left there they were a + # torn tail, and a torn tail is the one thing an open cannot + # tolerate without compacting: on a disk that was still full, + # every MCP server, `compartment status` and the panel then + # failed to open the vault at all, reads included, until space + # was freed. Truncating frees the space it took, so it succeeds + # on the full disk that caused it. If it cannot, the bytes stay + # a torn tail and the next open discards them, as before. + try: + os.ftruncate(fd, start) + os.fsync(fd) + except OSError: + pass + raise + finally: + os.close(fd) # --------------------------------------------------------------------------- diff --git a/tests/test_storage_durability.py b/tests/test_storage_durability.py new file mode 100644 index 0000000..3c578c4 --- /dev/null +++ b/tests/test_storage_durability.py @@ -0,0 +1,158 @@ +"""What reaches the disk, and what a failed write leaves behind. + +The journal's promise is that an acknowledged write survives and an +unacknowledged one costs nothing. These tests hold the second half to the +disk running out part way through, which used to leave a torn tail that no +open could get past while the disk stayed full. +""" +import errno +import os + +import pytest + +from compartment import vaultfile +from compartment.vault import Vault + +from conftest import PASS + + +class _With: + """A module as one other module sees it, with some functions replaced.""" + + def __init__(self, module, **replaced): + self._module = module + self.__dict__.update(replaced) + + def __getattr__(self, name): + return getattr(self._module, name) + + +class _Budget: + """A disk with `room` bytes left: writes past it land in part and then + fail with ENOSPC, the way a real full disk answers.""" + + def __init__(self, room: int): + self.room = room + + def take(self, data) -> int: + n = min(len(data), self.room) + self.room -= n + return n + + +def _disk_fills_during_appends(monkeypatch, budget: _Budget): + """Both ways an append can reach the file - os.write on a descriptor and + a Python file object - run out of room together.""" + real_write = os.write + real_open = open + + def write(fd, data): + n = budget.take(data) + if n == 0 and len(data): + raise OSError(errno.ENOSPC, "No space left on device") + real_write(fd, bytes(data[:n])) + if n < len(data): + raise OSError(errno.ENOSPC, "No space left on device") + return n + + class _File: + def __init__(self, f): + self._f = f + + def write(self, data): + n = budget.take(data) + self._f.write(data[:n]) + if n < len(data): + self._f.flush() + raise OSError(errno.ENOSPC, "No space left on device") + return n + + def __getattr__(self, name): + return getattr(self._f, name) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self._f.close() + + def opener(path, mode="r", *args, **kwargs): + f = real_open(path, mode, *args, **kwargs) + return _File(f) if "a" in mode else f + + monkeypatch.setattr(vaultfile, "os", _With(os, write=write)) + monkeypatch.setattr(vaultfile, "open", opener, raising=False) + + +def _disk_stays_full(monkeypatch): + """No save can be written. Patched where the vault calls it, never as + `_atomic_replace`: modules imported for the first time during the test + bind that name for good, and would carry the full disk into the next.""" + def full(*args, **kwargs): + raise OSError(errno.ENOSPC, "No space left on device") + monkeypatch.setattr(vaultfile, "write_vault_file", full) + + +def test_an_append_the_disk_refuses_leaves_no_torn_tail(vault_path, + monkeypatch): + """The partial entry stayed on the end of the file. Nothing had been + acknowledged, but every open after it had to compact the tail away, and + on a disk that was still full that compaction failed and the open with + it: every agent's server, `compartment status` and the panel lost the + vault, reads included, until someone freed space.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("The first memory reached the disk whole.", caller="t") + size = os.path.getsize(vault_path) + + with monkeypatch.context() as m: + _disk_fills_during_appends(m, _Budget(room=40)) + with pytest.raises(OSError, match="No space"): + v.store("The second memory ran out of disk part way.", caller="t") + + assert os.path.getsize(vault_path) == size, \ + "the refused bytes are taken back off the end" + loaded = vaultfile.read_vault_file(vault_path) + assert not loaded.truncated_tail + + _disk_stays_full(monkeypatch) + again = Vault.unlock(vault_path, passphrase=PASS) + texts = [r["text"] for r in again.peek_recent("t")["results"]] + assert len(texts) == 1 + assert texts[0].startswith("The first memory reached the disk whole.") + + +def test_an_append_whose_sync_fails_is_not_left_to_replay(vault_path, + monkeypatch): + """Every byte written, then the sync refused (EIO, a full quota on a + network share). The caller was told the store failed, so the entry must + not come back on the next open as a memory nobody stored.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory that was acknowledged.", caller="t") + size = os.path.getsize(vault_path) + calls = [] + + def fsync(fd): + calls.append(fd) + if len(calls) == 1: + raise OSError(errno.EIO, "Input/output error") + return os.fsync(fd) + + with monkeypatch.context() as m: + m.setattr(vaultfile, "os", _With(os, fsync=fsync)) + with pytest.raises(OSError, match="Input/output"): + v.store("A memory whose sync failed.", caller="t") + assert os.path.getsize(vault_path) == size + assert Vault.unlock(vault_path, passphrase=PASS).status()[ + "organic_records"] == 1 + + +def test_an_append_never_creates_a_vault_that_is_not_there(vault_path): + """An append to a missing file used to create it, holding journal + frames and no header: a file no build can open. The writer's own + staleness check refuses first; this is the floor under it.""" + v = Vault.create(vault_path, PASS, creator="t") + os.unlink(vault_path) + with pytest.raises(FileNotFoundError): + vaultfile.append_journal_entry(vault_path, v.header, 0, + {"op": "noop"}, v._master) + assert not os.path.exists(vault_path) From 8099b6fc3fc1ab929fc45d70d8b4ccb803f5f74c Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:36:57 -0700 Subject: [PATCH 004/135] A client config, Claude Desktop's config or Claude Code's settings saved with a byte order mark or as UTF-16 is read and updated instead of refused Windows users are sent to Notepad or PowerShell to edit these files, and both save encodings the readers did not accept. Notepad before Windows 10 1903 and PowerShell 5's `Out-File -Encoding utf8` put a UTF-8 byte order mark first, which json.loads and tomllib do not take for whitespace, so the file was refused as "not valid JSON" (or TOML) and the user got the block to paste into the very file just refused. PowerShell 5's `>` and `Out-File` write UTF-16, which did not decode at all; Claude Desktop's and the hook's readers let that escape as a bare UnicodeDecodeError. Codex's own parser skips a leading mark, so a config.toml Codex read happily was refused too. There is now one reader, clients.read_config_text, used for every client config, Codex's TOML, Claude Desktop's config and Claude Code's settings.json. A marked file is decoded by its mark (UTF-8, UTF-16 or UTF-32, either byte order); an unmarked one must be UTF-8, as before. Line endings come back as \n, as read_text gave them, so a CRLF file is not written back with \r\r\n on Windows. What is written is UTF-8 without a mark, which every client reads: Node's JSON.parse rejects the mark and cannot read UTF-16, so for Electron clients such a file was already broken. The backup is the original bytes, as always, and a file that is not text in any of these is still refused untouched. Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_desktop.py | 14 +-- src/compartment/claude_hooks.py | 13 +-- src/compartment/clients.py | 49 +++++++++- tests/test_integrations_encodings.py | 139 +++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 24 deletions(-) create mode 100644 tests/test_integrations_encodings.py diff --git a/src/compartment/claude_desktop.py b/src/compartment/claude_desktop.py index dea524d..0f8dd96 100644 --- a/src/compartment/claude_desktop.py +++ b/src/compartment/claude_desktop.py @@ -66,16 +66,10 @@ def is_registered(path: Path | None = None) -> bool: def _read(path: Path) -> dict: - if not path.exists(): - return {} - try: - text = path.read_text(encoding="utf-8").strip() - data = json.loads(text) if text else {} - except (json.JSONDecodeError, OSError): - raise ValueError(f"{path} is not valid JSON; refusing to touch it") - if not isinstance(data, dict): # null, a list: not a config - raise ValueError(f"{path} is not a JSON object; refusing to touch it") - return data + """The config as a dict, or ValueError: the same reader every other + client's config goes through, so a file Notepad or PowerShell saved + with a byte order mark, or as UTF-16, is read rather than refused.""" + return clients._read_json(path) def register(command: str, args: list[str], path: Path | None = None) -> dict: diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index f7a4954..addebde 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -71,16 +71,9 @@ def is_installed(settings: Path | None = None) -> bool: def _read(path: Path) -> dict: - if not path.exists(): - return {} - try: - text = path.read_text(encoding="utf-8").strip() - data = json.loads(text) if text else {} - except (json.JSONDecodeError, OSError): - raise ValueError(f"{path} is not valid JSON; refusing to touch it") - if not isinstance(data, dict): # null, a list: not settings - raise ValueError(f"{path} is not a JSON object; refusing to touch it") - return data + """settings.json as a dict, or ValueError, through the reader every + client config goes through (a byte order mark or UTF-16 is decoded).""" + return clients._read_json(path) def install(compartment_bin: str | None = None, vault: str | None = None, diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 8261aaf..3350dbc 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -34,6 +34,7 @@ """ from __future__ import annotations +import codecs import errno import json import os @@ -442,6 +443,46 @@ def snippet(client: Client, vault: str, caller: str | None = None) -> str: # --- reading and writing --------------------------------------------------- +#: Byte order marks, longest first: UTF-32's little-endian mark begins +#: with UTF-16's. +_BOMS = ((codecs.BOM_UTF8, "utf-8-sig"), (codecs.BOM_UTF32_LE, "utf-32"), + (codecs.BOM_UTF32_BE, "utf-32"), (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16")) + + +def read_config_text(path: Path) -> str: + """A config file's text, in whichever Unicode encoding it was saved. + + Windows editors do not all save UTF-8 without a mark. Notepad before + Windows 10 1903 and PowerShell 5's `Out-File -Encoding utf8` put a UTF-8 + byte order mark first; PowerShell 5's `>` and `Out-File` write UTF-16. + Those are the tools the Claude Desktop and Cursor guides send Windows + users to, and a file saved by them was refused as "not valid JSON" (the + mark is not whitespace to json.loads) or failed to decode at all, so + the user got a block to paste into the very file that had just been + refused. A marked file is decoded by its mark; anything unmarked must + be UTF-8, as before. What is written back is always UTF-8 without a + mark, which every client reads: Node's JSON.parse, which Electron + clients use, rejects the mark and cannot read UTF-16 at all, so a + marked file was already broken for most of them. + + Line endings come back as \\n, as read_text gave them: write_config + writes in text mode, and a \\r\\n kept in the text would be written + back as \\r\\r\\n on Windows. + + Raises OSError when the file cannot be read and UnicodeDecodeError (a + ValueError) when it is not text in any of these encodings. + """ + raw = Path(path).read_bytes() + for bom, codec in _BOMS: + if raw.startswith(bom): + text = raw.decode(codec) + break + else: + text = raw.decode("utf-8") + return text.lstrip("\ufeff").replace("\r\n", "\n").replace("\r", "\n") + + def _read_json(path: Path) -> dict: """The file as a dict, or a refusal. @@ -452,7 +493,7 @@ def _read_json(path: Path) -> dict: if not path.exists(): return {} try: - text = path.read_text(encoding="utf-8").strip() + text = read_config_text(path).strip() data = json.loads(text) if text else {} except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc: raise ValueError(f"{path} is not valid JSON ({exc}); " @@ -603,7 +644,7 @@ def is_registered(client: Client, path: Path | None = None) -> bool: return False if client.kind == "toml": try: - return _toml_has_ours(client, p.read_text(encoding="utf-8")) + return _toml_has_ours(client, read_config_text(p)) except (OSError, ValueError): # unreadable, not UTF-8, not TOML return False if client.kind != "json": @@ -671,7 +712,7 @@ def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: """ block = _toml_block(entry) try: - existing = p.read_text(encoding="utf-8") if p.exists() else "" + existing = read_config_text(p) if p.exists() else "" registered = _toml_has_ours(client, existing) except (OSError, UnicodeDecodeError) as exc: out["reason"] = f"cannot read {p} ({exc})" @@ -707,7 +748,7 @@ def unregister(client: Client, path: Path | None = None) -> bool: if client.kind == "toml": try: - text = p.read_text(encoding="utf-8") + text = read_config_text(p) if not _toml_has_ours(client, text): return False except ValueError: # not UTF-8 or not TOML: not ours to guess at diff --git a/tests/test_integrations_encodings.py b/tests/test_integrations_encodings.py new file mode 100644 index 0000000..1cc8e95 --- /dev/null +++ b/tests/test_integrations_encodings.py @@ -0,0 +1,139 @@ +"""Config files as Windows editors save them. + +Notepad before Windows 10 1903 and PowerShell 5's `Out-File -Encoding utf8` +put a UTF-8 byte order mark at the front of the file; PowerShell 5's `>` and +plain `Out-File` write UTF-16. The Claude Desktop and Cursor guides send +Windows users to exactly those tools, so these are real files on real +machines, and each of them has another server in it that must survive. +""" +import codecs +import json +import tomllib + +import pytest + +from compartment import claude_desktop, claude_hooks, clients + +VAULT = r"C:\Users\Ana\.compartment\memory.vault" +THEIRS = {"mcpServers": {"theirs": {"command": "npx", "args": ["-y", "x"]}}, + "globalShortcut": "Ctrl+Space"} + +ENCODINGS = { + "utf-8 with a mark (Notepad, Out-File -Encoding utf8)": + lambda t: codecs.BOM_UTF8 + t.encode("utf-8"), + "utf-16 (PowerShell 5's > and Out-File)": + lambda t: t.encode("utf-16"), # BOM + little-endian + "utf-16 big-endian with a mark": + lambda t: codecs.BOM_UTF16_BE + t.encode("utf-16-be"), + "utf-8 with CRLF line endings": + lambda t: t.replace("\n", "\r\n").encode("utf-8"), +} + + +def _write(path, text, how): + path.parent.mkdir(parents=True, exist_ok=True) + raw = ENCODINGS[how](text) + path.write_bytes(raw) + return raw + + +def _is_plain_utf8(raw: bytes) -> bool: + return not raw.startswith((codecs.BOM_UTF8, codecs.BOM_UTF16_LE, + codecs.BOM_UTF16_BE)) and b"\r\r" not in raw + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_a_client_config_saved_by_a_windows_editor_is_registered(how, + tmp_path): + """The mark is not whitespace to json.loads, so a marked file was + refused as "not valid JSON", and a UTF-16 one did not decode at all: + the user was handed a block to paste into the file just refused.""" + p = tmp_path / ".cursor" / "mcp.json" + before = _write(p, json.dumps(THEIRS, indent=2) + "\n", how) + + res = clients.register(clients.CLIENTS["cursor"], VAULT, path=p) + + assert res["written"] is True, res["reason"] + raw = p.read_bytes() + assert _is_plain_utf8(raw) + data = json.loads(raw.decode("utf-8")) + assert data["mcpServers"]["theirs"] == THEIRS["mcpServers"]["theirs"] + assert data["globalShortcut"] == "Ctrl+Space" + assert data["mcpServers"]["compartment"]["args"][1] == VAULT + assert open(res["backup"], "rb").read() == before # byte-exact + assert clients.is_registered(clients.CLIENTS["cursor"], path=p) + assert clients.unregister(clients.CLIENTS["cursor"], path=p) is True + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_claude_desktops_config_saved_by_a_windows_editor_is_registered( + how, tmp_path): + """Claude Desktop's own guide has Windows users open the file in + Notepad. Its reader refused a marked file, and a UTF-16 one escaped + as a bare UnicodeDecodeError.""" + p = tmp_path / "Claude" / claude_desktop.FILENAME + _write(p, json.dumps(THEIRS, indent=2) + "\n", how) + + claude_desktop.register(r"C:\x\compartment.exe", ["serve"], path=p) + + raw = p.read_bytes() + assert _is_plain_utf8(raw) + data = json.loads(raw.decode("utf-8")) + assert set(data["mcpServers"]) == {"theirs", "compartment"} + assert data["mcpServers"]["compartment"]["command"] == ( + r"C:\x\compartment.exe") + assert claude_desktop.is_registered(path=p) is True + assert claude_desktop.unregister(path=p) is True + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_claude_code_settings_saved_by_a_windows_editor_get_the_hook( + how, tmp_path): + p = tmp_path / ".claude" / "settings.json" + _write(p, json.dumps({"permissions": {"allow": ["Bash"]}}) + "\n", how) + + claude_hooks.install(compartment_bin=r"C:\x\compartment.exe", + settings=p) + + raw = p.read_bytes() + assert _is_plain_utf8(raw) + data = json.loads(raw.decode("utf-8")) + assert data["permissions"] == {"allow": ["Bash"]} + assert claude_hooks.is_installed(settings=p) is True + assert claude_hooks.uninstall(settings=p) is True + + +@pytest.mark.parametrize("how", sorted(ENCODINGS)) +def test_a_codex_config_saved_by_a_windows_editor_is_registered(how, + tmp_path): + """Codex's TOML parser skips a leading mark; tomllib does not, so a + config Codex reads happily was refused as "not valid TOML".""" + p = tmp_path / ".codex" / "config.toml" + _write(p, '# mine\nmodel = "o3"\n\n[mcp_servers.other]\ncommand = "x"\n', + how) + codex = clients.CLIENTS["codex"] + + res = clients.register(codex, VAULT, path=p) + + assert res["written"] is True, res["reason"] + raw = p.read_bytes() + assert _is_plain_utf8(raw) + doc = tomllib.loads(raw.decode("utf-8")) + assert doc["model"] == "o3" and doc["mcp_servers"]["other"] == { + "command": "x"} + assert doc["mcp_servers"]["compartment"]["args"][1] == VAULT + assert raw.decode("utf-8").startswith("# mine\n") + assert clients.is_registered(codex, path=p) is True + assert clients.unregister(codex, path=p) is True + assert "compartment" not in tomllib.loads( + p.read_bytes().decode("utf-8"))["mcp_servers"] + + +def test_a_file_that_is_not_text_in_any_encoding_is_still_refused(tmp_path): + p = tmp_path / ".cursor" / "mcp.json" + p.parent.mkdir() + p.write_bytes(b"\x00\xff\xfe\x80 not text") + before = p.read_bytes() + res = clients.register(clients.CLIENTS["cursor"], VAULT, path=p) + assert res["written"] is False and "refusing to touch it" in res["reason"] + assert p.read_bytes() == before From 549f85b0b3ffea1b1930ea55dbb0d0a1fe8350fd Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:37:25 -0700 Subject: [PATCH 005/135] Every download is built from the tag on GitHub's machines, Intel Macs included, and CI covers every declared Python and both ARM platforms Releases were built by hand on one laptop, so each one depended on that machine and could only produce installers for its architecture. The new Release workflow checks the tag against every version field, builds the wheel and sdist, builds the macOS app, .pkg and .dmg on an Apple silicon runner and on an Intel one (onnxruntime 1.23.2, its last x86_64 release), and attaches all of them with SHA256SUMS.txt to a draft release. Publishing the draft uploads those same wheel and sdist files to PyPI instead of a second build, so the release page and pip hand out identical bytes. CI now runs the suite on Python 3.11 through 3.14 (3.13 and 3.14 were classifiers nothing ran), on ARM64 Linux, Windows on ARM and an Intel Mac. A wheel-smoke job installs the built wheel with pipx on each system and uses it like a person and like an MCP client would (tools/mcp_smoke.py), so a file missing from the wheel or a server that only starts from source is caught. A lowest-dependencies job runs the suite at the lowest versions the bounds allow: it found that mcp>=1.0 was never true, since nothing before 1.10 can start the server, so the floor is now mcp>=1.10. Every action is on its current, Node 24 major. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 112 +++++++++++++-- .github/workflows/publish-registry.yml | 2 +- .github/workflows/publish.yml | 41 +++++- .github/workflows/release.yml | 184 +++++++++++++++++++++++++ RELEASING.md | 46 ++++--- pyproject.toml | 9 +- tools/mcp_smoke.py | 143 +++++++++++++++++++ 7 files changed, 502 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 tools/mcp_smoke.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0adc973..d22cf69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,11 +14,29 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python: ["3.11", "3.12"] + # Every Python the package declares. 3.13 and 3.14 were classifiers + # nothing ran, and 3.14 is what a fresh Homebrew, Fedora or python.org + # install hands people now. + python: ["3.11", "3.12", "3.13", "3.14"] + include: + # ARM64 Linux (Graviton, Ampere, Raspberry Pi 5, Apple silicon VMs) + # and Windows on ARM (Snapdragon laptops): onnxruntime ships wheels + # for both, and "it installed but the native library will not load" + # is the failure only a real machine of that architecture shows. + - os: ubuntu-24.04-arm + python: "3.12" + - os: windows-11-arm + python: "3.12" + # Intel Macs. onnxruntime's last macOS x86_64 wheels are 1.23.2, + # built for Python up to 3.13, so this is the newest pairing that + # installs there, and it proves the code still runs on that older + # onnxruntime as well as on the newest one above. + - os: macos-15-intel + python: "3.13" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python }} - name: Install (network allowed for dependency download only) @@ -36,7 +54,7 @@ jobs: - name: Coverage to Codecov # Never fail the build on a reporting service: the offline guard and the # test suite are the gate here, not the upload. - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v7 continue-on-error: true with: files: ./coverage.xml @@ -44,6 +62,82 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false + wheel-smoke: + # The suite runs against an editable checkout, which hides packaging + # mistakes: a model file or the starter pack missing from the wheel, an + # entry point that only resolves from source, a server that starts in a + # test but not as a console script. This builds the wheel, installs it + # the way the README tells people to (pipx), and uses it like a person + # and like an MCP client would, offline guard on. + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 25 + env: + COMPARTMENT_ASSERT_OFFLINE: "1" + COMPARTMENT_PASSPHRASE: ci-only-not-a-secret + PYTHONIOENCODING: utf-8 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Build the wheel + shell: bash + run: | + python -m pip install --upgrade build + python -m build --wheel + ls -l dist + - name: Install it with pipx, as the README says + shell: bash + env: + COMPARTMENT_ASSERT_OFFLINE: "" + run: | + pipx install --python python dist/*.whl + pipx ensurepath + - name: Create, write, read, verify + shell: bash + run: | + compartment --version + compartment init --passphrase ci-only-not-a-secret --no-app + compartment store --source "from chat" "Deploys go out Tuesday and Thursday mornings." + compartment search "when do deploys go out" | tee search.txt + grep -q "Tuesday and Thursday" search.txt + compartment selftest + compartment status + - name: An MCP client's handshake against the installed server + shell: bash + run: python tools/mcp_smoke.py --command compartment + + lowest-dependencies: + # pyproject.toml's lower bounds are a promise to everyone whose distro or + # lock file holds an older library. It used to promise mcp>=1.0, and no + # release before 1.10 can even start the server (FastMCP arrived in 1.2, + # tool titles in 1.10). This installs the lowest version of every direct + # dependency the bounds allow and runs the suite, so a bound is a tested + # fact rather than a guess. + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + - name: Install at the lowest versions the bounds allow + # mcp 1.10-1.12 import pydantic internals that pydantic 2.12 moved, a + # pairing their own metadata does not exclude; the constraint keeps + # the transitive pydantic to what those releases were built against. + run: | + python -m pip install uv + uv pip install --system --resolution lowest-direct -e ".[dev]" "pydantic<2.12" + python -m pip list + - name: Test suite with offline guard active + env: + COMPARTMENT_ASSERT_OFFLINE: "1" + run: pytest tests/ -q --ignore=tests/test_dash.py + linux-panel: # Linux has no notification area worth relying on, so it draws the same # panel as an ordinary window. "It opens and stays open" is the failure @@ -52,8 +146,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: The Tk package, which pip cannot install @@ -129,8 +223,8 @@ jobs: # locked, and the next command sat asking for input. timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" - name: Install with the tray extra @@ -285,7 +379,7 @@ jobs: [System.Drawing.Imaging.ImageFormat]::Png) Write-Host "panel: ${w}x${ht}" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: windows-tray-screenshot path: | diff --git a/.github/workflows/publish-registry.yml b/.github/workflows/publish-registry.yml index 601c4d1..e096dff 100644 --- a/.github/workflows/publish-registry.yml +++ b/.github/workflows/publish-registry.yml @@ -25,7 +25,7 @@ jobs: publish-registry: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install mcp-publisher run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0e8075b..71f72a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,17 +23,44 @@ jobs: environment: pypi permissions: id-token: write # required for Trusted Publishing (OIDC) + contents: read # to download the release's own files steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" - - name: Build wheel + sdist + # PyPI gets the very wheel and sdist attached to the GitHub release + # (release.yml builds them from the tag and lists their SHA-256 sums), + # so "pip install compartment" and the release page hand out identical + # bytes. A manual run, or a release drafted by hand without them, + # falls back to building from this checkout as before. + - name: Take the release's wheel + sdist, or build them + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + shell: bash run: | - python -m pip install --upgrade build - python -m build + mkdir -p dist + if [ -n "$TAG" ]; then + gh release download "$TAG" -R "$GITHUB_REPOSITORY" -D dist \ + -p "compartment-*.whl" -p "compartment-*.tar.gz" || true + fi + if ls dist/*.whl >/dev/null 2>&1 && ls dist/*.tar.gz >/dev/null 2>&1; then + echo "publishing the release's own files:" + else + rm -rf dist + python -m pip install --upgrade build + python -m build + echo "built from the checkout:" + fi + ls -l dist + if [ -n "$TAG" ]; then + v="${TAG#v}" + ls dist | grep -q "compartment-$v" || { + echo "::error::dist/ holds no compartment-$v files for tag $TAG"; exit 1; } + fi - name: Check metadata run: | @@ -68,9 +95,9 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5831383 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,184 @@ +name: Release + +# Every release ships every download (RELEASING.md). They used to be built by +# hand on one laptop, which tied each release to that laptop and could only +# ever produce installers for the laptop's own architecture. This builds all +# of them on GitHub's machines from the tagged commit - the wheel, the sdist, +# and the macOS .pkg and .dmg for Apple silicon AND for Intel - and attaches +# them, with their SHA-256 sums, to a DRAFT release. +# +# Publishing the draft stays a human decision. Publishing is what triggers +# publish.yml (PyPI, which uploads these very wheel and sdist files) and +# publish-registry.yml (the MCP registry). +# +# git tag v4.13.0 && git push origin v4.13.0 # then review the draft + +on: + push: + tags: ["v*.*.*"] + workflow_dispatch: + inputs: + tag: + description: "An existing tag to build or rebuild, e.g. v4.13.0" + required: true + +permissions: + contents: read + +concurrency: + group: release-${{ inputs.tag || github.ref_name }} + cancel-in-progress: false + +jobs: + check: + name: The tag and every version field agree + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.v.outputs.tag }} + version: ${{ steps.v.outputs.version }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag || github.ref }} + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - id: v + name: The tag names the package version + shell: bash + env: + TAG: ${{ inputs.tag || github.ref_name }} + run: | + version=$(python -c "import re, pathlib; print(re.search(r'__version__\s*=\s*\"([^\"]+)\"', pathlib.Path('src/compartment/__init__.py').read_text(encoding='utf-8')).group(1))") + if [ "$TAG" != "v$version" ]; then + echo "::error::tag $TAG does not match the package version $version" + exit 1 + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Every manifest carries that version + # The lockstep test reads files only, so it runs without the package's + # dependencies and without the suite's conftest. + run: | + python -m pip install pytest + python -m pytest -q -p no:cacheprovider --noconftest tests/test_version_lockstep.py + + python-dist: + name: Wheel and sdist + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.check.outputs.tag }} + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Build and check the metadata + run: | + python -m pip install --upgrade build twine + python -m build + python -m twine check --strict dist/* + ls -l dist + - name: The wheel installs and starts + env: + COMPARTMENT_ASSERT_OFFLINE: "1" + run: | + python -m pip install dist/*.whl + compartment --version | grep -F "${{ needs.check.outputs.version }}" + - uses: actions/upload-artifact@v7 + with: + name: python-dist + path: dist/* + if-no-files-found: error + + macos-app: + name: macOS app (${{ matrix.arch }}) + needs: check + strategy: + fail-fast: false + matrix: + include: + - runner: macos-latest + arch: apple-silicon + suffix: "" + # onnxruntime's last macOS x86_64 wheels are 1.23.2 for Python up + # to 3.13; pip resolves to it here on its own. + - runner: macos-15-intel + arch: intel + suffix: "-intel" + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.check.outputs.tag }} + - uses: astral-sh/setup-uv@v10 + - name: A standalone, non-framework Python to embed + # tools/build_macos_app.py explains why it must not be a framework + # build: the app would then report itself as org.python.python. + shell: bash + run: | + uv python install 3.12 + echo "EMBED_PY=$(uv python find --managed-python 3.12)" >> "$GITHUB_ENV" + - name: Build Compartment.app, the .pkg and the .dmg + shell: bash + run: | + "$EMBED_PY" tools/build_macos_app.py --python "$EMBED_PY" --pkg --dmg + ls -l build + - name: Name them for their architecture + shell: bash + env: + V: ${{ needs.check.outputs.version }} + SUFFIX: ${{ matrix.suffix }} + run: | + mkdir -p out + cp "build/Compartment-$V.pkg" "out/Compartment-$V$SUFFIX.pkg" + cp "build/Compartment-$V.dmg" "out/Compartment-$V$SUFFIX.dmg" + ls -l out + - uses: actions/upload-artifact@v7 + with: + name: macos-${{ matrix.arch }} + path: out/* + if-no-files-found: error + + draft: + name: Draft the release with every download + needs: [check, python-dist, macos-app] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.check.outputs.tag }} + - uses: actions/download-artifact@v8 + with: + path: assets + merge-multiple: true + - name: Checksums + working-directory: assets + run: | + sha256sum * > SHA256SUMS.txt + cat SHA256SUMS.txt + - name: Create the draft, or refresh its downloads on a rebuild + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.check.outputs.tag }} + VERSION: ${{ needs.check.outputs.version }} + shell: bash + run: | + notes=".github/release-notes/$TAG.md" + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" assets/* --clobber + echo "refreshed the downloads on the existing release $TAG" + else + if [ -f "$notes" ]; then + gh release create "$TAG" --draft --verify-tag \ + --title "Compartment $VERSION" --notes-file "$notes" assets/* + else + gh release create "$TAG" --draft --verify-tag \ + --title "Compartment $VERSION" --generate-notes assets/* + fi + echo "drafted $TAG; review it and press Publish" + fi diff --git a/RELEASING.md b/RELEASING.md index 14c6ea8..e38788b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -7,14 +7,19 @@ most convenient one. Someone arriving at the release page should not have to know which artifact they need, go hunting through PyPI, or find that the only link is for a platform they are not on. -Attach all four, and list them together in the notes: +Attach all of them, and list them together in the notes. The Release +workflow (`.github/workflows/release.yml`) builds every one from the tag on +GitHub's machines and attaches them to a draft, so nobody has to remember: | Artifact | For | |---|---| -| `Compartment-.pkg` | macOS, one click, with the optional menu bar utility | -| `Compartment-.dmg` | macOS, drag to Applications | +| `Compartment-.pkg` | macOS on Apple silicon, one click, with the optional menu bar utility | +| `Compartment-.dmg` | macOS on Apple silicon, drag to Applications | +| `Compartment--intel.pkg` | macOS on Intel, one click | +| `Compartment--intel.dmg` | macOS on Intel, drag to Applications | | `compartment--py3-none-any.whl` | pip / offline installs, any OS | | `compartment-.tar.gz` | source distribution, packagers, air-gapped builds | +| `SHA256SUMS.txt` | the checksums of everything above | GitHub adds "Source code (zip/tar.gz)" on its own. The notes should also carry the `pip install compartment` line for people who want the CLI and MCP @@ -53,18 +58,25 @@ server without the app. python -m pytest -q # must be green -# 2. python artifacts -rm -rf dist && python -m build - -# 3. macOS app + installers (needs a NON-framework Python; see the script) -python tools/build_macos_app.py --pkg --dmg +# 2. merge the release PR, then tag the merge commit and push the tag. +# That is the whole build: release.yml checks the tag against every +# version field, builds the wheel and sdist, builds the macOS app, .pkg +# and .dmg on an Apple silicon runner AND an Intel one, and attaches all +# of them plus SHA256SUMS.txt to a DRAFT release. Release notes come from +# .github/release-notes/vX.Y.Z.md when that file exists. +git tag vX.Y.Z && git push origin vX.Y.Z + +# 3. review the draft on the Releases page, then press Publish. Publishing +# triggers publish.yml, which uploads the release's own wheel and sdist to +# PyPI (the same bytes as the downloads), and publish-registry.yml. +# A rebuild of an existing tag: Actions -> Release -> Run workflow -> tag. +# +# Building by hand still works when it has to, from a Mac: +# rm -rf dist && python -m build +# python tools/build_macos_app.py --pkg --dmg # NON-framework Python; see the script +# gh release create vX.Y.Z --draft --notes-file NOTES.md build/*.pkg build/*.dmg dist/* -# 4. tag, publish, attach EVERYTHING -gh release create vX.Y.Z --title "…" --notes-file NOTES.md \ - build/Compartment-X.Y.Z.pkg \ - build/Compartment-X.Y.Z.dmg \ - dist/compartment-X.Y.Z-py3-none-any.whl \ - dist/compartment-X.Y.Z.tar.gz +# 4. nothing to do by hand for PyPI or the registry; see "Verify" below. # 5. the MCP registry entry tracks the version too. Three things bite here. # @@ -80,8 +92,10 @@ gh release create vX.Y.Z --title "…" --notes-file NOTES.md \ # c) Published versions are immutable. A wrong pin cannot be edited, only # superseded, so it is worth getting right the first time. # -# The registry token expires within minutes, so log in immediately before -# every publish or status call, even if you logged in one command ago. +# publish-registry.yml does this on Publish, authenticated by OIDC. By +# hand only when that workflow failed: the registry token expires within +# minutes, so log in immediately before every publish or status call, +# even if you logged in one command ago. mcp-publisher login github --token "$(gh auth token)" \ && mcp-publisher validate && mcp-publisher publish ``` diff --git a/pyproject.toml b/pyproject.toml index 798e873..d7109e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Security :: Cryptography", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Database :: Database Engines/Servers", @@ -41,8 +42,12 @@ dependencies = [ # Both majors. mcp 2.0 (2026-07-28) removed mcp.server.fastmcp in favour # of mcp.server.mcpserver; server.py imports whichever is present, so the # same wheel runs on 1.x and 2.x and single-version distros are not - # stranded on either side of the rename. - "mcp>=1.0,<3", + # stranded on either side of the rename. The floor is 1.10, not 1.0: + # FastMCP only arrived in 1.2 and the tool titles every tool is + # registered with in 1.10, so anything older crashed on import while + # pip called the install satisfied. CI's lowest-dependencies job runs + # the suite at exactly this floor. + "mcp>=1.10,<3", # The app ships with every install. It is not an add-on: unlocking, # locking and changing the passphrase are things people do through a # panel, not through a terminal, so an install that only lays down a CLI diff --git a/tools/mcp_smoke.py b/tools/mcp_smoke.py new file mode 100644 index 0000000..2e6f96e --- /dev/null +++ b/tools/mcp_smoke.py @@ -0,0 +1,143 @@ +"""Talk to `compartment serve` the way an MCP client does, and fail loudly. + + python tools/mcp_smoke.py [--command compartment] [--vault PATH] + +Starts the server over stdio, sends `initialize`, the `initialized` +notification and `tools/list`, then calls `memory_status`, and checks every +answer. CI runs it against the INSTALLED package on macOS, Windows and Linux: +the unit suite drives the server in-process, which cannot see the failures a +client sees first - an entry point that does not start, a stray line on +stdout that corrupts the JSON-RPC stream, a handshake that never answers. + +Standard library only, so it runs before or without the package's own +dependencies on the interpreter that launches it. +""" +from __future__ import annotations + +import argparse +import json +import os +import queue +import shutil +import subprocess +import sys +import threading + +#: Tools every build must expose. memory_unlock is off by default, so it is +#: not required here. +REQUIRED_TOOLS = { + "memory_search", "memory_store", "memory_store_many", "memory_get", + "memory_recent", "memory_forget", "memory_link", "memory_relations", + "memory_unlink", "memory_list_namespaces", "memory_status", "memory_lock", + "memory_selftest", +} +TIMEOUT_S = 120.0 # the first call may load the embedding model + + +def _reader(stream, q: queue.Queue) -> None: + for line in iter(stream.readline, b""): + q.put(line) + q.put(None) + + +def _expect(q: queue.Queue, want_id: int, stderr_tail: list[str]) -> dict: + """The response with this id. Every stdout line must be JSON-RPC: a line + that is not is exactly the stream corruption clients choke on.""" + while True: + try: + line = q.get(timeout=TIMEOUT_S) + except queue.Empty: + raise SystemExit(f"no answer to request {want_id} within " + f"{TIMEOUT_S:.0f}s\nstderr:\n" + "".join(stderr_tail)) + if line is None: + raise SystemExit(f"the server exited before answering {want_id}\n" + "stderr:\n" + "".join(stderr_tail)) + text = line.decode("utf-8", "replace").strip() + if not text: + continue + try: + msg = json.loads(text) + except ValueError: + raise SystemExit(f"non-JSON line on the server's stdout: {text!r}") + if msg.get("jsonrpc") != "2.0": + raise SystemExit(f"not a JSON-RPC 2.0 message: {text!r}") + if msg.get("id") == want_id: + if "error" in msg: + raise SystemExit(f"request {want_id} failed: {msg['error']}") + return msg["result"] + # notifications (logging, progress) are fine; keep reading + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--command", default="compartment", + help="the executable to start (default: compartment on PATH)") + ap.add_argument("--vault", help="pass --vault to the server") + args = ap.parse_args() + + exe = shutil.which(args.command) or args.command + cmd = [exe] + if args.vault: + cmd += ["--vault", args.vault] + cmd += ["serve"] + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=os.environ.copy()) + out_q: queue.Queue = queue.Queue() + err_tail: list[str] = [] + + def _drain_err(): + for line in iter(proc.stderr.readline, b""): + err_tail.append(line.decode("utf-8", "replace")) + del err_tail[:-40] + threading.Thread(target=_reader, args=(proc.stdout, out_q), daemon=True).start() + threading.Thread(target=_drain_err, daemon=True).start() + + def send(obj: dict) -> None: + proc.stdin.write((json.dumps(obj) + "\n").encode("utf-8")) + proc.stdin.flush() + + try: + send({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "compartment-smoke", "version": "1"}}}) + init = _expect(out_q, 1, err_tail) + name = init.get("serverInfo", {}).get("name", "") + print(f"initialize: server {name!r} {init.get('serverInfo', {}).get('version', '')}, " + f"protocol {init.get('protocolVersion')}") + if not init.get("instructions"): + raise SystemExit("the server sent no instructions in initialize") + send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + + send({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + tools = {t["name"]: t for t in _expect(out_q, 2, err_tail)["tools"]} + missing = REQUIRED_TOOLS - set(tools) + if missing: + raise SystemExit(f"tools missing from tools/list: {sorted(missing)}") + for t in tools.values(): + schema = t.get("inputSchema") or {} + if schema.get("type") != "object": + raise SystemExit(f"{t['name']}: inputSchema is not an object schema") + print(f"tools/list: {len(tools)} tools") + + send({"jsonrpc": "2.0", "id": 3, "method": "tools/call", + "params": {"name": "memory_status", "arguments": {}}}) + res = _expect(out_q, 3, err_tail) + if res.get("isError"): + raise SystemExit(f"memory_status reported an error: {res}") + print("tools/call memory_status: ok") + finally: + try: + proc.stdin.close() + except OSError: + pass + try: + proc.wait(timeout=20) + except subprocess.TimeoutExpired: + proc.kill() + raise SystemExit("the server did not exit after its stdin closed") + print("MCP smoke test passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 893afab14b74fd9d8f28b12fcbc18d34710d34a5 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:37:49 -0700 Subject: [PATCH 006/135] On macOS an acknowledged store survives a power cut, and a power cut no longer leaves a vault that refuses to open as tampered fsync on macOS hands the data to the drive and no further: the drive may keep it in its volatile cache and write it out in any order, and SSDs in current Macs are measured to lose fsync'd data on power loss. Apple documents F_FULLFSYNC as the call that flushes it. Without it the journal's promise (an acknowledged write survives kill -9 and power loss, as the docs say) did not hold on a Mac, and worse, an entry whose length reached the disk before its body read back as a complete entry failing authentication, which an open must treat as tampering: the vault refused to open after a power cut. The compaction's copy had the same exposure, with the rename able to land before the bytes it points at. Journal appends, the compaction's copy and the truncation of a refused append now go through one _sync: F_FULLFSYNC where Python defines it (macOS, iOS), falling back to fsync on a filesystem that refuses it (SMB, some FUSE and USB media), and plain fsync everywhere else. It costs about 4 ms per store on an Apple SSD, against roughly 40 ms of embedding. Co-Authored-By: Claude Opus 5.5 --- FORMAT.md | 3 +- src/compartment/vaultfile.py | 48 +++++++++++++++++++-- tests/test_storage_durability.py | 71 ++++++++++++++++++++++++++++++-- tests/test_vault_ops.py | 12 +++--- 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/FORMAT.md b/FORMAT.md index 092dcd2..cc18e1c 100644 --- a/FORMAT.md +++ b/FORMAT.md @@ -99,7 +99,8 @@ u32 section_count, then per section: Each entry: `AEAD_seal(master_key, canonical_json(entry), aad="compartment-journal:"+vault_id+":"+u64_be(seq))` where `seq` starts at 0 after each compaction and increments per entry. Entries are fsync'd on -append (acknowledged). An append the disk refuses part way (ENOSPC, EDQUOT, +append (acknowledged; F_FULLFSYNC on Apple platforms, where fsync stops at +the drive's volatile cache). An append the disk refuses part way (ENOSPC, EDQUOT, a failed sync) is truncated back to where it began, so an unacknowledged write leaves no bytes behind. Ops: diff --git a/src/compartment/vaultfile.py b/src/compartment/vaultfile.py index f26fd4b..091f987 100644 --- a/src/compartment/vaultfile.py +++ b/src/compartment/vaultfile.py @@ -8,7 +8,8 @@ Invariant I2: plaintext never touches disk. The payload and every journal entry are sealed before any byte is written. lock/save rewrites the file atomically (temp file → fsync → rename). Journal appends are fsync'd, so an -acknowledged write survives kill -9; a truncated *final* entry is an +acknowledged write survives kill -9 and power loss (see _sync for what that +takes on macOS); a truncated *final* entry is an unacknowledged write and is discarded on open with a notice - any other malformed byte is a tamper error. """ @@ -27,6 +28,15 @@ from . import crypto, wire from .crypto import CryptoError, TamperError +try: + import fcntl +except ImportError: # Windows + fcntl = None + +#: The fcntl that makes a write durable on Apple's platforms, or None where +#: Python does not define it (everywhere else). See _sync. +_FULLFSYNC = getattr(fcntl, "F_FULLFSYNC", None) + MAGIC = b"NUCV" FORMAT_VERSION = 1 @@ -141,6 +151,36 @@ def _atomic_replace(tmp: str, path: str) -> None: raise last if last is not None else OSError(f"could not replace {path}") +def _sync(fd: int) -> None: + """Make what was written to `fd` durable: on the disk itself, not in a + cache that a power cut empties. + + On macOS (and iOS) fsync only hands the data to the drive, which is free + to hold it in its own volatile cache and to write it out in any order; + Apple documents F_FULLFSYNC as the call that actually flushes it, and + SSDs in current Macs are measured to lose fsync'd data on power loss + without it. Two promises rest on this. An acknowledged store is meant to + survive power loss, not only kill -9. And a journal entry whose length + reached the disk before its body did reads back as a complete entry that + fails authentication, which an open has to treat as tampering: the vault + refused to open after a power cut, for no fault of the user's. The + compaction's copy is synced the same way, or the rename could land + before the bytes it points at. + + F_FULLFSYNC costs a few milliseconds per call (about 4 ms measured on an + Apple SSD, against roughly 40 ms for the embedding a store already + pays), and stores are not a hot loop, so it is paid on every append. A + filesystem that does not support it (SMB, some FUSE and USB media) + refuses it, and plain fsync is the best that disk offers.""" + if _FULLFSYNC is not None: + try: + fcntl.fcntl(fd, _FULLFSYNC) + return + except OSError: + pass + os.fsync(fd) + + def _payload_aad(vault_id: str) -> bytes: return wire.payload(vault_id)[0] @@ -361,7 +401,7 @@ def write_vault_file( f.write(hjson) f.write(payload_ct) f.flush() - os.fsync(f.fileno()) + _sync(f.fileno()) # O_CREAT only sets the mode when it actually creates the file, so a temp # left behind by an earlier run keeps its old mode. Set it either way. os.chmod(tmp, VAULT_MODE) @@ -412,7 +452,7 @@ def append_journal_entries(path: str, header: VaultHeader, seq: int, view = memoryview(frames) while view: view = view[os.write(fd, view):] - os.fsync(fd) + _sync(fd) except BaseException: # A write the disk refused part way (ENOSPC, EDQUOT, an I/O # error, a sync that failed) is a write nobody acknowledged, and @@ -426,7 +466,7 @@ def append_journal_entries(path: str, header: VaultHeader, seq: int, # a torn tail and the next open discards them, as before. try: os.ftruncate(fd, start) - os.fsync(fd) + _sync(fd) except OSError: pass raise diff --git a/tests/test_storage_durability.py b/tests/test_storage_durability.py index 3c578c4..e98ffb9 100644 --- a/tests/test_storage_durability.py +++ b/tests/test_storage_durability.py @@ -130,15 +130,16 @@ def test_an_append_whose_sync_fails_is_not_left_to_replay(vault_path, v.store("A memory that was acknowledged.", caller="t") size = os.path.getsize(vault_path) calls = [] + real_sync = vaultfile._sync - def fsync(fd): + def sync(fd): calls.append(fd) if len(calls) == 1: raise OSError(errno.EIO, "Input/output error") - return os.fsync(fd) + return real_sync(fd) with monkeypatch.context() as m: - m.setattr(vaultfile, "os", _With(os, fsync=fsync)) + m.setattr(vaultfile, "_sync", sync) with pytest.raises(OSError, match="Input/output"): v.store("A memory whose sync failed.", caller="t") assert os.path.getsize(vault_path) == size @@ -156,3 +157,67 @@ def test_an_append_never_creates_a_vault_that_is_not_there(vault_path): vaultfile.append_journal_entry(vault_path, v.header, 0, {"op": "noop"}, v._master) assert not os.path.exists(vault_path) + + +class _Fcntl: + """fcntl as the vault file sees it: records every call, and refuses + them all when `refuse` is set, as SMB and some USB media do.""" + + def __init__(self, refuse: bool = False): + self.calls, self.refuse = [], refuse + + def fcntl(self, fd, op, *args): + self.calls.append(op) + if self.refuse: + raise OSError(errno.ENOTSUP, "Operation not supported") + return 0 + + +def _record_fsyncs(monkeypatch) -> list: + fsyncs = [] + + def fsync(fd): + fsyncs.append(fd) + return os.fsync(fd) + monkeypatch.setattr(vaultfile, "os", _With(os, fsync=fsync)) + return fsyncs + + +def test_appends_and_compactions_reach_the_disk_on_macos(vault_path, + monkeypatch): + """fsync on macOS hands data to the drive's own cache and no further. + An acknowledged store could be lost to a power cut, and a journal entry + whose length landed before its body read back as tampering, so the + vault refused to open. F_FULLFSYNC is the call Apple documents for it.""" + v = Vault.create(vault_path, PASS, creator="t") + fake = _Fcntl() + monkeypatch.setattr(vaultfile, "fcntl", fake) + monkeypatch.setattr(vaultfile, "_FULLFSYNC", 51) # macOS's value + fsyncs = _record_fsyncs(monkeypatch) + v.store("A memory that has to survive a power cut.", caller="t") + assert fake.calls == [51], "the append is flushed past the drive cache" + v.save() + assert fake.calls[1] == 51, "and so is the compaction's copy" + assert not fsyncs, "F_FULLFSYNC does fsync's work and more" + + +def test_a_disk_that_refuses_a_full_sync_still_gets_fsync(vault_path, + monkeypatch): + """A vault on an SMB share or a USB stick, where the filesystem does not + implement F_FULLFSYNC: a store must not fail for want of it.""" + v = Vault.create(vault_path, PASS, creator="t") + monkeypatch.setattr(vaultfile, "fcntl", _Fcntl(refuse=True)) + monkeypatch.setattr(vaultfile, "_FULLFSYNC", 51) + fsyncs = _record_fsyncs(monkeypatch) + v.store("A memory on a share without F_FULLFSYNC.", caller="t") + assert len(fsyncs) == 1 + assert Vault.unlock(vault_path, passphrase=PASS).status()[ + "organic_records"] == 1 + + +def test_elsewhere_an_append_is_an_ordinary_fsync(vault_path, monkeypatch): + v = Vault.create(vault_path, PASS, creator="t") + monkeypatch.setattr(vaultfile, "_FULLFSYNC", None) + fsyncs = _record_fsyncs(monkeypatch) + v.store("A memory on Linux or Windows.", caller="t") + assert len(fsyncs) == 1 diff --git a/tests/test_vault_ops.py b/tests/test_vault_ops.py index 9ab2179..6af739b 100644 --- a/tests/test_vault_ops.py +++ b/tests/test_vault_ops.py @@ -574,19 +574,21 @@ def test_a_save_still_syncs_its_copy_before_the_rename(vault, vault_path, synced before it is renamed over the vault, and the vault stays 0600.""" vault.store("A memory saved the ordinary way.", caller="test") order = [] + real_sync = vaultfile._sync - def fsync(fd): - order.append("fsync") - return os.fsync(fd) + def sync(fd): + order.append("sync") + return real_sync(fd) def replace(src, dst): order.append("replace") return os.replace(src, dst) with monkeypatch.context() as m: - m.setattr(vaultfile, "os", _With(os, fsync=fsync, replace=replace)) + m.setattr(vaultfile, "_sync", sync) + m.setattr(vaultfile, "os", _With(os, replace=replace)) vault.save() - assert order == ["fsync", "replace"] + assert order[:2] == ["sync", "replace"] assert not os.path.exists(vault_path + ".tmp") if os.name != "nt": assert os.stat(vault_path).st_mode & 0o777 == 0o600 From 24dc490a3f05f5f9d5976cb3691f40e77e421853 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:38:07 -0700 Subject: [PATCH 007/135] Looking up a stored credential no longer writes to the session directory, so a vault opens where that directory cannot be made or re-moded Every vault open asks session.get() first, before the Keychain and before COMPARTMENT_PASSPHRASE. get() began by making the session directory and chmod-ing it 0700, on every call, whether or not a credential was there. Two kinds of process could not do that and so could not open a vault at all: - A container run as an arbitrary uid (`docker run --user 1000`, OpenShift, Kubernetes runAsUser), where HOME is `/` and the store would be /.compartment/session, or a read-only root. mkdir raised PermissionError out of find_credential, so even with the passphrase exported every open was a traceback. `compartment lock` failed the same way. - An agent's shell under macOS Seatbelt (Claude Code, Codex), which may read the user's 0700 session directory but answers chmod on it with EPERM; checked in this sandbox on an existing 0700 directory outside the write allowlist. So `compartment search` run by the agent died before it could read the credential the user had unlocked. get() and clear() now look at the credential's path without creating anything, and _session_dir() narrows the mode only when it is not already 0700, so a store that is wider is still narrowed on the next write. _forget_boot_secret() treats a store that cannot be made like one with no holder. Before the fix: test_finding_no_credential_needs_no_session_directory E NotADirectoryError: [Errno 20] Not a directory: '.../session' test_the_passphrase_variable_opens_a_vault_where_home_is_not_writable E NotADirectoryError: [Errno 20] Not a directory: '.../session' test_a_store_that_refuses_chmod_still_yields_its_credential E PermissionError: [Errno 1] Operation not permitted: '.../sess' Co-Authored-By: Claude Opus 5.5 --- src/compartment/session.py | 32 +++++++++--- tests/test_platform_session_store.py | 78 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 tests/test_platform_session_store.py diff --git a/src/compartment/session.py b/src/compartment/session.py index 5494c66..87719eb 100644 --- a/src/compartment/session.py +++ b/src/compartment/session.py @@ -106,9 +106,20 @@ def _session_dir() -> Path: + """The session store, created 0700 if it is not there yet. + + The mode is narrowed only when it is wrong, not on every call. A chmod + is a write, and the processes that most need to READ a stored credential + are often not allowed to write here: an agent's shell under Claude Code + or Codex runs in a macOS Seatbelt profile whose write allowlist is the + project and the temp directory, and there chmod on the user's own 0700 + session directory fails with EPERM. Calling it unconditionally made + every `compartment search` the agent ran die with PermissionError + before it could look at the credential the user had unlocked.""" d = Path(env("SESSION_DIR", home() / "session")) d.mkdir(parents=True, exist_ok=True) - os.chmod(d, 0o700) + if stat.S_IMODE(d.stat().st_mode) != 0o700: + os.chmod(d, 0o700) return d @@ -393,7 +404,9 @@ def _forget_boot_secret() -> bool: try: name = _holder() return _win_forget(name) if IS_WINDOWS else _posix_forget(name) - except CryptoError: + except (CryptoError, OSError): + # OSError: the session directory, which names the holder, cannot be + # made here. Then no credential was ever stored in it either. return False @@ -540,8 +553,15 @@ def get(vault_path: str) -> bytes | None: None means locked. A CryptoError means something is wrong that deleting the credential would not fix - an unreadable file, no boot clock - and in - that case nothing is deleted.""" - p = _file_for(vault_path) + that case nothing is deleted. + + Looking costs nothing on disk: with no credential there, nothing is + created. Every vault open starts here, before the Keychain and before + COMPARTMENT_PASSPHRASE are consulted, so a lookup that insisted on + making the session directory first took down every open where that + directory cannot be made - a container run as an arbitrary uid, whose + HOME is `/`, or a read-only root - even with the passphrase exported.""" + p = credential_path(vault_path) if not p.is_file(): return None try: @@ -590,10 +610,10 @@ def clear(vault_path: str) -> bool: If that was the last one in this store, destroy the per-boot secret too: keeping it would leave key material in memory that nothing on disk can use, and a panic-lock should not leave that lying about.""" - p = _file_for(vault_path) + p = credential_path(vault_path) existed = p.is_file() if existed: p.unlink() - if not any(_session_dir().glob("*.session")): + if not p.parent.is_dir() or not any(p.parent.glob("*.session")): _forget_boot_secret() return existed diff --git a/tests/test_platform_session_store.py b/tests/test_platform_session_store.py new file mode 100644 index 0000000..23c22a9 --- /dev/null +++ b/tests/test_platform_session_store.py @@ -0,0 +1,78 @@ +"""Where the session store cannot be written: a container run as an arbitrary +uid (HOME is `/`), a read-only root, and an agent's shell under macOS +Seatbelt, which may read the user's session directory but not write it. + +Reading a credential, or finding there is none, must not need a write.""" +import errno +import os + +import pytest + +from compartment import session +from compartment.vault import Vault + +from conftest import PASS + + +def _unmakeable_store(tmp_path, monkeypatch): + """A session directory nobody can create: its parent is a regular file. + mkdir refuses that for every user, root included, on every platform.""" + blocker = tmp_path / "home-is-not-writable" + blocker.write_text("") + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(blocker / "session")) + return blocker / "session" + + +def test_finding_no_credential_needs_no_session_directory(tmp_path, monkeypatch): + store = _unmakeable_store(tmp_path, monkeypatch) + vault = str(tmp_path / "v.vault") + assert session.get(vault) is None + assert session.clear(vault) is False # `compartment lock` + assert not store.exists() + + +def test_the_passphrase_variable_opens_a_vault_where_home_is_not_writable( + tmp_path, monkeypatch): + """`docker run --user 1000 -e COMPARTMENT_PASSPHRASE ...`: HOME is `/`, so + the session store would be /.compartment/session. Every open used to die + with PermissionError making it, before the variable was ever read.""" + vault = str(tmp_path / "v.vault") + Vault.create(vault, PASS, creator="test").lock() + _unmakeable_store(tmp_path, monkeypatch) + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + assert Vault.find_credential(vault) == (PASS, None) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX modes") +def test_a_store_that_refuses_chmod_still_yields_its_credential( + tmp_path, monkeypatch): + """The agent-sandbox case. The user unlocked in a terminal; the agent's + `compartment search` runs under Seatbelt, which lets it read the 0700 + session directory but answers chmod on it with EPERM (seen in Claude + Code's own sandbox). The credential is there and readable, so the + lookup must find it rather than die on a mode it did not need to set.""" + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + vault = str(tmp_path / "v.vault") + key = b"k" * 32 + session.store(vault, key) + real_chmod = os.chmod + + def seatbelt(path, mode, *a, **kw): + if os.fspath(path).startswith(str(tmp_path / "sess")): + raise PermissionError(errno.EPERM, "Operation not permitted", + os.fspath(path)) + return real_chmod(path, mode, *a, **kw) + monkeypatch.setattr(session.os, "chmod", seatbelt) + assert session.get(vault) == key + assert session.credential_path(vault).is_file() + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX modes") +def test_a_store_with_a_wide_mode_is_still_narrowed(tmp_path, monkeypatch): + """Skipping a chmod that is not needed must not skip one that is.""" + d = tmp_path / "sess" + d.mkdir(mode=0o755) + os.chmod(d, 0o755) + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(d)) + session.store(str(tmp_path / "v.vault"), b"k" * 32) + assert (d.stat().st_mode & 0o777) == 0o700 From 278a7c215c5c66f7ed9d5f9d8aa5f130e107f4bc Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:38:12 -0700 Subject: [PATCH 008/135] The macOS app is built with the wheels for the macOS it declares, and the build refuses a binary that needs a newer one The bundle declares macOS 13.0 (LSMinimumSystemVersion, the launcher's -mmacosx-version-min, the .pkg's allowed versions), but the build let pip choose the wheels, and pip chooses for the macOS it is running on. From onnxruntime 1.24 on, the macOS wheels are arm64 only and tagged macosx_14_0, and numpy ships a macosx_14_0 build beside its 11_0 one, so an app built on a current Mac or a macos-latest runner carried libraries that need macOS 14. Installer accepted it on macOS 13, and there the app could not load its embedding model, with no window to say so. The wheels are now chosen for the declared minimum: Compartment is built as a wheel, `pip download --platform macosx__ --only-binary=:all:` resolves everything into a wheelhouse, and the install takes only from it. On arm64 with 13.0 that is onnxruntime 1.23.2, the last release with a macosx_13_0 build; numpy and the rest get their macosx_11_0 or universal2 wheels. A Python that has no such set (onnxruntime 1.23.2 stops at 3.13) is refused with what to embed instead. The launcher is compiled for the interpreter's architecture. A new check, _verify_deployment_target, reads every Mach-O in the bundle (thin and universal, LC_BUILD_VERSION and LC_VERSION_MIN_MACOSX) and fails the build if any lacks the app's architecture or needs a newer macOS than the app declares. --min-macos (default 13.0, not below it, since SMAppService needs 13) sets the minimum in one place, and the .pkg's allowed-os-versions is read from the built app instead of a second hard-coded 13.0. Co-Authored-By: Claude Opus 5.5 --- tests/test_macos_bundle.py | 171 +++++++++++++++++++++++++++ tools/build_macos_app.py | 233 +++++++++++++++++++++++++++++++++++-- 2 files changed, 394 insertions(+), 10 deletions(-) diff --git a/tests/test_macos_bundle.py b/tests/test_macos_bundle.py index 6dafb14..28659ef 100644 --- a/tests/test_macos_bundle.py +++ b/tests/test_macos_bundle.py @@ -200,3 +200,174 @@ def test_a_system_python_is_never_embedded(): """The whole prefix gets copied in, so /usr would mean copying /usr.""" src = SPEC.read_text(encoding="utf-8") assert '("/usr", "/System", "/Library")' in src + + +# ------------------------------------- every binary runs where the app claims + +def _macho(arch="arm64", minos=(13, 0), legacy=False) -> bytes: + """A minimal 64-bit Mach-O header with the one load command that names + its minimum macOS: LC_BUILD_VERSION, or the older LC_VERSION_MIN_MACOSX.""" + import struct + cpu = {"arm64": 0x0100000C, "x86_64": 0x01000007}[arch] + v = (minos[0] << 16) | (minos[1] << 8) + cmd = (struct.pack(" bytes: + import struct + cpus = {"arm64": 0x0100000C, "x86_64": 0x01000007} + entries, body = b"", b"" + for arch, data in slices: + entries += struct.pack(">IIIII", cpus[arch], 0, 4096 + len(body), + len(data), 12) + body += data.ljust(4096, b"\0") + head = struct.pack(">II", 0xCAFEBABE, len(slices)) + entries + return head.ljust(4096, b"\0") + body + + +def test_the_mach_o_reader_finds_each_slice_and_its_minimum(tmp_path): + b = _builder() + thin = tmp_path / "thin.so" + thin.write_bytes(_macho("arm64", (14, 0))) + assert b._macho_slices(thin) == {"arm64": (14, 0)} + old = tmp_path / "old.so" + old.write_bytes(_macho("x86_64", (10, 9), legacy=True)) + assert b._macho_slices(old) == {"x86_64": (10, 9)} + fat = tmp_path / "fat.so" + fat.write_bytes(_fat(("x86_64", _macho("x86_64", (10, 13))), + ("arm64", _macho("arm64", (11, 0))))) + assert b._macho_slices(fat) == {"x86_64": (10, 13), "arm64": (11, 0)} + # Not Mach-O: text, and a Java class file, which shares the fat magic. + (tmp_path / "a.py").write_text("print('hi')\n") + (tmp_path / "A.class").write_bytes(b"\xca\xfe\xba\xbe\x00\x00\x00\x34") + assert b._macho_slices(tmp_path / "a.py") == {} + assert b._macho_slices(tmp_path / "A.class") == {} + + +def _fake_app(tmp_path, files: dict) -> pathlib.Path: + app = tmp_path / "Compartment.app" + for rel, data in files.items(): + p = app / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(data) + return app + + +def test_a_library_built_for_a_newer_macos_than_the_app_is_refused(tmp_path): + """onnxruntime 1.24 and later ship macOS wheels tagged macosx_14_0 only, + and pip on a current build machine takes them, so the bundle declaring + 13.0 carried an embedding model that macOS 13 cannot load.""" + lib = ("Contents/Resources/runtime/lib/python3.12/site-packages/" + "onnxruntime/capi/onnxruntime_pybind11_state.so") + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("arm64", (13, 0)), + lib: _macho("arm64", (14, 0)), + "Contents/Resources/runtime/lib/python3.12/os.py": b"# stdlib\n"}) + b = _builder() + with pytest.raises(SystemExit) as e: + b._verify_deployment_target(app, "arm64", "13.0") + assert "onnxruntime_pybind11_state.so needs macOS 14.0" in str(e.value) + b._verify_deployment_target(app, "arm64", "14.0") # must not raise + + +def test_an_intel_build_refuses_a_library_without_an_intel_slice(tmp_path): + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("x86_64", (13, 0)), + "Contents/Resources/runtime/lib/x.so": _macho("arm64", (11, 0)), + "Contents/Resources/runtime/lib/u.so": _fat( + ("x86_64", _macho("x86_64", (10, 13))), + ("arm64", _macho("arm64", (11, 0))))}) + b = _builder() + with pytest.raises(SystemExit) as e: + b._verify_deployment_target(app, "x86_64", "13.0") + assert "x.so (arm64)" in str(e.value) + assert "u.so" not in str(e.value), "a universal2 library is fine" + + +def test_the_build_runs_the_deployment_check_before_signing(): + src = SPEC.read_text(encoding="utf-8") + body = src.split("def build_app")[1] + assert body.index("_verify_deployment_target(app, arch, min_macos)") \ + < body.index('_run("codesign"') + + +def test_wheels_are_chosen_for_the_declared_minimum_macos(tmp_path, + monkeypatch): + """pip on its own picks wheels for the macOS it runs on. The build has to + ask for the macOS the app declares, binary only, and install nothing + else.""" + import subprocess + b = _builder() + b._EMBEDDED_VER[0] = "3.12" + calls = [] + + def fake_run(*args, **kw): + argv = [str(a) for a in args] + calls.append(argv) + if "wheel" in argv: + house = pathlib.Path(argv[argv.index("--wheel-dir") + 1]) + (house / "compartment-4.13.0-py3-none-any.whl").write_bytes(b"") + return subprocess.CompletedProcess(argv, 0) + + def fake_subprocess_run(argv, **kw): + calls.append([str(a) for a in argv]) + return subprocess.CompletedProcess(argv, 0, "", "") + + monkeypatch.setattr(b, "_run", fake_run) + monkeypatch.setattr(b.subprocess, "run", fake_subprocess_run) + b._install_packages(pathlib.Path("/rt/bin/python3.12"), "/src", + "arm64", "13.0") + download = next(c for c in calls if "download" in c) + assert download[download.index("--platform") + 1] == "macosx_13_0_arm64" + assert "--only-binary=:all:" in download + install = next(c for c in calls if "install" in c) + assert "--no-index" in install, "only the wheels chosen above" + assert not any("install" in c and "--no-index" not in c for c in calls) + + +def test_no_wheels_for_the_minimum_says_what_to_embed_instead(tmp_path, + monkeypatch): + import subprocess + b = _builder() + b._EMBEDDED_VER[0] = "3.14" + + def fake_run(*args, **kw): + argv = [str(a) for a in args] + if "wheel" in argv: + house = pathlib.Path(argv[argv.index("--wheel-dir") + 1]) + (house / "compartment-4.13.0-py3-none-any.whl").write_bytes(b"") + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(b, "_run", fake_run) + monkeypatch.setattr(b.subprocess, "run", lambda argv, **kw: ( + subprocess.CompletedProcess(argv, 1, "", "ERROR: No matching " + "distribution found for onnxruntime"))) + with pytest.raises(SystemExit) as e: + b._install_packages(pathlib.Path("/rt/bin/python3.14"), "/src", + "arm64", "13.0") + msg = str(e.value) + assert "Python 3.14 on macOS 13.0" in msg and "onnxruntime" in msg + assert "--min-macos 14.0" in msg and "3.12" in msg + + +def test_the_installer_accepts_only_the_macos_the_app_declares(tmp_path, + monkeypatch): + import plistlib + b = _builder() + app = tmp_path / "Compartment.app" + (app / "Contents").mkdir(parents=True) + with open(app / "Contents" / "Info.plist", "wb") as fh: + plistlib.dump({"LSMinimumSystemVersion": "14.0"}, fh) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + monkeypatch.setattr(b, "_run", lambda *a, **k: None) + b.build_pkg(app) + dist = (tmp_path / "build" / "distribution.xml").read_text() + assert '' in dist + + +def test_a_minimum_below_what_the_login_item_needs_is_refused(): + with pytest.raises(SystemExit, match="SMAppService"): + _builder().build_app(python="/nonexistent", min_macos="12.0") diff --git a/tools/build_macos_app.py b/tools/build_macos_app.py index 7a9284d..9782e61 100644 --- a/tools/build_macos_app.py +++ b/tools/build_macos_app.py @@ -49,6 +49,7 @@ import pathlib import plistlib import shutil +import struct import subprocess import sys import sysconfig @@ -59,7 +60,12 @@ BUILD = ROOT / "build" APP_NAME = "Compartment" BUNDLE_ID = "io.github.maxfreedompollard.compartment" -MIN_MACOS = "13.0" # also LSMinimumSystemVersion, below +# The oldest macOS the app is built for, and the default of --min-macos. It +# is LSMinimumSystemVersion, the launcher's -mmacosx-version-min, the .pkg's +# allowed-os-versions, and - the part that is easy to miss - the platform +# every wheel in the bundle is chosen for (_install_packages). 13 is also the +# floor: SMAppService, which registers the login item, is macOS 13 and up. +MIN_MACOS = "13.0" # Shown under the app's name in System Settings > General > Login Items. DESCRIPTION = ("Compartment keeps your AI agents' memory encrypted on this Mac. " "The menu bar item shows what it has remembered and lets you " @@ -147,6 +153,189 @@ def _python_version(exe: str) -> str: return out.stdout.strip() +def _python_arch(exe: str) -> str: + """The CPU the embedded interpreter is built for: arm64 or x86_64. + + Asked of that interpreter, not of this one or of the machine. It decides + which wheels go into the bundle, which architecture the launcher is + compiled for, and which Macs the installer will agree to run on, and all + three have to be the interpreter's or the app cannot start.""" + out = subprocess.run( + [exe, "-c", "import platform;print(platform.machine())"], + capture_output=True, text=True, check=True) + return out.stdout.strip() + + +def _version_tuple(text: str) -> tuple[int, int]: + parts = (str(text).split(".") + ["0"])[:2] + return int(parts[0]), int(parts[1]) + + +def _install_packages(rt_python: pathlib.Path, spec: str, arch: str, + min_macos: str) -> None: + """Install Compartment and its dependencies into the embedded runtime, + each as the wheel built for `min_macos` on `arch`. + + pip on its own picks the wheel for the macOS it is running on, and the + build machine is the newest macOS there is. From onnxruntime 1.24 on, + the macOS wheels are arm64 only and tagged macosx_14_0, and numpy ships + a macosx_14_0 build beside its macosx_11_0 one; pip prefers the newer + tag whenever the machine allows it. So an app built on a current runner + carried libraries that need macOS 14 inside a bundle declaring 13.0: + Installer accepted it on macOS 13, and the app could not load its own + embedding model there. + + So the wheels are chosen for the declared minimum instead. `pip + download --platform` resolves against that platform (and every older + one, so universal2 and macosx_11_0 wheels qualify), binary only, into a + wheelhouse; the install then takes only from the wheelhouse. On arm64 + with a minimum of 13.0 that resolves onnxruntime 1.23.2, the last + release with a macosx_13_0 build, and on Intel the last with any macOS + x86_64 build at all. _verify_deployment_target checks the result. + """ + major, minor = _version_tuple(min_macos) + platform_tag = f"macosx_{major}_{minor}_{arch}" + with tempfile.TemporaryDirectory(prefix="compartment-wheels-") as td: + house = pathlib.Path(td) + # Compartment itself, as a wheel, so the binary-only resolve below + # can take it: it is pure Python, built from this checkout (or + # fetched, for a --spec naming a release). + _run(rt_python, "-m", "pip", "wheel", "--quiet", "--no-deps", + "--wheel-dir", house, spec) + wheels = sorted(house.glob("compartment-*.whl")) + if not wheels: + raise SystemExit(f"error: building a wheel from {spec} produced " + "no compartment wheel") + wanted = [str(wheels[-1]), "pyobjc-framework-Cocoa>=10.0"] + got = subprocess.run( + [str(rt_python), "-m", "pip", "download", "--quiet", + "--only-binary=:all:", "--platform", platform_tag, + "--dest", str(house), *wanted], + capture_output=True, text=True) + if got.returncode != 0: + raise SystemExit( + f"error: no complete set of wheels for Python " + f"{_EMBEDDED_VER[0]} on macOS {min_macos} ({arch})\n" + f"{(got.stderr or got.stdout).strip()[-1500:]}\n" + " onnxruntime 1.23.2 is the last release with macOS 13 and " + "Intel wheels, and it\n goes up to Python 3.13: embed 3.12 " + "or 3.13 (uv python install 3.12), or, on\n Apple silicon " + "only, build for macOS 14 with --min-macos 14.0") + print(f"wheels chosen for {platform_tag}: " + + ", ".join(sorted(p.name.split("-")[0] + " " + + p.name.split("-")[1] + for p in house.glob("*.whl")))) + _run(rt_python, "-m", "pip", "install", "--quiet", "--no-index", + "--find-links", house, *wanted) + + +# Mach-O, read directly: what the loader reads, with no tool to install and +# a check that a test can feed bytes to on any OS. +_FAT_MAGICS = (b"\xca\xfe\xba\xbe", b"\xca\xfe\xba\xbf") +_THIN_MAGICS = (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe") # 64, 32 bit LE +_CPU_NAMES = {0x01000007: "x86_64", 0x0100000C: "arm64", 7: "i386", + 12: "arm"} +_LC_VERSION_MIN_MACOSX = 0x24 +_LC_BUILD_VERSION = 0x32 +_PLATFORM_MACOS = 1 + + +def _macho_thin(f, start: int) -> tuple[str, tuple[int, int] | None] | None: + """(architecture, minimum macOS) of the Mach-O image at `start`.""" + f.seek(start) + head = f.read(28) + if len(head) < 28 or head[:4] not in _THIN_MAGICS: + return None + cputype, _sub, _ftype, ncmds, sizeofcmds, _flags = struct.unpack( + " len(cmds): + break + cmd, size, a, b = struct.unpack_from("> 16, (b >> 8) & 0xFF) + elif cmd == _LC_VERSION_MIN_MACOSX: + minos = (a >> 16, (a >> 8) & 0xFF) + if size < 8: + break + pos += size + return _CPU_NAMES.get(cputype, hex(cputype)), minos + + +def _macho_slices(path: pathlib.Path) -> dict[str, tuple[int, int] | None]: + """{architecture: minimum macOS} for each slice of a Mach-O file, or {} + for anything that is not one. The minimum is None for an image that + records none.""" + try: + with open(path, "rb") as f: + magic = f.read(4) + if magic in _THIN_MAGICS: + one = _macho_thin(f, 0) + return dict([one]) if one else {} + if magic not in _FAT_MAGICS: + return {} + wide = magic == _FAT_MAGICS[1] + (count,) = struct.unpack(">I", f.read(4)) + # A Java class file starts with the same four bytes, followed by + # its version, 45 and up. No universal binary has that many + # slices. + if not 0 < count <= 16: + return {} + entry = ">IIQQII" if wide else ">IIIII" + offsets = [struct.unpack(entry, f.read(struct.calcsize(entry)))[2] + for _ in range(count)] + out = {} + for off in offsets: + one = _macho_thin(f, off) + if one is None: + return {} + out[one[0]] = one[1] + return out + except (OSError, struct.error): + return {} + + +def _verify_deployment_target(app: pathlib.Path, arch: str, + min_macos: str) -> None: + """Every binary in the bundle runs on `arch` under `min_macos`. + + The check that would have caught the macOS 14 onnxruntime in a bundle + declaring 13.0, and an arm64-only library in an Intel build. A Mach-O + built for a newer macOS than the bundle declares fails to load on the + older one with nothing on screen - the menu bar app has no window to + report it in - so this is fatal, as the signature check is.""" + floor = _version_tuple(min_macos) + too_new, wrong_arch, count = [], [], 0 + for p in sorted(app.rglob("*")): + if p.is_symlink() or not p.is_file(): + continue + slices = _macho_slices(p) + if not slices: + continue + count += 1 + rel = p.relative_to(app) + if arch not in slices: + wrong_arch.append(f"{rel} ({', '.join(sorted(slices))})") + elif slices[arch] is not None and slices[arch] > floor: + need = slices[arch] + too_new.append(f"{rel} needs macOS {need[0]}.{need[1]}") + problems = [] + if wrong_arch: + problems.append(f"built without {arch}, so it cannot load in this " + "app:\n " + "\n ".join(wrong_arch[:20])) + if too_new: + problems.append(f"built for a newer macOS than the {min_macos} the " + "app declares:\n " + "\n ".join(too_new[:20])) + if problems: + raise SystemExit("error: " + "\nerror: ".join(problems)) + print(f"{count} binaries, every one {arch} and none needing more than " + f"macOS {min_macos}") + + def _strip(lib: pathlib.Path) -> None: """Drop what a shipped app has no use for. @@ -284,12 +473,21 @@ def _verify_self_contained(app: pathlib.Path) -> None: def build_app(spec: str | None = None, out: pathlib.Path | None = None, - python: str | None = None) -> pathlib.Path: + python: str | None = None, + min_macos: str = MIN_MACOS) -> pathlib.Path: """spec: what to pip-install (default: this checkout, so the build always matches the source it came from).""" version = _version() + if _version_tuple(min_macos) < (13, 0): + raise SystemExit("error: --min-macos cannot go below 13.0: the login " + "item is registered with SMAppService, which is " + "macOS 13 and up") base_python = python or find_standalone_python() - print(f"embedding {base_python}") + arch = _python_arch(base_python) + if arch not in ("arm64", "x86_64"): + raise SystemExit(f"error: {base_python} is built for {arch!r}; the " + "app is built for arm64 or x86_64") + print(f"embedding {base_python} ({arch}), for macOS {min_macos} and up") app = (out or BUILD / f"{APP_NAME}.app") if app.exists(): shutil.rmtree(app) @@ -322,9 +520,10 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # bundle, and installing into it is the entire point. (runtime / "lib" / f"python{ver}" / "EXTERNALLY-MANAGED").unlink( missing_ok=True) - _run(rt_python, "-m", "pip", "install", "--quiet", spec or str(ROOT)) - _run(rt_python, "-m", "pip", "install", "--quiet", - "pyobjc-framework-Cocoa>=10.0") + if subprocess.run([str(rt_python), "-m", "pip", "--version"], + capture_output=True).returncode != 0: + _run(rt_python, "-m", "ensurepip", "--default-pip") # a pip-less build + _install_packages(rt_python, spec or str(ROOT), arch, min_macos) _strip(runtime / "lib") @@ -353,7 +552,7 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # an image and a window - but the menu bar never gives it a slot, and # the icon simply never appears. See tools/launcher.c. launcher = macos / APP_NAME - _run("cc", "-O2", f"-mmacosx-version-min={MIN_MACOS}", + _run("cc", "-O2", "-arch", arch, f"-mmacosx-version-min={min_macos}", "-o", str(launcher), str(pathlib.Path(__file__).with_name("launcher.c")), f"-I{runtime / 'include' / f'python{ver}'}", f"-L{runtime / 'lib'}", f"-lpython{ver}", @@ -375,7 +574,7 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, "CFBundleVersion": version, "CFBundleInfoDictionaryVersion": "6.0", "LSUIElement": True, # menu bar only, no dock icon - "LSMinimumSystemVersion": MIN_MACOS, + "LSMinimumSystemVersion": min_macos, "NSHumanReadableCopyright": "https://github.com/" "MaxFreedomPollard/Compartment", "NSHumanReadableDescription": DESCRIPTION, @@ -391,6 +590,7 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # Checked BEFORE signing: running the interpreter is what writes bytecode # into the bundle, and doing that afterwards is what broke the seal. _verify_self_contained(app) + _verify_deployment_target(app, arch, min_macos) _run("codesign", "--force", "--deep", "--sign", "-", "--identifier", BUNDLE_ID, app) @@ -464,9 +664,19 @@ def _login_plist(app: pathlib.Path) -> dict: } +def _app_min_macos(app: pathlib.Path) -> str: + """The minimum macOS the built app declares. The installers take it from + the app rather than from a constant of their own, so a .pkg can never + accept a macOS the app inside it was not built for.""" + with open(app / "Contents" / "Info.plist", "rb") as fh: + return str(plistlib.load(fh).get("LSMinimumSystemVersion") + or MIN_MACOS) + + def build_pkg(app: pathlib.Path) -> pathlib.Path: """A .pkg whose second, optional component starts it at login.""" version = _version() + min_macos = _app_min_macos(app) staging = BUILD / "pkgroot" login_root = BUILD / "pkgroot-login" for d in (staging, login_root): @@ -540,7 +750,7 @@ def _scripts(name: str, body: str) -> pathlib.Path: Compartment - + @@ -611,11 +821,14 @@ def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--spec", help="what to pip install (default: this repo)") ap.add_argument("--python", help="standalone interpreter to embed") + ap.add_argument("--min-macos", default=MIN_MACOS, + help=f"oldest macOS the app runs on (default {MIN_MACOS}); " + "every wheel is chosen for it") ap.add_argument("--dmg", action="store_true") ap.add_argument("--pkg", action="store_true") args = ap.parse_args() BUILD.mkdir(exist_ok=True) - app = build_app(args.spec, python=args.python) + app = build_app(args.spec, python=args.python, min_macos=args.min_macos) if args.dmg: build_dmg(app) if args.pkg: From 04d222ec95f39d6afc7b0e88a6736fc3cc8ae544 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:38:14 -0700 Subject: [PATCH 009/135] `integrate codex` writes a table Codex can always read, and leaves a config.toml alone when a table cannot be added to it without breaking it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex refuses to start on a config.toml it cannot parse, and two things let `integrate codex` produce one. The values were escaped with json.dumps, whose escaping resembles TOML's but writes a character outside the Basic Multilingual Plane as a UTF-16 surrogate pair (🔒). TOML allows only Unicode scalar values in an escape, so a vault in a folder with an emoji in its name, or under a user name with a CJK Extension B character, appended a table that made the whole file unreadable. Strings are now escaped as TOML defines them, with everything else written as itself in the file's UTF-8. And the append was never read back. A config that declares `mcp_servers = { ... }` inline, or as an array, cannot be extended by a [mcp_servers.compartment] header, so a working config with every other server in it became one Codex could not load. The appended document is now parsed before it is written: it has to hold exactly our entry and everything else exactly as before, or the file is left untouched, no backup is taken, and the block to paste is printed with the reason. Dotted keys for another server still take the table, as they should. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 76 ++++++++++++++++++++++++++++++-- tests/test_integrations_codex.py | 67 ++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 tests/test_integrations_codex.py diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 3350dbc..f38ca27 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -547,13 +547,46 @@ def write_config(path: Path, text: str) -> Path | None: return backup +#: TOML's short escapes; every other control character is \\uXXXX. +_TOML_ESCAPES = {'"': '\\"', "\\": "\\\\", "\b": "\\b", "\t": "\\t", + "\n": "\\n", "\f": "\\f", "\r": "\\r"} + + +def _toml_string(s: str) -> str: + """A TOML basic string. + + Not json.dumps, whose escaping looks like TOML's and is not: it writes a + character outside the Basic Multilingual Plane as a UTF-16 surrogate + pair, \\ud83d\\ude00, and TOML allows only Unicode scalar values in an + escape. A vault in a folder named with an emoji, or with a CJK Extension + B character in a user name, appended a table that made the whole of + config.toml unreadable, and Codex refuses to start on a config it cannot + read. Here only what TOML requires is escaped, and the rest is written + as itself, in the UTF-8 the file is. A lone surrogate (an undecodable + byte in a POSIX path) cannot be written in UTF-8 at all and raises + ValueError. + """ + out = [] + for ch in s: + o = ord(ch) + if ch in _TOML_ESCAPES: + out.append(_TOML_ESCAPES[ch]) + elif o < 0x20 or o == 0x7F: + out.append(f"\\u{o:04x}") + elif 0xD800 <= o <= 0xDFFF: + raise ValueError(f"{s!r} cannot be written as UTF-8 TOML") + else: + out.append(ch) + return '"' + "".join(out) + '"' + + def _toml_value(v) -> str: """Just enough TOML to emit one table of strings and string arrays.""" if isinstance(v, list): return "[" + ", ".join(_toml_value(x) for x in v) + "]" if isinstance(v, bool): return "true" if v else "false" - return json.dumps(str(v)) # JSON string escaping is TOML's too + return _toml_string(str(v)) def _toml_block(entry: dict) -> str: @@ -701,6 +734,31 @@ def register(client: Client, vault: str, caller: str | None = None, return out +def _toml_breaks(client: Client, before: str, after: str, + entry: dict) -> str | None: + """Why `after` must not be written in place of `before`, or None. + + Appending a table is the whole edit, and an appended table is not + always a valid one: a config that declares `mcp_servers = { ... }` + inline, or as an array, cannot be extended by a [mcp_servers.x] header, + and the result is a file Codex will not start with. So the document is + read back as Codex will read it: it has to parse, hold exactly our + entry, and hold everything else exactly as it did before.""" + try: + doc = tomllib.loads(after) + was = tomllib.loads(before) + except tomllib.TOMLDecodeError as exc: + return f"unreadable ({exc})" + servers = doc.get(client.root) + if not isinstance(servers, dict) or servers.get(NAME) != entry: + return f"without a [{client.root}.{NAME}] table Codex can read" + rest = dict(doc) + rest[client.root] = {k: v for k, v in servers.items() if k != NAME} + if not rest[client.root] and client.root not in was: + del rest[client.root] + return None if rest == was else "with its other settings changed" + + def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: """Codex keeps TOML, and TOML tables are order-independent. @@ -710,7 +768,12 @@ def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: that is already there is reported, not duplicated. A file that does not parse is refused, as a JSON one is. """ - block = _toml_block(entry) + try: + block = _toml_block(entry) + except ValueError as exc: + out["reason"] = f"cannot be written as TOML ({exc})" + out["snippet"] = json.dumps({client.root: {NAME: entry}}, indent=2) + return out try: existing = read_config_text(p) if p.exists() else "" registered = _toml_has_ours(client, existing) @@ -729,8 +792,15 @@ def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: return out joiner = "" if not existing else ("\n" if existing.endswith("\n") else "\n\n") + text = existing + joiner + block + "\n" + broken = _toml_breaks(client, existing, text, entry) + if broken: + out["reason"] = (f"adding the table would leave {p} {broken}; " + "refusing to touch it") + out["snippet"] = block + return out try: - backup = write_config(p, existing + joiner + block + "\n") + backup = write_config(p, text) except OSError as exc: out["reason"] = f"cannot write {p} ({exc})" out["snippet"] = block diff --git a/tests/test_integrations_codex.py b/tests/test_integrations_codex.py new file mode 100644 index 0000000..3a51c7c --- /dev/null +++ b/tests/test_integrations_codex.py @@ -0,0 +1,67 @@ +"""Codex reads one TOML file, and refuses to start at all when it cannot +read it. Whatever `integrate codex` appends has to leave a document Codex +can read, with every other setting in it exactly as it was.""" +import tomllib + +import pytest + +from compartment import clients + +CODEX = clients.CLIENTS["codex"] + + +@pytest.mark.parametrize("vault", [ + "/Users/ana/Vaults \U0001F512/memory.vault", # an emoji in a folder + "/home/\U00020BB7\u5409/memory.vault", # CJK Extension B + "C:\\Users\\Jos\u00e9\\.compartment\\memory.vault", + '/tmp/a "quoted"\tname\x7f/memory.vault', +]) +def test_the_codex_table_reads_back_whatever_the_vault_path_holds(vault, + tmp_path): + """The values were escaped with json.dumps, which writes a character + outside the Basic Multilingual Plane as a UTF-16 surrogate pair; TOML + allows only scalar values in an escape, so a vault in a folder with an + emoji in its name left config.toml unreadable and Codex would not + start.""" + p = tmp_path / "config.toml" + p.write_text('model = "o3"\n', encoding="utf-8") + + res = clients.register(CODEX, vault, path=p) + + assert res["written"] is True, res["reason"] + doc = tomllib.loads(p.read_text(encoding="utf-8")) + assert doc["model"] == "o3" + assert doc["mcp_servers"]["compartment"]["args"][1] == vault + assert tomllib.loads(clients.snippet(CODEX, vault))["mcp_servers"][ + "compartment"]["args"][1] == vault + + +@pytest.mark.parametrize("existing", [ + 'mcp_servers = { other = { command = "x" } }\n', + 'mcp_servers = []\n', + '[[mcp_servers]]\nname = "x"\ncommand = "y"\n', +]) +def test_a_config_a_table_cannot_be_added_to_is_left_alone(existing, + tmp_path): + """An inline `mcp_servers = { ... }` cannot be extended by a + [mcp_servers.compartment] header, so the append turned a working config + into one Codex could not read, every server in it gone with it.""" + p = tmp_path / "config.toml" + p.write_text(existing, encoding="utf-8") + + res = clients.register(CODEX, "/v/memory.vault", path=p) + + assert res["written"] is False + assert "refusing to touch it" in res["reason"] + assert "[mcp_servers.compartment]" in res["snippet"] + assert p.read_text(encoding="utf-8") == existing + assert not p.with_name(p.name + clients.BACKUP_SUFFIX).exists() + + +def test_dotted_keys_for_another_server_still_take_the_table(tmp_path): + p = tmp_path / "config.toml" + p.write_text('mcp_servers.other.command = "x"\n', encoding="utf-8") + res = clients.register(CODEX, "/v/memory.vault", path=p) + assert res["written"] is True, res["reason"] + servers = tomllib.loads(p.read_text(encoding="utf-8"))["mcp_servers"] + assert servers["other"] == {"command": "x"} and "compartment" in servers From 312d94a27e283837d7449ecfc48829c729f926cf Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:38:29 -0700 Subject: [PATCH 010/135] Bug reports ask for the CPU architecture, WSL, containers and sandboxed shells, and list the stores and clients people now install from Co-Authored-By: Claude Opus 5.5 --- .github/ISSUE_TEMPLATE/bug_report.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 33d4e98..81ddf00 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -23,6 +23,7 @@ body: - pip / pipx / uv (PyPI) - Compartment.pkg (macOS one-click) - Compartment.dmg + - A plugin or extension store (Claude Code, Gemini CLI, Hermes, Claude Desktop bundle...) - Docker - From source validations: @@ -30,8 +31,9 @@ body: - type: input id: platform attributes: - label: OS and Python - placeholder: "macOS 26.2 / Python 3.12.4" + label: OS, CPU and Python + description: Include the architecture (Apple silicon / Intel / ARM64 / x86_64), and say so if it runs under WSL, in a container, or from an agent's sandboxed shell. + placeholder: "macOS 26.2, Apple silicon / Python 3.12.4" validations: required: true - type: dropdown @@ -47,6 +49,12 @@ body: - Cline / Roo Code - Codex CLI - Gemini CLI + - VS Code / GitHub Copilot + - Windsurf + - Zed + - OpenCode + - Goose + - LM Studio - Other MCP client - CLI only - The menu bar / tray app From 6f9f7752224ece45954d03e4988dd4bea2104d3c Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:38:58 -0700 Subject: [PATCH 011/135] A compaction's rename is synced through the vault's folder, so the stores acknowledged after it survive a power cut The compaction synced its new copy and renamed it over the vault, but a rename is a change to the folder, and on POSIX only syncing the folder makes it durable. After a power cut the old file could come back, and with it every store acknowledged since the compaction was gone, because each had been appended (and synced) to the new file that no name pointed at any more. ext4's ordering usually hid this; btrfs, XFS, APFS and network or FUSE filesystems give no such promise. write_vault_file now syncs the containing folder after the rename (with F_FULLFSYNC on macOS, like every other sync). It is best effort: a mount that refuses to sync a directory does not fail a save that has already happened. Skipped on Windows, where a directory cannot be opened for this and NTFS journals the rename itself. Co-Authored-By: Claude Opus 5.5 --- FORMAT.md | 3 +- src/compartment/vaultfile.py | 37 ++++++++++++++-- tests/test_storage_durability.py | 76 ++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 5 deletions(-) diff --git a/FORMAT.md b/FORMAT.md index cc18e1c..7ec8813 100644 --- a/FORMAT.md +++ b/FORMAT.md @@ -149,7 +149,8 @@ requires the chain to extend the anchor. The anchor is mandatory: its absence is reported as tampering, not as an older vault. Compaction (`save`/`lock`): serialize → seal → write `path.tmp` → fsync → -atomic rename; journal restarts empty. +atomic rename → fsync of the containing folder (POSIX), so the rename itself +survives power loss; journal restarts empty. ### Manifest (signed vaults) diff --git a/src/compartment/vaultfile.py b/src/compartment/vaultfile.py index 091f987..39211b3 100644 --- a/src/compartment/vaultfile.py +++ b/src/compartment/vaultfile.py @@ -7,10 +7,10 @@ Invariant I2: plaintext never touches disk. The payload and every journal entry are sealed before any byte is written. lock/save rewrites the file -atomically (temp file → fsync → rename). Journal appends are fsync'd, so an -acknowledged write survives kill -9 and power loss (see _sync for what that -takes on macOS); a truncated *final* entry is an -unacknowledged write and is discarded on open with a notice - any other +atomically (temp file → fsync → rename → fsync of the folder). Journal +appends are fsync'd, so an acknowledged write survives kill -9 and power +loss (see _sync for what that takes on macOS); a truncated *final* entry is +an unacknowledged write and is discarded on open with a notice - any other malformed byte is a tamper error. """ from __future__ import annotations @@ -181,6 +181,34 @@ def _sync(fd: int) -> None: os.fsync(fd) +def _sync_dir(path: str) -> None: + """Make the rename that just put `path` in place durable. + + A rename is a change to the FOLDER, and on POSIX it is the folder that + has to be synced for it to survive a power cut: syncing the file makes + its bytes durable, not the name pointing at them. Lose the rename and + the old vault comes back, and with it every store acknowledged since the + compaction is gone, because each was appended to the new file. Best + effort: some filesystems (certain FUSE and network mounts) refuse to + sync a directory, and the save has already succeeded by then. + + Not on Windows, where a directory cannot be opened for this at all and + NTFS journals the rename itself.""" + if os.name == "nt": + return + try: + fd = os.open(os.path.dirname(os.path.abspath(path)), + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + except OSError: + return + try: + _sync(fd) + except OSError: + pass + finally: + os.close(fd) + + def _payload_aad(vault_id: str) -> bytes: return wire.payload(vault_id)[0] @@ -415,6 +443,7 @@ def write_vault_file( except OSError: pass raise + _sync_dir(path) def append_journal_entry(path: str, header: VaultHeader, seq: int, entry: dict, master_key: bytes) -> None: diff --git a/tests/test_storage_durability.py b/tests/test_storage_durability.py index e98ffb9..064eb81 100644 --- a/tests/test_storage_durability.py +++ b/tests/test_storage_durability.py @@ -221,3 +221,79 @@ def test_elsewhere_an_append_is_an_ordinary_fsync(vault_path, monkeypatch): fsyncs = _record_fsyncs(monkeypatch) v.store("A memory on Linux or Windows.", caller="t") assert len(fsyncs) == 1 + + +def _record_save(monkeypatch, name: str = "posix") -> list: + """What a save does to the disk, in order: syncs (with what they + synced), the rename, and every folder opened.""" + order = [] + real_sync, real_open = vaultfile._sync, os.open + + def sync(fd): + order.append(("sync", None)) + return real_sync(fd) + + def replace(src, dst): + order.append(("replace", None)) + return os.replace(src, dst) + + def opener(path, flags, *args): + if os.path.isdir(path): + order.append(("open folder", path)) + return real_open(path, flags, *args) + + monkeypatch.setattr(vaultfile, "_sync", sync) + monkeypatch.setattr(vaultfile, "os", _With(os, name=name, replace=replace, + open=opener)) + return order + + +@pytest.mark.skipif(os.name == "nt", reason="a folder cannot be opened " + "for syncing on Windows; the test below covers it") +def test_the_rename_of_a_compaction_is_synced_through_its_folder( + vault_path, monkeypatch): + """Syncing the file makes its bytes durable, not the name pointing at + them. After a power cut the old vault could come back, and every store + acknowledged since the compaction - each appended to the NEW file - + went with it.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory that the compaction folds in.", caller="t") + order = _record_save(monkeypatch) + v.save() + kinds = [k for k, _ in order] + assert kinds == ["sync", "replace", "open folder", "sync"] + assert order[2][1] == os.path.dirname(os.path.abspath(vault_path)) + + +def test_on_windows_no_folder_is_opened(vault_path, monkeypatch): + """A directory cannot be opened for syncing on Windows, and NTFS + journals the rename itself.""" + v = Vault.create(vault_path, PASS, creator="t") + order = _record_save(monkeypatch, name="nt") + v.save() + assert [k for k, _ in order] == ["sync", "replace"] + + +def test_a_folder_that_cannot_be_synced_does_not_fail_the_save( + vault_path, monkeypatch): + """Some FUSE and network mounts refuse to sync a directory. The save + has already succeeded by then and must say so.""" + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory on a mount that cannot sync folders.", caller="t") + real_sync, real_replace = vaultfile._sync, os.replace + renamed = [] + + def sync(fd): + if renamed: # the folder's sync, after the rename + raise OSError(errno.EINVAL, "Invalid argument") + return real_sync(fd) + + def replace(src, dst): + renamed.append(dst) + return real_replace(src, dst) + + monkeypatch.setattr(vaultfile, "_sync", sync) + monkeypatch.setattr(vaultfile, "os", _With(os, replace=replace)) + v.save() + assert Vault.unlock(vault_path, passphrase=PASS).status()[ + "organic_records"] == 1 From f6905aad2b5e711f5ece7ea70f6de1d3fe4c8a72 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:39:23 -0700 Subject: [PATCH 012/135] The compartment command inside the macOS app runs wherever the app is installed, so agents connected from the app get a working memory server pip writes the absolute path of the interpreter that installed a console script into its first line, and the build installs with the copy of Python inside build/Compartment.app. So Contents/Resources/runtime/bin/ compartment in the shipped app named an interpreter on the build machine (checked: pip 26 writes `#!/runtime/bin/python3.12`). That script is what clients.executable puts into every agent's config when the panel's Connect button or `integrate` runs inside the app, and what the capture hook runs. On anybody else's Mac the agent's memory server failed to start with "bad interpreter", while the panel reported the agent connected. The build now rewrites every console script that names the runtime being built, plain shebang or pip's exec form for a path with a space, into the relocatable form uv gives its own scripts: run the interpreter beside the script, found through realpath (macOS 13 and up), with -B so nothing is written into the signed bundle. A new check fails the build if any script still names the build directory, and runs the bundle's compartment --version from a clean environment before signing. Co-Authored-By: Claude Opus 5.5 --- tests/test_macos_bundle.py | 65 +++++++++++++++++++++++++ tools/build_macos_app.py | 97 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/tests/test_macos_bundle.py b/tests/test_macos_bundle.py index 28659ef..b4f8b0d 100644 --- a/tests/test_macos_bundle.py +++ b/tests/test_macos_bundle.py @@ -371,3 +371,68 @@ def test_the_installer_accepts_only_the_macos_the_app_declares(tmp_path, def test_a_minimum_below_what_the_login_item_needs_is_refused(): with pytest.raises(SystemExit, match="SMAppService"): _builder().build_app(python="/nonexistent", min_macos="12.0") + + +# ---------------------------- the CLI inside the bundle runs on another Mac + +def _pip_written(runtime: pathlib.Path, name: str, long_form=False) -> None: + """A console script as pip writes it: the absolute path of the + interpreter that installed it, plain or in the exec form pip uses for a + path with a space in it.""" + py = runtime / "bin" / "python3.12" + head = (f"#!/bin/sh\n'''exec' \"{py}\" \"$0\" \"$@\"\n' '''\n" + if long_form else f"#!{py}\n") + (runtime / "bin" / name).write_text( + head + "# -*- coding: utf-8 -*-\nimport sys\n" + "print('ran', sys.argv[1:])\n", encoding="utf-8") + (runtime / "bin" / name).chmod(0o755) + + +@pytest.mark.skipif(sys.platform == "win32", reason="the bundle is POSIX") +def test_console_scripts_still_run_after_the_bundle_moves(tmp_path): + """pip names the build directory's interpreter in every script it writes, + and `compartment` inside the app is what the panel writes into each + agent's config. Installed anywhere else - which is everywhere but the + build machine - it failed with "bad interpreter".""" + import shutil + import subprocess + if not shutil.which("realpath"): + pytest.skip("no realpath here") + build = tmp_path / "build dir" / "runtime" + (build / "bin").mkdir(parents=True) + (build / "bin" / "python3.12").symlink_to(sys.executable) + _pip_written(build, "compartment") + _pip_written(build, "cmpt", long_form=True) + relocatable = "#!/bin/sh\n'''exec' \"$(dirname -- \"$0\")/python3.12\" \"$0\"\n' '''\nprint(1)\n" + (build / "bin" / "pip").write_text(relocatable, encoding="utf-8") + assert sorted(_builder()._relocate_scripts(build)) == ["cmpt", + "compartment"] + assert (build / "bin" / "pip").read_text() == relocatable, \ + "a script that is relocatable already is left alone" + installed = tmp_path / "Applications" / "runtime" + installed.parent.mkdir() + shutil.move(str(build), str(installed)) + for name in ("compartment", "cmpt"): + script = installed / "bin" / name + assert str(build) not in script.read_text() + r = subprocess.run([str(script), "--version"], capture_output=True, + text=True, timeout=60) + assert r.returncode == 0, r.stderr + assert r.stdout.strip() == "ran ['--version']" + + +def test_a_script_naming_the_build_directory_fails_the_build(tmp_path): + app = tmp_path / "Compartment.app" + runtime = app / "Contents" / "Resources" / "runtime" + (runtime / "bin").mkdir(parents=True) + _pip_written(runtime, "compartment") + with pytest.raises(SystemExit, match="name the build directory"): + _builder()._verify_console_scripts(app) + + +def test_the_build_makes_its_scripts_relocatable_and_checks_them(): + src = SPEC.read_text(encoding="utf-8") + body = src.split("def build_app")[1] + assert body.index("_relocate_scripts(runtime)") < body.index("compileall") + assert body.index("_verify_console_scripts(app)") \ + < body.index('_run("codesign"') diff --git a/tools/build_macos_app.py b/tools/build_macos_app.py index 9782e61..3d05f2d 100644 --- a/tools/build_macos_app.py +++ b/tools/build_macos_app.py @@ -48,6 +48,7 @@ import os import pathlib import plistlib +import re import shutil import struct import subprocess @@ -363,6 +364,99 @@ def _strip(lib: pathlib.Path) -> None: print(f"stripped {freed / 1024 / 1024:.0f} MB of test and build files") +#: What a console script in the bundle starts with: the interpreter beside +#: the script, wherever the bundle is, the form uv gives its own Pythons' +#: scripts. Valid sh and valid Python at once: sh runs the exec line, and +#: Python reads lines two and three as one string and moves on. -B because +#: the bundle is signed, and bytecode written into it breaks the seal. +#: realpath is in macOS 13 and up, which is MIN_MACOS's floor. +_RELOCATABLE_SHEBANG = ( + "#!/bin/sh\n" + "'''exec' \"$(dirname -- \"$(realpath -- \"$0\")\")/{python}\" -B " + "\"$0\" \"$@\"\n" + "' '''\n") + + +def _relocate_scripts(runtime: pathlib.Path) -> list[str]: + """Rewrite the console scripts pip put in runtime/bin so they run from + wherever the bundle ends up. + + pip writes the absolute path of the interpreter that installed them into + each script's first line, and that interpreter is the copy inside the + BUILD directory. So `compartment` inside the app, which is what the + panel's Connect buttons write into every agent's config (clients. + executable picks the console script beside the running interpreter) and + what the capture hook runs, named a Python on the build machine: on + anybody else's Mac the agent's memory server failed with "bad + interpreter" on its first start. The long-path form pip uses for a + folder with a space in it names the same absolute path, inside an exec + line, and is rewritten the same way.""" + bin_dir = runtime / "bin" + here = {str(runtime), str(runtime.resolve())} + done = [] + for script in sorted(bin_dir.iterdir()): + if script.is_symlink() or not script.is_file(): + continue + try: + lines = script.read_text(encoding="utf-8").split("\n") + except (UnicodeDecodeError, OSError): + continue # a binary + first = lines[0] + if (first.startswith("#!") and len(lines) > 2 + and first.strip() == "#!/bin/sh" + and lines[1].startswith("'''exec' ") + and lines[2] == "' '''"): + named, rest = lines[1], lines[3:] + elif first.startswith("#!"): + named, rest = first, lines[1:] + else: + continue + if not any(h in named for h in here): + continue # not ours, or relocatable already + # The interpreter's own name, python3.12, from wherever it sat: the + # path itself may hold spaces, so it is not split on them. + found = re.search(r"/bin/(python[0-9.]*)", named) + python = found.group(1) if found else "python3" + script.write_text(_RELOCATABLE_SHEBANG.format(python=python) + + "\n".join(rest), encoding="utf-8") + done.append(script.name) + return done + + +def _verify_console_scripts(app: pathlib.Path) -> None: + """No script in the bundle names the build machine's paths, and the CLI + an agent is given really runs, from a clean environment.""" + runtime = app / "Contents" / "Resources" / "runtime" + built_at = {str(app), str(app.resolve())} + strays = [] + for script in sorted((runtime / "bin").iterdir()): + if script.is_symlink() or not script.is_file(): + continue + try: + head = script.read_text(encoding="utf-8")[:1024] + except (UnicodeDecodeError, OSError): + continue + if any(p in head for p in built_at): + strays.append(script.name) + if strays: + raise SystemExit( + "error: these scripts name the build directory and would fail " + "on any other Mac:\n " + "\n ".join(strays)) + cli = runtime / "bin" / "compartment" + if not cli.is_file(): + raise SystemExit("error: the bundle has no compartment command for " + "agents to run") + with tempfile.TemporaryDirectory(prefix="compartment-buildprobe-") as tmp: + out = subprocess.run([str(cli), "--version"], capture_output=True, + text=True, timeout=120, + env={"PATH": "/usr/bin:/bin", "HOME": tmp}) + if out.returncode != 0: + raise SystemExit("error: the bundle's compartment command does not " + "run\n" + (out.stderr or out.stdout).strip()[:2000]) + print(f"the compartment command agents are given runs: " + f"{out.stdout.strip()}") + + def _unseal_hazards(contents: pathlib.Path) -> None: """Delete files that make the bundle unsignable. @@ -524,6 +618,8 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, capture_output=True).returncode != 0: _run(rt_python, "-m", "ensurepip", "--default-pip") # a pip-less build _install_packages(rt_python, spec or str(ROOT), arch, min_macos) + fixed = _relocate_scripts(runtime) + print(f"console scripts made relocatable: {', '.join(fixed) or 'none'}") _strip(runtime / "lib") @@ -590,6 +686,7 @@ def build_app(spec: str | None = None, out: pathlib.Path | None = None, # Checked BEFORE signing: running the interpreter is what writes bytecode # into the bundle, and doing that afterwards is what broke the seal. _verify_self_contained(app) + _verify_console_scripts(app) _verify_deployment_target(app, arch, min_macos) _run("codesign", "--force", "--deep", "--sign", "-", From e395bac201b8aef767a26e146f9611b66a901c5b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:39:30 -0700 Subject: [PATCH 013/135] Running `integrate codex` again refreshes an entry whose command or vault has changed, keeping whatever else the user put in it Re-registering Codex reported "already registered; left as it is" whatever the existing [mcp_servers.compartment] table said, while every JSON client's entry is refreshed in place. So a Codex entry naming a bare `compartment` that Codex could not start, a console script an upgrade had moved, or another vault could not be put right by running `integrate codex` (or `integrate --all`) again; the user had to remove it first, and nothing said so. An entry with the same command and arguments is still left untouched. One that differs is taken out exactly as `unregister` takes it out, header, keys and sub-tables, and appended again with the new command and arguments; anything else the user had added to it (startup_timeout_sec, an env table, enabled = false) is carried over. The result goes through the same read-back check as a first registration. An entry written in a form that cannot be taken out cleanly, an inline table or dotted keys, is left as it is with the block to paste. Taking a table out now also takes the blank line that set it off when another follows, so refreshing or removing it does not leave the file one blank line longer each time. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 95 +++++++++++++++++++++++++++----- tests/test_clients.py | 3 + tests/test_integrations_codex.py | 67 ++++++++++++++++++++++ 3 files changed, 151 insertions(+), 14 deletions(-) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index f38ca27..42cfb1b 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -580,18 +580,42 @@ def _toml_string(s: str) -> str: return '"' + "".join(out) + '"' +def _toml_key(k: str) -> str: + """A key, bare when TOML allows it bare.""" + return k if k and all(c.isascii() and (c.isalnum() or c in "-_") + for c in k) else _toml_string(k) + + def _toml_value(v) -> str: - """Just enough TOML to emit one table of strings and string arrays.""" - if isinstance(v, list): - return "[" + ", ".join(_toml_value(x) for x in v) + "]" + """A TOML value for anything tomllib reads back: our own strings and + string arrays, and whatever else the user put in our table (a + startup_timeout_sec, an env table, enabled = false), which a refresh + carries over.""" if isinstance(v, bool): return "true" if v else "false" + if isinstance(v, int): + return str(v) + if isinstance(v, float): + if v != v: + return "nan" + if v in (float("inf"), float("-inf")): + return "inf" if v > 0 else "-inf" + return repr(v) + if isinstance(v, list): + return "[" + ", ".join(_toml_value(x) for x in v) + "]" + if isinstance(v, dict): + if not v: + return "{}" + return "{ " + ", ".join(f"{_toml_key(k)} = {_toml_value(x)}" + for k, x in v.items()) + " }" + if hasattr(v, "isoformat"): # datetime, date, time + return v.isoformat() return _toml_string(str(v)) def _toml_block(entry: dict) -> str: lines = [f"[mcp_servers.{NAME}]"] - lines += [f"{k} = {_toml_value(v)}" for k, v in entry.items()] + lines += [f"{_toml_key(k)} = {_toml_value(v)}" for k, v in entry.items()] return "\n".join(lines) @@ -662,6 +686,13 @@ def _toml_without_ours(client: Client, text: str) -> str | None: j += 1 for k in range(i, last + 1): keep[k] = False + # The blank line that set our table off goes with it when another + # follows, so taking it out (or refreshing it) leaves one, not two. + after = next((k for k in range(last + 1, len(stmts)) if keep[k]), + None) + if (i > 0 and keep[i - 1] and not stmts[i - 1][0].strip() + and (after is None or not stmts[after][0].strip())): + keep[i - 1] = False i = j out = "".join(s for (s, _), k in zip(stmts, keep) if k) try: @@ -764,9 +795,22 @@ def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: So the whole edit is appending one table, which leaves every comment, every ordering and every hand-made formatting decision in the file - exactly as its owner left it. Re-registering rewrites nothing: an entry - that is already there is reported, not duplicated. A file that does - not parse is refused, as a JSON one is. + exactly as its owner left it. Re-registering with the same command and + arguments rewrites nothing. A file that does not parse is refused, as a + JSON one is. + + Re-registering with a different command or vault refreshes the entry, + as it does for every JSON client. It used to report "already + registered; left as it is" whatever the entry said, so a config that + named a bare `compartment` Codex could not find, a console script that + an upgrade had moved, or last month's vault could not be put right by + running `integrate codex` again: the user had to remove it first, + which nothing told them. Our table is taken out exactly as + `unregister` takes it out and appended again with the new command and + arguments, and whatever else the user had put in it (a + startup_timeout_sec, an env table, enabled = false) is carried over. + When the table cannot be taken out cleanly (an inline entry, dotted + keys), the file is left as it is and the block is printed. """ try: block = _toml_block(entry) @@ -787,15 +831,38 @@ def _register_toml(client: Client, p: Path, entry: dict, out: dict) -> dict: return out if registered: - out["reason"] = "already registered; left as it is" - out["snippet"] = block - return out + old = tomllib.loads(existing)[client.root][NAME] + if not isinstance(old, dict) or all(old.get(k) == v + for k, v in entry.items()): + out["reason"] = "already registered; left as it is" + out["snippet"] = block + return out + rest = _toml_without_ours(client, existing) + if rest is None: + out["reason"] = ("registered with another command, written in a " + "form that cannot be updated without breaking " + f"{p}; left as it is. Replace the entry with " + "this one") + out["snippet"] = block + return out + entry = {**entry, **{k: v for k, v in old.items() if k not in entry}} + try: + refreshed = _toml_block(entry) + except ValueError as exc: + out["reason"] = f"cannot be written as TOML ({exc})" + out["snippet"] = block + return out + before, verb = rest, "updating" + else: + before, refreshed, verb = existing, block, "adding" - joiner = "" if not existing else ("\n" if existing.endswith("\n") else "\n\n") - text = existing + joiner + block + "\n" - broken = _toml_breaks(client, existing, text, entry) + # One blank line before the table, however many times it is refreshed. + joiner = ("" if not before or before.endswith("\n\n") + else "\n" if before.endswith("\n") else "\n\n") + text = before + joiner + refreshed + "\n" + broken = _toml_breaks(client, before, text, entry) if broken: - out["reason"] = (f"adding the table would leave {p} {broken}; " + out["reason"] = (f"{verb} the table would leave {p} {broken}; " "refusing to touch it") out["snippet"] = block return out diff --git a/tests/test_clients.py b/tests/test_clients.py index 0dbcd2b..ec9a2fe 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -189,10 +189,13 @@ def test_codex_gets_a_table_appended_and_keeps_the_rest(tmp_path): def test_codex_is_not_registered_twice(tmp_path): p = tmp_path / "config.toml" p.write_text('[mcp_servers.compartment]\ncommand = "compartment"\n') + clients.register(clients.CLIENTS["codex"], VAULT, path=p) res = clients.register(clients.CLIENTS["codex"], VAULT, path=p) assert res["written"] is False assert p.read_text().count("[mcp_servers.compartment]") == 1 + entry = tomllib.loads(p.read_text())["mcp_servers"]["compartment"] + assert entry["args"][:2] == ["--vault", VAULT] # refreshed once def test_codex_unregister_takes_only_our_table(tmp_path): diff --git a/tests/test_integrations_codex.py b/tests/test_integrations_codex.py index 3a51c7c..9077a68 100644 --- a/tests/test_integrations_codex.py +++ b/tests/test_integrations_codex.py @@ -65,3 +65,70 @@ def test_dotted_keys_for_another_server_still_take_the_table(tmp_path): assert res["written"] is True, res["reason"] servers = tomllib.loads(p.read_text(encoding="utf-8"))["mcp_servers"] assert servers["other"] == {"command": "x"} and "compartment" in servers + + +def _registered(p): + return tomllib.loads(p.read_text(encoding="utf-8"))["mcp_servers"][ + "compartment"] + + +def test_registering_codex_again_refreshes_a_stale_entry(tmp_path, + monkeypatch): + """Re-registering said "already registered; left as it is" whatever the + entry held, so an entry naming a bare `compartment` that Codex could + not find, a script an upgrade had moved, or another vault was never + put right by running `integrate codex` again, which is what every JSON + client does.""" + p = tmp_path / "config.toml" + p.write_text( + '# my settings\nmodel = "o3"\n\n' + '[mcp_servers.compartment]\ncommand = "compartment"\n' + 'args = ["--vault", "/old.vault", "--caller", "codex", "serve"]\n' + 'startup_timeout_sec = 30\n\n' + '[mcp_servers.compartment.env]\nCOMPARTMENT_EMBED_DAEMON = "0"\n\n' + "# Bob's server\n[mcp_servers.bob]\ncommand = \"bob\"\n", + encoding="utf-8") + monkeypatch.setattr(clients, "executable", lambda: "/opt/bin/compartment") + + res = clients.register(CODEX, "/new.vault", path=p) + + assert res["written"] is True, res["reason"] + text = p.read_text(encoding="utf-8") + doc = tomllib.loads(text) + ours = doc["mcp_servers"]["compartment"] + assert ours["command"] == "/opt/bin/compartment" + assert ours["args"][:2] == ["--vault", "/new.vault"] + # What the user added to our table stays with it. + assert ours["startup_timeout_sec"] == 30 + assert ours["env"] == {"COMPARTMENT_EMBED_DAEMON": "0"} + assert doc["mcp_servers"]["bob"] == {"command": "bob"} + assert doc["model"] == "o3" + assert text.startswith("# my settings\n") and "# Bob's server" in text + assert text.count("[mcp_servers.compartment]") == 1 + backup = p.with_name(p.name + clients.BACKUP_SUFFIX) + assert "/old.vault" in backup.read_text(encoding="utf-8") + + # Refreshing again changes nothing and does not grow the file. + again = clients.register(CODEX, "/new.vault", path=p) + assert again["written"] is False + assert again["reason"] == "already registered; left as it is" + assert p.read_text(encoding="utf-8") == text + monkeypatch.setattr(clients, "executable", lambda: "/usr/bin/compartment") + clients.register(CODEX, "/new.vault", path=p) + clients.register(CODEX, "/other.vault", path=p) + after = p.read_text(encoding="utf-8") + assert "\n\n\n" not in after + assert _registered(p)["args"][1] == "/other.vault" + + +def test_a_stale_codex_entry_that_cannot_be_taken_out_is_left_for_the_user( + tmp_path): + p = tmp_path / "config.toml" + text = ('[mcp_servers]\ncompartment = { command = "compartment", ' + 'args = ["serve"] }\nother = { command = "x" }\n') + p.write_text(text, encoding="utf-8") + res = clients.register(CODEX, "/v/memory.vault", path=p) + assert res["written"] is False + assert "left as it is" in res["reason"] + assert "[mcp_servers.compartment]" in res["snippet"] + assert p.read_text(encoding="utf-8") == text From 528008643399580f806c0561e637be7932d6e67a Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:39:36 -0700 Subject: [PATCH 014/135] A vault on a filesystem that cannot lock says so at once instead of waiting ten seconds and blaming another process FileLock took every OSError from flock (msvcrt.locking on Windows) for "held by someone else" and retried until its timeout, then raised "Vault is busy: another process holds the write lock (waited 10s)". A filesystem that does not lock at all fails flock with something else: ENOLCK on NFS whose lock service is not running or that is mounted nolock, ENOTSUP/EOPNOTSUPP on a macOS NFS mount with nolocks and on some FUSE and SMB mounts. Anyone with a home directory or vault on such a share waited ten seconds on every write and was then told to look for a process that did not exist. Only EWOULDBLOCK/EAGAIN/EACCES (EACCES/EDEADLOCK from msvcrt) now mean busy. Any other error stops at once with "Cannot lock : the filesystem it is on does not support file locks (...)", naming network shares as the usual cause and saying to keep the vault on a local disk or enable locking on the mount. Locking is never skipped: a vault written without the lock can lose a concurrent write. Before the fix: test_a_filesystem_without_locks_is_named_at_once_not_called_busy [ENOLCK, EOPNOTSUPP, EINVAL] E assert (93361.48 - 93358.43) < 1.5 (it waited the full 3 s) Co-Authored-By: Claude Opus 5.5 --- src/compartment/platforms.py | 43 ++++++++++++++++++++++++++----- tests/test_platform_filelock.py | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 tests/test_platform_filelock.py diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index 7471f97..5310b88 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -8,6 +8,7 @@ from __future__ import annotations from .home import env, home +import errno import json import os import platform @@ -27,15 +28,26 @@ # Advisory exclusive file lock (context manager over an open file handle) # --------------------------------------------------------------------------- +# Only these errors mean "someone else holds it". Anything else means the +# filesystem cannot lock at all - NFS whose lock service is not running or +# that is mounted `nolock`/`nolocks` (ENOLCK, ENOTSUP), some FUSE and SMB +# mounts (EOPNOTSUPP, EINVAL) - and no amount of waiting changes that. +# Treating those as busy made every write on such a vault wait out the +# timeout and then blame "another process", which sent people hunting for a +# process that did not exist. if IS_WINDOWS: import msvcrt + _BUSY = frozenset({errno.EACCES, getattr(errno, "EDEADLOCK", 36)}) + def _lock_nb(fh) -> bool: try: msvcrt.locking(fh.fileno(), msvcrt.LK_NBLCK, 1) return True - except OSError: - return False + except OSError as exc: + if exc.errno in _BUSY: + return False + raise def _unlock(fh) -> None: try: @@ -45,12 +57,16 @@ def _unlock(fh) -> None: else: import fcntl + _BUSY = frozenset({errno.EAGAIN, errno.EWOULDBLOCK, errno.EACCES}) + def _lock_nb(fh) -> bool: try: fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) return True - except OSError: - return False + except OSError as exc: + if exc.errno in _BUSY: + return False + raise def _unlock(fh) -> None: try: @@ -70,15 +86,30 @@ def __init__(self, path: str, timeout: float = 10.0): def __enter__(self): self._fh = open(self.path, "a+") deadline = time.time() + self.timeout - while not _lock_nb(self._fh): + while True: + try: + if _lock_nb(self._fh): + return self + except OSError as exc: + self._fh.close() + self._fh = None + from .crypto import CryptoError + raise CryptoError( + f"Cannot lock {self.path}: the filesystem it is on does " + f"not support file locks ({exc.strerror or exc}). That is " + f"usually a network share - NFS without its lock service " + f"or mounted nolock, or an SMB mount without byte-range " + f"locks. Keep the vault on a local disk (--vault or " + f"COMPARTMENT_VAULT), or enable locking on the " + f"mount.") from exc if time.time() > deadline: self._fh.close() + self._fh = None from .crypto import CryptoError raise CryptoError( f"Vault is busy: another process holds the write lock " f"(waited {self.timeout:.0f}s)") time.sleep(0.05) - return self def __exit__(self, *exc): if self._fh is not None: diff --git a/tests/test_platform_filelock.py b/tests/test_platform_filelock.py new file mode 100644 index 0000000..7bf1bf3 --- /dev/null +++ b/tests/test_platform_filelock.py @@ -0,0 +1,45 @@ +"""FileLock on filesystems that cannot lock: NFS without its lock service or +mounted nolock, some SMB and FUSE mounts. That is not "busy", and waiting +does not change it.""" +import errno +import os +import time + +import pytest + +from compartment import platforms +from compartment.crypto import CryptoError + +pytestmark = pytest.mark.skipif(os.name == "nt", reason="the fcntl branch") + + +@pytest.mark.parametrize("code", [errno.ENOLCK, errno.EOPNOTSUPP, errno.EINVAL]) +def test_a_filesystem_without_locks_is_named_at_once_not_called_busy( + tmp_path, monkeypatch, code): + """Before: every write waited out the timeout and then reported "Vault + is busy: another process holds the write lock", on a vault nobody else + had open.""" + def no_locks(fh, op): + raise OSError(code, os.strerror(code)) + monkeypatch.setattr(platforms.fcntl, "flock", no_locks) + started = time.monotonic() + with pytest.raises(CryptoError) as err: + with platforms.FileLock(str(tmp_path / "v.vault.flock"), timeout=3): + pass + assert time.monotonic() - started < 1.5 + assert "busy" not in str(err.value) + assert "does not support file locks" in str(err.value) + assert "local disk" in str(err.value) + + +@pytest.mark.parametrize("code", [errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES]) +def test_a_lock_someone_holds_is_still_waited_for_and_called_busy( + tmp_path, monkeypatch, code): + """BSD flock answers EWOULDBLOCK, and Linux's NFS emulation can answer + EAGAIN or EACCES: all three are another holder.""" + def held(fh, op): + raise OSError(code, os.strerror(code)) + monkeypatch.setattr(platforms.fcntl, "flock", held) + with pytest.raises(CryptoError, match="busy"): + with platforms.FileLock(str(tmp_path / "v.vault.flock"), timeout=0.2): + pass From fba25132d70d8040fae080bf83ad99368a0d1279 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:39:52 -0700 Subject: [PATCH 015/135] A process that may not use Unix domain sockets embeds with its own model at once, instead of starting a daemon that can never listen Claude Code and Codex run an agent's shell commands on macOS under a Seatbelt profile that refuses bind() on an AF_UNIX path with EPERM. Every `compartment search` or `store` an agent ran there started the shared embedding daemon, waited for it to import numpy and onnxruntime and exit, and only then built the in-process model it was always going to use: a wasted process and a quarter of a second per command, again at every reconnect in a server, and `compartment embed-daemon run` exited 1 without a word. A sandbox that refused connect() to a daemon started outside it, or refused starting a process, was asked again on every call. The client now asks the kernel with one bind() on a probe name before it starts anything, treats PermissionError from socket(), connect() or the spawn as a refusal, and remembers it for the life of the process, so later embedders go straight to their own model with no system call at all. `embed-daemon status` says why nothing runs, `embed-daemon run` says why it cannot listen, and status and stop no longer reach for a uid on Windows or raise when the session directory cannot be made. The daemon tests that need a real listening socket now skip with the reason on a machine that refuses bind(), and the refusal paths are tested everywhere by refusing bind() and connect() in-process. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed_daemon.py | 136 ++++++++++++++++++++++++--- tests/test_embed_daemon.py | 161 ++++++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+), 11 deletions(-) diff --git a/src/compartment/embed_daemon.py b/src/compartment/embed_daemon.py index ad06edf..31ea79c 100644 --- a/src/compartment/embed_daemon.py +++ b/src/compartment/embed_daemon.py @@ -31,6 +31,9 @@ Windows has no AF_UNIX in Python, so there every process keeps its own model as it always did; `enabled()` says no and `get_embedder` never comes here. +Neither does a process the operating system refuses Unix domain sockets, as +an agent's sandbox does: after the first refusal it keeps its own model too, +without starting a daemon that could never listen (see `_refused`). """ from __future__ import annotations @@ -115,6 +118,31 @@ class Unavailable(RuntimeError): """The shared daemon cannot be used; the caller embeds locally.""" +#: Why the operating system will not let this process use a Unix domain +#: socket at all, once it has said so; None until then. +#: +#: An agent's own shell is often sandboxed: on macOS, Claude Code and Codex +#: run their shell commands under a Seatbelt profile that refuses bind() on +#: an AF_UNIX path with EPERM, and other sandboxes refuse connect() or +#: starting a process as well. There, every `compartment search` and `store` an agent ran started a +#: daemon that could only fail to bind and exit, and waited for it to do so: +#: a fresh interpreter, numpy and onnxruntime imported, a quarter of a +#: second or more, at every command, and again at every reconnect in a +#: server, before the in-process model it was always going to use. A +#: sandbox does not lift while a process runs, so the answer is kept for +#: the life of the process: the first refusal costs one failed system call, +#: and every later embedder goes straight to its own model. +_refused: str | None = None + + +def _note_refused(what: str, exc: OSError) -> None: + global _refused + if _refused is None: + _refused = (f"this process may not {what} ({exc.strerror or exc})," + " as inside an agent's sandbox: it embeds with a model " + "of its own") + + # --------------------------------------------------------------- where and if def supported() -> bool: @@ -272,7 +300,14 @@ def run(self) -> int: srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: srv.bind(str(self.path)) - except OSError: + except OSError as exc: + # Said for `compartment embed-daemon run`; a daemon a client + # started has no stderr, and the client goes on without it. + print(f"compartment: the embedding daemon cannot listen at " + f"{self.path}: {exc.strerror or exc}", file=sys.stderr) + srv.close() + fd, self._lock_fd = self._lock_fd, None + os.close(fd) # releases the flock return 1 os.chmod(self.path, 0o600) self._inode = os.stat(self.path).st_ino @@ -521,10 +556,23 @@ def run(path: Path | None = None, idle: int | None = None) -> int: def _connect(path: Path, timeout: float = 5.0) -> socket.socket | None: if not _owned_socket(path): return None - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except PermissionError as exc: + _note_refused("create a Unix domain socket", exc) + return None + except OSError: + return None s.settimeout(timeout) try: s.connect(str(path)) + except PermissionError as exc: + # The socket is this user's own (checked above) in this user's own + # directory, so a refusal is not its permissions: it is a sandbox + # that will not let this process reach any socket file. + s.close() + _note_refused("connect to a Unix domain socket", exc) + return None except OSError: s.close() return None @@ -535,6 +583,46 @@ def _connect(path: Path, timeout: float = 5.0) -> socket.socket | None: return s +def _bind_refused(path: Path) -> bool: + """Whether the operating system refuses this process a listening Unix + domain socket beside `path`, which is what the daemon it would start + needs: the daemon inherits whatever sandbox this process is in. + + Asked with one bind() on a name of its own, which is removed again, + instead of with a daemon: a refusal is EPERM or EACCES at once, while a + daemon has to be started, import numpy and onnxruntime, and exit before + anyone learns the same thing. Any other error (a path too long, a name + in use) is left for the daemon to meet, as before.""" + if _refused is not None: + return True + probe = str(path) + ".p" + try: + os.unlink(probe) # one a killed probe left behind + except OSError: + pass + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + except PermissionError as exc: + _note_refused("create a Unix domain socket", exc) + return True + except OSError: + return False + try: + s.bind(probe) + except PermissionError as exc: + _note_refused("bind a Unix domain socket", exc) + return True + except OSError: + return False + finally: + s.close() + try: + os.unlink(probe) + except OSError: + pass + return False + + def _spawn(path: Path, idle: int) -> subprocess.Popen: """Start the daemon detached: its own session, so neither the MCP client's process-group cleanup nor a closing terminal takes it down, @@ -570,9 +658,14 @@ def connect_or_spawn(path: Path, idle: int, *, spawn: bool = True, daemon, the others wait on the lock and connect to it. Whoever spawned it polls until the socket accepts, which is a few hundred milliseconds - the daemon binds first and loads the model on the first request. + + Where the operating system refuses this process Unix domain sockets + (see `_refused`), nothing is started and None comes back at once. """ + if _refused is not None: + return None s = _connect(path) - if s is not None or not spawn: + if s is not None or not spawn or _refused is not None: return s path.parent.mkdir(parents=True, exist_ok=True) fd = os.open(str(path) + ".spawn", os.O_RDWR | os.O_CREAT, 0o600) @@ -587,10 +680,17 @@ def connect_or_spawn(path: Path, idle: int, *, spawn: bool = True, return None time.sleep(0.05) s = _connect(path) - if s is not None: + if s is not None or _refused is not None: return s + if _bind_refused(path): + return None try: child = _spawn(path, idle) + except PermissionError as exc: + # A sandbox that forbids starting processes forbids it every + # time: one try per process, not one per vault opened. + _note_refused("start the embedding daemon", exc) + return None except OSError: return None while time.monotonic() < deadline: @@ -826,14 +926,25 @@ def embed_record(self, text: str) -> np.ndarray: # ------------------------------------------------------- status, stop, main def status(path: Path | None = None) -> dict: - """What the daemon at `path` reports, or that nothing is running.""" - path = path or socket_path() + """What the daemon at `path` reports, or that nothing is running, and + when nothing can run for this process, why.""" if not supported(): - return {"running": False, "socket": str(path), - "reason": "not supported on this platform"} + # Before socket_path(): its fallback names the file after the uid, + # which Windows does not have. + return {"running": False, "socket": None, + "reason": "not supported on this platform: every process " + "embeds with a model of its own"} + try: + path = path or socket_path() + except OSError as exc: + return {"running": False, "socket": None, + "reason": f"no session directory to hold its socket ({exc})"} s = _connect(path, timeout=2.0) if s is None: - return {"running": False, "socket": str(path)} + out = {"running": False, "socket": str(path)} + if _bind_refused(path): + out["reason"] = _refused + return out try: w, r = s.makefile("wb"), s.makefile("rb") _write(w, {"op": "status"}) @@ -853,9 +964,12 @@ def stop(path: Path | None = None, wait: float = 5.0) -> dict: gets SIGTERM. Stale files a crashed daemon left are removed so the next client starts cleanly. """ - path = path or socket_path() if not supported(): - return {"stopped": False, "socket": str(path)} + return {"stopped": False, "socket": None} + try: + path = path or socket_path() + except OSError: + return {"stopped": False, "socket": None} pid = None s = _connect(path, timeout=2.0) if s is not None: diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index 79eaaf5..ea3800f 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -6,6 +6,7 @@ away on its own, and that a vault never notices when it is missing. """ import contextlib +import errno import json import os import shutil @@ -33,6 +34,45 @@ reason="the daemon needs Unix domain sockets") +def _bind_refused_here() -> str | None: + """Why this machine refuses this process a listening AF_UNIX socket, or + None. An agent's sandbox does (macOS Seatbelt as Claude Code and Codex + apply it to their shell commands refuses bind() with EPERM), and a test + that needs a real daemon cannot run there; what such a sandbox gets + instead, a model of its own at once, is tested below on every machine + by refusing bind() in-process.""" + if not embed_daemon.supported(): + return None + d = tempfile.mkdtemp(prefix="cmpt-") + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind(os.path.join(d, "probe")) + except PermissionError as exc: + return str(exc) + except OSError: + return None + finally: + s.close() + shutil.rmtree(d, ignore_errors=True) + return None + + +_BIND_REFUSED = _bind_refused_here() + +needs_bind = pytest.mark.skipif( + _BIND_REFUSED is not None, + reason=f"this machine refuses AF_UNIX bind() ({_BIND_REFUSED}), as an " + "agent's sandbox does, so no daemon can run here; the fallback " + "such a sandbox gets is tested by test_a_sandbox_*") + + +@pytest.fixture(autouse=True) +def no_refusal_carried_over(monkeypatch): + """embed_daemon remembers a refusal for the life of the process; each + test starts as a fresh process would.""" + monkeypatch.setattr(embed_daemon, "_refused", None) + + @pytest.fixture() def daemon_env(monkeypatch): """A daemon of this test's own, on a short socket path, gone afterwards. @@ -64,6 +104,7 @@ def _wait_until(pred, timeout=15.0): # ------------------------------------------------------------- same vectors -- +@needs_bind def test_the_daemon_answers_with_the_in_process_vectors(daemon_env, local): r = RemoteEmbedder.connect() assert r is not None and r.shared and r.daemon_pid @@ -81,6 +122,7 @@ def test_the_daemon_answers_with_the_in_process_vectors(daemon_env, local): # ------------------------------------------------------------ one of them -- +@needs_bind def test_several_clients_share_one_daemon(daemon_env): clients = [RemoteEmbedder.connect() for _ in range(4)] assert all(c is not None for c in clients) @@ -92,6 +134,7 @@ def test_several_clients_share_one_daemon(daemon_env): assert _wait_until(lambda: embed_daemon.status(daemon_env).get("clients") == 0) +@needs_bind def test_clients_arriving_together_start_exactly_one_daemon(daemon_env): """Six MCP servers start within the same second when a client comes up; the spawn lock makes one of them start the daemon and the rest wait.""" @@ -110,6 +153,7 @@ def go(): assert all(r.embed_query("x").shape == (r.dim,) for r in results) +@needs_bind def test_the_daemon_starts_fast_and_loads_the_model_on_demand(daemon_env): t0 = time.monotonic() r = RemoteEmbedder.connect() @@ -123,6 +167,7 @@ def test_the_daemon_starts_fast_and_loads_the_model_on_demand(daemon_env): # ------------------------------------------------------------- lifecycle -- +@needs_bind def test_the_daemon_exits_when_nobody_is_connected(daemon_env): r = RemoteEmbedder.connect() assert r is not None @@ -131,6 +176,7 @@ def test_the_daemon_exits_when_nobody_is_connected(daemon_env): assert _wait_until(lambda: not daemon_env.exists(), 5) +@needs_bind def test_a_client_arriving_as_an_idle_daemon_exits_starts_the_next_one( daemon_env, monkeypatch): """An idle daemon takes its socket file away first, so that a client @@ -151,6 +197,7 @@ def test_a_client_arriving_as_an_idle_daemon_exits_starts_the_next_one( assert late is not None and late.shared and late.daemon_pid != pid +@needs_bind def test_a_client_survives_the_daemon_being_killed(daemon_env): r = RemoteEmbedder.connect() first = r.daemon_pid @@ -172,6 +219,7 @@ def _returns_within(seconds, fn): return out[0] +@needs_bind def test_a_daemon_that_stops_answering_does_not_hang_its_clients( daemon_env, monkeypatch, local): """A daemon that is alive but not serving (suspended with Ctrl-Z, stuck @@ -196,6 +244,7 @@ def test_a_daemon_that_stops_answering_does_not_hang_its_clients( assert isinstance(fresh, Embedder) +@needs_bind def test_a_text_the_model_rejects_is_an_error_not_a_lost_daemon(daemon_env): """A text the tokenizer cannot encode, such as one with a lone surrogate from a cut emoji, comes back from the daemon as an error reply. The @@ -213,6 +262,7 @@ def test_a_text_the_model_rejects_is_an_error_not_a_lost_daemon(daemon_env): assert r.shared +@needs_bind def test_a_long_list_of_passages_is_embedded_once_through_the_daemon( monkeypatch): """A reply carries 2 KB a text, so from 32,768 texts on it was longer @@ -258,6 +308,7 @@ def embed_passages(self, texts, batch=64): shutil.rmtree(d, ignore_errors=True) +@needs_bind def test_a_dead_socket_file_is_replaced(daemon_env): daemon_env.parent.mkdir(parents=True, exist_ok=True) s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -268,6 +319,7 @@ def test_a_dead_socket_file_is_replaced(daemon_env): assert r is not None and r.embed_query("x").shape == (r.dim,) +@needs_bind def test_stop_ends_it_and_status_says_so(daemon_env): r = RemoteEmbedder.connect() pid = r.daemon_pid @@ -278,6 +330,7 @@ def test_stop_ends_it_and_status_says_so(daemon_env): assert r.embed_query("after stop").shape == (r.dim,) # starts a new one +@needs_bind def test_status_and_stop_from_the_command_line(daemon_env, capsys): r = RemoteEmbedder.connect() assert r is not None @@ -292,6 +345,7 @@ def test_status_and_stop_from_the_command_line(daemon_env, capsys): # ------------------------------------------------------------ versions -- +@needs_bind def test_an_older_client_leaves_a_newer_daemon_alone(daemon_env, monkeypatch): keep = RemoteEmbedder.connect() # the "newer" daemon pid = keep.daemon_pid @@ -300,6 +354,7 @@ def test_an_older_client_leaves_a_newer_daemon_alone(daemon_env, monkeypatch): assert embed_daemon.status(daemon_env)["pid"] == pid +@needs_bind def test_a_newer_client_replaces_an_older_daemon(daemon_env, monkeypatch): old = RemoteEmbedder.connect() pid = old.daemon_pid @@ -340,6 +395,7 @@ def cleanup(self): """ +@needs_bind def test_a_newer_client_takes_over_from_an_older_daemon_at_once(daemon_env): """Every upgrade meets the daemon the older install left running. The newer client told it to go and started its own as soon as its socket @@ -389,6 +445,7 @@ def _hello(path): s.close() +@needs_bind def test_the_daemon_a_guarded_process_starts_is_guarded(daemon_env, guarded_by_the_flag): """The flag guarded the server and the CLI, and the daemon they started, @@ -432,6 +489,7 @@ def serve(): srv.close() +@needs_bind def test_a_guarded_client_keeps_its_own_model_beside_an_unguarded_daemon( daemon_env, guarded_by_the_flag): """The daemon is shared, so a guarded server could send its texts to a @@ -464,6 +522,7 @@ def test_disabled_means_a_local_encoder(monkeypatch): assert isinstance(get_embedder(), Embedder) +@needs_bind def test_enabled_means_the_shared_one(daemon_env): e = get_embedder() assert isinstance(e, RemoteEmbedder) and e.shared @@ -508,6 +567,7 @@ def test_the_peer_credential_is_this_user(): # ------------------------------------------------------------- the vault -- +@needs_bind def test_a_vault_searches_identically_through_the_daemon(daemon_env, tmp_path, monkeypatch): vp = str(tmp_path / "shared.vault") v = Vault.create(vp, PASS, creator="test") @@ -575,3 +635,104 @@ def test_a_home_that_cannot_hold_the_socket_embeds_with_its_own_model( # A model that is not installed is still the error that names the remedy. with pytest.raises(ModelError, match="download-model"): get_embedder("multilingual-e5-small-int8") + + +# ------------------------------------------------------- inside a sandbox -- +class _RefusingSocket(socket.socket): + """A socket the way an agent's sandbox hands them out: it can be made, + but bind() and connect() on a path are refused with EPERM.""" + + def bind(self, address): + if self.family == socket.AF_UNIX: + raise PermissionError(errno.EPERM, os.strerror(errno.EPERM)) + return super().bind(address) + + def connect(self, address): + if self.family == socket.AF_UNIX: + raise PermissionError(errno.EPERM, os.strerror(errno.EPERM)) + return super().connect(address) + + +@pytest.fixture() +def sandboxed(monkeypatch): + """Unix domain sockets refused as a sandbox refuses them, the daemon + switched on, and every attempt to start one recorded.""" + d = tempfile.mkdtemp(prefix="cmpt-") + monkeypatch.setenv("COMPARTMENT_EMBED_DAEMON", "1") + monkeypatch.setenv("COMPARTMENT_EMBED_SOCKET", str(Path(d) / "embed.sock")) + monkeypatch.setattr(socket, "socket", _RefusingSocket) + spawned = [] + + def spawn(path, idle): + spawned.append(path) + raise AssertionError("started a daemon that could never listen") + monkeypatch.setattr(embed_daemon, "_spawn", spawn) + yield Path(d), spawned + shutil.rmtree(d, ignore_errors=True) + + +def test_a_sandbox_that_refuses_binding_gets_a_local_model_at_once(sandboxed): + """Claude Code and Codex run an agent's shell commands under a sandbox + that refuses bind() on a Unix domain socket. Every `compartment search` + an agent ran there started a daemon, waited for it to import numpy and + onnxruntime and fail, and only then used a model of its own: a wasted + process and a quarter of a second or more per command, and again at + every reconnect in a server.""" + d, spawned = sandboxed + for _ in range(3): + t0 = time.monotonic() + assert isinstance(get_embedder(), Embedder) + assert time.monotonic() - t0 < 5 + assert spawned == [] + assert "bind" in embed_daemon._refused + assert not any(p.name.endswith(".p") for p in d.iterdir()) + st = embed_daemon.status() + assert st["running"] is False and "sandbox" in st["reason"] + + +def test_a_sandbox_that_refuses_connecting_starts_no_daemon(sandboxed, + monkeypatch): + """A daemon an unsandboxed server started is listening, and the sandbox + refuses this process the connection: starting a second daemon for the + same socket cannot help, and used to be tried on every call.""" + d, spawned = sandboxed + monkeypatch.setattr(embed_daemon, "_owned_socket", lambda p: True) + assert RemoteEmbedder.connect() is None + assert spawned == [] + assert "connect" in embed_daemon._refused + assert isinstance(get_embedder(), Embedder) + assert spawned == [] + + +def test_a_sandbox_that_refuses_starting_processes_is_asked_once(monkeypatch): + """Some sandboxes refuse to start a process at all; every vault opened + asked again.""" + d = tempfile.mkdtemp(prefix="cmpt-") + monkeypatch.setenv("COMPARTMENT_EMBED_DAEMON", "1") + monkeypatch.setenv("COMPARTMENT_EMBED_SOCKET", str(Path(d) / "embed.sock")) + monkeypatch.setattr(embed_daemon, "_bind_refused", lambda path: False) + tries = [] + + def popen(*a, **k): + tries.append(a) + raise PermissionError(errno.EPERM, os.strerror(errno.EPERM)) + monkeypatch.setattr(embed_daemon.subprocess, "Popen", popen) + try: + for _ in range(3): + assert isinstance(get_embedder(), Embedder) + assert len(tries) == 1 + finally: + shutil.rmtree(d, ignore_errors=True) + + +def test_a_daemon_run_where_it_cannot_listen_says_why(sandboxed, monkeypatch, + capsys): + """`compartment embed-daemon run` in a sandbox exited 1 and said + nothing.""" + d, _ = sandboxed + with pytest.raises(SystemExit) as e: + cli.main(["embed-daemon", "run", "--socket", str(d / "embed.sock"), + "--idle", "1"]) + assert e.value.code == 1 + err = capsys.readouterr().err + assert "cannot listen" in err and os.strerror(errno.EPERM) in err From 0ac9743c958566c94f7d6daedce1da3b5f8b434f Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:40:00 -0700 Subject: [PATCH 016/135] The macOS installer runs only on the Macs its app can start on, and an Intel build gets its own Compartment--intel name The .pkg's distribution file said hostArchitectures="arm64,x86_64" for every build, and every build is single-architecture: the embedded standalone Python is one or the other (python-build-standalone has no universal2 build), and so is every wheel chosen for it, onnxruntime's included (its last universal2 wheel was 1.20.1). So an Intel Mac installed the Apple silicon app, which could never launch, and being a menu bar app with no window it said nothing. The architecture is now read from the built app's executable, and the installer allows only that one. An x86_64 build, which choosing the wheels for the declared minimum (two commits back) makes possible on an Intel runner with Python 3.12 or 3.13 and onnxruntime 1.23.2, is written as Compartment--intel.pkg and .dmg, so a release can carry both without the second upload replacing the first; the Apple silicon names stay Compartment-.pkg and .dmg, which the README and every earlier release link. The .dmg's READ ME names the .pkg of its own architecture. Co-Authored-By: Claude Opus 5.5 --- tests/test_macos_bundle.py | 40 ++++++++++++++++++++++++++++++++++++-- tools/build_macos_app.py | 33 +++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/tests/test_macos_bundle.py b/tests/test_macos_bundle.py index b4f8b0d..0b24e1b 100644 --- a/tests/test_macos_bundle.py +++ b/tests/test_macos_bundle.py @@ -357,8 +357,8 @@ def test_the_installer_accepts_only_the_macos_the_app_declares(tmp_path, monkeypatch): import plistlib b = _builder() - app = tmp_path / "Compartment.app" - (app / "Contents").mkdir(parents=True) + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("arm64", (14, 0))}) with open(app / "Contents" / "Info.plist", "wb") as fh: plistlib.dump({"LSMinimumSystemVersion": "14.0"}, fh) monkeypatch.setattr(b, "BUILD", tmp_path / "build") @@ -436,3 +436,39 @@ def test_the_build_makes_its_scripts_relocatable_and_checks_them(): assert body.index("_relocate_scripts(runtime)") < body.index("compileall") assert body.index("_verify_console_scripts(app)") \ < body.index('_run("codesign"') + + +# ------------------------------- an Intel build is an installer of its own + +def test_release_names_keep_apple_silicon_where_it_was_and_mark_intel(): + b = _builder() + assert b.artifact_name("4.13.0", "arm64", "pkg") == "Compartment-4.13.0.pkg" + assert b.artifact_name("4.13.0", "arm64", "dmg") == "Compartment-4.13.0.dmg" + assert b.artifact_name("4.13.0", "x86_64", "pkg") == \ + "Compartment-4.13.0-intel.pkg" + + +@pytest.mark.parametrize("arch,suffix", [("arm64", ""), + ("x86_64", "-intel")]) +def test_the_installer_runs_only_on_the_macs_the_app_can_start_on( + tmp_path, monkeypatch, arch, suffix): + """It said arm64,x86_64 whatever was inside, so an Intel Mac installed + the Apple silicon app, which could never launch.""" + import plistlib + b = _builder() + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho(arch, (13, 0))}) + with open(app / "Contents" / "Info.plist", "wb") as fh: + plistlib.dump({"LSMinimumSystemVersion": "13.0"}, fh) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + ran = [] + monkeypatch.setattr(b, "_run", lambda *a, **k: ran.append(a)) + out = b.build_pkg(app) + dist = (tmp_path / "build" / "distribution.xml").read_text() + assert f'hostArchitectures="{arch}"' in dist + version = b._version() + assert out.name == f"Compartment-{version}{suffix}.pkg" + dmg = b.build_dmg(app) + assert dmg.name == f"Compartment-{version}{suffix}.dmg" + readme = (tmp_path / "build" / "dmg" / "READ ME FIRST.txt").read_text() + assert f"Compartment-{version}{suffix}.pkg" in readme diff --git a/tools/build_macos_app.py b/tools/build_macos_app.py index 3d05f2d..ae61657 100644 --- a/tools/build_macos_app.py +++ b/tools/build_macos_app.py @@ -761,6 +761,24 @@ def _login_plist(app: pathlib.Path) -> dict: } +def _app_arch(app: pathlib.Path) -> str: + """The architecture of the built app, read from its executable.""" + slices = _macho_slices(app / "Contents" / "MacOS" / APP_NAME) + if not slices: + raise SystemExit(f"error: {app} has no executable to read") + return ",".join(sorted(slices)) + + +def artifact_name(version: str, arch: str, ext: str) -> str: + """build/: Compartment-.pkg for Apple silicon, the build + the README and every release so far have linked, and + Compartment--intel.pkg for an Intel build. The two cannot share + a name: a release carries both, and whichever was uploaded second would + replace the first.""" + suffix = "" if arch == "arm64" else "-intel" + return f"{APP_NAME}-{version}{suffix}.{ext}" + + def _app_min_macos(app: pathlib.Path) -> str: """The minimum macOS the built app declares. The installers take it from the app rather than from a constant of their own, so a .pkg can never @@ -774,6 +792,11 @@ def build_pkg(app: pathlib.Path) -> pathlib.Path: """A .pkg whose second, optional component starts it at login.""" version = _version() min_macos = _app_min_macos(app) + # The installer agrees to run only on the Macs this build can start on. + # It used to say arm64,x86_64 whatever was inside, so an Intel Mac + # installed the Apple silicon app - which could never launch, and being + # a menu bar app said nothing about it. + arch = _app_arch(app) staging = BUILD / "pkgroot" login_root = BUILD / "pkgroot-login" for d in (staging, login_root): @@ -846,7 +869,7 @@ def _scripts(name: str, body: str) -> pathlib.Path: dist.write_text(f""" Compartment - + @@ -866,7 +889,7 @@ def _scripts(name: str, body: str) -> pathlib.Path: """, encoding="utf-8") - out = BUILD / f"{APP_NAME}-{version}.pkg" + out = BUILD / artifact_name(version, arch, "pkg") _run("productbuild", "--distribution", dist, "--package-path", BUILD, out) print(f"built {out}") return out @@ -874,6 +897,7 @@ def _scripts(name: str, body: str) -> pathlib.Path: def build_dmg(app: pathlib.Path) -> pathlib.Path: version = _version() + arch = _app_arch(app) stage = BUILD / "dmg" if stage.exists(): shutil.rmtree(stage) @@ -888,7 +912,8 @@ def build_dmg(app: pathlib.Path) -> pathlib.Path: (stage / "READ ME FIRST.txt").write_text( "Compartment " + version + "\n" "=================\n\n" - "Easiest install: use Compartment-" + version + ".pkg from the release page\n" + "Easiest install: use " + artifact_name(version, arch, "pkg") + + " from the release page\n" "instead of this disk image. It sets everything up in one click and\n" "skips the warning below entirely.\n\n" "If you would rather drag the app across:\n\n" @@ -903,7 +928,7 @@ def build_dmg(app: pathlib.Path) -> pathlib.Path: "just open Compartment.app again: it will show its panel in a window.\n\n" "Source, and every other way to install:\n" "https://github.com/MaxFreedomPollard/Compartment\n", encoding="utf-8") - out = BUILD / f"{APP_NAME}-{version}.dmg" + out = BUILD / artifact_name(version, arch, "dmg") out.unlink(missing_ok=True) _run("hdiutil", "create", "-volname", f"{APP_NAME} {version}", "-srcfolder", stage, "-ov", "-format", "UDZO", out) From 0df2e8e826f5888c6831174716591eb21fce86d2 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:40:29 -0700 Subject: [PATCH 017/135] On OpenBSD the embedding daemon reads the connecting user's uid from OpenBSD's own peer-credential layout, so it no longer turns its own user away The daemon accepts a connection only from its own uid, asked through SO_PEERCRED where the platform has it. OpenBSD has SO_PEERCRED, but its struct sockpeercred is { uid; gid; pid }, while the code unpacked Linux's struct ucred { pid; uid; gid }. On OpenBSD the "uid" it compared was the peer's gid: every client of a user whose primary group id differs from their user id was hung up on, so every agent there fell back to loading its own model, and a peer whose gid happened to equal the daemon's uid would have been let in on that basis. The uid is now read from the field OpenBSD puts it in; Linux, musl and Android keep struct ucred. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed_daemon.py | 11 ++++++++++- tests/test_embed_daemon.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/compartment/embed_daemon.py b/src/compartment/embed_daemon.py index 31ea79c..e83cf93 100644 --- a/src/compartment/embed_daemon.py +++ b/src/compartment/embed_daemon.py @@ -244,7 +244,16 @@ def _peer_uid(conn: socket.socket) -> int | None: if hasattr(socket, "SO_PEERCRED"): raw = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("iII")) - _pid, uid, _gid = struct.unpack("iII", raw) + if sys.platform.startswith("openbsd"): + # OpenBSD has SO_PEERCRED too, but its struct sockpeercred + # is { uid_t uid; gid_t gid; pid_t pid; }, not Linux's + # struct ucred { pid; uid; gid; }. Read as Linux's, the + # "uid" was the peer's gid, so the daemon turned away every + # client of a user whose group id is not their user id, and + # would have let in one whose group id happened to be ours. + uid, _gid, _pid = struct.unpack("IIi", raw) + else: + _pid, uid, _gid = struct.unpack("iII", raw) return uid if sys.platform == "darwin" or "bsd" in sys.platform: # struct xucred { u_int cr_version; uid_t cr_uid; short diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index ea3800f..565c07e 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -12,6 +12,7 @@ import shutil import signal import socket +import struct import subprocess import sys import tempfile @@ -566,6 +567,34 @@ def test_the_peer_credential_is_this_user(): b.close() +class _PeerCred: + """A connection whose SO_PEERCRED answer is `raw`.""" + + def __init__(self, raw): + self.raw = raw + + def getsockopt(self, level, option, size): + assert size >= len(self.raw) + return self.raw + + +@pytest.mark.parametrize("platform, raw", [ + # Linux, musl and Android: struct ucred { pid_t pid; uid_t uid; gid_t gid; } + ("linux", struct.pack("iII", 4242, 1000, 1001)), + # OpenBSD: struct sockpeercred { uid_t uid; gid_t gid; pid_t pid; } + ("openbsd7", struct.pack("IIi", 1000, 1001, 4242)), +], ids=["linux", "openbsd"]) +def test_the_peer_uid_is_read_from_the_platforms_own_struct(monkeypatch, + platform, raw): + """OpenBSD answers SO_PEERCRED with its fields in another order, and + the daemon read the peer's group id as its user id: every client of a + user whose gid is not their uid was turned away, and the daemon was + useless there.""" + monkeypatch.setattr(socket, "SO_PEERCRED", 17, raising=False) + monkeypatch.setattr(embed_daemon.sys, "platform", platform) + assert embed_daemon._peer_uid(_PeerCred(raw)) == 1000 + + # ------------------------------------------------------------- the vault -- @needs_bind def test_a_vault_searches_identically_through_the_daemon(daemon_env, tmp_path, monkeypatch): From 2eeb17657acddb44aa6584796a8d165c3a9ab125 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:40:32 -0700 Subject: [PATCH 018/135] `compartment init` on a Python whose SQLite lacks FTS5 says which interpreter is the problem and where a working one comes from Some Python builds have a SQLite without FTS5: Python compiled against the SQLite of CentOS/RHEL 7 or Amazon Linux 2, and some minimal or embedded builds. The store had a friendly refusal for that, but it ran after the schema was created, and creating the schema's full-text table is exactly where such a SQLite first fails. A new vault therefore died on a bare "sqlite3.OperationalError: no such module: fts5", and the message was only ever reached when an existing vault was opened. The schema, the column migration and the probe now run under one handler that turns a missing fts5 module into the StoreError, and the message names the SQLite version and the interpreter in use, lists where a Python with FTS5 comes from, and says how to reinstall with it. There is no degraded mode to fall back to: the full-text table is part of the vault image and keyword search is half of every recall. Other schema errors are untouched. Co-Authored-By: Claude Opus 5.5 --- src/compartment/store.py | 48 +++++++++---- tests/test_storage_sqlite_builds.py | 106 ++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 tests/test_storage_sqlite_builds.py diff --git a/src/compartment/store.py b/src/compartment/store.py index c16c6ab..7efdbb0 100644 --- a/src/compartment/store.py +++ b/src/compartment/store.py @@ -10,6 +10,7 @@ import json import math import sqlite3 +import sys import time import uuid @@ -112,6 +113,27 @@ class StoreError(CryptoError): pass +def _no_fts5_error() -> StoreError: + """The refusal for a Python whose SQLite was built without FTS5. + + Rare, but real: Python compiled against the SQLite of an older + enterprise distribution (CentOS/RHEL 7, Amazon Linux 2) and some + minimal or embedded builds. Keyword search is half of every recall and + the full-text table is part of the vault's image, so there is no + degraded mode to fall back to; what the user needs is to know which + interpreter is the problem and where a working one comes from.""" + return StoreError( + f"This Python's SQLite ({sqlite3.sqlite_version}, used by " + f"{sys.executable or 'this interpreter'}) was built without FTS5, " + "the full-text index Compartment's keyword search and vault format " + "require. Run Compartment under a Python whose SQLite has it: the " + "python.org installers, Homebrew, conda, `uv python install 3.12`, " + "and the python3 of any current Linux distribution all do. " + "Reinstall Compartment with that interpreter ( -m pip " + "install compartment, or uv tool install --python 3.12 " + "compartment); your vault is untouched.") + + class Store: def __init__(self, image: bytes | None = None): # Autocommit: the DB lives only in RAM - durability comes from the @@ -154,21 +176,23 @@ def __init__(self, image: bytes | None = None): # GROUP BY over every record. Dropped by whatever adds a record, # removes one or moves one to another namespace. self._ns_cache: list[dict] | None = None - if image is not None: - self.conn.deserialize(image) - # idempotent schema upgrade: vaults sealed by older versions gain - # any new tables (e.g. relations) the moment they are opened - self.conn.executescript(SCHEMA) - else: - self.conn.executescript(SCHEMA) - self._migrate_columns() try: + if image is not None: + self.conn.deserialize(image) + # Idempotent schema upgrade: vaults sealed by older versions gain + # any new tables (e.g. relations) the moment they are opened. + # Creating the full-text table is also where a SQLite without + # FTS5 first fails, for a new vault: the friendly refusal below + # used to come after it and was reached only when an existing + # vault was opened, so `compartment init` died on a bare + # "no such module: fts5". + self.conn.executescript(SCHEMA) + self._migrate_columns() self.conn.execute("SELECT count(*) FROM fts") except sqlite3.OperationalError as exc: - raise StoreError( - "This Python's SQLite lacks FTS5, which Compartment requires " - "for hybrid search. Install a Python built with full SQLite." - ) from exc + if "fts5" in str(exc).lower(): + raise _no_fts5_error() from exc + raise # New COLUMNS on an existing table, unlike new tables, do not arrive for # free: `CREATE TABLE IF NOT EXISTS` is a no-op the moment the table diff --git a/tests/test_storage_sqlite_builds.py b/tests/test_storage_sqlite_builds.py new file mode 100644 index 0000000..de71742 --- /dev/null +++ b/tests/test_storage_sqlite_builds.py @@ -0,0 +1,106 @@ +"""Compartment on a Python whose SQLite was built without FTS5. + +Python compiled against an older enterprise distribution's SQLite (CentOS +and RHEL 7, Amazon Linux 2) and some minimal builds have no FTS5. There is no +degraded mode to offer - the full-text table is part of the vault image - so +the requirement is a refusal that names the interpreter and the fix, on the +first command anyone runs, which is `compartment init`. +""" +import sqlite3 + +import pytest + +from compartment import store as store_mod +from compartment.crypto import CryptoError +from compartment.store import Store, StoreError +from compartment.vault import Vault + +from conftest import PASS + + +class _NoFts5: + """A connection to a SQLite with no fts5 module: everything else works, + and anything that needs the module fails the way that SQLite fails.""" + + def __init__(self, conn): + object.__setattr__(self, "_conn", conn) + + @staticmethod + def _needs_fts5(sql: str) -> bool: + low = sql.lower() + return "fts5" in low or "from fts" in low or "into fts" in low + + def execute(self, sql, *args): + if self._needs_fts5(sql): + raise sqlite3.OperationalError("no such module: fts5") + return self._conn.execute(sql, *args) + + def executescript(self, sql): + if self._needs_fts5(sql): + raise sqlite3.OperationalError("no such module: fts5") + return self._conn.executescript(sql) + + def __getattr__(self, name): + return getattr(self._conn, name) + + def __setattr__(self, name, value): + setattr(self._conn, name, value) + + +def _store_connects_through(monkeypatch, proxy) -> None: + """The store's sqlite3, handing out connections wrapped in `proxy`.""" + real = sqlite3.connect + + class _Module: + Row = sqlite3.Row + OperationalError = sqlite3.OperationalError + sqlite_version = sqlite3.sqlite_version + + @staticmethod + def connect(*args, **kwargs): + return proxy(real(*args, **kwargs)) + + monkeypatch.setattr(store_mod, "sqlite3", _Module) + + +@pytest.fixture() +def sqlite_without_fts5(monkeypatch): + _store_connects_through(monkeypatch, _NoFts5) + + +def test_a_new_vault_is_refused_with_the_fix_not_a_bare_sqlite_error( + sqlite_without_fts5): + """The friendly refusal sat after the schema was created, and creating + the schema is where such a SQLite first fails, so a new vault died on + "no such module: fts5" and the message was only ever reached when an + existing vault was opened.""" + with pytest.raises(StoreError) as caught: + Store() + msg = str(caught.value) + assert "FTS5" in msg and sqlite3.sqlite_version in msg + assert "python.org" in msg, "it says where a working Python comes from" + + +def test_init_reports_it_as_a_vault_error(sqlite_without_fts5, vault_path): + """What `compartment init` and the panel's Create button catch.""" + with pytest.raises(CryptoError, match="FTS5"): + Vault.create(vault_path, PASS, creator="t") + + +def test_an_existing_vault_is_refused_the_same_way(monkeypatch): + image = Store().serialize() + _store_connects_through(monkeypatch, _NoFts5) + with pytest.raises(StoreError, match="FTS5"): + Store(image) + + +def test_other_schema_errors_are_not_called_a_missing_fts5(monkeypatch): + """Only the missing module earns that message; anything else is still + the error it was.""" + class _Broken(_NoFts5): + def executescript(self, sql): + raise sqlite3.OperationalError("database is locked") + + _store_connects_through(monkeypatch, _Broken) + with pytest.raises(sqlite3.OperationalError, match="locked"): + Store() From 1575b01aa19abcf92c00bc22f658b2ffa3e105fa Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:40:40 -0700 Subject: [PATCH 019/135] Unlock refuses OpenBSD's on-disk shared memory, and every "no place for the per-boot secret" error names what to use instead The boot-session credential is safe only because its per-boot secret is in a holder no disk backs. OpenBSD's libc implements shm_open as an ordinary file, /tmp/.shm (lib/libc/gen/shm_open.c, read on GitHub), and /tmp is a disk partition on a default install. There the secret sat on the same disk as the session file, and after a power cut it stayed there until the next boot pruned /tmp: a copy of that disk opened the credential, the exact case session.py says it protects against. Storing now refuses on OpenBSD the way it does where shm_open is missing, before anything is written. FreeBSD (kernel object) and NetBSD (libc requires /var/shm to be tmpfs and fails ENOTSUP otherwise, lib/librt/shm.c) are unchanged. There is no onnxruntime build for OpenBSD today, so this closes the hole before anyone can fall into it. Where shm_open exists but fails - glibc's /dev/shm missing in a chroot or build container (ENOENT), read-only or full (EROFS, ENOSPC), refused by a sandbox (EACCES/EPERM) - unlock said only "Cannot create this boot's secret (/cmpt...): [Errno 2] No such file or directory". It now says that POSIX shared memory is not usable here and why it usually is not, and names COMPARTMENT_PASSPHRASE, `unlock --once`, `unlock --keychain` and `init --no-session`, as the missing-shm_open error already did. Before the fix: test_a_holder_that_cannot_be_made_names_the_way_round_it [ENOENT, EACCES, EROFS, ENOSPC] E assert '/dev/shm' in "Cannot create this boot's secret (/cmpt.84fc...): [Errno 2] No such file or directory: ..." test_openbsd_does_not_keep_the_secret_in_a_file_on_disk E Failed: DID NOT RAISE CryptoError Co-Authored-By: Claude Opus 5.5 --- src/compartment/session.py | 42 ++++++++++++++++++++++++---- tests/test_platform_session_store.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/compartment/session.py b/src/compartment/session.py index 87719eb..72e55be 100644 --- a/src/compartment/session.py +++ b/src/compartment/session.py @@ -21,7 +21,9 @@ uid, mode 0600. It outlives the process that created it, is shared by every later process of this user, and dies with the kernel that holds it. On Linux that object is a file on the /dev/shm tmpfs; on macOS it has no - filesystem presence at all. + filesystem presence at all. FreeBSD keeps it in the kernel and NetBSD on + the /var/shm tmpfs. OpenBSD is refused: its shm_open is a plain file in + /tmp, which is on disk. * Windows: a REG_OPTION_VOLATILE key under HKEY_CURRENT_USER. Volatile keys are never written to the hive on disk and vanish when the hive unloads, which is at logoff and at every restart. Windows therefore also relocks @@ -147,6 +149,11 @@ def _uid() -> str: _libc_cache = None +#: What to do instead, for every error that means "no holder here". +_ALTERNATIVES = ( + "Use COMPARTMENT_PASSPHRASE, `compartment unlock --once`, `compartment " + "unlock --keychain` (macOS), or `compartment init --no-session`.") + def _libc(): """libc/librt with shm_open, argtypes set for the variadic call. @@ -159,6 +166,18 @@ def _libc(): global _libc_cache if _libc_cache is not None: return _libc_cache + if sys.platform.startswith("openbsd"): + # OpenBSD's libc implements shm_open as an ordinary file, + # /tmp/.shm, on whatever /tmp is - a disk + # partition on a default install (lib/libc/gen/shm_open.c). The + # secret would then be on the same disk as the session file, and + # stay there after a power cut until the next boot prunes /tmp: a + # copy of that disk opens the credential, which is the one thing + # this holder exists to prevent. + raise CryptoError( + "OpenBSD keeps POSIX shared memory as a file in /tmp, on disk, " + "so it cannot hold a per-boot secret that a copy of the disk " + "would not carry. " + _ALTERNATIVES) for name in (None, "rt", "c"): try: lib = ctypes.CDLL(ctypes.util.find_library(name) if name else None, @@ -177,9 +196,18 @@ def _libc(): "No POSIX shared memory (shm_open) on this system, so there is " "nowhere to hold a per-boot secret that a restart destroys. " "Compartment will not fall back to a credential that anyone holding " - "the session file could recompute. Use `compartment unlock " - "--keychain` (macOS), COMPARTMENT_PASSPHRASE, `compartment unlock " - "--once`, or `compartment init --no-session`.") + "the session file could recompute. " + _ALTERNATIVES) + + +#: Why shm_open fails where it exists at all. glibc keeps its objects in +#: /dev/shm and fails ENOENT where that is not mounted (chroots, some build +#: and CI containers, Android); a container's /dev/shm can be read-only or +#: full; and sandboxes refuse names outside their own prefix (snap, the +#: macOS App Sandbox) with EACCES or EPERM. +_NO_SHM_HERE = ( + "POSIX shared memory is not usable for this process: /dev/shm may be " + "missing, read-only or full (some containers and chroots), or a " + "sandbox may refuse it.") def _shm_open(name: str, flags: int, mode: int | None = None) -> int: @@ -206,7 +234,8 @@ def _posix_read(name: str) -> tuple[str, bytes | None]: if exc.errno == errno.ENOENT: return "absent", None raise CryptoError( - f"Cannot open this boot's secret ({name}): {exc}") from exc + f"Cannot open this boot's secret ({name}): {exc}. " + f"{_NO_SHM_HERE} {_ALTERNATIVES}") from exc try: st = os.fstat(fd) if st.st_uid != os.getuid(): @@ -237,7 +266,8 @@ def _posix_create(name: str) -> bytes | None: if exc.errno == errno.EEXIST: return None raise CryptoError( - f"Cannot create this boot's secret ({name}): {exc}") from exc + f"Cannot create this boot's secret ({name}): {exc}. " + f"{_NO_SHM_HERE} {_ALTERNATIVES}") from exc try: os.ftruncate(fd, SECRET_LEN) secret = secrets.token_bytes(SECRET_LEN) diff --git a/tests/test_platform_session_store.py b/tests/test_platform_session_store.py index 23c22a9..3c05f29 100644 --- a/tests/test_platform_session_store.py +++ b/tests/test_platform_session_store.py @@ -76,3 +76,45 @@ def test_a_store_with_a_wide_mode_is_still_narrowed(tmp_path, monkeypatch): monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(d)) session.store(str(tmp_path / "v.vault"), b"k" * 32) assert (d.stat().st_mode & 0o777) == 0o700 + + +# --- no usable holder: the refusal has to say what to do instead ---------- + +@pytest.mark.skipif(session.IS_WINDOWS, reason="POSIX holder") +@pytest.mark.parametrize("code", [errno.ENOENT, errno.EACCES, errno.EROFS, + errno.ENOSPC]) +def test_a_holder_that_cannot_be_made_names_the_way_round_it( + tmp_path, monkeypatch, code): + """glibc keeps shm objects in /dev/shm and fails ENOENT where it is not + mounted (chroots, some CI and build containers); a container's can be + read-only or full, and a sandbox can refuse it. The bare "[Errno 2] No + such file or directory" left nobody knowing what to do.""" + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + + def refuse(name, flags, mode=None): + if flags & os.O_CREAT: + raise OSError(code, os.strerror(code), name) + raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), name) + monkeypatch.setattr(session, "_shm_open", refuse) + with pytest.raises(session.CryptoError) as err: + session.store(str(tmp_path / "v.vault"), b"k" * 32) + msg = str(err.value) + assert "/dev/shm" in msg + assert "COMPARTMENT_PASSPHRASE" in msg and "--once" in msg + + +@pytest.mark.skipif(session.IS_WINDOWS, reason="POSIX holder") +def test_openbsd_does_not_keep_the_secret_in_a_file_on_disk( + tmp_path, monkeypatch): + """OpenBSD's libc implements shm_open as /tmp/.shm, an ordinary + file on disk (lib/libc/gen/shm_open.c), so the secret would travel with + any copy of the disk the session file is on. Storing must refuse there, + the way it does where shm_open is missing, and say what to use.""" + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + monkeypatch.setattr(session, "_libc_cache", None) + monkeypatch.setattr(session.sys, "platform", "openbsd7") + with pytest.raises(session.CryptoError) as err: + session.store(str(tmp_path / "v.vault"), b"k" * 32) + assert "OpenBSD" in str(err.value) + assert "COMPARTMENT_PASSPHRASE" in str(err.value) + assert not session.credential_path(str(tmp_path / "v.vault")).exists() From e4958a6e506d1157039529566147568f459d5978 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:41:03 -0700 Subject: [PATCH 020/135] A red CI job says why on its summary page: pytest's failures and last lines become annotations A job's log is only readable when signed in, and pytest's verdict sits at the end of thousands of lines. tools/ci_report.py runs pytest unchanged and, when it fails under GitHub Actions, also emits the short summary and the tail as error annotations, which show on the run page, inline on a pull request and through the check-runs API. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 6 ++--- tools/ci_report.py | 50 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 tools/ci_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d22cf69..71f6e5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,12 +44,12 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: pytest tests/ -q --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/ -q --cov=compartment --cov-report=xml --cov-append - name: Dashboard tests (loopback socket; offline guard intentionally off) # `compartment dash` binds 127.0.0.1 by design and is never run under the # offline guard, so its tests run here with the guard off. - run: pytest tests/test_dash.py -q --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/test_dash.py -q --cov=compartment --cov-report=xml --cov-append - name: Coverage to Codecov # Never fail the build on a reporting service: the offline guard and the @@ -136,7 +136,7 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: pytest tests/ -q --ignore=tests/test_dash.py + run: python tools/ci_report.py tests/ -q --ignore=tests/test_dash.py linux-panel: # Linux has no notification area worth relying on, so it draws the same diff --git a/tools/ci_report.py b/tools/ci_report.py new file mode 100644 index 0000000..369d82e --- /dev/null +++ b/tools/ci_report.py @@ -0,0 +1,50 @@ +"""Run pytest and, when it fails, surface the reason where it can be read. + + python tools/ci_report.py [pytest args...] + +A red CI job's log is only readable by someone signed in to GitHub, and +pytest's verdict (which test, which assertion, which import failed during +collection) sits at the end of thousands of lines. This runs pytest with the +given arguments, streams its output unchanged, and on failure also emits the +short test summary and the last lines as GitHub Actions error annotations: +they appear on the run's summary page, inline on a pull request, and in the +check-run annotations API. Outside GitHub Actions it is plain pytest. +""" +from __future__ import annotations + +import os +import subprocess +import sys + +TAIL_LINES = 60 + + +def _escape(text: str) -> str: + # Workflow-command data escaping, so a multi-line message stays one + # annotation. + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def main(argv: list[str]) -> int: + proc = subprocess.Popen([sys.executable, "-m", "pytest", "-rfE", *argv], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + lines: list[str] = [] + assert proc.stdout is not None + for raw in proc.stdout: + line = raw.decode("utf-8", "replace") + sys.stdout.write(line) + lines.append(line.rstrip("\n")) + rc = proc.wait() + if rc != 0 and os.environ.get("GITHUB_ACTIONS") == "true": + summary = [ln for ln in lines if ln.startswith(("FAILED ", "ERROR "))] + if summary: + print("::error title=pytest failures::" + + _escape("\n".join(summary[:50]))) + print(f"::error title=pytest exited {rc} (last lines)::" + + _escape("\n".join(lines[-TAIL_LINES:]))) + sys.stdout.flush() + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 8e7f67d28bc91ac13e43c77c7fe98630ae1271b5 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:41:27 -0700 Subject: [PATCH 021/135] A Windows process started without USERNAME is still recognised as its user and no longer deletes everyone's unlock credential MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows the session module identifies the user by the USERNAME variable, for both the holder's registry name and the wrap key's context, and fell back to the literal "user" when it was missing. A process can be started without it: an MCP client that replaces the environment with the `env` block of its config (older MCP SDKs did exactly that), a service wrapper, a scheduler. That process derived a different holder name, found no secret there, took the credential for one from an earlier boot, and get() deleted it - the file every other process of the user was opening - so one misconfigured client relocked the vault for all of them. USERNAME still comes first, so every existing holder keeps its name; when it is missing the account name is asked of Windows (GetUserNameW, which returns the logon name Windows itself puts in USERNAME) before falling back to "user". POSIX is unchanged. Before the fix: test_a_windows_process_started_without_username_is_still_the_same_user E AssertionError: assert 'user' == 'Zoë' Co-Authored-By: Claude Opus 5.5 --- src/compartment/session.py | 30 ++++++++++++++++++++++++++-- tests/test_platform_session_store.py | 27 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/compartment/session.py b/src/compartment/session.py index 72e55be..3267ce1 100644 --- a/src/compartment/session.py +++ b/src/compartment/session.py @@ -132,9 +132,35 @@ def _boot_time() -> str: def _uid() -> str: - # os.getuid() is POSIX-only; on Windows fall back to the username. + """Who this is, for the holder's name and the wrap key's context. + + os.getuid() is POSIX-only; Windows uses the account name. USERNAME is + where that has always been read from, and it stays first so every + existing credential keeps its holder. But it is only an environment + variable, and a process can be started without it: an MCP client that + replaces the environment with the `env` block of its config, a service + wrapper, a scheduler. Such a process used to call itself "user", derive + a different holder name, find no secret under it, and delete the one + credential file every other process of the user was opening. The + account name the variable normally carries is asked of Windows + instead.""" getuid = getattr(os, "getuid", None) - return str(getuid()) if getuid else os.environ.get("USERNAME", "user") + if getuid: + return str(getuid()) + return os.environ.get("USERNAME") or _windows_account_name() or "user" + + +def _windows_account_name() -> str | None: + """GetUserNameW: the logon name Windows itself puts in USERNAME.""" + try: + from ctypes import wintypes + size = wintypes.DWORD(257) # UNLEN + 1 + buf = ctypes.create_unicode_buffer(size.value) + if ctypes.windll.advapi32.GetUserNameW(buf, ctypes.byref(size)): + return buf.value or None + except (AttributeError, OSError, ValueError): + pass + return None # --------------------------------------------------------------------------- diff --git a/tests/test_platform_session_store.py b/tests/test_platform_session_store.py index 3c05f29..7f8c870 100644 --- a/tests/test_platform_session_store.py +++ b/tests/test_platform_session_store.py @@ -118,3 +118,30 @@ def test_openbsd_does_not_keep_the_secret_in_a_file_on_disk( assert "OpenBSD" in str(err.value) assert "COMPARTMENT_PASSPHRASE" in str(err.value) assert not session.credential_path(str(tmp_path / "v.vault")).exists() + + +# --- Windows: who the user is must not hang on one environment variable --- + +def test_a_windows_process_started_without_username_is_still_the_same_user( + monkeypatch): + """The holder's name and the wrap key's context both fold in the user. + A process started without USERNAME - an MCP client that replaces the + environment with its config's `env` block, a service wrapper - called + itself "user", found no secret under that holder, and get() deleted the + credential every other process of the user was opening.""" + monkeypatch.delattr(session.os, "getuid", raising=False) + monkeypatch.setattr(session, "_windows_account_name", lambda: "Zoë", + raising=False) + monkeypatch.setenv("USERNAME", "Zoë") + with_variable = session._uid() + monkeypatch.delenv("USERNAME") + assert session._uid() == with_variable == "Zoë" + + +def test_the_username_variable_still_wins_so_existing_holders_keep_their_name( + monkeypatch): + monkeypatch.delattr(session.os, "getuid", raising=False) + monkeypatch.setattr(session, "_windows_account_name", lambda: "other", + raising=False) + monkeypatch.setenv("USERNAME", "Zoë") + assert session._uid() == "Zoë" From 2ab5bb044519383cc5269dbb299186b3cbf994f4 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:41:46 -0700 Subject: [PATCH 022/135] The embedding daemon's lock and spawn files are never opened through a link, so a link another user leaves in the temp directory cannot make it overwrite a file A home directory too deep for an AF_UNIX path moves the daemon's socket, and the .lock and .spawn files beside it, into the shared temp directory, under names anyone can derive from the home path. Both were opened with O_CREAT and no O_NOFOLLOW, and the daemon truncates its lock file and writes its pid there: a symlink or hard link another local user left at that name made the daemon replace the target, the user's own shell profile for instance, with a number. Linux refuses to follow a stranger's symlink in /tmp (fs.protected_symlinks) and hard links to files one does not own (protected_hardlinks), but FreeBSD, OpenBSD, NetBSD and others do not. The files are now opened with O_NOFOLLOW and accepted only when they are a regular file owned by this user with a single link; anything else is refused, the daemon exits 1 and says why, and the client embeds with its own model as it does whenever no daemon can be had. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed_daemon.py | 42 +++++++++++++++++++++++++++++---- tests/test_embed_daemon.py | 27 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/compartment/embed_daemon.py b/src/compartment/embed_daemon.py index e83cf93..7386da0 100644 --- a/src/compartment/embed_daemon.py +++ b/src/compartment/embed_daemon.py @@ -269,6 +269,32 @@ def _peer_uid(conn: socket.socket) -> int | None: return None +def _open_private(name: str, create: bool = True) -> int: + """A descriptor on this user's own regular file `name`, made 0600 if + it is not there, for the daemon's lock and spawn files. + + They sit beside the socket, and a home too deep for an AF_UNIX path puts + the socket in the shared temp directory (see socket_path), where the + names are predictable from the home path. Opened plainly, a symlink or a + hard link another user left at the lock file's name led the daemon to + truncate whatever it pointed at and write its pid there: the user's own + shell profile, say. Linux refuses to follow a stranger's link in /tmp + (fs.protected_symlinks), the BSDs and others do not. So a link is never + followed, and a file that is not this user's alone is refused.""" + flags = os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + fd = os.open(name, flags | (os.O_CREAT if create else 0), 0o600) + try: + st = os.fstat(fd) + except OSError: + os.close(fd) + raise + if (not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() + or st.st_nlink != 1): + os.close(fd) + raise PermissionError(f"{name} is not a file of this user's own") + return fd + + def _owned_socket(p: Path) -> bool: """True if `p` is a socket file that belongs to this user.""" try: @@ -297,8 +323,14 @@ def __init__(self, path: Path, idle: int): # -- lifecycle --------------------------------------------------------- def run(self) -> int: - self.path.parent.mkdir(parents=True, exist_ok=True) - if not self._take_lock(): + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + locked = self._take_lock() + except OSError as exc: + print(f"compartment: the embedding daemon cannot take its lock " + f"file beside {self.path}: {exc}", file=sys.stderr) + return 1 + if not locked: return 0 # another daemon owns this socket try: self.path.unlink() # a socket file a dead daemon left @@ -347,7 +379,7 @@ def run(self) -> int: return 0 def _take_lock(self) -> bool: - fd = os.open(str(self.path) + ".lock", os.O_RDWR | os.O_CREAT, 0o600) + fd = _open_private(str(self.path) + ".lock") try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: @@ -677,7 +709,7 @@ def connect_or_spawn(path: Path, idle: int, *, spawn: bool = True, if s is not None or not spawn or _refused is not None: return s path.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(str(path) + ".spawn", os.O_RDWR | os.O_CREAT, 0o600) + fd = _open_private(str(path) + ".spawn") try: deadline = time.monotonic() + wait while True: @@ -1023,7 +1055,7 @@ def _connect_probe(path: Path) -> bool: def _locked_pid(path: Path) -> int | None: """The pid written into the lock file, if a live daemon holds the lock.""" try: - fd = os.open(str(path) + ".lock", os.O_RDWR) + fd = _open_private(str(path) + ".lock", create=False) except OSError: return None try: diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index 565c07e..2f7739a 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -765,3 +765,30 @@ def test_a_daemon_run_where_it_cannot_listen_says_why(sandboxed, monkeypatch, assert e.value.code == 1 err = capsys.readouterr().err assert "cannot listen" in err and os.strerror(errno.EPERM) in err + + +# ------------------------------------------------- the shared temp directory -- +@pytest.mark.parametrize("link", ["symlink", "hardlink"]) +def test_a_link_left_at_the_lock_files_name_is_not_written_through( + tmp_path, link, capsys): + """A home too deep for an AF_UNIX path puts the socket, and its lock and + spawn files, in the shared temp directory under names anyone can work + out from the home path. Another user's symlink or hard link there led + the daemon to truncate what it pointed at, the user's own shell profile + say, and write its pid into it. Linux refuses to follow a stranger's + link in /tmp; the BSDs do not.""" + profile = tmp_path / "profile" + profile.write_text("export PATH=/opt/bin:$PATH\n") + sock = tmp_path / "compartment-1000-abc.sock" + for suffix in (".lock", ".spawn"): + target = Path(str(sock) + suffix) + if link == "symlink": + target.symlink_to(profile) + else: + os.link(profile, target) + code = embed_daemon._Daemon(sock, idle=1).run() + assert profile.read_text() == "export PATH=/opt/bin:$PATH\n" + assert code == 1 and "lock file" in capsys.readouterr().err + assert RemoteEmbedder.connect(path=sock, wait=0.5) is None + assert embed_daemon._locked_pid(sock) is None + assert profile.read_text() == "export PATH=/opt/bin:$PATH\n" From 2b5348a160371880cd814f270a1baf83a8744928 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:42:21 -0700 Subject: [PATCH 023/135] `integrate zed` and `integrate vscode` write into a settings file that has comments in it, keeping every comment, instead of refusing it Zed writes its settings.json on first run with a block of // comments above the opening brace, and VS Code reads mcp.json as JSON with comments and trailing commas. The writer accepted only strict JSON, so `integrate zed` (and `--all`) refused the file Zed itself creates on every machine and printed a block to paste, and a VS Code mcp.json with a comment in it went the same way. Re-serialising would have parsed, and deleted everything the user wrote in comments. So for clients that read their file as JSON with comments (Client.jsonc), a file the strict reader refuses is now edited as text: it is parsed with the position of every object member, our member is spliced in after the last one in the file's own indentation (or its value is replaced, when it is already there), and the result is parsed again and must equal the original data with exactly that change, or nothing is written. Comments, trailing commas, ordering and formatting stay byte for byte. Taking the entry out works the same way and takes only its own lines and the comma that separated it. A duplicate key, a root that is not an object, an unterminated comment or anything else that cannot be proven is refused as before, with the reason and the block to paste, and a registration that cannot be taken out is reported as such. Clients that read strict JSON, such as Cursor, still refuse a file with comments. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 383 ++++++++++++++++++++++++++++++- tests/test_integrations_jsonc.py | 179 +++++++++++++++ 2 files changed, 556 insertions(+), 6 deletions(-) create mode 100644 tests/test_integrations_jsonc.py diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 42cfb1b..f169a0d 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -38,6 +38,7 @@ import errno import json import os +import re import secrets import shutil import sys @@ -129,6 +130,9 @@ class Client: writes: bool = True #: whether we are willing to edit it note: str = "" #: anything the user needs to know after aliases: tuple[str, ...] = field(default=()) + #: Whether the client reads its file as JSON with comments, so a file + #: holding comments or trailing commas is edited as text, not refused. + jsonc: bool = False def config_path(self) -> Path: return self.path() @@ -186,16 +190,16 @@ def present(self) -> bool: # -- coding agents that spell the key differently ---------------------- Client("vscode", "VS Code", lambda: _vscode_user() / "mcp.json", - root="servers", entry=_entry_vscode), + root="servers", entry=_entry_vscode, jsonc=True), Client("vscode-insiders", "VS Code Insiders", lambda: _vscode_user("Code - Insiders") / "mcp.json", - root="servers", entry=_entry_vscode), + root="servers", entry=_entry_vscode, jsonc=True), + # Zed writes its settings.json with a block of // comments on first + # run, so it is edited as JSON with comments: comments kept. Client("zed", "Zed", lambda: (_appdata() / "Zed" / "settings.json" if os.name == "nt" else _xdg() / "zed" / "settings.json"), - root="context_servers", - note="Zed allows comments in settings.json; if yours has any, " - "the block is printed rather than written"), + root="context_servers", jsonc=True), Client("opencode", "OpenCode", lambda: _xdg() / "opencode" / "opencode.json", root="mcp", entry=_entry_opencode), @@ -503,6 +507,330 @@ def _read_json(path: Path) -> dict: return data +# --- JSONC: the comments and trailing commas some clients allow ------------ +# +# VS Code (and Insiders and VSCodium), Zed, Gemini CLI, Qwen Code and +# OpenCode read their config as JSON with comments: // and /* */, and a +# comma after the last member. Zed writes its own settings.json that way on +# first run, with a block of // lines above the opening brace, so a strict +# JSON reader refused every Zed user's file and printed the block instead. +# +# Re-serialising would parse, but it would delete every comment the user +# wrote. So the edit is textual: the document is parsed with the position +# of every object member, our member is spliced in (or out) as text, and +# the result is parsed again and must equal the original data with exactly +# our change applied. Anything this cannot do cleanly - a duplicate key, a +# root that is not an object, a result that does not read back - is +# refused, and the user gets the block to paste, as before. + +_JSONC_SCALAR = re.compile( + r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?|true|false|null") + + +class _JsoncObject(dict): + """A parsed object that remembers where it is in the text: its braces, + and for each member the start of its key and the span of its value.""" + + def __init__(self, open_at: int): + super().__init__() + self.open = open_at + self.close = -1 + self.spans: dict[str, tuple[int, int, int]] = {} + + +class _JsoncParser: + def __init__(self, text: str): + self.t, self.i = text, 0 + + def fail(self, what: str): + line = self.t.count("\n", 0, self.i) + 1 + raise ValueError(f"{what} at line {line}") + + def skip(self) -> None: + t, n = self.t, len(self.t) + while self.i < n: + c = t[self.i] + if c in " \t\r\n": + self.i += 1 + elif t.startswith("//", self.i): + j = t.find("\n", self.i) + self.i = n if j < 0 else j + 1 + elif t.startswith("/*", self.i): + j = t.find("*/", self.i + 2) + if j < 0: + self.fail("unterminated comment") + self.i = j + 2 + else: + return + + def at(self, ch: str) -> bool: + return self.t.startswith(ch, self.i) + + def document(self): + self.skip() + value = self.value() + self.skip() + if self.i != len(self.t): + self.fail("text after the end of the document") + return value + + def value(self): + self.skip() + if self.at("{"): + return self.object() + if self.at("["): + return self.array() + if self.at('"'): + return self.string() + m = _JSONC_SCALAR.match(self.t, self.i) + if not m: + self.fail("unexpected character") + self.i = m.end() + return json.loads(m.group()) + + def string(self) -> str: + try: + s, self.i = json.decoder.scanstring(self.t, self.i + 1, True) + except json.JSONDecodeError as exc: + raise ValueError(str(exc)) from exc + return s + + def object(self) -> _JsoncObject: + obj = _JsoncObject(self.i) + self.i += 1 + while True: + self.skip() + if self.at("}"): + break + if not self.at('"'): + self.fail("expected a quoted key") + key_at = self.i + key = self.string() + self.skip() + if not self.at(":"): + self.fail("expected ':'") + self.i += 1 + self.skip() + value_at = self.i + value = self.value() + if key in obj: + # Which one a client honours is the client's business; an + # edit that guessed could change the one it reads. + self.fail(f"duplicate key {key!r}") + obj[key] = value + obj.spans[key] = (key_at, value_at, self.i) + self.skip() + if self.at(","): + self.i += 1 + continue + if not self.at("}"): + self.fail("expected ',' or '}'") + break + obj.close = self.i + self.i += 1 + return obj + + def array(self) -> list: + out = [] + self.i += 1 + while True: + self.skip() + if self.at("]"): + break + out.append(self.value()) + self.skip() + if self.at(","): + self.i += 1 + continue + if not self.at("]"): + self.fail("expected ',' or ']'") + break + self.i += 1 + return out + + +def _jsonc_parse(text: str): + """The data in a JSONC document. Raises ValueError.""" + return _JsoncParser(text).document() + + +def _plain(value): + """`value` as plain dicts and lists, for comparing and copying.""" + if isinstance(value, dict): + return {k: _plain(v) for k, v in value.items()} + if isinstance(value, list): + return [_plain(v) for v in value] + return value + + +def _line_start(text: str, i: int) -> int: + return text.rfind("\n", 0, i) + 1 + + +def _own_line_indent(text: str, i: int) -> str | None: + """The indentation before position `i`, when nothing else is before it + on its line.""" + lead = text[_line_start(text, i):i] + return lead if not lead.strip() else None + + +def _line_indent(text: str, i: int) -> str: + """The indentation of the line holding position `i`.""" + start = _line_start(text, i) + line = text[start:] + return line[:len(line) - len(line.lstrip(" \t"))] + + +def _indent_unit(text: str, root: _JsoncObject) -> str: + """One level of indentation, as the document's own first member has.""" + for key_at, _, _ in sorted(root.spans.values()): + lead = _own_line_indent(text, key_at) + if lead: + return lead + break + return " " + + +def _jsonc_dump(value, indent: str, unit: str) -> str: + """`value` as JSON, continuation lines indented to sit at `indent`.""" + return json.dumps(value, indent=unit).replace("\n", "\n" + indent) + + +def _jsonc_insert(text: str, obj: _JsoncObject, key: str, value, + unit: str) -> str: + """`text` with `"key": value` added as the last member of `obj`.""" + if not obj.spans: + close_indent = _line_indent(text, obj.open) + member_indent = close_indent + unit + member = f"{json.dumps(key)}: {_jsonc_dump(value, member_indent, unit)}" + inside = text[obj.open + 1:obj.close] + if not inside.strip(): + return (text[:obj.open + 1] + "\n" + member_indent + member + + "\n" + close_indent + text[obj.close:]) + return (text[:obj.open + 1] + "\n" + member_indent + member + + text[obj.open + 1:]) + + first_key = min(k for k, _, _ in obj.spans.values()) + last_key, _, last_end = max(obj.spans.values()) + member_indent = _own_line_indent(text, first_key) + if member_indent is None: # a one-line object + compact = json.dumps(value) + return (text[:last_end] + f", {json.dumps(key)}: {compact}" + + text[last_end:]) + member = f"{json.dumps(key)}: {_jsonc_dump(value, member_indent, unit)}" + eol = text.find("\n", last_end) + tail = text[last_end:] if eol < 0 else text[last_end:eol] + after = tail.lstrip(" \t") + trailing_comma = after.startswith(",") + if trailing_comma: + after = after[1:].lstrip(" \t") + if eol >= 0 and (not after or after.startswith("//")): + # The last member ends its line, perhaps with a comma or a comment + # after it: the new member goes on a line of its own below. + comma = "" if trailing_comma else "," + return (text[:last_end] + comma + text[last_end:eol] + "\n" + + member_indent + member + ("," if trailing_comma else "") + + text[eol:]) + compact = json.dumps(value) + return (text[:last_end] + f", {json.dumps(key)}: {compact}" + + text[last_end:]) + + +def _jsonc_set(text: str, root: str, name: str, entry: dict) -> str: + """`text` with root[name] set to `entry`, every comment and every other + byte left where it was. Raises ValueError when that cannot be done and + proven.""" + doc = _jsonc_parse(text) + if not isinstance(doc, _JsoncObject): + raise ValueError("not a JSON object") + unit = _indent_unit(text, doc) + expected = _plain(doc) + if root in doc: + servers = doc[root] + if not isinstance(servers, _JsoncObject): + raise ValueError(f"{root} is not an object") + if name in servers: + key_at, value_at, value_end = servers.spans[name] + indent = _line_indent(text, key_at) + new = (text[:value_at] + _jsonc_dump(entry, indent, unit) + + text[value_end:]) + else: + new = _jsonc_insert(text, servers, name, entry, unit) + expected[root][name] = entry + else: + new = _jsonc_insert(text, doc, root, {name: entry}, unit) + expected[root] = {name: entry} + if _plain(_jsonc_parse(new)) != expected: + raise ValueError("the edited file would not read back as intended") + return new + + +def _skip_blank(text: str, i: int) -> int: + """The first position at or after `i` that is not whitespace or a + comment.""" + p = _JsoncParser(text) + p.i = i + p.skip() + return p.i + + +def _jsonc_delete(text: str, root: str, name: str) -> str | None: + """`text` with root[name] taken out and nothing else changed, or None + when it is not there. Raises ValueError when it cannot be taken out + cleanly.""" + doc = _jsonc_parse(text) + if not isinstance(doc, _JsoncObject): + raise ValueError("not a JSON object") + servers = doc.get(root) + if not isinstance(servers, _JsoncObject) or name not in servers: + return None + order = sorted(servers.spans, key=lambda k: servers.spans[k][0]) + idx = order.index(name) + key_at, _, value_end = servers.spans[name] + cuts: list[tuple[int, int]] = [] + + def member_span(start: int, end: int) -> tuple[int, int]: + """Widen [start, end) to whole lines when the member has its lines + to itself.""" + eol = text.find("\n", end) + rest = text[end:] if eol < 0 else text[end:eol] + if _own_line_indent(text, start) is not None and not rest.strip(): + return _line_start(text, start), (len(text) if eol < 0 + else eol + 1) + return start, end + + if idx < len(order) - 1: + comma = _skip_blank(text, value_end) + if not text.startswith(",", comma): + raise ValueError("no comma after the entry") + cuts.append(member_span(key_at, comma + 1)) + elif idx > 0: + prev_end = servers.spans[order[idx - 1]][2] + comma = _skip_blank(text, prev_end) + if not text.startswith(",", comma): + raise ValueError("no comma before the entry") + end = value_end + trailing = _skip_blank(text, value_end) + if text.startswith(",", trailing): # it had a trailing comma + end = trailing + 1 + cuts.append((comma, comma + 1)) + cuts.append(member_span(key_at, end)) + else: + end = value_end + trailing = _skip_blank(text, value_end) + if text.startswith(",", trailing): + end = trailing + 1 + cuts.append(member_span(key_at, end)) + new = text + for start, end in sorted(cuts, reverse=True): + new = new[:start] + new[end:] + expected = _plain(doc) + del expected[root][name] + if _plain(_jsonc_parse(new)) != expected: + raise ValueError("the edited file would not read back as intended") + return new + + def write_config(path: Path, text: str) -> Path | None: """Replace a file the user or another program owns with `text`. Returns the byte-exact backup taken first, beside it, or None when there was no @@ -716,6 +1044,13 @@ def is_registered(client: Client, path: Path | None = None) -> bool: try: return NAME in (_read_json(p).get(client.root) or {}) except ValueError: + if not client.jsonc: + return False + try: + doc = _jsonc_parse(read_config_text(p)) + servers = doc.get(client.root) if isinstance(doc, dict) else None + return isinstance(servers, dict) and NAME in servers + except (OSError, ValueError): return False @@ -744,6 +1079,9 @@ def register(client: Client, vault: str, caller: str | None = None, try: data = _read_json(p) except ValueError as exc: + if client.jsonc: + return _register_jsonc(client, p, entry, out, str(exc), + snippet(client, vault, caller)) out["reason"] = str(exc) out["snippet"] = snippet(client, vault, caller) return out @@ -765,6 +1103,30 @@ def register(client: Client, vault: str, caller: str | None = None, return out +def _register_jsonc(client: Client, p: Path, entry: dict, out: dict, + strict_error: str, block: str) -> dict: + """Register in a file that is not strict JSON but that this client + reads as JSON with comments: spliced in as text, every comment kept, + and proven by reading it back. When it cannot be, the strict reader's + refusal stands and the block is printed.""" + try: + new = _jsonc_set(read_config_text(p), client.root, NAME, entry) + except (OSError, ValueError) as exc: + out["reason"] = (f"{strict_error}; nor could it be edited as JSON " + f"with comments ({exc})") + out["snippet"] = block + return out + try: + backup = write_config(p, new) + except OSError as exc: + out["reason"] = f"cannot write {p} ({exc})" + out["snippet"] = block + return out + out["backup"] = str(backup) if backup else None + out["written"] = True + return out + + def _toml_breaks(client: Client, before: str, after: str, entry: dict) -> str | None: """Why `after` must not be written in place of `before`, or None. @@ -899,7 +1261,16 @@ def unregister(client: Client, path: Path | None = None) -> bool: try: data = _read_json(p) except ValueError: - return False + if not client.jsonc: + return False + try: + rest = _jsonc_delete(read_config_text(p), client.root, NAME) + except (OSError, ValueError): # left for the user to take out + return False + if rest is None: + return False + write_config(p, rest) + return True servers = data.get(client.root) or {} if not isinstance(servers, dict) or NAME not in servers: return False diff --git a/tests/test_integrations_jsonc.py b/tests/test_integrations_jsonc.py new file mode 100644 index 0000000..ed31816 --- /dev/null +++ b/tests/test_integrations_jsonc.py @@ -0,0 +1,179 @@ +"""Clients that read their config as JSON with comments. + +Zed writes settings.json on first run with a block of // comments above the +opening brace, so a strict reader refused every Zed user's file. VS Code's +mcp.json is JSON with comments too. Re-serialising would delete what the +user wrote, so the edit is textual and proven by reading it back: every +comment, every trailing comma, every other byte stays where it was. +""" +import json + +import pytest + +from compartment import cli, clients + +VAULT = "/v/memory.vault" + +#: Zed's own first-run settings.json, with a server and a comment the user +#: added since. +ZED_FIRST_RUN = """\ +// Zed settings +// +// For information on how to configure Zed, see the Zed +// documentation: https://zed.dev/docs/configuring-zed +// +// To see all of Zed's default settings without changing your +// custom settings, run `zed: open default settings` from the +// command palette (cmd-shift-p / ctrl-shift-p) +{ + "ui_font_size": 16, + "buffer_font_size": 15, + "theme": { + "mode": "system", + "light": "One Light", + "dark": "One Dark" // the one I use + }, + /* servers */ + "context_servers": { + "github": { "command": "gh-mcp", "args": [] }, // keep this one + }, +} +""" + +COMMENTS = ["// Zed settings", "// the one I use", "/* servers */", + "// keep this one", "(cmd-shift-p / ctrl-shift-p)"] + + +def _lenient(text): + return clients._plain(clients._jsonc_parse(text)) + + +def test_a_zed_settings_file_with_comments_is_written_keeping_them(tmp_path): + """The strict reader refused the file Zed itself writes, so `integrate + zed` printed a block on every machine instead of connecting Zed.""" + p = tmp_path / "settings.json" + p.write_text(ZED_FIRST_RUN, encoding="utf-8") + zed = clients.CLIENTS["zed"] + + res = clients.register(zed, VAULT, path=p) + + assert res["written"] is True, res["reason"] + text = p.read_text(encoding="utf-8") + for comment in COMMENTS: + assert comment in text + data = _lenient(text) + assert data["context_servers"]["github"] == {"command": "gh-mcp", + "args": []} + assert data["context_servers"]["compartment"]["args"][:2] == [ + "--vault", VAULT] + assert data["theme"]["dark"] == "One Dark" + assert open(res["backup"], encoding="utf-8").read() == ZED_FIRST_RUN + assert clients.is_registered(zed, path=p) is True + + # Again: refreshed in place, not duplicated, comments still there. + clients.register(zed, "/other.vault", path=p) + again = p.read_text(encoding="utf-8") + assert again.count('"compartment"') == 1 + assert _lenient(again)["context_servers"]["compartment"]["args"][1] == ( + "/other.vault") + for comment in COMMENTS: + assert comment in again + + # And out again: only ours goes. + assert clients.unregister(zed, path=p) is True + out = p.read_text(encoding="utf-8") + assert _lenient(out) == _lenient(ZED_FIRST_RUN) + for comment in COMMENTS: + assert comment in out + assert clients.is_registered(zed, path=p) is False + + +@pytest.mark.parametrize("text", [ + "// mine\n{}\n", + "{\n // nothing here yet\n}\n", + '{\n\t"theme": "dark", // tabs\n}\n', + '{"theme": "dark", /* one line */ "vim_mode": true}', + '{\n "context_servers": {}, // none yet\n "vim_mode": true\n}\n', + '{\n "context_servers": {\n // none yet\n }\n}\n', + '{\n "a": [1, 2,], // trailing in an array\n "b": "//not a comment"\n}', +]) +def test_every_shape_of_jsonc_reads_back_with_our_entry_and_out_again( + text, tmp_path): + p = tmp_path / "settings.json" + p.write_text(text, encoding="utf-8") + zed = clients.CLIENTS["zed"] + before = _lenient(text) + + assert clients.register(zed, VAULT, path=p)["written"] is True + after = _lenient(p.read_text(encoding="utf-8")) + assert after["context_servers"]["compartment"]["args"][1] == VAULT + del after["context_servers"]["compartment"] + if "context_servers" not in before: + assert after.pop("context_servers") == {} + assert after == before + + assert clients.unregister(zed, path=p) is True + restored = _lenient(p.read_text(encoding="utf-8")) + if "context_servers" not in before: + assert restored.pop("context_servers") == {} + assert restored == before + + +@pytest.mark.parametrize("text, why", [ + ('// c\n{"theme": "a", "theme": "b"}', "duplicate key"), + ("// c\n[1, 2]", "not a JSON object"), + ('// c\n{"context_servers": []}', "not an object"), + ("{'single': 'quotes'}", "expected a quoted key"), + ('{"a": 1 /* never closed', "unterminated comment"), +]) +def test_jsonc_this_cannot_edit_safely_is_left_alone(text, why, tmp_path): + p = tmp_path / "settings.json" + p.write_text(text, encoding="utf-8") + res = clients.register(clients.CLIENTS["zed"], VAULT, path=p) + assert res["written"] is False + assert why in res["reason"] and "refusing to touch it" in res["reason"] + assert "compartment" in res["snippet"] + assert p.read_text(encoding="utf-8") == text + + +def test_a_vscode_mcp_json_with_comments_is_written(tmp_path): + p = tmp_path / "mcp.json" + text = ('{\n\t// my servers\n\t"servers": {\n\t\t"x": {"type": "stdio", ' + '"command": "x"},\n\t},\n\t"inputs": [],\n}\n') + p.write_text(text, encoding="utf-8") + res = clients.register(clients.CLIENTS["vscode"], VAULT, path=p) + assert res["written"] is True, res["reason"] + out = p.read_text(encoding="utf-8") + assert "// my servers" in out + entry = _lenient(out)["servers"]["compartment"] + assert entry["type"] == "stdio" and entry["args"][1] == VAULT + + +def test_a_strict_json_client_still_refuses_comments(tmp_path): + """Cursor reads strict JSON: a file with comments in it is not one + Cursor can read either, so it is not ours to rewrite.""" + p = tmp_path / "mcp.json" + text = '{"mcpServers": {} // a comment\n}' + p.write_text(text, encoding="utf-8") + res = clients.register(clients.CLIENTS["cursor"], VAULT, path=p) + assert res["written"] is False + assert p.read_text(encoding="utf-8") == text + + +def test_removing_a_zed_entry_that_cannot_be_taken_out_says_so( + tmp_path, monkeypatch, capsys): + """A registration this cannot take out cleanly is reported, with the + entry to remove, rather than claimed never to have been there.""" + zed = clients.CLIENTS["zed"] + p = tmp_path / "settings.json" + p.write_text('// mine\n{"context_servers": {"compartment": ' + '{"command": "x"}}}\n', encoding="utf-8") + assert clients.is_registered(zed, path=p) is True + monkeypatch.setattr(clients, "_jsonc_delete", + lambda *a: (_ for _ in ()).throw(ValueError("no"))) + monkeypatch.setattr(zed.__class__, "config_path", lambda self: p) + cli._integrate_client(zed, type("A", (), {"remove": True, + "vault": VAULT})()) + out = capsys.readouterr().out + assert "left as it is" in out and "was not in" not in out + assert json.loads(p.read_text(encoding="utf-8").split("\n", 1)[1]) From ea23a44165d0c2c21188cc09d99d94c6d2503290 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:42:31 -0700 Subject: [PATCH 024/135] A memory or a search holding half an emoji is stored and answered, with the broken character shown as U+FFFD, instead of failing Python's json decoder turns an unpaired "\ud83d" escape into a lone surrogate without complaint, and agents send them: a JavaScript client that cuts a string to length cuts emoji in half, and Windows hands back unpaired UTF-16 from file names and the clipboard. Such a string has no UTF-8 encoding, so the tokenizer refused it ("TypeError: TextInputSequence must be str") and SQLite refused it after that (UnicodeEncodeError): memory_store, memory_search, memory_link and memory_relations failed outright with an error that said nothing about the cause, and the memory was lost. The vault now cleans the free-text fields at its entry points (store: text, source, tags, discovered, namespace; search: query, tags, namespace and the discovery bounds; link and relations: subject, predicate, object, entity, namespace). A lone surrogate becomes U+FFFD, what any UTF-8 decoder shows for the same damage, and a pair that arrived as two code points is joined into its character. Text that already encodes is returned untouched. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 41 ++++++++++++++++++++++++++++++ tests/test_storage_text.py | 52 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tests/test_storage_text.py diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 87d5935..6fd8aef 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -120,6 +120,33 @@ def discovery_date(value=None) -> str: return s[:40] +def clean_text(value): + """Text as UTF-8 can carry it: every lone surrogate replaced by U+FFFD, + and a surrogate pair that arrived as two code points joined into the + character it encodes. Anything that is not a str (None, a number) and a + list of strings come back the same shape. + + Python's json decoder turns "\\ud83d" into a lone surrogate without + complaint, and lone surrogates are common in what agents send: a + JavaScript client that cuts a string to length cuts emoji in half, and + Windows hands back unpaired UTF-16 from file names and the clipboard. + Such a string cannot be encoded as UTF-8, so the tokenizer refused it + ("TextInputSequence must be str") and SQLite would have refused it next: + the store or search failed outright, with an error that said nothing + about why. One replacement character is a far smaller loss than the + memory, and it is what every UTF-8 decoder shows for the same damage.""" + if isinstance(value, list): + return [clean_text(v) for v in value] + if not isinstance(value, str): + return value + try: + value.encode("utf-8") + return value + except UnicodeEncodeError: + return value.encode("utf-16", "surrogatepass").decode("utf-16", + "replace") + + def strip_provenance(text: str) -> str: """The claim on its own, without its provenance clause. @@ -911,6 +938,10 @@ def store(self, text: str, caller: str, namespace: str | None = None, _expiry_strict: bool = True, _gate: bool = True, _split_hint: str | None = None) -> dict: self._require_open() + text, source, discovered, namespace = ( + clean_text(text), clean_text(source), clean_text(discovered), + clean_text(namespace)) + tags = clean_text(tags) ns = namespace or self.config.default_namespace(caller) if pack is None: self.config.check(caller, ns, write=True) @@ -1281,6 +1312,10 @@ def search(self, query: str, caller: str, namespace: str | None = None, An explicit `top_k` still means exactly that many, unchanged, because callers that page or benchmark need a fixed window.""" self._require_open() + query, namespace, tags = (clean_text(query), clean_text(namespace), + clean_text(tags)) + discovered_since, discovered_until = (clean_text(discovered_since), + clean_text(discovered_until)) self._maybe_expire() if namespace is not None: self.config.grant_for(caller, namespace) # raises if no access @@ -1724,6 +1759,9 @@ def link(self, subject: str, predicate: str, obj: str, caller: str, host model (or the user), exactly like memory curation. Idempotent: re-linking the same triple returns the existing relation.""" self._require_open() + subject, predicate, obj, namespace = ( + clean_text(subject), clean_text(predicate), clean_text(obj), + clean_text(namespace)) ns = namespace or self.config.default_namespace(caller) self.config.check(caller, ns, write=True) for label, val in (("subject", subject), ("predicate", predicate), @@ -1774,6 +1812,9 @@ def relations(self, caller: str, entity: str | None = None, subject OR object (case-insensitive); `as_of` keeps relations whose validity window covers that instant.""" self._require_open() + entity, subject, predicate, obj, namespace = ( + clean_text(entity), clean_text(subject), clean_text(predicate), + clean_text(obj), clean_text(namespace)) if namespace is not None: self.config.grant_for(caller, namespace) allowed = {namespace} diff --git a/tests/test_storage_text.py b/tests/test_storage_text.py new file mode 100644 index 0000000..8a00e78 --- /dev/null +++ b/tests/test_storage_text.py @@ -0,0 +1,52 @@ +"""Text the encoder and SQLite cannot take as it arrives. + +A JSON string may carry "\\ud83d" with no partner, and Python's decoder hands +it on as a lone surrogate. JavaScript clients that cut a string to length cut +emoji in half; Windows returns unpaired UTF-16 from file names and the +clipboard. Such a string has no UTF-8 encoding, so every layer below the +vault refuses it. +""" +import json + +import pytest + +from compartment.vault import Vault, clean_text + +HALF_EMOJI = json.loads('"cut here \\ud83d"') # what a JSON client sends +PAIR_APART = chr(0xD83D) + chr(0xDE00) # a pair, as two points + + +def test_clean_text_replaces_only_what_cannot_be_encoded(): + assert clean_text("plain, ünïcode, 🦄 and עברית") == \ + "plain, ünïcode, 🦄 and עברית" + assert clean_text(HALF_EMOJI) == "cut here \ufffd" + assert clean_text(PAIR_APART) == "\N{GRINNING FACE}", "a split pair is joined, not lost" + assert clean_text(["a\udc80b", "ok"]) == ["a\ufffdb", "ok"] + assert clean_text(None) is None and clean_text(3) == 3 + + +def test_a_memory_with_half_an_emoji_is_stored_and_found(vault): + """It failed with "TextInputSequence must be str" from the tokenizer, + and the memory was not stored.""" + text = "The deploy channel is #ops-alerts " + HALF_EMOJI + out = vault.store(text, caller="t", source="chat \udc81", + tags=["ops\ud800"]) + got = vault.get(out["id"], caller="t") + assert "\ufffd" in got["text"] and got["source"] == "chat \ufffd" + assert got["tags"] == ["ops\ufffd"] + hits = vault.search("deploy channel " + HALF_EMOJI, caller="t") + assert [h["id"] for h in hits["results"]][:1] == [out["id"]] + + +def test_the_memory_survives_a_reopen(vault, vault_path): + out = vault.store("Half an emoji at the end " + HALF_EMOJI, caller="t") + again = Vault.unlock(vault_path, passphrase="CorrectHorse") + assert again.get(out["id"], caller="t")["text"].startswith( + "Half an emoji at the end cut here \ufffd") + + +def test_the_graph_takes_and_finds_the_same_text(vault): + vault.link("Ana " + HALF_EMOJI, "works on", "the \udcffrelease", caller="t") + rels = vault.relations("t", entity="Ana " + HALF_EMOJI)["relations"] + assert [(r["subject"], r["object"]) for r in rels] == [ + ("Ana cut here \ufffd", "the \ufffdrelease")] From 20b179fb5e8f9be700724a62a942ce0d31e7e2fc Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:42:39 -0700 Subject: [PATCH 025/135] The embedding daemon starts even when the home directory holds a script named like a module it imports The daemon is started as `python -m compartment.embed_daemon` with the home directory as its working directory, and `-m` puts the working directory first on sys.path. A tempfile.py, random.py, json.py, socket.py or compartment.py in the home directory, the kind of scratch file people learning Python leave there, was imported in place of the real module: the daemon died at start every time, and every agent on the machine quietly loaded a model of its own and paid for a failed start at each connect. The daemon is now started with -P (Python 3.11+, which Compartment requires), so the working directory is not put on the path; the package is still found through the PYTHONPATH the client sets. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed_daemon.py | 6 +++++- tests/test_embed_daemon.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/compartment/embed_daemon.py b/src/compartment/embed_daemon.py index 7386da0..4903e89 100644 --- a/src/compartment/embed_daemon.py +++ b/src/compartment/embed_daemon.py @@ -669,7 +669,11 @@ def _spawn(path: Path, idle: int) -> subprocess.Popen: client's process-group cleanup nor a closing terminal takes it down, and no inherited stdio, because stdout of an MCP server IS the transport.""" - cmd = [sys.executable, "-m", "compartment.embed_daemon", + # -P: `-m` puts the working directory first on sys.path, and that is the + # home directory, so a random.py, json.py or compartment.py anyone keeps + # there was imported in place of the real one and the daemon died at + # start, every time, leaving every agent to load a model of its own. + cmd = [sys.executable, "-P", "-m", "compartment.embed_daemon", "--socket", str(path), "--idle", str(idle)] # The daemon must run the same code as the client that starts it, so it # imports the package from wherever this module was imported from - a diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index 2f7739a..d2b371c 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -792,3 +792,31 @@ def test_a_link_left_at_the_lock_files_name_is_not_written_through( assert RemoteEmbedder.connect(path=sock, wait=0.5) is None assert embed_daemon._locked_pid(sock) is None assert profile.read_text() == "export PATH=/opt/bin:$PATH\n" + + +def test_a_module_in_the_home_directory_does_not_replace_the_daemons_own( + tmp_path, monkeypatch): + """The daemon starts in the home directory with `python -m`, which puts + that directory first on sys.path: a tempfile.py, random.py, json.py or + compartment.py kept there was imported instead of the real module, the + daemon died at start, and every agent loaded a model of its own. The + command the client builds is run here with --help, from such a home.""" + monkeypatch.setenv("HOME", str(tmp_path)) + (tmp_path / "tempfile.py").write_text( + "raise SystemExit('the home directory shadowed tempfile')\n") + built = {} + real_popen = subprocess.Popen + + def popen(cmd, **kwargs): + built.update(cmd=cmd, **kwargs) + raise OSError("not started in this test") + monkeypatch.setattr(subprocess, "Popen", popen) + with pytest.raises(OSError): + embed_daemon._spawn(tmp_path / "embed.sock", 1) + monkeypatch.setattr(subprocess, "Popen", real_popen) + assert built["cwd"] == str(tmp_path) + cmd = built["cmd"][:built["cmd"].index("--socket")] + ["--help"] + out = subprocess.run(cmd, cwd=built["cwd"], env=built["env"], + capture_output=True, text=True, timeout=120) + assert "shadowed" not in out.stdout + out.stderr + assert out.returncode == 0 and "--socket" in out.stdout From 74139ee72fc2234dd867c73fc8ad81f8815d94f8 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:42:43 -0700 Subject: [PATCH 026/135] On Windows, start at sign-in runs the tray app with pythonw.exe, so no console window sits behind it for the whole session The Run key and the scheduled task launched the console script when one was on PATH, and only fell back to pythonw.exe without one. But compartment.exe, which pip and uv write for the console script, is a console program just like python.exe. Started at sign-in it opened a console window, and since that process is the tray app the window stayed for the whole session; closing it quit the app. Every pip, pipx and uv install has the console script on PATH, so this was the normal case. The docstring said the opposite of what the code did, and the existing test only checked that python.exe was not named. pythonw.exe beside the running interpreter now comes first (every venv, uv tool environment, python.org and Store install has one, and it runs the same environment's compartment); the console script is the fallback for a build without one. The running-copy detection matches either form, since it reads the command line, not the image name. Co-Authored-By: Claude Opus 5.5 --- src/compartment/systray.py | 30 +++++++++++++++++++++--------- tests/test_status_bar_install.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/compartment/systray.py b/src/compartment/systray.py index dff85ca..1655487 100644 --- a/src/compartment/systray.py +++ b/src/compartment/systray.py @@ -175,11 +175,12 @@ def _winreg(): def _autostart_command(vault: str | None = None) -> str: """What Windows runs at sign-in. - Two things this has to get right. It launches through the console script - or pythonw.exe rather than python.exe, because python.exe opens a console - window behind a tray app that has no window, at every single sign-in. And - it carries the vault path, because a user running a non-default vault - would otherwise silently get the default one back after a reboot. + Two things this has to get right. It launches through pythonw.exe, the + one interpreter that opens no console, because anything else opens a + console window behind a tray app that has no window, at every single + sign-in. And it carries the vault path, because a user running a + non-default vault would otherwise silently get the default one back + after a reboot. """ exe, args = _autostart_parts(vault) return f'"{exe}" {args}' @@ -188,14 +189,25 @@ def _autostart_command(vault: str | None = None) -> str: def _autostart_parts(vault: str | None = None) -> tuple[str, str]: """The same command, split. A scheduled task wants the program and its arguments in separate XML elements, and re-parsing a quoted command line - to get them back would be a second place for the quoting to be wrong.""" + to get them back would be a second place for the quoting to be wrong. + + pythonw.exe beside this interpreter comes first, and the console script + only when there is none. compartment.exe, which pip and uv write for the + console script, is a console program like python.exe: started at + sign-in it opened a console window that stayed for the whole session, + since that process IS the tray app, and closing the window quit it. It + used to come first. Every venv, uv tool environment and python.org + install has a pythonw.exe beside python.exe, and it runs the same + environment's compartment. + """ vault = vault or env("VAULT") or str(home() / "memory.vault") + pyw = Path(sys.executable).with_name("pythonw.exe") + if pyw.is_file(): + return str(pyw), f'-m compartment.cli --vault "{vault}" tray' exe = shutil.which("compartment") if exe: return exe, f'--vault "{vault}" tray' - pyw = Path(sys.executable).with_name("pythonw.exe") - runner = str(pyw) if pyw.is_file() else sys.executable - return runner, f'-m compartment.cli --vault "{vault}" tray' + return sys.executable, f'-m compartment.cli --vault "{vault}" tray' # --- the Linux application entry -------------------------------------------- diff --git a/tests/test_status_bar_install.py b/tests/test_status_bar_install.py index efebba7..2af3ec0 100644 --- a/tests/test_status_bar_install.py +++ b/tests/test_status_bar_install.py @@ -187,6 +187,35 @@ def test_windows_autostart_avoids_a_console_window_and_keeps_the_vault(): assert "python.exe" not in cmd.lower() or "pythonw.exe" in cmd.lower() +def test_windows_autostart_prefers_pythonw_over_the_console_script( + tmp_path, monkeypatch): + """compartment.exe, the console script pip and uv write, is a console + program. Run at sign-in it opened a console window that stayed for the + whole session - that process is the tray app - and closing the window + quit the app. pythonw.exe beside the interpreter opens none.""" + from compartment import systray + scripts = tmp_path / "venv" / "Scripts" + scripts.mkdir(parents=True) + for name in ("python.exe", "pythonw.exe", "compartment.exe"): + (scripts / name).write_bytes(b"MZ") + monkeypatch.setattr(systray.sys, "executable", str(scripts / "python.exe")) + monkeypatch.setattr(systray.shutil, "which", + lambda name, *a, **k: str(scripts / "compartment.exe")) + exe, args = systray._autostart_parts("C:/Users/Zoë Ann/v.vault") + assert exe == str(scripts / "pythonw.exe") + assert args == '-m compartment.cli --vault "C:/Users/Zoë Ann/v.vault" tray' + assert systray._autostart_command("C:/v.vault").startswith( + f'"{scripts / "pythonw.exe"}" -m compartment.cli') + # The scheduled task runs the same thing. + xml = systray.scheduled_task_xml("C:/v.vault") + assert f"{scripts / 'pythonw.exe'}" in xml + # With no pythonw.exe (an embeddable or unusual build) the console + # script is still better than nothing. + (scripts / "pythonw.exe").unlink() + assert systray._autostart_parts("C:/v.vault")[0] == \ + str(scripts / "compartment.exe") + + # --- opting out of the GUI at install time --------------------------------- def test_a_scripted_install_never_waits_for_a_keypress(monkeypatch, capsys): From b54b4a560feddf31ace0c6be797f15cea2990c96 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:42:52 -0700 Subject: [PATCH 027/135] The macOS app build runs unattended in CI with only uv's standalone Python, exits non-zero naming the failed step, and lists what it built Three things stood between tools/build_macos_app.py and a CI runner: - The icon was drawn at build time by tools/make_icon.py, which needs Pillow in whatever Python runs the script (and iconutil). A runner given only uv's standalone Python has no Pillow, so the build died at the icon after every other step had passed. The package already ships that same drawing as data/app.icns (the pip install's login bundle uses it), so the build copies it, and only a checkout missing it is drawn. - A failed step surfaced as a CalledProcessError traceback through _run. main() now prints one line, "error: `` exited N", and returns 1; the checks that refuse a broken bundle already exit 1 with their reason. On success it prints `artifact: ` for the app and each installer. - hdiutil on hosted runners intermittently fails with "Resource busy"; it is tried three times, five seconds apart. find_standalone_python also looks where UV_PYTHON_INSTALL_DIR points. The docstring gives the exact CI invocation. Co-Authored-By: Claude Opus 5.5 --- tests/test_macos_bundle.py | 68 +++++++++++++++++++++++++++++++++++ tools/build_macos_app.py | 74 +++++++++++++++++++++++++++++++++----- 2 files changed, 133 insertions(+), 9 deletions(-) diff --git a/tests/test_macos_bundle.py b/tests/test_macos_bundle.py index 0b24e1b..8d716e3 100644 --- a/tests/test_macos_bundle.py +++ b/tests/test_macos_bundle.py @@ -472,3 +472,71 @@ def test_the_installer_runs_only_on_the_macs_the_app_can_start_on( assert dmg.name == f"Compartment-{version}{suffix}.dmg" readme = (tmp_path / "build" / "dmg" / "READ ME FIRST.txt").read_text() assert f"Compartment-{version}{suffix}.pkg" in readme + + +# ----------------------------------------------------- a build CI can run + +def test_the_icon_needs_no_pillow_in_the_python_running_the_build( + tmp_path, monkeypatch): + """A runner given only uv's standalone Python has no Pillow, and the + build died at the icon after every other step had passed.""" + monkeypatch.setitem(sys.modules, "PIL", None) # import PIL fails + b = _builder() + out = b.build_icon(tmp_path / "Resources" / "Compartment.icns") + shipped = ROOT / "src" / "compartment" / "data" / "app.icns" + assert out.read_bytes() == shipped.read_bytes() + + +def _main(b, monkeypatch, tmp_path, argv): + monkeypatch.setattr(b.sys, "platform", "darwin") + monkeypatch.setattr(b.sys, "argv", ["build_macos_app.py", *argv]) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + return b.main() + + +def test_a_failed_step_exits_non_zero_naming_the_step(tmp_path, monkeypatch, + capsys): + import subprocess + b = _builder() + + def fail(*a, **k): + raise subprocess.CalledProcessError(2, ["pkgbuild", "--root", "x"]) + + monkeypatch.setattr(b, "build_app", fail) + assert _main(b, monkeypatch, tmp_path, ["--pkg"]) == 1 + err = capsys.readouterr().err + assert "error: `pkgbuild --root x` exited 2" in err + assert "Traceback" not in err + + +def test_the_build_lists_what_it_made(tmp_path, monkeypatch, capsys): + b = _builder() + app = tmp_path / "build" / "Compartment.app" + monkeypatch.setattr(b, "build_app", lambda *a, **k: app) + monkeypatch.setattr(b, "build_pkg", lambda a: a.with_name("C-1.pkg")) + monkeypatch.setattr(b, "build_dmg", lambda a: a.with_name("C-1.dmg")) + assert _main(b, monkeypatch, tmp_path, ["--pkg", "--dmg"]) == 0 + listed = [ln.split(": ", 1)[1] for ln in capsys.readouterr().out + .splitlines() if ln.startswith("artifact: ")] + assert listed == [str(app), str(app.with_name("C-1.dmg")), + str(app.with_name("C-1.pkg"))] + + +def test_a_busy_disk_image_is_tried_again(tmp_path, monkeypatch): + import subprocess + b = _builder() + app = _fake_app(tmp_path, { + "Contents/MacOS/Compartment": _macho("arm64", (13, 0))}) + monkeypatch.setattr(b, "BUILD", tmp_path / "build") + monkeypatch.setattr(b.time, "sleep", lambda s: None) + tries = [] + + def run(*args, **kw): + if args[0] == "hdiutil": + tries.append(args) + if len(tries) < 3: + raise subprocess.CalledProcessError(1, list(args)) + + monkeypatch.setattr(b, "_run", run) + b.build_dmg(app) + assert len(tries) == 3 diff --git a/tools/build_macos_app.py b/tools/build_macos_app.py index ae61657..f681389 100644 --- a/tools/build_macos_app.py +++ b/tools/build_macos_app.py @@ -5,6 +5,17 @@ python tools/build_macos_app.py --pkg # …and a .pkg with an optional # "start at login" component +From CI, on a GitHub macOS runner (arm64, or macos-15-intel for x86_64), +with uv's standalone Python and nothing else installed: + + uv python install 3.12 + PY="$(uv python find --managed-python 3.12)" + "$PY" tools/build_macos_app.py --python "$PY" --pkg --dmg + +It exits non-zero on any failure, with one line saying which step, and +writes build/Compartment-.pkg and .dmg (-intel before the +extension for an x86_64 build), listing them as `artifact: `. + Why the app has to embed its own Python --------------------------------------- A status bar item belongs to whatever process created it, and macOS decides @@ -55,6 +66,7 @@ import sys import sysconfig import tempfile +import time ROOT = pathlib.Path(__file__).resolve().parents[1] _EMBEDDED_VER = [""] # X.Y of the interpreter actually embedded @@ -85,8 +97,25 @@ def _run(*args, **kw) -> subprocess.CompletedProcess: def build_icon(dest: pathlib.Path) -> pathlib.Path: + """The app's icon: the .icns the package already ships, which + tools/make_icon.py draws and the pip install's login bundle uses too. + + Drawing it again here needed Pillow in whatever Python ran this script + and iconutil, and a CI runner given only uv's standalone Python has no + Pillow: the build died at the icon after every other step had passed. + The copy is the same picture, and only a checkout missing it is drawn. + """ + shipped = ROOT / "src" / "compartment" / "data" / "app.icns" + if shipped.is_file(): + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(shipped, dest) + return dest sys.path.insert(0, str(ROOT / "tools")) - from make_icon import build as build_icns + try: + from make_icon import build as build_icns + except ImportError as exc: + raise SystemExit(f"error: {shipped} is missing, and drawing it needs " + f"Pillow ({exc}): pip install pillow") from exc return build_icns(dest) @@ -127,7 +156,11 @@ def ok(exe: str) -> bool: if ok(sys.executable): return sys.executable - uv_root = pathlib.Path.home() / ".local" / "share" / "uv" / "python" + # Where uv keeps its Pythons, which UV_PYTHON_INSTALL_DIR moves (CI + # caches often do). + uv_root = pathlib.Path(os.environ.get("UV_PYTHON_INSTALL_DIR") + or pathlib.Path.home() / ".local" / "share" / "uv" + / "python") for d in sorted(uv_root.glob("cpython-3.*-macos-*"), reverse=True): for exe in sorted(d.glob("bin/python3.*"), reverse=True): if exe.is_file() and ok(str(exe)): @@ -930,8 +963,20 @@ def build_dmg(app: pathlib.Path) -> pathlib.Path: "https://github.com/MaxFreedomPollard/Compartment\n", encoding="utf-8") out = BUILD / artifact_name(version, arch, "dmg") out.unlink(missing_ok=True) - _run("hdiutil", "create", "-volname", f"{APP_NAME} {version}", - "-srcfolder", stage, "-ov", "-format", "UDZO", out) + # hdiutil on a CI runner now and then fails with "Resource busy" while + # something (Spotlight, the previous image) still has the volume; the + # same command a few seconds later succeeds. Three tries, then the + # error stands. + for attempt in range(3): + try: + _run("hdiutil", "create", "-volname", f"{APP_NAME} {version}", + "-srcfolder", stage, "-ov", "-format", "UDZO", out) + break + except subprocess.CalledProcessError: + if attempt == 2: + raise + print("hdiutil failed; trying again in 5 s") + time.sleep(5) print(f"built {out}") return out @@ -950,11 +995,22 @@ def main() -> int: ap.add_argument("--pkg", action="store_true") args = ap.parse_args() BUILD.mkdir(exist_ok=True) - app = build_app(args.spec, python=args.python, min_macos=args.min_macos) - if args.dmg: - build_dmg(app) - if args.pkg: - build_pkg(app) + try: + artifacts = [build_app(args.spec, python=args.python, + min_macos=args.min_macos)] + if args.dmg: + artifacts.append(build_dmg(artifacts[0])) + if args.pkg: + artifacts.append(build_pkg(artifacts[0])) + except subprocess.CalledProcessError as exc: + # A step that failed says which, in one line, instead of a + # traceback through _run that a CI log buries the command in. + cmd = " ".join(str(a) for a in exc.cmd) + print(f"error: `{cmd[:300]}` exited {exc.returncode}", + file=sys.stderr) + return 1 + for path in artifacts: + print(f"artifact: {path}") return 0 From c40f5c1a928e746b2ed2ac3e92d396f69a5031fc Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:43:00 -0700 Subject: [PATCH 028/135] The boot instant is read on macOS and the BSDs without starting a process on every vault open Two forks were left on the path c557f1a meant to make in-process: - The in-process kern.boottime read found libc with ctypes.util.find_library("c"), which on FreeBSD, OpenBSD and DragonFly runs /sbin/ldconfig -r and on NetBSD runs the C compiler, in a subprocess, on every call. It now uses the process's own symbols (CDLL(None)), where libc's sysctlbyname already is. - On macOS, when the in-process read is refused (Seatbelt, under Claude Code and Codex), the sysctl binary was forked as a fallback. It makes the same call and is refused the same way, so every vault open in an agent's shell paid a process and a sandbox-violation report for nothing. macOS now goes straight to the clock fallback, which the binary was only ever meant to back up on OpenBSD (no sysctlbyname). The clock fallback's docstring said macOS CLOCK_MONOTONIC is mach_continuous_time. It is not: libc computes it as the wall clock minus the kernel's boot time from the commpage, so wall clock minus it is kern.boottime to the microsecond (checked in this sandbox: constant across calls, and 8.6 s from CLOCK_MONOTONIC_RAW). The conclusion, that it counts through sleep and agrees with the kernel record, stands, and the docstring now says why. A Linux test now checks the CLOCK_BOOTTIME estimate against /proc/stat's btime; the existing comparison only ever ran where sysctl was readable, which no CI runner is. Before the fix: test_a_refused_kern_boottime_on_macos_does_not_fork_sysctl E Left contains one more item: ['sysctl', '-n', 'kern.boottime'] test_the_in_process_read_runs_no_helper_to_find_libc [FreeBSD, NetBSD, DragonFly, Darwin] E Left contains one more item: 'c' Co-Authored-By: Claude Opus 5.5 --- src/compartment/platforms.py | 29 +++++++++++--- tests/test_platform_boot.py | 75 ++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 tests/test_platform_boot.py diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index 5310b88..15d915d 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -197,8 +197,12 @@ def _sysctl_boot_estimate() -> tuple[float, float] | None: import struct try: import ctypes - import ctypes.util - libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + # The process's own symbols, which include libc's. Not + # ctypes.util.find_library("c"): on FreeBSD, OpenBSD and DragonFly + # that runs /sbin/ldconfig -r, and on NetBSD it runs the C compiler, + # in a subprocess, on every call - a fork on every vault open, which + # is the thing reading this in-process was meant to avoid. + libc = ctypes.CDLL(None, use_errno=True) buf = ctypes.create_string_buffer(32) size = ctypes.c_size_t(len(buf)) if libc.sysctlbyname(b"kern.boottime", buf, ctypes.byref(size), None, 0) == 0: @@ -208,8 +212,16 @@ def _sysctl_boot_estimate() -> tuple[float, float] | None: sec = float(struct.unpack_from(fmt, buf.raw)[0]) if sec > 0: return sec, time.time() - sec - except (ImportError, OSError, AttributeError, ValueError, struct.error): + except (ImportError, OSError, AttributeError, ValueError, TypeError, + struct.error): pass + if platform.system() == "Darwin": + # macOS always has sysctlbyname, and the binary makes the same call: + # where the call above was refused - Seatbelt, under Claude Code and + # Codex - the binary is refused too. Forking it anyway cost every + # vault open in an agent's shell a process and a sandbox-violation + # report, for an answer that could not come. + return None try: out = subprocess.run(["sysctl", "-n", "kern.boottime"], capture_output=True, text=True, check=True, @@ -229,9 +241,14 @@ def _clock_boot_estimate() -> tuple[float, float] | None: Only clocks that keep counting through sleep qualify, or every resume would read as a reboot: CLOCK_BOOTTIME on Linux, Android and OpenBSD, and - CLOCK_MONOTONIC on macOS, where it is mach_continuous_time and includes - sleep. The result feeds _pinned_boot_token like any other estimate, so - the first value seen this boot is the one every process agrees on.""" + CLOCK_MONOTONIC on macOS. There libc computes CLOCK_MONOTONIC as the wall + clock minus the kernel's boot time, read from the commpage, which no + sandbox gates; so wall clock minus it IS kern.boottime, to the + microsecond (checked under Seatbelt: constant across calls, and 8.6 s + away from mach_continuous_time, CLOCK_MONOTONIC_RAW, on a machine up + for two days). The result feeds _pinned_boot_token like any other + estimate, so the first value seen this boot is the one every process + agrees on.""" names = ("CLOCK_MONOTONIC",) if platform.system() == "Darwin" else ( "CLOCK_BOOTTIME",) for name in names: diff --git a/tests/test_platform_boot.py b/tests/test_platform_boot.py new file mode 100644 index 0000000..c9beff3 --- /dev/null +++ b/tests/test_platform_boot.py @@ -0,0 +1,75 @@ +"""Reading the boot instant on macOS and the BSDs: in-process, without a +fork on every vault open, and from every sysctl dialect.""" +import ctypes +import ctypes.util +import platform +import subprocess + +import pytest + +from compartment import platforms + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "sess")) + + +class _RefusedLibc: + """libc as Seatbelt leaves it: sysctlbyname is there and answers EPERM.""" + + def sysctlbyname(self, *a): + return -1 + + +def _record_subprocesses(monkeypatch, stdout="", fail=False): + calls = [] + + def run(cmd, *a, **kw): + calls.append(cmd) + if fail: + raise subprocess.CalledProcessError(1, cmd) + return subprocess.CompletedProcess(cmd, 0, stdout, "") + monkeypatch.setattr(platforms.subprocess, "run", run) + return calls + + +def test_a_refused_kern_boottime_on_macos_does_not_fork_sysctl(monkeypatch): + """Under Seatbelt (Claude Code, Codex) the in-process sysctl is refused, + and the sysctl binary makes the same call and is refused as well. It was + forked anyway on every vault open from the agent's shell.""" + monkeypatch.setattr(platforms.platform, "system", lambda: "Darwin") + monkeypatch.setattr(ctypes, "CDLL", lambda *a, **kw: _RefusedLibc()) + calls = _record_subprocesses(monkeypatch, fail=True) + assert platforms._sysctl_boot_estimate() is None + assert calls == [] + + +@pytest.mark.parametrize("system", ["FreeBSD", "NetBSD", "DragonFly", "Darwin"]) +def test_the_in_process_read_runs_no_helper_to_find_libc(monkeypatch, system): + """ctypes.util.find_library("c") runs /sbin/ldconfig -r on FreeBSD, + OpenBSD and DragonFly and the C compiler on NetBSD, in a subprocess, on + every call: the in-process read forked anyway.""" + looked_up = [] + monkeypatch.setattr(ctypes.util, "find_library", + lambda name: looked_up.append(name) or None) + monkeypatch.setattr(platforms.platform, "system", lambda: system) + _record_subprocesses(monkeypatch, fail=True) + platforms._sysctl_boot_estimate() + assert looked_up == [] + + +@pytest.mark.skipif(platform.system() != "Linux", + reason="compares against /proc/stat") +def test_the_linux_clock_estimate_matches_the_kernel_btime(): + """The CLOCK_BOOTTIME fallback (Termux, a sandbox hiding /proc/stat) must + name the boot instant the kernel reports, or a process that can read + /proc/stat and one that cannot would pin different tokens.""" + try: + with open("/proc/stat") as f: + btime = next(float(line.split()[1]) for line in f + if line.startswith("btime ")) + except (OSError, StopIteration): + pytest.skip("/proc/stat btime not readable here") + clock = platforms._clock_boot_estimate() + assert clock is not None and abs(clock[0] - btime) < 2 From 18ae36b50a43db1ea5115f90112ae78855b152b4 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:43:19 -0700 Subject: [PATCH 029/135] OpenBSD's kern.boottime is read from the sysctl binary instead of being missed OpenBSD has no sysctlbyname, so `sysctl -n kern.boottime` is its only way to the kernel's boot record, and there it prints just the seconds (sbin/sysctl/sysctl.c, read on GitHub). The parser only knew FreeBSD's "{ sec = N, usec = M }" form, so on OpenBSD the kernel record was always missed and the boot token came from the CLOCK_BOOTTIME estimate instead. A bare integer is now read as well. FreeBSD's form is unchanged. Before the fix: test_every_sysctl_dialect_of_kern_boottime_is_read[1700000000\n] E assert (None is not None) Co-Authored-By: Claude Opus 5.5 --- src/compartment/platforms.py | 6 +++++- tests/test_platform_boot.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index 15d915d..fa9db98 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -227,7 +227,11 @@ def _sysctl_boot_estimate() -> tuple[float, float] | None: capture_output=True, text=True, check=True, timeout=_SUBPROCESS_TIMEOUT).stdout import re - m = re.search(r"sec = (\d+)", out) + # FreeBSD: "{ sec = 1700000000, usec = 0 } Tue Nov 14 ...". + # OpenBSD, the one that needs this path, prints only the seconds + # under -n (sbin/sysctl/sysctl.c), which the first pattern missed. + m = (re.search(r"sec = (\d+)", out) + or re.fullmatch(r"\s*(\d+)\s*", out)) if m: sec = float(m.group(1)) return sec, time.time() - sec diff --git a/tests/test_platform_boot.py b/tests/test_platform_boot.py index c9beff3..f740f06 100644 --- a/tests/test_platform_boot.py +++ b/tests/test_platform_boot.py @@ -73,3 +73,22 @@ def test_the_linux_clock_estimate_matches_the_kernel_btime(): pytest.skip("/proc/stat btime not readable here") clock = platforms._clock_boot_estimate() assert clock is not None and abs(clock[0] - btime) < 2 + + +@pytest.mark.parametrize("stdout", [ + "1700000000\n", # OpenBSD -n + "{ sec = 1700000000, usec = 123456 } Tue Nov 14 22:13:20 2023\n", # FreeBSD +]) +def test_every_sysctl_dialect_of_kern_boottime_is_read(monkeypatch, stdout): + """OpenBSD has no sysctlbyname, so the binary is its only kernel record, + and `sysctl -n kern.boottime` prints the bare seconds there + (sbin/sysctl/sysctl.c). Only FreeBSD's "sec = N" was understood, so + OpenBSD always fell through to the clock.""" + monkeypatch.setattr(platforms.platform, "system", lambda: "OpenBSD") + + class NoSysctlbyname: + pass + monkeypatch.setattr(ctypes, "CDLL", lambda *a, **kw: NoSysctlbyname()) + _record_subprocesses(monkeypatch, stdout=stdout) + est = platforms._sysctl_boot_estimate() + assert est is not None and est[0] == 1_700_000_000.0 From 389dfc8e1892678d280217f330e7114862f1acc9 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:44:21 -0700 Subject: [PATCH 030/135] On Windows, the panel's buttons start the CLI without a console window, and Dashboard no longer keeps one open while it runs The tray app runs with no console (pythonw.exe at sign-in, and DETACHED_PROCESS when `init` starts it), and every CLI it starts is a console program. A console program started without CREATE_NO_WINDOW from a process that has no console gets a window of its own. Only the vault view and the JSON status calls passed the flag; Unlock, Lock, Create vault, Change password and the Connect buttons each flashed a console window, and Dashboard kept one open for as long as the dashboard ran, where closing it stopped the dashboard. Every child the panel starts now goes through one helper, _no_console, which gives the flag on Windows and nothing elsewhere (creationflags is an error on POSIX). Grandchildren, such as the agent's own CLI that `integrate` runs, inherit the hidden console and open none either. Co-Authored-By: Claude Opus 5.5 --- src/compartment/menubar.py | 33 ++++++---- tests/test_apps_windows_children.py | 96 +++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 tests/test_apps_windows_children.py diff --git a/src/compartment/menubar.py b/src/compartment/menubar.py index 1e0f1aa..588252c 100644 --- a/src/compartment/menubar.py +++ b/src/compartment/menubar.py @@ -324,7 +324,8 @@ def _run(args: list[str], timeout: int = 60) -> tuple[int, str]: try: p = subprocess.run(args, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}) + env={**os.environ, "PATH": user_path()}, + **_no_console()) return p.returncode, (p.stdout or "") + (p.stderr or "") except (OSError, subprocess.SubprocessError) as exc: return 1, str(exc) @@ -350,13 +351,11 @@ def _json_cmd(vault: str, *sub: str) -> dict | None: def _json_run(argv: list[str], timeout: float = 60) -> dict | None: """Run a command and parse the JSON it prints: _json_cmd's runner, for any command line. None if it failed, timed out, or printed no JSON.""" - kwargs = {} - if sys.platform == "win32": - kwargs["creationflags"] = _CREATE_NO_WINDOW try: p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}, **kwargs) + env={**os.environ, "PATH": user_path()}, + **_no_console()) except (OSError, subprocess.SubprocessError): return None if p.returncode != 0: @@ -442,6 +441,17 @@ def set_setting(vault: str, key: str, value) -> dict: #: and the vault view would keep its open for as long as it runs. _CREATE_NO_WINDOW = 0x08000000 + +def _no_console() -> dict: + """subprocess keyword arguments for a child the panel starts: on Windows, + no console window. Every child, not only the two that had it. Unlock, + Lock, Create, Change password and Connect each ran the CLI as a console + program, so from the tray - pythonw.exe, which has no console of its own + - each click flashed a console window, and Dashboard kept one open for + as long as the dashboard ran, where closing it stopped the dashboard.""" + return ({"creationflags": _CREATE_NO_WINDOW} if sys.platform == "win32" + else {}) + NO_VAULT_NOTE = "no vault yet - choose a passphrase below to create one" CREDENTIAL_REFUSED_NOTE = ("The saved unlock no longer opens this vault. " "Enter the passphrase to unlock it again.") @@ -534,15 +544,12 @@ def __init__(self, vault: str): self._next = 0 def _start(self): - kwargs = {} - if sys.platform == "win32": - kwargs["creationflags"] = _CREATE_NO_WINDOW proc = subprocess.Popen( _view_argv(self.vault), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None if env("MENUBAR_DEBUG") else subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace", bufsize=1, - env={**os.environ, "PATH": user_path()}, **kwargs) + env={**os.environ, "PATH": user_path()}, **_no_console()) lines: queue.Queue = queue.Queue() def pump(): @@ -948,7 +955,7 @@ def open_dashboard(vault: str) -> str | None: [*_cli_argv(), "--vault", vault, "dash"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}) + env={**os.environ, "PATH": user_path()}, **_no_console()) except (OSError, subprocess.SubprocessError): return None _DASH["proc"], _DASH["url"] = proc, None @@ -1011,7 +1018,7 @@ def unlock_vault(vault: str, passphrase: str) -> tuple[bool, str]: p = subprocess.run( [*_cli_argv(), "--vault", vault, "unlock", "--passphrase-stdin"], input=passphrase + "\n", capture_output=True, text=True, - timeout=180, encoding="utf-8", errors="replace") + timeout=180, encoding="utf-8", errors="replace", **_no_console()) except (OSError, subprocess.SubprocessError) as exc: return False, str(exc) out = " ".join(((p.stdout or "") + (p.stderr or "")).split()) @@ -1064,7 +1071,7 @@ def create_vault(vault: str, passphrase: str, repeat: str) -> tuple[bool, str]: "--no-app"], input=passphrase + "\n", capture_output=True, text=True, timeout=900, encoding="utf-8", errors="replace", - env={**os.environ, "PATH": user_path()}) + env={**os.environ, "PATH": user_path()}, **_no_console()) except subprocess.TimeoutExpired as exc: if os.path.exists(vault): # init writes the vault in its first seconds. What ran out of @@ -1106,7 +1113,7 @@ def change_passphrase(vault: str, new: str, repeat: str) -> tuple[bool, str]: p = subprocess.run( [*_cli_argv(), "--vault", vault, "rekey", "--new-passphrase-stdin"], input=new + "\n", capture_output=True, text=True, - timeout=180, encoding="utf-8", errors="replace") + timeout=180, encoding="utf-8", errors="replace", **_no_console()) except (OSError, subprocess.SubprocessError) as exc: return False, str(exc) out = " ".join(((p.stdout or "") + (p.stderr or "")).split()) diff --git a/tests/test_apps_windows_children.py b/tests/test_apps_windows_children.py new file mode 100644 index 0000000..905c067 --- /dev/null +++ b/tests/test_apps_windows_children.py @@ -0,0 +1,96 @@ +"""On Windows, nothing the panel starts opens a console window. + +The tray app runs under pythonw.exe, which has no console. Every CLI it +starts is a console program, and a console program started without +CREATE_NO_WINDOW from a process with no console gets a window of its own: +a flash for each click on Unlock, Lock, Create, Change password or Connect, +and, for Dashboard, a window that stayed open for as long as the dashboard +ran, where closing it stopped the dashboard. Only the vault view and the +JSON status calls had the flag. +""" +from __future__ import annotations + +import io +import subprocess + +import pytest + +from compartment import menubar + +CREATE_NO_WINDOW = 0x08000000 + + +@pytest.fixture() +def children(monkeypatch, tmp_path): + """Every child the panel starts, with the keyword arguments it got.""" + seen: list[tuple[list, dict]] = [] + + def run(argv, **kw): + seen.append((list(argv), kw)) + return subprocess.CompletedProcess(argv, 0, "", "") + + class Proc: + def __init__(self, argv, **kw): + seen.append((list(argv), kw)) + self.stdout = io.StringIO( + "Compartment dashboard: http://127.0.0.1:1/t/\n") + self.stdin = io.StringIO() + + def poll(self): + return None + + def terminate(self): + pass + + def wait(self, timeout=None): + return 0 + + def kill(self): + pass + + monkeypatch.setattr(menubar.sys, "platform", "win32") + monkeypatch.setattr(menubar.subprocess, "run", run) + monkeypatch.setattr(menubar.subprocess, "Popen", Proc) + monkeypatch.setattr(menubar, "user_path", lambda: "") + monkeypatch.setattr(menubar, "agent_present", lambda target: True) + monkeypatch.setattr(menubar, "integration_status", + lambda vault: {"claude": True}) + yield seen + menubar._DASH.update(proc=None, url=None) + + +def _flags(kw: dict) -> int: + return int(kw.get("creationflags", 0)) + + +@pytest.mark.parametrize("click", [ + lambda v: menubar.unlock_vault(v, "pw"), + lambda v: menubar.lock_vault(v), + lambda v: menubar.create_vault(v + ".new", "pw", "pw"), + lambda v: menubar.change_passphrase(v, "pw", "pw"), + lambda v: menubar.integrate(v, "claude"), + lambda v: menubar.open_dashboard(v), +], ids=["unlock", "lock", "create", "change-password", "connect", + "dashboard"]) +def test_every_button_starts_its_cli_without_a_console(children, tmp_path, + click): + click(str(tmp_path / "memory.vault")) + cli = [(argv, kw) for argv, kw in children if "compartment.cli" in argv] + assert cli, "the click ran the CLI" + for argv, kw in cli: + assert _flags(kw) & CREATE_NO_WINDOW, argv + + +def test_the_vault_view_and_status_calls_keep_theirs(children, tmp_path): + vault = str(tmp_path / "memory.vault") + menubar.ViewProcess(vault)._start() + menubar._json_run(["x"]) + assert all(_flags(kw) & CREATE_NO_WINDOW for _argv, kw in children) + + +def test_elsewhere_no_windows_flag_is_passed(monkeypatch): + """creationflags is Windows-only; passing it on POSIX raises.""" + monkeypatch.setattr(menubar.sys, "platform", "darwin") + assert menubar._no_console() == {} + monkeypatch.setattr(menubar.sys, "platform", "linux") + assert menubar._no_console() == {} From e427c92cd40ff5a44bdd3c7f764cd820b1e4d98f Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:45:26 -0700 Subject: [PATCH 031/135] Every memory tool tells the client it works on this machine alone, and the ones safe to repeat say so The MCP spec reads an absent openWorldHint as true: the tool "may interact with an open world of external entities", as a web search does. No tool set it, so every client that weighs the hint (to group tools, flag them, or ask before running them) was told a fully offline, encrypted vault might reach the internet with each search and store. Every tool now declares openWorldHint=False, and memory_link, memory_unlink, memory_forget, memory_lock and memory_unlock declare idempotentHint=True, since repeating one with the same arguments changes nothing more. Names, titles, input schemas and results are unchanged. New tests list the tools through whichever SDK major is installed (checked on mcp 2.3.0 and 1.30.0) and hold the hints, and the input-schema shapes strict clients (Gemini, Codex, Cursor) accept, in place. Co-Authored-By: Claude Opus 5.5 --- src/compartment/server.py | 74 +++++++++++++++++++++------------- tests/test_server_tools.py | 81 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 tests/test_server_tools.py diff --git a/src/compartment/server.py b/src/compartment/server.py index 22ed303..4cec57f 100644 --- a/src/compartment/server.py +++ b/src/compartment/server.py @@ -409,6 +409,19 @@ def _call(): return wrapper +def _hints(title: str, **hints) -> ToolAnnotations: + """A tool's annotations, with openWorldHint=False on every one. + + The spec's value for an absent openWorldHint is true: "this tool may + interact with an open world of external entities", the way a web search + does. Left out, every memory tool told the client it might reach beyond + this machine, which is the one thing Compartment never does: clients + that weigh the hint (to group, flag or confirm such tools) treated a + fully offline vault as an internet tool. Every tool works on the local + vault alone, so all of them say so.""" + return ToolAnnotations(title=title, openWorldHint=False, **hints) + + IMPORTANCE_DEFAULT = 0.5 # Records per slice of a background retag pass. Sized so the gate is held for @@ -433,8 +446,8 @@ def _importance(value: float) -> float: return min(1.0, max(0.0, v)) -@mcp.tool(title="Store a memory", annotations=ToolAnnotations( - title="Store a memory", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Store a memory", annotations=_hints( + "Store a memory", readOnlyHint=False, destructiveHint=False)) @_offload def memory_store(text: str, source: str, namespace: str | None = None, tags: list[str] | None = None, importance: float = 0.5, @@ -560,8 +573,8 @@ def _prevalidate_facts(v: Vault, facts: list, namespace: str | None = None, raise MemoryToolError(f"facts[{i}]: {exc}") from None -@mcp.tool(title="Store several memories", annotations=ToolAnnotations( - title="Store several memories", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Store several memories", annotations=_hints( + "Store several memories", readOnlyHint=False, destructiveHint=False)) @_offload def memory_store_many(facts: list[dict], source: str, namespace: str | None = None, @@ -636,8 +649,8 @@ def _store_all(v): return json.dumps({"stored": results, "count": len(results)}) -@mcp.tool(title="Search memory", annotations=ToolAnnotations( - title="Search memory", readOnlyHint=True)) +@mcp.tool(title="Search memory", annotations=_hints( + "Search memory", readOnlyHint=True)) @_offload def memory_search(query: str, namespace: str | None = None, tags: list[str] | None = None, top_k: int = 8, @@ -668,8 +681,9 @@ def memory_search(query: str, namespace: str | None = None, raise _fail(exc) from exc -@mcp.tool(title="Link two entities", annotations=ToolAnnotations( - title="Link two entities", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Link two entities", annotations=_hints( + "Link two entities", readOnlyHint=False, destructiveHint=False, + idempotentHint=True)) @_offload def memory_link(subject: str, predicate: str, object: str, src_id: str | None = None, valid_from: float | None = None, @@ -691,8 +705,8 @@ def memory_link(subject: str, predicate: str, object: str, raise _fail(exc) from exc -@mcp.tool(title="Query the memory graph", annotations=ToolAnnotations( - title="Query the memory graph", readOnlyHint=True)) +@mcp.tool(title="Query the memory graph", annotations=_hints( + "Query the memory graph", readOnlyHint=True)) @_offload def memory_relations(entity: str | None = None, subject: str | None = None, predicate: str | None = None, object: str | None = None, @@ -719,8 +733,9 @@ def memory_relations(entity: str | None = None, subject: str | None = None, return json.dumps(out) -@mcp.tool(title="Remove a relation", annotations=ToolAnnotations( - title="Remove a relation", readOnlyHint=False, destructiveHint=True)) +@mcp.tool(title="Remove a relation", annotations=_hints( + "Remove a relation", readOnlyHint=False, destructiveHint=True, + idempotentHint=True)) @_offload def memory_unlink(relation_id: str) -> str: """Remove one relation from the memory graph (memories stay untouched).""" @@ -731,8 +746,8 @@ def memory_unlink(relation_id: str) -> str: raise _fail(exc) from exc -@mcp.tool(title="Fetch a memory", annotations=ToolAnnotations( - title="Fetch a memory", readOnlyHint=True)) +@mcp.tool(title="Fetch a memory", annotations=_hints( + "Fetch a memory", readOnlyHint=True)) @_offload def memory_get(record_id: str) -> str: """Fetch one memory by id.""" @@ -743,8 +758,9 @@ def memory_get(record_id: str) -> str: raise _fail(exc) from exc -@mcp.tool(title="Forget a memory", annotations=ToolAnnotations( - title="Forget a memory", readOnlyHint=False, destructiveHint=True)) +@mcp.tool(title="Forget a memory", annotations=_hints( + "Forget a memory", readOnlyHint=False, destructiveHint=True, + idempotentHint=True)) @_offload def memory_forget(record_id: str, shred: bool = False) -> str: """Delete a memory. shred=True crypto-shreds it (unrecoverable from this vault).""" @@ -755,8 +771,8 @@ def memory_forget(record_id: str, shred: bool = False) -> str: raise _fail(exc) from exc -@mcp.tool(title="List namespaces", annotations=ToolAnnotations( - title="List namespaces", readOnlyHint=True)) +@mcp.tool(title="List namespaces", annotations=_hints( + "List namespaces", readOnlyHint=True)) @_offload def memory_list_namespaces() -> str: """List namespaces and record counts.""" @@ -766,8 +782,8 @@ def memory_list_namespaces() -> str: raise _fail(exc) from exc -@mcp.tool(title="Recent memories", annotations=ToolAnnotations( - title="Recent memories", readOnlyHint=True)) +@mcp.tool(title="Recent memories", annotations=_hints( + "Recent memories", readOnlyHint=True)) @_offload def memory_recent(limit: int = 20, namespace: str | None = None, include_seeded: bool = False) -> str: @@ -785,8 +801,8 @@ def memory_recent(limit: int = 20, namespace: str | None = None, raise _fail(exc) from exc -@mcp.tool(title="Vault status", annotations=ToolAnnotations( - title="Vault status", readOnlyHint=True)) +@mcp.tool(title="Vault status", annotations=_hints( + "Vault status", readOnlyHint=True)) @_offload def memory_status() -> str: """Vault status: lock state, counts, packs, model, index, RAM, audit head.""" @@ -805,8 +821,8 @@ def memory_status() -> str: raise _fail(exc) from exc -@mcp.tool(title="Memory self-test", annotations=ToolAnnotations( - title="Memory self-test", readOnlyHint=True)) +@mcp.tool(title="Memory self-test", annotations=_hints( + "Memory self-test", readOnlyHint=True)) @_offload def memory_selftest() -> str: """Health check: canned queries against the built-in seed pack, with latencies.""" @@ -834,8 +850,9 @@ def _drop_key(v: Vault) -> None: v._id_by_ikey = {} -@mcp.tool(title="Lock the vault", annotations=ToolAnnotations( - title="Lock the vault", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Lock the vault", annotations=_hints( + "Lock the vault", readOnlyHint=False, destructiveHint=False, + idempotentHint=True)) @_offload def memory_lock() -> str: """PANIC LOCK: flush, seal, and drop key material now. Always available. @@ -882,8 +899,9 @@ def memory_lock() -> str: return json.dumps(out) -@mcp.tool(title="Unlock the vault", annotations=ToolAnnotations( - title="Unlock the vault", readOnlyHint=False, destructiveHint=False)) +@mcp.tool(title="Unlock the vault", annotations=_hints( + "Unlock the vault", readOnlyHint=False, destructiveHint=False, + idempotentHint=True)) @_offload def memory_unlock(passphrase: str) -> str: """Unlock the vault for this session so the other memory tools can use it diff --git a/tests/test_server_tools.py b/tests/test_server_tools.py new file mode 100644 index 0000000..199f96b --- /dev/null +++ b/tests/test_server_tools.py @@ -0,0 +1,81 @@ +"""What `tools/list` tells a client about each memory tool: the hints it +uses to decide how to present and gate a tool, and input schemas that the +strictest clients' validators take. + +Listed through the SDK the server runs on (mcp 1.x or 2.x), without an +event loop, as test_server.py calls tools: listing awaits nothing.""" +from compartment import server + + +def _listed(): + call = server.mcp.list_tools() + try: + call.send(None) + except StopIteration as done: + return done.value + finally: + call.close() + raise AssertionError("list_tools waited for an event loop") + + +def _dump(tool) -> dict: + return tool.model_dump(by_alias=True, exclude_none=True) + + +def test_every_tool_says_it_stays_on_this_machine(): + """openWorldHint absent means true in the spec: "may interact with an + open world of external entities". Every tool left it out, so a client + that weighs the hint treated a fully offline vault as an internet + tool.""" + tools = [_dump(t) for t in _listed()] + assert len(tools) == 14 + for t in tools: + hints = t["annotations"] + assert hints["openWorldHint"] is False, t["name"] + assert hints["title"] == t["title"], t["name"] + assert isinstance(hints["readOnlyHint"], bool), t["name"] + + +def test_the_hints_match_what_each_tool_does(): + hints = {t["name"]: t["annotations"] for t in map(_dump, _listed())} + for name in ("memory_search", "memory_get", "memory_recent", + "memory_relations", "memory_list_namespaces", + "memory_status", "memory_selftest"): + assert hints[name]["readOnlyHint"] is True, name + for name in ("memory_forget", "memory_unlink"): + assert hints[name]["destructiveHint"] is True, name + for name in ("memory_link", "memory_unlink", "memory_forget", + "memory_lock", "memory_unlock"): + assert hints[name].get("idempotentHint") is True, name + for name in ("memory_store", "memory_store_many"): + assert not hints[name].get("idempotentHint"), name + + +def _typed(schema: dict, where: str) -> None: + """Every schema node carries a type, directly or in each branch of an + anyOf: Gemini's and older Codex's converters dropped or defaulted an + untyped node, and none of them resolve $ref.""" + assert "$ref" not in schema and "$defs" not in schema, where + if "anyOf" in schema: + assert schema["anyOf"], where + for i, branch in enumerate(schema["anyOf"]): + _typed(branch, f"{where}.anyOf[{i}]") + return + assert "type" in schema, where + if schema["type"] == "array": + _typed(schema["items"], f"{where}.items") + if schema["type"] == "object": + for key, sub in schema.get("properties", {}).items(): + _typed(sub, f"{where}.{key}") + + +def test_every_input_schema_is_one_strict_clients_accept(): + """An object at the top, with no union there (OpenAI and Gemini refuse + one), every property typed, and every required name defined.""" + for t in map(_dump, _listed()): + schema = t["inputSchema"] + assert schema["type"] == "object", t["name"] + for key in ("anyOf", "oneOf", "allOf", "not", "$ref"): + assert key not in schema, (t["name"], key) + _typed(schema, t["name"]) + assert set(schema.get("required", [])) <= set(schema["properties"]) From 9603864304f27568d489625b3b379a93c4c5fbc2 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:45:27 -0700 Subject: [PATCH 032/135] The offline guard refuses every socket family that can reach another machine, not only AF_INET and AF_INET6 The guard's promise is that nothing in the process can put a byte on the wire, but it refused only AF_INET and AF_INET6 and let "every other local family" through. Several are not local: AF_PACKET writes raw Ethernet frames (root in a container has CAP_NET_RAW by default), AF_BLUETOOTH RFCOMM/L2CAP reaches a paired device with no privilege at all, AF_VSOCK and AF_HYPERV reach the host of a VM - WSL2's included - and AF_CAN, AF_TIPC, AF_RDS and AF_QIPCRTR are buses and cluster or modem links. The check is now an allow list: AF_UNIX, and the kernel's own interfaces (AF_NETLINK/AF_ROUTE, AF_ALG, macOS AF_SYSTEM), none of which has a far end. Every other family, including one this list has never heard of, is refused, through socket.socket, the C type, and the audit hook alike. The socketpair window for Windows' loopback AF_INET pair is unchanged, and so is everything Compartment itself uses (AF_UNIX for the embedding daemon and asyncio's self-pipe on POSIX). Before the fix, under the guard, the operating system answered instead: test_a_socket_that_can_reach_another_machine_is_refused_whatever_its_family [AF_APPLETALK, AF_IPX, AF_DECnet, AF_SNA, AF_UNSPEC] (macOS) E PermissionError: [Errno 1] Operation not permitted (AF_PACKET, AF_BLUETOOTH, AF_VSOCK, AF_CAN, ... run where Python has them) Co-Authored-By: Claude Opus 5.5 --- src/compartment/offline_guard.py | 39 ++++++++++++++++++++------- tests/test_offline_guard_paths.py | 45 +++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/compartment/offline_guard.py b/src/compartment/offline_guard.py index 34e0a08..ac1212a 100644 --- a/src/compartment/offline_guard.py +++ b/src/compartment/offline_guard.py @@ -4,11 +4,16 @@ that could put a byte of ours on the wire raises OfflineViolation and the process aborts loudly. That is three doors, not one: - 1. Sockets. Creating an AF_INET/AF_INET6 socket raises. AF_UNIX and every - other local family keep working, because local IPC is not networking. - So does socket.socketpair, which on Windows is two AF_INET sockets on - loopback and which asyncio's event loop needs, for the length of that - call and in that thread only. + 1. Sockets. Creating a socket of any family that can reach another + machine raises. That is not only AF_INET/AF_INET6: AF_PACKET writes raw + frames, AF_BLUETOOTH reaches a paired device without any privilege, + AF_VSOCK and AF_HYPERV reach the host of a VM (WSL2's included), AF_CAN, + AF_TIPC and AF_RDS are buses and cluster links. So the families are an + allow list: AF_UNIX, because local IPC is not networking, and the + kernel's own interfaces (netlink/route, AF_ALG, macOS AF_SYSTEM), which + have no far end. socket.socketpair, which on Windows is two AF_INET + sockets on loopback and which asyncio's event loop needs, is let + through for the length of that call and in that thread only. 2. Name resolution. getaddrinfo, gethostbyname, gethostbyname_ex, gethostbyaddr, getnameinfo, getfqdn and create_connection raise for any host that is not already local. A DNS lookup needs no socket object at @@ -47,6 +52,13 @@ _real_raw_socket = _socket.socket _original_socketpair = socket.socketpair _INET_FAMILIES = (socket.AF_INET, socket.AF_INET6) +#: The only families a socket may have while the guard is on: none of them +#: has a far end on another machine. Anything not listed - a family this +#: platform has and this list never heard of included - is refused. +_LOCAL_FAMILIES = frozenset( + getattr(socket, name) + for name in ("AF_UNIX", "AF_NETLINK", "AF_ROUTE", "AF_ALG", "AF_SYSTEM") + if hasattr(socket, name)) _RESOLVERS = ("getaddrinfo", "gethostbyname", "gethostbyname_ex", "gethostbyaddr", "getnameinfo", "getfqdn", "create_connection") @@ -108,11 +120,20 @@ def _reject_lookup(host: object) -> None: def _reject_inet(family: object) -> None: + """Refuse every family but the local ones. -1 is "the default", which is + AF_INET; with a fileno it means "whatever that descriptor is", which the + audit hook cannot see, so it is refused too, as it always was.""" fam = family if family not in (-1, None) else socket.AF_INET - if fam in _INET_FAMILIES and not getattr(_state, "socketpair", 0): - raise _violation( - "something attempted to create a network socket" - ) + try: + if fam in _LOCAL_FAMILIES: + return + except TypeError: # unhashable: not a family at all + pass + if fam in _INET_FAMILIES and getattr(_state, "socketpair", 0): + return + raise _violation( + "something attempted to create a network socket" + ) # ------------------------------------------------------------ socket patches diff --git a/tests/test_offline_guard_paths.py b/tests/test_offline_guard_paths.py index 02af643..e188300 100644 --- a/tests/test_offline_guard_paths.py +++ b/tests/test_offline_guard_paths.py @@ -97,6 +97,51 @@ def test_local_ipc_still_works(guarded, tmp_path, monkeypatch): server.close() +#: Families with a far end on another machine, by the name each platform +#: knows them by. Each one runs where its platform has it. +_FAR_FAMILIES = ["AF_PACKET", "AF_BLUETOOTH", "AF_VSOCK", "AF_HYPERV", + "AF_CAN", "AF_TIPC", "AF_RDS", "AF_QIPCRTR", "AF_APPLETALK", + "AF_IPX", "AF_DECnet", "AF_SNA", "AF_IRDA", "AF_X25", + "AF_AX25", "AF_UNSPEC"] + + +@pytest.mark.parametrize("name", _FAR_FAMILIES) +def test_a_socket_that_can_reach_another_machine_is_refused_whatever_its_family( + guarded, name): + """Only AF_INET and AF_INET6 were refused, and "every other local family" + let through - but AF_PACKET writes raw frames, AF_BLUETOOTH reaches a + paired device with no privilege at all, and AF_VSOCK/AF_HYPERV reach the + host of a VM (WSL2's included). The refusal comes before the operating + system is asked, so where it would refuse the family anyway the answer + must still be the guard's.""" + family = getattr(socket, name, None) + if family is None: + pytest.skip(f"{name} does not exist here") + for make in (socket.socket, _socket.socket): + with pytest.raises(offline_guard.OfflineViolation): + make(family, socket.SOCK_DGRAM).close() + + +@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="no AF_UNIX here") +def test_a_socketpair_and_the_kernel_interfaces_are_still_local(guarded): + """What stays open: the AF_UNIX pair asyncio builds its self-pipe from on + POSIX, and the kernel's own sockets, which have no far end.""" + a, b = socket.socketpair() + with a, b: + a.sendall(b"x") + assert b.recv(1) == b"x" + for name in ("AF_NETLINK", "AF_ROUTE", "AF_SYSTEM"): + family = getattr(socket, name, None) + if family is None: + continue + try: + socket.socket(family, socket.SOCK_RAW).close() + except offline_guard.OfflineViolation: + raise + except OSError: + pass # the platform's own answer, not the guard's + + def test_literal_and_localhost_lookups_still_resolve(guarded): """Resolving something already local asks no one anything, so it stays allowed: blocking it would break loopback code paths for no gain.""" From 3377012f7a2d68c734ce37c028cfa944c179b0f7 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:45:36 -0700 Subject: [PATCH 033/135] The macOS app declares macOS 13.4, the minimum its bundled onnxruntime really needs A real build with the new deployment check (earlier commit) stopped at: onnxruntime/capi/libonnxruntime.1.23.2.dylib needs macOS 13.4 onnxruntime/capi/onnxruntime_pybind11_state.so needs macOS 13.4 onnxruntime 1.23.2 is the newest release a macOS 13 build can have (from 1.24 the macOS wheels are macosx_14_0 and arm64 only), and its wheels are tagged macosx_13_0 while the libraries inside record a minimum of 13.4, on arm64 and on x86_64 (checked by unpacking the x86_64 wheel too). So declaring 13.0 let macOS 13.0 to 13.3 install an app that could not load its embedding model there. MIN_MACOS, the default of --min-macos, is now 13.4: LSMinimumSystemVersion, the launcher's deployment target and the .pkg's allowed versions follow it. pip's tags for macOS 11 and later are major versions only, so the wheels chosen are the same macosx_13_0 set. With it the build in this sandbox ran to the end of the app: 55 binaries, all arm64, none above 13.4, signed, verified, run, and the copied bundle's compartment command and onnxruntime 1.23.2 both load. 13.4 is a free update for every Mac that runs 13. Co-Authored-By: Claude Opus 5.5 --- tests/test_macos_bundle.py | 13 +++++++++++++ tools/build_macos_app.py | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_macos_bundle.py b/tests/test_macos_bundle.py index 8d716e3..1aa880c 100644 --- a/tests/test_macos_bundle.py +++ b/tests/test_macos_bundle.py @@ -540,3 +540,16 @@ def run(*args, **kw): monkeypatch.setattr(b, "_run", run) b.build_dmg(app) assert len(tries) == 3 + + +def test_the_default_minimum_is_what_the_bundled_onnxruntime_needs(): + """onnxruntime 1.23.2's macosx_13_0 wheels hold libraries that record a + minimum of macOS 13.4 (arm64 and x86_64), and 1.23.2 is the newest + release a macOS 13 build can have. A bundle declaring 13.0 installed on + 13.0 to 13.3 and could not load its model there.""" + b = _builder() + assert b._version_tuple(b.MIN_MACOS) >= (13, 4) + # pip's tags for macOS 11 and later are major versions only, so 13.4 + # still selects the macosx_13_0 wheels it has to. + major, _minor = b._version_tuple(b.MIN_MACOS) + assert major == 13 diff --git a/tools/build_macos_app.py b/tools/build_macos_app.py index f681389..e398bb0 100644 --- a/tools/build_macos_app.py +++ b/tools/build_macos_app.py @@ -78,7 +78,14 @@ # allowed-os-versions, and - the part that is easy to miss - the platform # every wheel in the bundle is chosen for (_install_packages). 13 is also the # floor: SMAppService, which registers the login item, is macOS 13 and up. -MIN_MACOS = "13.0" +# +# 13.4, not 13.0. onnxruntime 1.23.2 is the last release with macOS 13 (and +# Intel) wheels, and its wheels are tagged macosx_13_0 while the libraries +# inside record a minimum of 13.4, on arm64 and x86_64 alike (found by +# _verify_deployment_target on a real build). Declaring 13.0 let macOS 13.0 +# to 13.3 install an app that could not load its embedding model. 13.4 is a +# free update every macOS 13 Mac can take. +MIN_MACOS = "13.4" # Shown under the app's name in System Settings > General > Login Items. DESCRIPTION = ("Compartment keeps your AI agents' memory encrypted on this Mac. " "The menu bar item shows what it has remembered and lets you " From a882be7ecf102df3a664e40dc6a9b7f95ff5ca38 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:45:45 -0700 Subject: [PATCH 034/135] A memory imported with its date as text, milliseconds, NaN or infinity no longer breaks every search, and a vault already broken that way is repaired when it opens `created` and `affirmed` reach the vault from files on the restore paths (`compartment import`, curation plans) and were stored as they came. An ISO string such as "2024-05-01T09:30:00Z", which is how most other tools export a date, went into the REAL column as text, and every later search failed with "could not convert string to float" while building the recency population, which reads every live record: the whole vault stopped answering, for good, and the bad record could not be found to remove because finding it needed a search. An infinite value was stored and then raised OverflowError from every reader of the record, and NaN failed the import half way through with a NOT NULL error, the records before it left in RAM unsaved. store() now reads both values with instant(): seconds, numeric text, an ISO date or datetime (local time when it has no zone), or milliseconds as JavaScript's Date.now() writes them; anything else, NaN and infinity included, becomes now (or no affirmed time). local_stamp shows an unrepresentable value as no date instead of raising. Opening a vault runs a one-time repair over records an older build stored this way, reading each bad value the same way or falling back to when the row was written, audits it, and the open's compaction commits it. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 89 +++++++++++++++++++++++++++++- tests/test_storage_dates.py | 107 ++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 tests/test_storage_dates.py diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 6fd8aef..89fe6b9 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -87,10 +87,59 @@ def local_stamp(created: float | None) -> str: try: return datetime.datetime.fromtimestamp( float(created)).strftime("%Y-%m-%d %H:%M") - except (TypeError, ValueError, OSError): + except (TypeError, ValueError, OSError, OverflowError): + # OverflowError: an infinite or vast value, which a JSONL import + # could carry in, raised past every reader of the record, so one + # such memory failed every search that would have returned it. return "" +#: The span a stored instant may take: what a calendar date can be made of +#: (years 1 to 9999, in UTC), checked as a number so the answer does not +#: depend on what the platform's C library will convert. +_MIN_INSTANT = -62135596800.0 +_MAX_INSTANT = 253402300799.0 + + +def instant(value) -> float | None: + """A moment as unix seconds, or None when `value` does not name one. + + `created` and `affirmed` reach the vault from outside on the restore + paths - a JSONL import, a curation plan - and whatever a file holds was + stored as it came: an ISO string such as "2024-05-01T09:30:00Z" went + into the REAL column as text, and the next search failed converting it + for the recency prior. Every search, for good, since every search reads + the whole population. So the value is read the way a person would read + it: a number of seconds; numeric text; an ISO date or datetime (one + without a zone is local time, as a person writing it means); or a count + of milliseconds, which is what JavaScript's Date.now() writes and which + as seconds would be a date thousands of years away. Anything else, + including NaN and infinity, is None, and the caller falls back to now + rather than keep a date nothing can read.""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, (int, float)): + t = float(value) + else: + s = str(value).strip() + if not s: + return None + try: + t = float(s) + except ValueError: + try: + t = datetime.datetime.fromisoformat(s).timestamp() + except (ValueError, OverflowError, OSError): + return None + if not math.isfinite(t): + return None + if _MIN_INSTANT <= t <= _MAX_INSTANT: + return t + if _MIN_INSTANT <= t / 1000.0 <= _MAX_INSTANT: + return t / 1000.0 + return None + + # A memory's own account of where it came from, written at the end of its text # as "[method, YYYY-MM-DD]". Recognised on the way back IN so that re-importing # an exported memory does not stack a second clause onto the first. @@ -467,7 +516,9 @@ def _unlock_once(cls, path: str, passphrase: str | None, v._replay(e) merged = v._merge_legacy_starter() rewired = v._migrate_wire_format() - if entries or loaded.truncated_tail or merged or rewired or slots_migrated: + repaired = v._repair_instants() + if (entries or loaded.truncated_tail or merged or rewired + or slots_migrated or repaired): if loaded.truncated_tail: # Reaching here means the framing proved this was an # interrupted append: an intact, checksummed length prefix @@ -540,6 +591,37 @@ def _migrate_wire_format(self) -> int: self.header.extra["wire"] = wire.WIRE_FORMAT return n or 1 # truthy even for an empty vault: the marker must save + def _repair_instants(self) -> int: + """Give every record a `created` (and `affirmed`) that is a number. + + Builds before `instant` stored an imported date as whatever the file + held, so a vault that took one line with an ISO string, a NaN or a + word in it failed every search from then on, and nothing in it could + be fixed from the outside because finding the record needed a + search. Each bad value is read as `instant` reads it; one that names + no moment becomes the time the row was written (`accessed` starts + there, and a record no search could return was never touched since), + and a bad `affirmed` is dropped. Writes nothing itself: the caller's + save commits it, as with the other migrations here.""" + rows = self.db.conn.execute( + "SELECT id, created, affirmed, accessed FROM records" + " WHERE typeof(created) NOT IN ('integer', 'real')" + " OR (affirmed IS NOT NULL" + " AND typeof(affirmed) NOT IN ('integer', 'real'))").fetchall() + for r in rows: + created = (instant(r["created"]) or instant(r["accessed"]) + or time.time()) + self.db.conn.execute( + "UPDATE records SET created = ?, affirmed = ? WHERE id = ?", + (created, instant(r["affirmed"]), r["id"])) + if rows: + # Direct SQL, so the cache of the population's times is stale. + self.db._invalidate_recency() + self._audit_and_capture( + "system", "repair", + f"{len(rows)} records given a readable stored date") + return len(rows) + def _merge_legacy_starter(self) -> int: """One-time reorganization: fold the legacy read-only packs/starter section into "main", so starting memories are ordinary memories - @@ -999,6 +1081,9 @@ def store(self, text: str, caller: str, namespace: str | None = None, # excluded so the vector represents the claim itself. vec = self.embedder.embed_record(strip_provenance(text)) vec = np.atleast_2d(np.asarray(vec, dtype=np.float32)) + # Read as moments, or dropped: see instant. Stored as they came, an + # ISO string or a NaN from an import broke every later search. + created, affirmed = instant(created), instant(affirmed) # Kept to 0..1 as the server's tools keep it, so a value on another # scale (1-10, say) lands in a dashboard bucket; a NaN is the default. importance = float(importance) diff --git a/tests/test_storage_dates.py b/tests/test_storage_dates.py new file mode 100644 index 0000000..0d823c8 --- /dev/null +++ b/tests/test_storage_dates.py @@ -0,0 +1,107 @@ +"""Stored instants that arrive from outside the vault. + +`created` and `affirmed` reach the vault from files on the restore paths. A +value stored as it came - an ISO string, a NaN, an infinity, milliseconds - +used to break every later search, because every search reads every live +record's time for the recency prior. +""" +import datetime +import json +import time + +import pytest + +from compartment.vault import Vault, instant, local_stamp + +from conftest import PASS + + +def _line(**fields) -> str: + return json.dumps(fields) + "\n" + + +def test_instant_reads_what_a_person_would_mean(): + t = datetime.datetime(2024, 5, 1, 9, 30, + tzinfo=datetime.timezone.utc).timestamp() + assert instant(t) == t + assert instant(str(t)) == t + assert instant("2024-05-01T09:30:00Z") == t + assert instant("2024-05-01T09:30:00+00:00") == t + assert instant(t * 1000) == t, "JavaScript's Date.now() is milliseconds" + local_midnight = datetime.datetime(2024, 5, 1).timestamp() + assert instant("2024-05-01") == local_midnight + for nonsense in (None, "", "yesterday", True, float("nan"), + float("inf"), -float("inf"), 1e300, [1]): + assert instant(nonsense) is None, nonsense + + +def test_an_infinite_date_is_shown_as_no_date_instead_of_raising(): + assert local_stamp(float("inf")) == "" + assert local_stamp(1e300) == "" + + +@pytest.mark.parametrize("created", [ + "2024-05-01T09:30:00Z", float("inf"), float("nan"), "last tuesday", + 1714555800000]) +def test_an_imported_date_never_breaks_search(vault, created): + """An ISO string went into the REAL column as text and the next search + failed converting it; infinity was stored and failed every reader of + the record; NaN failed the import half way.""" + vault.store("Tea is kept in the left cupboard.", caller="t") + before = time.time() + n = vault.import_jsonl( + json.dumps({"text": "Coffee beans are kept in the freezer.", + "created": created}, allow_nan=True) + "\n", + caller="t") + assert n == 1 + hits = vault.search("where are the coffee beans kept", caller="t") + top = hits["results"][0] + assert top["text"].startswith("Coffee beans are kept in the freezer.") + assert isinstance(top["created"], float) + assert top["created_local"], "the date reads as a date" + want = instant(created) + if want is None: + assert top["created"] >= before, "an unreadable date becomes now" + else: + assert top["created"] == want + assert vault.recent("t")["results"] + + +def test_an_unreadable_affirmed_time_is_dropped(vault): + vault.import_jsonl(_line(text="Prefers tabs over spaces.", + kind="opinion", affirmed="recently"), + caller="t") + row = vault.db.conn.execute("SELECT affirmed FROM records").fetchone() + assert row["affirmed"] is None + assert vault.search("tabs or spaces", caller="t")["results"] + + +def test_a_vault_an_older_build_broke_is_repaired_when_it_opens(vault, + vault_path): + """A vault that took such a line before this build failed every search + for good, and the record could not be found to remove it.""" + good = vault.store("Tea is kept in the left cupboard.", caller="t")["id"] + bad = vault.store("Coffee beans are kept in the freezer.", + caller="t")["id"] + vault.db.conn.execute( + "UPDATE records SET created = ?, affirmed = ? WHERE id = ?", + ("2024-05-01T09:30:00Z", "soon", bad)) + vault.db._invalidate_recency() + with pytest.raises(ValueError): # what an older build did + vault.search("coffee", caller="t") + vault.save() + + again = Vault.unlock(vault_path, passphrase=PASS) + hits = again.search("where are the coffee beans kept", caller="t") + assert hits["results"][0]["id"] == bad + assert hits["results"][0]["created"] == instant("2024-05-01T09:30:00Z") + row = again.db.get_row(bad) + assert row["affirmed"] is None + assert again.db.get_row(good)["created"] == vault.db.get_row(good)[ + "created"], "a record with a good date is untouched" + ops = [r["op"] for r in again.db.conn.execute("SELECT op FROM audit")] + assert "repair" in ops + # and the repair was compacted in: the next open has nothing to do + third = Vault.unlock(vault_path, passphrase=PASS) + assert [r["op"] for r in third.db.conn.execute( + "SELECT op FROM audit")].count("repair") == 1 From 1fa447acc623f53a946b3903f919e2d30cfb1b0a Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:46:25 -0700 Subject: [PATCH 035/135] Claude Code settings whose hooks are not the shape Claude Code reads are refused with the file's name instead of a traceback The capture hook's install, uninstall and status walked hooks.PostToolUse as if it were always an object holding a list of matcher groups, each with a list of hooks. A settings.json with a list where the hooks object belongs, or a string where a group or its hooks belong, died with AttributeError, which `integrate claude`, `compartment hook status` and the uninstaller do not catch, so the user got a traceback instead of what to fix; and an event that was an object was silently replaced by a list. The shape is now checked first, and anything else is refused with ValueError naming the file and the part that is wrong, exactly as a file that is not JSON is, and left untouched. Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_hooks.py | 46 ++++++++++++++++++++++++++++---- tests/test_integrations_hooks.py | 27 +++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 tests/test_integrations_hooks.py diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index addebde..fd97923 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -62,9 +62,11 @@ def is_ours(command: object) -> bool: def is_installed(settings: Path | None = None) -> bool: - data = _read(settings or SETTINGS) - for group in (data.get("hooks", {}) or {}).get(EVENT, []) or []: - for h in group.get("hooks", []) or []: + """Whether compartment's capture hook is in settings.json. Raises + ValueError when the file cannot be read as settings.""" + path = Path(settings) if settings else SETTINGS + for group in _groups(_read(path), path): + for h in group.get("hooks") or []: if is_ours(h.get("command")): return True return False @@ -76,14 +78,48 @@ def _read(path: Path) -> dict: return clients._read_json(path) +def _groups(data: dict, path: Path) -> list: + """The PostToolUse matcher groups, checked for the shape Claude Code + reads: `hooks` an object, the event a list of objects, each one's + `hooks` a list of objects. + + Anything else is somebody's file in a shape this does not understand, + and it is refused as a malformed one is. It used to be walked as if it + were right: a list where the object belongs died with AttributeError, + which `integrate claude`, `hook status` and the uninstaller do not + catch, so the user got a traceback instead of the file's name and what + to fix, and an event that was an object was replaced by a list.""" + hooks = data.get("hooks") + if hooks is None: + return [] + if not isinstance(hooks, dict): + raise ValueError(f"{path}: \"hooks\" is not an object; refusing to " + "touch it") + groups = hooks.get(EVENT) + if groups is None: + return [] + if not isinstance(groups, list): + raise ValueError(f"{path}: \"hooks.{EVENT}\" is not a list; " + "refusing to touch it") + for group in groups: + inner = group.get("hooks") if isinstance(group, dict) else None + if not isinstance(group, dict) or not ( + inner is None or (isinstance(inner, list) and all( + isinstance(h, dict) for h in inner))): + raise ValueError(f"{path}: an entry in \"hooks.{EVENT}\" is not " + "a matcher with a list of hooks; refusing to " + "touch it") + return groups + + def install(compartment_bin: str | None = None, vault: str | None = None, settings: Path | None = None) -> dict: """Add (or refresh) compartment's capture hook. Never removes anyone else's.""" path = Path(settings) if settings else SETTINGS data = _read(path) # raises on malformed + groups = _groups(data, path) # and on a bad shape hooks = data.setdefault("hooks", {}) - groups = hooks.setdefault(EVENT, []) # drop only our own previous entries, so re-installing updates in place kept = [] for group in groups: @@ -106,7 +142,7 @@ def install(compartment_bin: str | None = None, vault: str | None = None, def uninstall(settings: Path | None = None) -> bool: path = Path(settings) if settings else SETTINGS data = _read(path) - groups = (data.get("hooks", {}) or {}).get(EVENT) + groups = _groups(data, path) if not groups: return False kept, removed = [], False diff --git a/tests/test_integrations_hooks.py b/tests/test_integrations_hooks.py new file mode 100644 index 0000000..ec30546 --- /dev/null +++ b/tests/test_integrations_hooks.py @@ -0,0 +1,27 @@ +"""Claude Code's settings.json, and the capture hook's place in it.""" +import pytest + +from compartment import claude_hooks + + +@pytest.mark.parametrize("settings", [ + '{"hooks": []}', + '{"hooks": {"PostToolUse": {"a": 1}}}', + '{"hooks": {"PostToolUse": ["x"]}}', + '{"hooks": {"PostToolUse": [{"matcher": "Edit", "hooks": "x"}]}}', +]) +def test_settings_whose_hooks_are_the_wrong_shape_are_refused_untouched( + settings, tmp_path): + """A list where the hooks object belongs, or a string where a matcher + group does, was walked as if it were right and died with + AttributeError, which `integrate claude`, `hook status` and the + uninstaller do not catch: a traceback instead of the file's name and + what is wrong with it. An event that was an object was replaced.""" + p = tmp_path / "settings.json" + p.write_text(settings, encoding="utf-8") + for call in (lambda: claude_hooks.install("compartment", settings=p), + lambda: claude_hooks.uninstall(settings=p), + lambda: claude_hooks.is_installed(settings=p)): + with pytest.raises(ValueError, match="refusing to touch it"): + call() + assert p.read_text(encoding="utf-8") == settings From a4b9c5a896d80a3b77f342f6a162407acefad745 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:46:41 -0700 Subject: [PATCH 036/135] The capture hook's command reaches Git Bash on Windows intact, and a path with a shell character in it stays one word everywhere Claude Code runs hook commands through bash on every platform, Git Bash on Windows since 2.1.47, and to bash an unquoted backslash is an escape character. The hook quoted the executable only when its path held a space, so the usual Windows install path, C:\Users\ana\AppData\Roaming\Python\Python312\Scripts\compartment.exe, reached Git Bash as C:UsersanaAppDataRoaming...compartment.exe: "command not found" after every Write and Edit, and nothing was captured. Inside the vault's double quotes $ and ` still expanded, so a vault path holding either named some other file. On Windows both paths are now written with forward slashes, which Git Bash, cmd and PowerShell all read as the same path and none treats as an escape; the vault opens the same session either way, since the session store normalises the path. Characters still special inside double quotes are escaped, and an executable path that needs no quoting stays bare, which PowerShell also runs as a command. POSIX output for an ordinary path is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_hooks.py | 42 ++++++++++++++-- tests/test_integrations_hooks.py | 83 +++++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index fd97923..2759365 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -42,16 +42,50 @@ def _entry(command: str) -> dict: "timeout": TIMEOUT}]} +#: Characters a word can hold and still mean itself to a POSIX shell. +_SHELL_SAFE = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789_./:@%+=,-") + + +def _windows() -> bool: + return os.name == "nt" + + +def _shell_word(s: str, always: bool = False) -> str: + """`s` as one word for the shell Claude Code runs hooks in. + + That shell is bash on every platform, Git Bash on Windows (Claude Code + 2.1.47 and later), and an unquoted backslash is an escape character to + it. The hook used to quote the executable only when its path held a + space, so the usual Windows path (C:\\Users\\ana\\...\\compartment.exe) + reached bash as C:Usersana...compartment.exe: "command not found" after + every Write and Edit, and nothing was ever captured. On Windows the path is + written with forward slashes, which Git Bash, cmd and PowerShell all + take as a path and none of them treats as an escape; it is still the + same file to Windows, and the vault it names opens the same session, + since the session store normalises the path. Anything still special + inside double quotes ($, `, ", \\) is escaped, so a path with one of + those in it is not expanded either. A word that needs no quoting is + left bare unless `always`: a bare executable path is also one + PowerShell runs, where a quoted one is a string expression to it. + """ + if _windows(): + s = s.replace("\\", "/") + if not always and s and all(c in _SHELL_SAFE for c in s): + return s + for ch in ("\\", '"', "$", "`"): + s = s.replace(ch, "\\" + ch) + return f'"{s}"' + + def hook_command(compartment_bin: str | None = None, vault: str | None = None) -> str: """The exact shell command the hook runs. The compartment beside this interpreter, as every MCP registration names it: a bare name only works where `compartment` is on the PATH Claude Code's shell gets, and the .app install puts it on none.""" - exe = compartment_bin or clients.executable() - if " " in exe: - exe = f'"{exe}"' - vault_part = f' --vault "{vault}"' if vault else "" + exe = _shell_word(compartment_bin or clients.executable()) + vault_part = f" --vault {_shell_word(vault, always=True)}" if vault else "" return f"{exe}{vault_part} hook capture" diff --git a/tests/test_integrations_hooks.py b/tests/test_integrations_hooks.py index ec30546..b9dbc20 100644 --- a/tests/test_integrations_hooks.py +++ b/tests/test_integrations_hooks.py @@ -1,8 +1,89 @@ -"""Claude Code's settings.json, and the capture hook's place in it.""" +"""The capture hook's command, as the shell Claude Code runs it in reads it. + +Claude Code runs hook commands through bash on every platform: Git Bash on +Windows since 2.1.47. To bash an unquoted backslash is an escape character, +and $ and ` expand inside double quotes.""" +import os +import subprocess +import sys + import pytest from compartment import claude_hooks +WIN_EXE = r"C:\Users\ana\AppData\Roaming\Python\Python312\Scripts\compartment.exe" +WIN_VAULT = r"C:\Users\ana\.compartment\memory.vault" + + +def _bash_words(cmd): + """The words bash makes of `cmd`, as bash itself splits them.""" + ran = subprocess.run(["bash", "-c", f'set -- {cmd}; for w in "$@"; do ' + 'printf "%s\\0" "$w"; done'], + capture_output=True, text=True, timeout=30) + assert ran.returncode == 0, ran.stderr + return ran.stdout.split("\0")[:-1] + + +needs_bash = pytest.mark.skipif(sys.platform == "win32" or not any( + os.access(os.path.join(d, "bash"), os.X_OK) + for d in os.environ.get("PATH", "").split(os.pathsep)), + reason="needs a bash to split the command") + + +@needs_bash +def test_a_windows_executable_path_reaches_git_bash_whole(monkeypatch): + """Quoted only when it held a space, the usual Windows path reached + Git Bash with its backslashes taken as escapes, + C:UsersanaAppDataRoaming...compartment.exe, so the hook failed with + "command not found" after every edit and captured nothing.""" + monkeypatch.setattr(claude_hooks, "_windows", lambda: True, + raising=False) + cmd = claude_hooks.hook_command(WIN_EXE, WIN_VAULT) + words = _bash_words(cmd) + assert words[0].replace("/", "\\") == WIN_EXE + assert words[1:3] == ["--vault", WIN_VAULT.replace("\\", "/")] + assert words[3:] == ["hook", "capture"] + assert claude_hooks.is_ours(cmd) + + +@needs_bash +@pytest.mark.parametrize("exe, vault", [ + ("/Applications/My Tools/compartment", "/v/m.vault"), + ("/home/ana/$HOME/bin/compartment", "/home/ana/a `b` $c \"d\"/m.vault"), + ("/opt/R&D (x86)/compartment", "/v/it's mine/m.vault"), +]) +def test_a_path_with_shell_characters_in_it_is_one_word(exe, vault): + cmd = claude_hooks.hook_command(exe, vault) + assert _bash_words(cmd) == [exe, "--vault", vault, "hook", "capture"] + assert claude_hooks.is_ours(cmd) + + +def test_a_windows_command_has_no_backslash_for_any_shell_to_eat(monkeypatch): + monkeypatch.setattr(claude_hooks, "_windows", lambda: True, + raising=False) + cmd = claude_hooks.hook_command(WIN_EXE, WIN_VAULT) + assert "\\" not in cmd + # A plain path stays bare, which PowerShell also runs as a command. + assert cmd.startswith("C:/Users/ana/AppData/Roaming/Python/Python312/" + "Scripts/compartment.exe --vault ") + spaced = claude_hooks.hook_command( + r"C:\Program Files (x86)\Compartment\compartment.exe", WIN_VAULT) + assert spaced.startswith('"C:/Program Files (x86)/Compartment/' + 'compartment.exe" --vault ') + + +def test_the_forward_slash_vault_opens_the_same_session(monkeypatch): + """The session store keys a vault by its normalised absolute path, so + the forward-slash spelling the hook now writes on Windows is the same + vault, unlocked by the same session, as the backslash one.""" + import ntpath + from compartment import session + with monkeypatch.context() as m: + m.setattr(session.os, "path", ntpath) + a = session._canon(WIN_VAULT) + b = session._canon(WIN_VAULT.replace("\\", "/")) + assert a == b + @pytest.mark.parametrize("settings", [ '{"hooks": []}', From f08b5958729ac6ce8815cc1f8e68806bdcfd9c35 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:46:46 -0700 Subject: [PATCH 037/135] `compartment dash` serves while the browser opens, opens the Windows browser from WSL, and on a machine with no display says how to reach the page dash called webbrowser.open before serve_forever. On Linux with no display (SSH, a container, WSL without WSLg) and TERM set, webbrowser's only candidates are text browsers - www-browser, w3m, lynx - which it runs in the same terminal and waits for. So the text browser asked for a page the server could not serve until the browser exited, and the dashboard hung until someone quit a browser they never asked for. On WSL the user's browser is a Windows program that xdg-open does not know about, and on a headless box nothing at all said the page had not opened. menubar.open_in_browser (the module with no heavy imports, so the panel can use it too) now decides: BROWSER, when set, is honoured as webbrowser honours it; on WSL, wslview or else explorer.exe opens the Windows default browser, which reaches 127.0.0.1 through WSL 2's localhost forwarding; on Linux and the BSDs with neither DISPLAY nor WAYLAND_DISPLAY nothing is opened; everywhere else webbrowser.open as before. It never raises. dash runs it on a thread beside the server, and when it opens nothing prints that the address above is to be opened by hand, with the ssh -L line for reaching it from another machine. The panel's second Dashboard click opens the page off the UI thread for the same reason. Co-Authored-By: Claude Opus 5.5 --- src/compartment/dash.py | 20 +++-- src/compartment/menubar.py | 65 +++++++++++++++- tests/test_apps_browser.py | 148 +++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 tests/test_apps_browser.py diff --git a/src/compartment/dash.py b/src/compartment/dash.py index 80a764a..734055e 100644 --- a/src/compartment/dash.py +++ b/src/compartment/dash.py @@ -418,9 +418,18 @@ def start(ref: _VaultRef, host: str = "127.0.0.1", return httpd, url +def _open_browser(url: str) -> None: + """Open the page, or say how to reach it when nothing here can.""" + from .menubar import open_in_browser + if not open_in_browser(url): + port = urllib.parse.urlsplit(url).port + print(" no browser to open here: open the address above in one. " + "From another machine, forward the port first: " + f"ssh -L {port}:127.0.0.1:{port} ", flush=True) + + def run(path: str, vault: Vault) -> None: """CLI entry: serve until Ctrl-C. Zero flags, zero configuration.""" - import webbrowser ref = _VaultRef(path, vault) httpd, url = start(ref) # Flushed, and before the browser: the panel learns where the page is @@ -430,10 +439,11 @@ def run(path: str, vault: Vault) -> None: print(" serving from RAM, 127.0.0.1 only, GET only - viewing the vault " "changes nothing in it", flush=True) print(" Ctrl-C to stop", flush=True) - try: - webbrowser.open(url) - except Exception: - pass + # Beside the server, not before it. A browser that runs in the + # foreground - a text browser in this terminal - was waited for here, + # while the page it was asking for could not be served until it exited. + threading.Thread(target=_open_browser, args=(url,), daemon=True, + name="compartment-dash-browser").start() try: httpd.serve_forever() except KeyboardInterrupt: diff --git a/src/compartment/menubar.py b/src/compartment/menubar.py index 588252c..511b008 100644 --- a/src/compartment/menubar.py +++ b/src/compartment/menubar.py @@ -926,6 +926,66 @@ def dashboard_url(line: str) -> str | None: return None +def _is_wsl() -> bool: + """Linux under Windows (WSL 1 or 2), where the user's browser is a + Windows program.""" + if os.environ.get("WSL_DISTRO_NAME") or os.environ.get("WSL_INTEROP"): + return True + try: + with open("/proc/sys/kernel/osrelease", encoding="utf-8", + errors="replace") as fh: + return "microsoft" in fh.read().lower() + except OSError: + return False + + +def open_in_browser(url: str) -> bool: + """Open `url` in the user's own browser. False when there is none to + open here, so the caller can say to open the address by hand. + + webbrowser.open on its own got three places wrong: + + * A Linux machine with no desktop (SSH, a container, WSL without WSLg) + has no graphical browser, and with TERM set webbrowser falls back to + a text browser - w3m, lynx, www-browser - run in the same terminal and + waited for. `compartment dash` called it before serving, so the text + browser sat waiting on a server that would not answer until the + browser exited. Without a display nothing is opened. + * WSL: the browser is on the Windows side, and xdg-open there opens + nothing or a stray Linux one. wslview (wslu) or explorer.exe opens + the Windows default browser, which reaches 127.0.0.1 in WSL 2 through + its localhost forwarding. + * BROWSER, when set, is the user's own choice and is honoured as + webbrowser honours it, display or not. + + Never raises. Can block if the browser it starts is one that runs in + the foreground, so a caller with other work runs it on a thread. + """ + try: + if os.environ.get("BROWSER"): + return bool(webbrowser.open(url)) + if sys.platform not in ("win32", "darwin"): + if _is_wsl(): + for opener in ("wslview", "explorer.exe"): + exe = shutil.which(opener) + if exe: + # explorer.exe exits 1 even when it opened the page, + # so having started it is the answer. + subprocess.Popen([exe, url], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True) + return True + return False + if not (os.environ.get("DISPLAY") + or os.environ.get("WAYLAND_DISPLAY")): + return False + return bool(webbrowser.open(url)) + except Exception: # noqa: BLE001 + return False + + def _drain(proc) -> None: # `dash` says a few more lines over its life; read them so it can never # block on a full pipe. @@ -948,7 +1008,10 @@ def open_dashboard(vault: str) -> str | None: """ proc, url = _DASH["proc"], _DASH["url"] if proc is not None and proc.poll() is None and url: - webbrowser.open(url) + # On a thread: this is the UI thread, and a browser that runs in the + # foreground would hold it for as long as it is open. + threading.Thread(target=open_in_browser, args=(url,), daemon=True, + name="compartment-open-browser").start() return url try: proc = subprocess.Popen( diff --git a/tests/test_apps_browser.py b/tests/test_apps_browser.py new file mode 100644 index 0000000..9acea6e --- /dev/null +++ b/tests/test_apps_browser.py @@ -0,0 +1,148 @@ +"""Opening the dashboard in a browser, on every kind of machine. + +`compartment dash` and the panel's Dashboard button open the page +themselves. webbrowser.open alone got headless Linux and WSL wrong: with no +display it falls back to a text browser run in the same terminal and waits +for it, and `dash` did that before serving, so the browser waited on a +server that could not answer until the browser exited. +""" +from __future__ import annotations + +import threading +import time + +import pytest + +from compartment import dash, menubar + +URL = "http://127.0.0.1:5555/tok/" + + +@pytest.fixture() +def linux(monkeypatch): + monkeypatch.setattr(menubar.sys, "platform", "linux") + for name in ("BROWSER", "DISPLAY", "WAYLAND_DISPLAY", "WSL_DISTRO_NAME", + "WSL_INTEROP"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(menubar, "_is_wsl", lambda: False) + opened, started = [], [] + monkeypatch.setattr(menubar.webbrowser, "open", + lambda url, *a, **k: opened.append(url) or True) + monkeypatch.setattr(menubar.subprocess, "Popen", + lambda argv, **k: started.append(argv)) + return opened, started + + +def test_a_headless_linux_box_opens_nothing(linux): + """No display: the only browsers webbrowser could find are text ones + that take over this terminal.""" + opened, started = linux + assert menubar.open_in_browser(URL) is False + assert opened == [] and started == [] + + +def test_a_desktop_session_gets_its_browser(linux, monkeypatch): + opened, _ = linux + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + assert menubar.open_in_browser(URL) is True + assert opened == [URL] + + +def test_browser_is_honoured_even_without_a_display(linux, monkeypatch): + opened, _ = linux + monkeypatch.setenv("BROWSER", "w3m") + assert menubar.open_in_browser(URL) is True + assert opened == [URL] + + +@pytest.mark.parametrize("on_path,expected", [ + ({"wslview", "explorer.exe"}, "wslview"), + ({"explorer.exe"}, "explorer.exe"), +]) +def test_wsl_opens_the_windows_browser(linux, monkeypatch, on_path, expected): + opened, started = linux + monkeypatch.setattr(menubar, "_is_wsl", lambda: True) + monkeypatch.setenv("DISPLAY", ":0") # WSLg: still the Windows one + monkeypatch.setattr(menubar.shutil, "which", + lambda name, *a, **k: (f"/usr/bin/{name}" + if name in on_path else None)) + assert menubar.open_in_browser(URL) is True + assert started == [[f"/usr/bin/{expected}", URL]] + assert opened == [] + + +def test_wsl_without_interop_says_it_could_not(linux, monkeypatch): + opened, started = linux + monkeypatch.setattr(menubar, "_is_wsl", lambda: True) + monkeypatch.setattr(menubar.shutil, "which", lambda *a, **k: None) + assert menubar.open_in_browser(URL) is False + assert opened == [] and started == [] + + +def test_wsl_is_recognised(monkeypatch, tmp_path): + monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") + assert menubar._is_wsl() is True + + +def test_a_browser_that_fails_never_raises(monkeypatch): + monkeypatch.setattr(menubar.sys, "platform", "darwin") + + def boom(*a, **k): + raise OSError("LSOpenURLsWithRole() failed") + + monkeypatch.setattr(menubar.webbrowser, "open", boom) + assert menubar.open_in_browser(URL) is False + + +class _FakeServer: + def __init__(self): + self.served = threading.Event() + + def serve_forever(self): + self.served.set() + + def server_close(self): + pass + + +def test_dash_serves_while_the_browser_is_still_open(monkeypatch, capsys, + tmp_path): + """A text browser in the foreground used to be waited for before the + server started, so it waited on a page nobody would serve.""" + server = _FakeServer() + release = threading.Event() + seen = {} + + def foreground_browser(url, *a, **k): + seen["served_before_return"] = server.served.wait(5) + release.wait(5) + return True + + monkeypatch.setattr(dash, "start", lambda ref: (server, URL)) + monkeypatch.setattr(menubar, "open_in_browser", foreground_browser) + monkeypatch.setattr(menubar.webbrowser, "open", foreground_browser) + began = time.monotonic() + dash.run(str(tmp_path / "v.vault"), vault=None) + assert server.served.is_set() + assert time.monotonic() - began < 4, "serving waited on the browser" + release.set() + deadline = time.monotonic() + 5 + while "served_before_return" not in seen and time.monotonic() < deadline: + time.sleep(0.01) + assert seen.get("served_before_return") is True + + +def test_dash_says_how_to_reach_the_page_when_nothing_opens(monkeypatch, + capsys, tmp_path): + server = _FakeServer() + monkeypatch.setattr(dash, "start", lambda ref: (server, URL)) + monkeypatch.setattr(menubar, "open_in_browser", lambda url: False) + dash.run(str(tmp_path / "v.vault"), vault=None) + deadline = time.monotonic() + 5 + out = "" + while "no browser" not in out and time.monotonic() < deadline: + out += capsys.readouterr().out + time.sleep(0.01) + assert out.splitlines()[0] == f"Compartment dashboard: {URL}" + assert "open the address above" in out + assert "ssh -L 5555:127.0.0.1:5555" in out From f660fc92df54a9a201f01844676cf20197cb65f7 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:46:57 -0700 Subject: [PATCH 038/135] A discovery date sent in milliseconds is read as the day it names, and an expiry no calendar can hold is refused with a reason instead of a raw date error discovery_date passed a number straight to date.fromtimestamp, so a JavaScript client sending Date.now() for `discovered` (milliseconds) failed the whole store with "ValueError: year 56759 is out of range". It now goes through instant(), which reads milliseconds as the moment they name, and a number that names no day at all is kept as written, as the field already does for any text it cannot parse. expiry_date let date arithmetic errors escape: `expires="9999y"` raised "ValueError: year 12025 is out of range", and a NaN or infinite number raised from int(). Each is now a CryptoError saying what to give instead, which every surface reports as a refusal, before anything is stored. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 39 +++++++++++++++++++++++++++++-------- tests/test_storage_dates.py | 34 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 89fe6b9..298a0d5 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -157,8 +157,18 @@ def discovery_date(value=None) -> str: time and lives in `created`.""" if value in (None, ""): return datetime.date.today().isoformat() - if isinstance(value, (int, float)): - return datetime.date.fromtimestamp(float(value)).isoformat() + if isinstance(value, (int, float)) and not isinstance(value, bool): + # Through instant, so milliseconds (what a JavaScript client sends) + # read as the day they name, and a number that names no day is kept + # as written below instead of raising "year 56759 is out of range" + # out of the store. + t = instant(value) + if t is not None: + try: + return datetime.date.fromtimestamp(t).isoformat() + except (ValueError, OverflowError, OSError): + pass + return str(value)[:40] s = str(value).strip() try: return datetime.date.fromisoformat(s[:10]).isoformat() @@ -280,6 +290,11 @@ def expiry_date(value, today: datetime.date | None = None, if value in (None, ""): return None today = today or datetime.date.today() + if isinstance(value, float) and not math.isfinite(value): + raise CryptoError( + f"expires={value!r} is not a date or a duration. Give the last " + f"day it is true (2026-09-03), or how long it lasts (14d, 2w, " + f"3m, 1y).") if isinstance(value, (int, float)) and not isinstance(value, bool): s = str(int(value)) else: @@ -293,12 +308,20 @@ def expiry_date(value, today: datetime.date | None = None, raise CryptoError( f"expires={value!r}: a duration has to be at least one day") unit = (m.group(2) or "d").lower() - if unit == "m": - return _add_months(today, n).isoformat() - if unit == "y": - return _add_months(today, n * 12).isoformat() - return (today + datetime.timedelta( - days=n * (7 if unit == "w" else 1))).isoformat() + try: + if unit == "m": + return _add_months(today, n).isoformat() + if unit == "y": + return _add_months(today, n * 12).isoformat() + return (today + datetime.timedelta( + days=n * (7 if unit == "w" else 1))).isoformat() + except (ValueError, OverflowError): + # `9999y` lands past the year 9999, the last a date can name. + # A fact meant to outlast that is a fact with no expiry. + raise CryptoError( + f"expires={value!r} runs past the year 9999. A fact that " + f"lasts that long does not expire: leave the expiry off." + ) from None try: day = datetime.date.fromisoformat(s[:10]) except ValueError: diff --git a/tests/test_storage_dates.py b/tests/test_storage_dates.py index 0d823c8..3aeeecb 100644 --- a/tests/test_storage_dates.py +++ b/tests/test_storage_dates.py @@ -105,3 +105,37 @@ def test_a_vault_an_older_build_broke_is_repaired_when_it_opens(vault, third = Vault.unlock(vault_path, passphrase=PASS) assert [r["op"] for r in third.db.conn.execute( "SELECT op FROM audit")].count("repair") == 1 + + +def test_a_discovery_date_in_milliseconds_is_the_day_it_names(vault): + """A JavaScript client sends Date.now(). The store raised "year 56759 is + out of range" and the memory was lost.""" + from compartment.vault import discovery_date + ms = datetime.datetime(2024, 5, 1, 12).timestamp() * 1000 + assert discovery_date(ms) == "2024-05-01" + out = vault.store("The office moved to the fourth floor.", caller="t", + discovered=ms) + assert out["discovered"] == "2024-05-01" + + +def test_a_discovery_number_that_names_no_day_is_kept_as_written(): + """The field's rule for anything unparseable: keep what the caller + meant, never raise, never invent a date that reads as verified.""" + from compartment.vault import discovery_date + assert discovery_date(1e300) == "1e+300" + assert discovery_date(float("nan")) == "nan" + + +@pytest.mark.parametrize("expires", ["9999y", float("nan"), + float("inf")]) +def test_an_expiry_nothing_can_date_is_refused_with_a_reason(vault, expires): + """These escaped as ValueError and OverflowError from date arithmetic, + which no surface reports as a refusal with a fix.""" + from compartment.crypto import CryptoError + from compartment.vault import expiry_date + with pytest.raises(CryptoError, match="expires="): + expiry_date(expires) + with pytest.raises(CryptoError, match="expires="): + vault.store("The trial licence lapses eventually.", caller="t", + expires=expires) + assert vault.db.count() == 0 From 9a6398040a20673f87005c16a9873d10bd5e8ff7 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:47:35 -0700 Subject: [PATCH 039/135] A CLI command that meets a missing file, a closed pipe or Ctrl-C ends with one line and a non-zero exit instead of a traceback The CLI caught CryptoError and nothing else. `compartment import missing.jsonl`, `pack install missing.mpack`, and `verify` or `2fa status` on a vault that is not there ended in a FileNotFoundError traceback; `compartment recent --all | head` ended in BrokenPipeError twice over, the second from the flush at exit, with exit code 120; Ctrl-C printed a KeyboardInterrupt traceback. With no credential stored, `search`, `recent` and every other command that opens the vault reported a vault that does not exist as locked and sent the user to `compartment unlock`, which cannot help. Agents run these commands from their own shells and read the output. Now a missing vault says "no vault at PATH (run `compartment init`, or pass --vault PATH)" before any credential is looked for; any other OSError ends the command with "error: : " and exit 1; a reader that closed the pipe ends it quietly with exit 1, stdout pointed at the null device so the exit flush cannot raise again; Ctrl-C prints "interrupted" and exits 130. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 42 +++++++++++++++++ tests/test_cli_errors.py | 98 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 tests/test_cli_errors.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 9fc9224..9fb7754 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -89,6 +89,11 @@ def _can_prompt() -> bool: def _open_vault(args) -> Vault: + if not os.path.exists(args.vault): + # Before the credential: with none stored, a missing vault was + # reported as a locked one, and `compartment unlock` cannot help. + _die(f"no vault at {args.vault} (run `compartment init`, or pass " + "--vault PATH)") try: pw, key = Vault.resolve_credential(args.vault) except CryptoError: @@ -2620,8 +2625,45 @@ def main(argv: list[str] | None = None) -> None: offline_guard.activate() try: args.fn(args) + # Inside the try: buffered output reaches a pipe here, and a reader + # that has gone away fails this flush, not one at interpreter exit. + sys.stdout.flush() except CryptoError as exc: _die(str(exc)) + except BrokenPipeError: + # `compartment recent --all | head`: the reader took what it wanted + # and closed the pipe. Not a failure of the command, and not a + # traceback; stdout goes to the null device so that the flush at + # exit does not raise the same error again. + try: + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + except (OSError, ValueError): + pass + sys.exit(1) + except KeyboardInterrupt: + print("\ninterrupted", file=sys.stderr) + sys.exit(130) + except OSError as exc: + # A file named on the command line that is not there or cannot be + # read, a folder that cannot be written, a full disk: said in one + # line naming the path, instead of a traceback an agent running + # the CLI has to read to the bottom of to find the same thing. + _die(_os_error_message(exc, args)) + + +def _os_error_message(exc: OSError, args) -> str: + """One line for an OSError that ended a command, naming the file.""" + where = exc.filename + vault = getattr(args, "vault", None) + if (isinstance(exc, FileNotFoundError) and where is not None and vault + and os.path.abspath(os.fsdecode(where)) == os.path.abspath(vault)): + return (f"no vault at {vault} (run `compartment init`, or pass " + "--vault PATH)") + if where is None: + return str(exc) + if exc.filename2 is not None: + where = f"{os.fsdecode(where)} -> {os.fsdecode(exc.filename2)}" + return f"{exc.strerror or exc}: {os.fsdecode(where)}" if __name__ == "__main__": diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py new file mode 100644 index 0000000..1639252 --- /dev/null +++ b/tests/test_cli_errors.py @@ -0,0 +1,98 @@ +"""How a CLI command ends when it cannot do what was asked: one line on +stderr that names the file or the remedy, and a non-zero exit, never a +traceback. Agents run the CLI from their own shells and read what it says; +a traceback is forty lines to read to the bottom of for the same fact.""" +import os + +import pytest + +from compartment import cli + +from conftest import PASS + + +@pytest.fixture() +def no_credential(monkeypatch): + from compartment import vault as vault_module + monkeypatch.setattr(vault_module, "keychain_get", lambda path: None) + for name in ("COMPARTMENT_PASSPHRASE", "ENGRAM_PASSPHRASE"): + monkeypatch.delenv(name, raising=False) + + +def _ends(argv, capsys, code=1): + with pytest.raises(SystemExit) as e: + cli.main(argv) + assert e.value.code == code + return capsys.readouterr().err + + +@pytest.mark.parametrize("command", [["search", "anything"], ["recent"], + ["status"], ["verify"], + ["2fa", "status"]]) +def test_a_vault_that_is_not_there_is_said_to_be_missing( + tmp_path, capsys, no_credential, command): + """With no vault at the path, `search` and the rest said the vault was + locked and to run `compartment unlock`, which cannot help, and + `verify` and `2fa status` raised FileNotFoundError.""" + missing = str(tmp_path / "nowhere" / "memory.vault") + err = _ends(["--vault", missing, *command], capsys) + assert f"no vault at {missing}" in err and "compartment init" in err + assert "Traceback" not in err and "locked" not in err + + +def test_a_file_named_on_the_command_line_that_is_not_there_is_named( + vault, vault_path, tmp_path, capsys, monkeypatch): + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + vault.lock() + for argv in (["import", str(tmp_path / "missing.jsonl")], + ["pack", "install", str(tmp_path / "missing.mpack")]): + err = _ends(["--vault", vault_path, *argv], capsys) + assert err.startswith("error: ") and "missing." in err + assert len(err.strip().splitlines()) == 1 + + +class _ReaderGone: + """stdout once the program reading it has exited: `| head`.""" + + def __init__(self, fd): + self.fd = fd + + def write(self, text): + raise BrokenPipeError(32, "Broken pipe") + + def flush(self): + raise BrokenPipeError(32, "Broken pipe") + + def fileno(self): + return self.fd + + +def test_output_cut_short_by_its_reader_is_not_a_traceback(tmp_path, capsys, + monkeypatch): + """`compartment recent --all | head` ended in BrokenPipeError twice + over: once from the print, again from the flush at exit.""" + with open(tmp_path / "stdout", "w") as stand_in: + monkeypatch.setattr("sys.stdout", _ReaderGone(stand_in.fileno())) + err = _ends(["embed-daemon", "status"], capsys) + assert err == "" + + +def test_ctrl_c_ends_a_command_without_a_traceback(monkeypatch, capsys): + def interrupted(args): + raise KeyboardInterrupt + monkeypatch.setattr(cli, "cmd_embed_daemon", interrupted) + err = _ends(["embed-daemon", "status"], capsys, code=130) + assert err.strip() == "interrupted" + + +def test_a_vault_found_but_unreadable_names_the_file(vault, vault_path, + capsys, monkeypatch): + if getattr(os, "geteuid", lambda: 1)() == 0: + pytest.skip("root reads any file") + vault.lock() + os.chmod(vault_path, 0) + try: + err = _ends(["--vault", vault_path, "verify"], capsys) + finally: + os.chmod(vault_path, 0o600) + assert vault_path in err and "Traceback" not in err From d4afe5c6cb4f42f8cd3b37e8b32c0c905d332252 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:48:01 -0700 Subject: [PATCH 040/135] `integrate claude` writes Claude Desktop's config where the Windows MSIX app reads it, and finds a Windows install that has never been opened Claude Desktop on Windows is an MSIX package now: the Microsoft Store build (AnthropicPBC.Claude_fnn82j28hfe8t), and the direct download, whose Claude Setup.exe became an MSIX bootstrapper that removes the older installer (Claude_pzs8sxrjxfjjc). Windows redirects a packaged app's writes under %APPDATA% into the package's own %LOCALAPPDATA%\Packages\\LocalCache\Roaming, and the app reads that private copy first. Compartment is not inside the package, so it wrote the real %APPDATA%\Claude\claude_desktop_config.json, which the app had stopped reading the first time it saved its settings: `integrate claude` and the Connect button reported Claude Desktop connected, and the app never saw the server. config_path() now returns the package's private copy once it exists (of two packages, the one saved last). Until it exists the app still reads the real file through Windows' merged view, so that one is written: a new private file holding only our server would hide every server already in the real one. With neither, the private path is created. With no package, %APPDATA%\Claude is used as before. present() looked only for %APPDATA%\Claude, which a package that has never been opened does not have, so `integrate claude` said Claude Desktop was not installed. An installed package directory, or the older installer's %LOCALAPPDATA%\AnthropicClaude, now counts. Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_desktop.py | 91 +++++++++++++++-- tests/test_integrations_claude_desktop.py | 114 ++++++++++++++++++++++ 2 files changed, 198 insertions(+), 7 deletions(-) create mode 100644 tests/test_integrations_claude_desktop.py diff --git a/src/compartment/claude_desktop.py b/src/compartment/claude_desktop.py index 0f8dd96..afa1c01 100644 --- a/src/compartment/claude_desktop.py +++ b/src/compartment/claude_desktop.py @@ -20,21 +20,87 @@ NAME = "compartment" FILENAME = "claude_desktop_config.json" +#: The MSIX package families Claude Desktop is published under on Windows: +#: the direct download (whose Claude Setup.exe has been an MSIX bootstrapper +#: since 2026, removing any older install) and the Microsoft Store build. +#: Taken from `Get-AppxPackage` output and the app's own main.log quoted in +#: anthropics/claude-ai-mcp#1150 and shraga100/claude-desktop-rtl-patch#45. +MSIX_FAMILIES = ("Claude_pzs8sxrjxfjjc", "AnthropicPBC.Claude_fnn82j28hfe8t") + + +def _windows() -> bool: + return os.name == "nt" + + +def _local_appdata() -> Path: + return Path(os.environ.get("LOCALAPPDATA") + or Path.home() / "AppData" / "Local") + + +def _roaming_config() -> Path: + """%APPDATA%\\Claude\\claude_desktop_config.json, the real one.""" + base = os.environ.get("APPDATA") or str(Path.home() / "AppData" + / "Roaming") + return Path(base) / "Claude" / FILENAME + + +def _msix_configs() -> list[Path]: + """The private config of each MSIX package of Claude Desktop installed + for this user, installed meaning its package directory exists.""" + packages = _local_appdata() / "Packages" + out = [] + for family in MSIX_FAMILIES: + try: + if (packages / family).is_dir(): + out.append(packages / family / "LocalCache" / "Roaming" + / "Claude" / FILENAME) + except OSError: + pass + return out + def config_path() -> Path: """Claude Desktop's config file on this platform. - The three locations are the app's own, not ours: macOS keeps it under - Application Support, Windows under %APPDATA%, Linux under the XDG config - directory. + The locations are the app's own, not ours: macOS keeps it under + Application Support, Linux (the official beta since June 2026, and the + community builds before it) under the XDG config directory, and Windows + under %APPDATA% - except that an MSIX app does not write %APPDATA%. + + Claude Desktop on Windows is an MSIX package now, from the Store or + from the direct download, and Windows redirects a packaged app's + writes under %APPDATA% into the package's own + %LOCALAPPDATA%\\Packages\\\\LocalCache\\Roaming. The app reads + that private copy first. Compartment is not inside the package, so it + wrote the real %APPDATA%\\Claude file, which the app had stopped + reading the first time it saved its settings: `integrate claude` + reported Claude Desktop connected and the app never saw the entry. + + So with a package installed, its private copy is the file, once it + exists. Until it does, the app still reads the real file through + Windows' merged view, and that is the one written: a new private file + holding only our server would hide every server already in the real + one. With neither, the private path is created, which the app will + read first. Of two packages with a private copy each, the one saved + last is the one in use. """ if sys.platform == "darwin": return (Path.home() / "Library" / "Application Support" / "Claude" / FILENAME) - if os.name == "nt": - base = os.environ.get("APPDATA") or str(Path.home() / "AppData" - / "Roaming") - return Path(base) / "Claude" / FILENAME + if _windows(): + roaming = _roaming_config() + packaged = _msix_configs() + if not packaged: + return roaming + written = [] + for p in packaged: + try: + written.append((p.stat().st_mtime, p)) + except OSError: + pass + if written: + return max(written)[1] + return roaming if roaming.exists() else packaged[0] base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") return Path(base) / "Claude" / FILENAME @@ -46,6 +112,11 @@ def present(path: Path | None = None) -> bool: copy that has never been opened is recognised by its application directory instead. Writing a config for an app that is not installed would leave litter behind and connect nothing. + + On Windows that is an MSIX package directory, or the older installer's + %LOCALAPPDATA%\\AnthropicClaude. A package that has never been opened + has no %APPDATA%\\Claude to find, so it was reported as not installed + and `integrate claude` skipped it. """ p = Path(path) if path else config_path() if p.exists() or p.parent.is_dir(): @@ -53,6 +124,12 @@ def present(path: Path | None = None) -> bool: if sys.platform == "darwin": return (Path("/Applications/Claude.app").exists() or (Path.home() / "Applications" / "Claude.app").exists()) + if _windows() and path is None: + try: + return (bool(_msix_configs()) + or (_local_appdata() / "AnthropicClaude").is_dir()) + except OSError: + return False return False diff --git a/tests/test_integrations_claude_desktop.py b/tests/test_integrations_claude_desktop.py new file mode 100644 index 0000000..9645358 --- /dev/null +++ b/tests/test_integrations_claude_desktop.py @@ -0,0 +1,114 @@ +"""Claude Desktop on Windows is an MSIX package now, from the Store or from +the direct download, and Windows keeps a packaged app's %APPDATA% writes in +the package's own LocalCache. The app reads that private copy first, so a +config written to the real %APPDATA%\\Claude is one it never sees.""" +import json + +import pytest + +from compartment import claude_desktop + +DIRECT, STORE = claude_desktop.MSIX_FAMILIES +ARGS = ["--vault", r"C:\Users\ana\.compartment\memory.vault", "--caller", + "claude-desktop", "serve"] + + +@pytest.fixture() +def windows(monkeypatch, tmp_path): + """A Windows profile under tmp_path: %APPDATA% and %LOCALAPPDATA%.""" + monkeypatch.setattr(claude_desktop, "_windows", lambda: True, + raising=False) + monkeypatch.setattr(claude_desktop.sys, "platform", "win32") + roaming = tmp_path / "AppData" / "Roaming" + local = tmp_path / "AppData" / "Local" + roaming.mkdir(parents=True) + local.mkdir(parents=True) + monkeypatch.setenv("APPDATA", str(roaming)) + monkeypatch.setenv("LOCALAPPDATA", str(local)) + + class Profile: + real = roaming / "Claude" / claude_desktop.FILENAME + + @staticmethod + def package(family): + (local / "Packages" / family).mkdir(parents=True, exist_ok=True) + return (local / "Packages" / family / "LocalCache" / "Roaming" + / "Claude" / claude_desktop.FILENAME) + + @staticmethod + def write(p, servers): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps({"mcpServers": servers}), encoding="utf-8") + + return Profile + + +def test_an_msix_install_is_written_where_the_app_reads( + windows, real_claude_desktop_config_path): + """The package had saved its settings to its private copy, so the app + no longer read %APPDATA%\\Claude, and that is where the entry went: + `integrate claude` reported Claude Desktop connected, and it never saw + the server.""" + private = windows.package(DIRECT) + windows.write(private, {"theirs": {"command": "npx"}}) + windows.write(windows.real, {"stale": {"command": "old"}}) + stale = windows.real.read_text(encoding="utf-8") + + p = real_claude_desktop_config_path() + assert p == private + claude_desktop.register(r"C:\x\compartment.exe", ARGS, path=p) + + servers = json.loads(private.read_text(encoding="utf-8"))["mcpServers"] + assert set(servers) == {"theirs", "compartment"} + assert windows.real.read_text(encoding="utf-8") == stale + + +def test_the_store_build_is_found_too(windows, + real_claude_desktop_config_path): + private = windows.package(STORE) + windows.write(private, {}) + assert real_claude_desktop_config_path() == private + + +def test_a_package_that_has_not_saved_yet_reads_the_real_file( + windows, real_claude_desktop_config_path): + """Until the package saves its own copy, Windows shows it the real file; + a private file holding only our server would hide every server in + it.""" + windows.package(DIRECT) + windows.write(windows.real, {"theirs": {"command": "npx"}}) + assert real_claude_desktop_config_path() == windows.real + + +def test_a_package_with_no_config_anywhere_gets_its_private_one( + windows, real_claude_desktop_config_path): + private = windows.package(DIRECT) + assert real_claude_desktop_config_path() == private + + +def test_of_two_packages_the_one_saved_last_is_written( + windows, real_claude_desktop_config_path): + import os + old, new = windows.package(DIRECT), windows.package(STORE) + windows.write(old, {}) + windows.write(new, {}) + os.utime(old, (1_000_000, 1_000_000)) + assert real_claude_desktop_config_path() == new + + +def test_without_a_package_the_regular_install_uses_appdata( + windows, real_claude_desktop_config_path): + assert real_claude_desktop_config_path() == windows.real + + +@pytest.mark.parametrize("installed", ["direct", "store", "squirrel"]) +def test_an_install_that_was_never_opened_is_present(windows, installed, + tmp_path): + """A package that has never run has no %APPDATA%\\Claude, so it was + reported as not installed and `integrate claude` skipped it.""" + assert claude_desktop.present() is False + if installed == "squirrel": + (tmp_path / "AppData" / "Local" / "AnthropicClaude").mkdir() + else: + windows.package(DIRECT if installed == "direct" else STORE) + assert claude_desktop.present() is True From 1994186e56d3c06f8013927357ed1065f53f92dd Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:48:46 -0700 Subject: [PATCH 041/135] `compartment update` and `uninstall` recognise a uv tool install on Windows and a pipx install anywhere, and use that tool's own commands How a copy was installed was read from the path of the running script: "uv/tools" in it meant uv, anything else meant pip. On Windows uv's launcher is an .exe of its own in ~/.local/bin, not a symlink into uv's tools directory, and the path has backslashes, so a `uv tool install` there was taken for pip and `compartment update` ran `python -m pip install --upgrade` in an environment uv builds without pip: "No module named pip", update failed. A `pipx install`, which the README offers, was taken for pip on every platform: update left pipx's own record stale, and the last step uninstall printed, `python -m pip uninstall compartment`, left pipx listing a broken install whose launcher points at nothing. The environment the interpreter runs in is now asked first: uv leaves uv-receipt.toml at its root and pipx leaves pipx_metadata.json, on every platform. A pipx install updates with `pipx upgrade` (`pipx install --force` for --source) and uninstalls with `pipx uninstall`. The path test stays as the fallback, now with forward slashes on every platform. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 26 +++++++++-- tests/test_cli_install_kind.py | 84 ++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/test_cli_install_kind.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 9fb7754..a22f408 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -433,10 +433,26 @@ def _wait_for_status_bar_app(timeout: float = 15.0) -> bool: def _install_kind() -> tuple[str, list[str]]: - """How this copy was installed, and the command that upgrades it.""" - exe = Path(sys.argv[0]).resolve() - if "uv/tools" in str(exe) or shutil.which("uv") and "uv/tools" in str( - Path(shutil.which("compartment") or "").resolve()): + """How this copy was installed, and the command that upgrades it. + + Asked of the environment this interpreter runs in, which each tool marks + the same way on every platform: `uv tool install` leaves uv-receipt.toml + at its root and `pipx install` leaves pipx_metadata.json. The path of the + running script, which is all this looked at before, says "uv/tools" only + where the launcher is a symlink into uv's directory and the separators + are forward slashes: on Windows, where uv's launcher is an .exe of its + own in ~/.local/bin, a uv install was taken for pip, and `update` ran + `python -m pip` in an environment uv builds without pip. A pipx install + was taken for pip everywhere: `update` left pipx's record of it stale, + and the last step `uninstall` printed left pipx a broken install.""" + prefix = Path(sys.prefix) + if (prefix / "uv-receipt.toml").is_file(): + return "uv", ["uv", "tool", "upgrade", "compartment"] + if (prefix / "pipx_metadata.json").is_file(): + return "pipx", ["pipx", "upgrade", "compartment"] + exe = Path(sys.argv[0]).resolve().as_posix() + if "uv/tools" in exe or shutil.which("uv") and "uv/tools" in Path( + shutil.which("compartment") or "").resolve().as_posix(): return "uv", ["uv", "tool", "upgrade", "compartment"] return "pip", [sys.executable, "-m", "pip", "install", "--upgrade", "compartment"] @@ -448,6 +464,7 @@ def cmd_update(args) -> None: if args.source: target = "git+https://github.com/MaxFreedomPollard/Compartment@main" cmd = (["uv", "tool", "install", "--force", target] if kind == "uv" + else ["pipx", "install", "--force", target] if kind == "pipx" else [sys.executable, "-m", "pip", "install", "--upgrade", "--force-reinstall", target]) print(f"Compartment {__version__} - updating ({kind} install)") @@ -670,6 +687,7 @@ def cmd_uninstall(args) -> None: print(" delete it yourself, or re-run with --purge, once you are sure") kind, _ = _install_kind() final = ("uv tool uninstall compartment" if kind == "uv" + else "pipx uninstall compartment" if kind == "pipx" else f"{sys.executable} -m pip uninstall compartment") print("\nOne step left, which cannot remove itself while it is running:") print(f" {final}") diff --git a/tests/test_cli_install_kind.py b/tests/test_cli_install_kind.py new file mode 100644 index 0000000..98a8111 --- /dev/null +++ b/tests/test_cli_install_kind.py @@ -0,0 +1,84 @@ +"""Which tool installed this copy, so that `update` upgrades it with that +tool and `uninstall` names that tool's command for the last step.""" +import subprocess +import sys +import types + +import pytest + +from compartment import cli + + +@pytest.fixture() +def env_root(tmp_path, monkeypatch): + """sys.prefix an environment of this test's own, and nothing on PATH or + in argv that points into a tool's directory.""" + root = tmp_path / "env" + root.mkdir() + monkeypatch.setattr(sys, "prefix", str(root)) + monkeypatch.setattr(sys, "argv", [str(tmp_path / "bin" / "compartment")]) + monkeypatch.setattr(cli.shutil, "which", lambda name: None) + return root + + +def test_a_uv_tool_install_on_windows_is_upgraded_with_uv(env_root, + monkeypatch): + """uv's launcher on Windows is an .exe of its own in ~/.local/bin, not a + symlink into uv's tools directory, and its paths have backslashes: the + "uv/tools" test never matched there, and `update` ran `python -m pip` + in an environment uv builds without pip.""" + (env_root / "uv-receipt.toml").write_text("[tool]\n") + monkeypatch.setattr(sys, "argv", + [r"C:\Users\me\.local\bin\compartment.exe"]) + assert cli._install_kind() == ("uv", ["uv", "tool", "upgrade", + "compartment"]) + + +def test_a_pipx_install_is_upgraded_and_removed_with_pipx(env_root, + monkeypatch, capsys): + (env_root / "pipx_metadata.json").write_text("{}") + assert cli._install_kind() == ("pipx", ["pipx", "upgrade", "compartment"]) + ran = [] + + def run(cmd, *a, **k): + ran.append(list(cmd)) + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + monkeypatch.setattr(cli.subprocess, "run", run) + from compartment import embed_daemon + monkeypatch.setattr(embed_daemon, "stop", lambda *a, **k: {}) + cli.cmd_update(types.SimpleNamespace(source=True, no_app=True, + vault="unused")) + assert ran[0][:3] == ["pipx", "install", "--force"] + assert ran[0][3].startswith("git+https://github.com/") + + +def test_a_plain_pip_install_is_still_upgraded_with_its_own_pip(env_root): + kind, cmd = cli._install_kind() + assert kind == "pip" and cmd[:3] == [sys.executable, "-m", "pip"] + + +def test_uninstall_names_the_pipx_command_for_a_pipx_install(env_root, + monkeypatch, + capsys): + """`python -m pip uninstall` inside a pipx environment removes the + package and leaves pipx listing a broken install whose launcher points + at nothing.""" + (env_root / "pipx_metadata.json").write_text("{}") + monkeypatch.setattr(cli, "_tray_app", + lambda: (_ for _ in ()).throw(RuntimeError("no app"))) + monkeypatch.setattr(subprocess, "run", lambda argv, **k: + subprocess.CompletedProcess(argv, 1, "", "")) + for name in ("unregister", "is_registered"): + monkeypatch.setattr(cli.clients, name, lambda *a, **k: False) + monkeypatch.setattr(cli.claude_desktop, "unregister", lambda: False) + monkeypatch.setattr(cli.claude_hooks, "is_installed", lambda: False) + monkeypatch.setattr(cli.agent_skill, "SKILL_TARGETS", {}) + monkeypatch.setattr(cli, "_remove_managed_claude_md", lambda: None) + from compartment import menubar + monkeypatch.setattr(menubar, "_is_generated", lambda path: False) + monkeypatch.setenv("OPENCLAW_HOME", str(env_root / "openclaw")) + monkeypatch.setenv("HERMES_HOME", str(env_root / "hermes")) + cli.cmd_uninstall(types.SimpleNamespace(purge=False, + vault=str(env_root / "v.vault"))) + out = capsys.readouterr().out + assert out.rstrip().endswith("pipx uninstall compartment") From 6c6183ea8273b9ccc7b9ece6523013f5ec438424 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:49:24 -0700 Subject: [PATCH 042/135] A panel that cannot start names the command that fixes it on this machine: the Tk package for each Linux family and BSD, XWayland for a Wayland session, and pip lines that install something When tkinter was missing, the panel said to install "your distribution's Python Tk package (Debian/Ubuntu: python3-tk, Fedora: python3-tkinter)". Arch, openSUSE, Alpine, Void, Gentoo, the BSDs and Termux all call it something else, and for a Python the distribution did not provide - pyenv, asdf, conda, a build in /usr/local - the package changes nothing, since _tkinter is built with the Python against Tk's headers. tk_install_hint now reads /etc/os-release (ID, then ID_LIKE, so Mint, Rocky, Manjaro and the openSUSE variants map to their family) and sys.base_prefix, and gives one command: apt, dnf, zypper, pacman, apk or xbps; pkg, pkg_add or pkgin with the right py3XY name on the BSDs; `pkg install python-tkinter` on Termux; `conda install tk`; `brew install python-tk@3.X`; or, for a Python built from source, the Tk headers and a rebuild. The message also names which Python it means. Every TclError from Tk() was reported as "no graphical display". Tk draws through X11 only, so a Wayland session without XWayland (DISPLAY unset, WAYLAND_DISPLAY set) is now told to turn XWayland on, and a Python whose Tcl files are missing ("Can't find a usable init.tcl") is told to reinstall it; a machine with no display gets the old advice. The tray's missing-pystray error said `pip install 'compartment[tray]'` and the menu bar's missing-PyObjC error `pip install 'compartment[menubar]'`; both extras have been empty since their packages became base dependencies, so following the advice installed nothing. They now name the packages, installed into the interpreter that is running. Co-Authored-By: Claude Opus 5.5 --- src/compartment/menubar.py | 9 +- src/compartment/systray.py | 175 +++++++++++++++++++++++++++--- tests/test_apps_panel_messages.py | 170 +++++++++++++++++++++++++++++ 3 files changed, 338 insertions(+), 16 deletions(-) create mode 100644 tests/test_apps_panel_messages.py diff --git a/src/compartment/menubar.py b/src/compartment/menubar.py index 511b008..66c470c 100644 --- a/src/compartment/menubar.py +++ b/src/compartment/menubar.py @@ -2060,8 +2060,13 @@ def run(vault: str | None = None, show: bool = False, from Foundation import (NSBundle, NSDistributedNotificationCenter, NSObject) except ImportError: - print("error: the menu bar app needs PyObjC.\n" - " pip install 'compartment[menubar]'", file=sys.stderr) + # Into this interpreter, by name. The [menubar] extra this used to + # name has been empty since PyObjC became a base dependency, so + # installing it changed nothing. + print("error: the menu bar app needs PyObjC, which this Python does " + "not have.\n" + f' "{sys.executable}" -m pip install pyobjc-framework-Cocoa', + file=sys.stderr) return 1 vault_path = vault or default_vault() diff --git a/src/compartment/systray.py b/src/compartment/systray.py index 1655487..464e677 100644 --- a/src/compartment/systray.py +++ b/src/compartment/systray.py @@ -791,6 +791,155 @@ def set_login(enabled: bool, vault: str | None = None) -> str: return f"error: {exc}" +# --- when the panel cannot draw --------------------------------------------- +# pip cannot install tkinter: it comes with the Python, or with a package of +# the distribution's. "Install your distribution's Python Tk package" left +# the user to find out what that is called, which differs on every family, +# and is the wrong advice altogether for a Python the distribution did not +# provide (pyenv, asdf, conda, one built from source), where the package +# changes nothing. + +#: Per distribution family: the package that adds Tk to the distribution's +#: own Python, and the headers a Python built from source needs to build it. +_TK_BY_FAMILY = { + "debian": ("sudo apt install python3-tk", "sudo apt install tk-dev"), + "fedora": ("sudo dnf install python3-tkinter", + "sudo dnf install tk-devel"), + "suse": ("sudo zypper install python3-tk", "sudo zypper install tk-devel"), + "arch": ("sudo pacman -S tk", "sudo pacman -S tk"), + "alpine": ("sudo apk add python3-tkinter", "sudo apk add tk-dev"), + "void": ("sudo xbps-install python3-tkinter", + "sudo xbps-install tk-devel"), + "gentoo": ("enable the tk USE flag for dev-lang/python, then: " + "sudo emerge --oneshot dev-lang/python", + "sudo emerge dev-lang/tk"), +} + +#: os-release ID and ID_LIKE words, to a family above. +_TK_FAMILY_OF = { + "debian": "debian", "ubuntu": "debian", "linuxmint": "debian", + "pop": "debian", "raspbian": "debian", "kali": "debian", + "elementary": "debian", "zorin": "debian", "neon": "debian", + "fedora": "fedora", "rhel": "fedora", "centos": "fedora", + "rocky": "fedora", "almalinux": "fedora", "amzn": "fedora", + "ol": "fedora", "nobara": "fedora", + "suse": "suse", "opensuse": "suse", "sles": "suse", + "arch": "arch", "manjaro": "arch", "endeavouros": "arch", + "cachyos": "arch", "garuda": "arch", "steamos": "arch", + "alpine": "alpine", "postmarketos": "alpine", + "void": "void", "gentoo": "gentoo", +} + + +def _os_release() -> str: + for path in ("/etc/os-release", "/usr/lib/os-release"): + try: + with open(path, encoding="utf-8", errors="replace") as fh: + return fh.read() + except OSError: + continue + return "" + + +def _distro_family(os_release: str) -> str | None: + fields = {} + for line in os_release.splitlines(): + key, sep, value = line.partition("=") + if sep: + fields[key.strip()] = value.strip().strip("\"'").lower() + words = [fields.get("ID", "")] + fields.get("ID_LIKE", "").split() + for word in words: + if not word: + continue + # opensuse-tumbleweed, opensuse-leap, sles_sap and the like + for name in (word, word.split("-")[0], word.split("_")[0]): + if name in _TK_FAMILY_OF: + return _TK_FAMILY_OF[name] + return None + + +def tk_install_hint(os_release: str | None = None, + base_prefix: str | None = None, + platform: str | None = None) -> str: + """The command that gives this Python tkinter, as exactly as this + machine allows. Pure given its arguments, which default to this + machine's, so every family is testable anywhere.""" + platform = platform or sys.platform + prefix = base_prefix if base_prefix is not None else sys.base_prefix + ver = f"{sys.version_info[0]}{sys.version_info[1]}" + if "com.termux" in prefix: + return "pkg install x11-repo && pkg install python-tkinter" + if platform.startswith("freebsd"): + return f"sudo pkg install py{ver}-tkinter" + if platform.startswith("openbsd"): + return "doas pkg_add python-tkinter" + if platform.startswith("netbsd"): + return f"sudo pkgin install py{ver}-Tk" + if os.path.isdir(os.path.join(prefix, "conda-meta")): + return "conda install tk" + if "linuxbrew" in prefix or "homebrew" in prefix.lower(): + return ("brew install " + f"python-tk@{sys.version_info[0]}.{sys.version_info[1]}") + if "/uv/python/" in prefix.replace(os.sep, "/"): + # uv's Pythons carry Tk; one without it is a damaged install. + return ("uv python install --reinstall " + f"{sys.version_info[0]}.{sys.version_info[1]}") + family = _distro_family(_os_release() if os_release is None + else os_release) + system, headers = _TK_BY_FAMILY.get(family, (None, None)) + if prefix.rstrip("/") not in ("/usr", ""): + # Not the distribution's Python: pyenv, asdf, a source build in + # /usr/local. Its _tkinter is built when the Python is, against + # Tk's headers, so the headers go in first and the Python is + # built again. + build = ("then build this Python again (pyenv: " + "pyenv install --force " + f"{sys.version_info[0]}.{sys.version_info[1]}." + f"{sys.version_info[2]})") + return (f"{headers}, {build}" if headers else + f"install Tk's development headers, {build}") + return system or ("install your distribution's Tk package for Python " + "(often python3-tk or python3-tkinter)") + + +def tk_missing_message(what: str = "panel") -> str: + return ("error: this Python has no tkinter, which the " + f"{what} needs.\n" + f" This Python is {sys.executable}. To give it tkinter:\n" + f" {tk_install_hint()}\n" + " Or install Compartment with uv, which brings its own Python " + "with tkinter in it:\n" + " uv tool install compartment\n" + " Everything else works without it: the CLI, the MCP server, " + "and `compartment dash`.") + + +def no_display_message(exc: Exception) -> str: + """Why Tk could not open a window, in words that say what to do.""" + text = str(exc) + low = text.lower() + if "init.tcl" in low or "tk.tcl" in low: + return (f"error: this Python's Tcl/Tk files are missing or not where " + f"it looks for them ({text}).\n" + " Reinstall this Python, or install Compartment with uv, " + "whose Python carries its own:\n" + " uv tool install compartment") + if (sys.platform not in ("win32", "darwin") + and os.environ.get("WAYLAND_DISPLAY") + and not os.environ.get("DISPLAY")): + # Tk draws through X11 only. A Wayland desktop runs XWayland for + # programs like it, usually on demand; this one has none running. + return ("error: the panel is drawn with Tk, which needs X11, and " + "this Wayland session has no X server (DISPLAY is not set).\n" + " Turn XWayland on in your compositor (sway: `xwayland " + "enable`; GNOME and KDE start it by themselves when it is\n" + " installed: package xwayland or xorg-x11-server-Xwayland), " + "or use `compartment dash` to read the vault in a browser.") + return (f"error: no graphical display to draw the panel on ({text}).\n" + " On a headless machine use the CLI, or `compartment dash` to\n" + " read the vault in a browser.") + + def panel_geometry(content: int, maximum: int) -> tuple[int, bool]: """How tall to draw the panel, and whether it needs a scrollbar. @@ -856,13 +1005,7 @@ def run(vault: str | None = None, show: bool = False, import tkinter as tk from tkinter import ttk except ImportError: - print("error: this Python has no tkinter, which the " - f"{'tray panel' if tray else 'panel'} needs.\n" - " The quickest fix is to install Compartment with uv, which\n" - " brings its own Python with tkinter already in it:\n" - " uv tool install compartment\n" - " Otherwise install your distribution's Python Tk package\n" - " (Debian/Ubuntu: python3-tk, Fedora: python3-tkinter).", + print(tk_missing_message("tray panel" if tray else "panel"), file=sys.stderr) return 3 pystray = Image = None @@ -871,8 +1014,13 @@ def run(vault: str | None = None, show: bool = False, import pystray from PIL import Image except ImportError: - print("error: the tray app needs pystray and Pillow.\n" - " pip install 'compartment[tray]'", file=sys.stderr) + # Into this interpreter, by name. The [tray] extra this used to + # name has been empty since the two became base dependencies, + # so installing it changed nothing. + print("error: the tray app needs pystray and Pillow, which this " + "Python does not have.\n" + f' "{sys.executable}" -m pip install pystray pillow', + file=sys.stderr) return 3 # Read the DPI (and declare awareness) before the first window exists. @@ -885,11 +1033,10 @@ def run(vault: str | None = None, show: bool = False, try: root = tk.Tk() except tk.TclError as exc: - # A machine with no desktop: SSH, a container, a CI runner. Say so - # in one line rather than showing a Tk stack trace. - print(f"error: no graphical display to draw the panel on ({exc}).\n" - " On a headless machine use the CLI, or `compartment dash` to\n" - " read the vault in a browser.", file=sys.stderr) + # A machine with no desktop (SSH, a container, a CI runner), a + # Wayland session with no XWayland, or a Python whose Tcl files are + # not where it looks. Say which, rather than show a Tk stack trace. + print(no_display_message(exc), file=sys.stderr) return 4 if S != 1.0: # Tk sizes fonts in points; this is what turns a point into a pixel. diff --git a/tests/test_apps_panel_messages.py b/tests/test_apps_panel_messages.py new file mode 100644 index 0000000..02b0296 --- /dev/null +++ b/tests/test_apps_panel_messages.py @@ -0,0 +1,170 @@ +"""When the panel cannot start, it says the one command that fixes it. + +pip cannot install tkinter, and "install your distribution's Python Tk +package" left the user to find out what that is called on their system, +which is different on nearly every family - and is the wrong advice for a +Python the distribution did not provide. The tray and menu bar named +`pip install 'compartment[tray]'` and `[menubar]`, extras that have been +empty since their packages became base dependencies. +""" +from __future__ import annotations + +import sys +import types + +import pytest + +from compartment import menubar, systray + +UBUNTU = 'NAME="Ubuntu"\nID=ubuntu\nID_LIKE=debian\nVERSION_ID="24.04"\n' +MINT = 'ID=linuxmint\nID_LIKE="ubuntu debian"\n' +FEDORA = "ID=fedora\nVERSION_ID=42\n" +ROCKY = 'ID="rocky"\nID_LIKE="rhel centos fedora"\n' +ARCH = "ID=arch\n" +MANJARO = "ID=manjaro\nID_LIKE=arch\n" +TUMBLEWEED = 'ID="opensuse-tumbleweed"\nID_LIKE="opensuse suse"\n' +ALPINE = "ID=alpine\n" +VOID = 'ID="void"\n' + + +@pytest.mark.parametrize("release,command", [ + (UBUNTU, "sudo apt install python3-tk"), + (MINT, "sudo apt install python3-tk"), + (FEDORA, "sudo dnf install python3-tkinter"), + (ROCKY, "sudo dnf install python3-tkinter"), + (ARCH, "sudo pacman -S tk"), + (MANJARO, "sudo pacman -S tk"), + (TUMBLEWEED, "sudo zypper install python3-tk"), + (ALPINE, "sudo apk add python3-tkinter"), + (VOID, "sudo xbps-install python3-tkinter"), +]) +def test_each_family_gets_its_own_package(release, command): + assert systray.tk_install_hint(release, base_prefix="/usr", + platform="linux") == command + + +def test_a_python_the_distribution_did_not_provide_is_told_to_rebuild(): + """The distribution's package does nothing for pyenv's Python: its + _tkinter is built with the Python, against Tk's headers.""" + hint = systray.tk_install_hint(UBUNTU, + base_prefix="/home/u/.pyenv/versions/3.12.4", + platform="linux") + assert hint.startswith("sudo apt install tk-dev") + assert "pyenv install --force" in hint + assert "python3-tk" not in hint + + +def test_conda_and_the_bsds_and_termux(tmp_path): + (tmp_path / "conda-meta").mkdir() + assert systray.tk_install_hint("", base_prefix=str(tmp_path), + platform="linux") == "conda install tk" + ver = f"{sys.version_info[0]}{sys.version_info[1]}" + assert systray.tk_install_hint("", base_prefix="/usr/local", + platform="freebsd14") == \ + f"sudo pkg install py{ver}-tkinter" + assert systray.tk_install_hint("", base_prefix="/usr/local", + platform="openbsd7") == \ + "doas pkg_add python-tkinter" + assert "python-tkinter" in systray.tk_install_hint( + "", base_prefix="/data/data/com.termux/files/usr", platform="linux") + + +def test_an_unknown_distribution_still_gets_a_useful_line(): + hint = systray.tk_install_hint("ID=somethingnew\n", base_prefix="/usr", + platform="linux") + assert "python3-tk" in hint and "python3-tkinter" in hint + + +def _stand_alone(monkeypatch, platform: str) -> None: + """run() up to the point where it imports its toolkit.""" + monkeypatch.setattr(systray.sys, "platform", platform) + monkeypatch.setattr(systray, "acquire_instance_lock", + lambda vault: (None, True)) + monkeypatch.setattr(systray, "hand_over_to_supervisor", lambda vault: False) + + +def test_a_panel_without_tkinter_prints_this_machines_command( + monkeypatch, capsys, tmp_path): + _stand_alone(monkeypatch, "linux") + monkeypatch.setitem(sys.modules, "tkinter", None) + monkeypatch.setattr(systray, "_os_release", lambda: ARCH) + monkeypatch.setattr(systray.sys, "base_prefix", "/usr") + assert systray.run(str(tmp_path / "v.vault")) == 3 + err = capsys.readouterr().err + assert "sudo pacman -S tk" in err + assert sys.executable in err, "say which Python it is about" + assert "uv tool install compartment" in err and "dash" in err + + +def _fake_tk(monkeypatch, error: str): + tk = types.ModuleType("tkinter") + + class TclError(Exception): + pass + + def Tk(): # noqa: N802 + raise TclError(error) + + tk.TclError, tk.Tk = TclError, Tk + tk.ttk = types.ModuleType("tkinter.ttk") + monkeypatch.setitem(sys.modules, "tkinter", tk) + monkeypatch.setitem(sys.modules, "tkinter.ttk", tk.ttk) + + +def test_a_wayland_session_without_xwayland_is_told_so(monkeypatch, capsys, + tmp_path): + """Tk draws through X11 only. "No graphical display" on a desktop that + is plainly running sends the user looking for the wrong problem.""" + _stand_alone(monkeypatch, "linux") + _fake_tk(monkeypatch, "no display name and no $DISPLAY environment " + "variable") + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-1") + assert systray.run(str(tmp_path / "v.vault")) == 4 + err = capsys.readouterr().err + assert "Wayland" in err and "XWayland" in err + assert "Traceback" not in err + + +def test_a_headless_box_still_gets_the_headless_advice(monkeypatch, capsys, + tmp_path): + _stand_alone(monkeypatch, "linux") + _fake_tk(monkeypatch, "no display name and no $DISPLAY environment " + "variable") + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + assert systray.run(str(tmp_path / "v.vault")) == 4 + err = capsys.readouterr().err + assert "no graphical display" in err and "compartment dash" in err + + +def test_a_python_with_broken_tcl_files_is_not_called_headless( + monkeypatch, capsys, tmp_path): + _stand_alone(monkeypatch, "linux") + _fake_tk(monkeypatch, "Can't find a usable init.tcl in the following " + "directories: /install/lib/tcl8.6") + monkeypatch.setenv("DISPLAY", ":0") + assert systray.run(str(tmp_path / "v.vault")) == 4 + err = capsys.readouterr().err + assert "Tcl/Tk files" in err and "no graphical display" not in err + + +def test_the_tray_names_packages_that_install_something(monkeypatch, capsys, + tmp_path): + _stand_alone(monkeypatch, "win32") + _fake_tk(monkeypatch, "unused") + monkeypatch.setitem(sys.modules, "pystray", None) + assert systray.run(str(tmp_path / "v.vault")) == 3 + err = capsys.readouterr().err + assert f'"{sys.executable}" -m pip install pystray pillow' in err + assert "[tray]" not in err, "an empty extra installs nothing" + + +def test_the_menu_bar_names_a_package_that_installs_something(monkeypatch, + capsys): + monkeypatch.setattr(menubar.sys, "platform", "darwin") + monkeypatch.setitem(sys.modules, "objc", None) + assert menubar.run("/tmp/v.vault") == 1 + err = capsys.readouterr().err + assert f'"{sys.executable}" -m pip install pyobjc-framework-Cocoa' in err + assert "[menubar]" not in err From 593fb3d876d61a370bb5866a5fdf90d953e162c2 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:49:33 -0700 Subject: [PATCH 043/135] The capture hook, the memory import and the /compartmentalize skill follow CLAUDE_CONFIG_DIR to wherever Claude Code keeps its files Claude Code keeps settings.json, the per-project memory under projects/ and skills/ in ~/.claude, and moves all of it when CLAUDE_CONFIG_DIR is set, which is how people run a second account or keep the directory in a dotfiles tree. Compartment only ever looked in ~/.claude, so for them `integrate claude` and `hook install` put the capture hook into a settings.json Claude Code never read, `import-claude` found no memories, and the skill was written where no session loads skills. Even with the hook in the right file, it passed over every memory write, because it recognised a memory file by a `.claude` directory in its path. claude_memory.claude_config_dir() is $CLAUDE_CONFIG_DIR when it is an absolute path (expanded from ~), as Claude Code's own VS Code extension requires, and ~/.claude otherwise. The hook's settings file, the import's default root and the skill's home come from it, and a memory directory under it counts as Claude Code's whatever the directory is called. Without the variable nothing changes. Co-Authored-By: Claude Opus 5.5 --- src/compartment/agent_skill.py | 7 +- src/compartment/claude_hooks.py | 19 ++++- src/compartment/claude_memory.py | 29 +++++++- tests/test_integrations_claude_config_dir.py | 78 ++++++++++++++++++++ 4 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 tests/test_integrations_claude_config_dir.py diff --git a/src/compartment/agent_skill.py b/src/compartment/agent_skill.py index db67abe..748fc30 100644 --- a/src/compartment/agent_skill.py +++ b/src/compartment/agent_skill.py @@ -15,7 +15,7 @@ `/skills//SKILL.md`, YAML frontmatter, markdown body - so one packaged file serves all three and the only thing that differs is the root: - claude ~/.claude/skills/compartmentalize/SKILL.md + claude $CLAUDE_CONFIG_DIR or ~/.claude/skills/compartmentalize/SKILL.md hermes $HERMES_HOME or ~/.hermes/skills/compartmentalize/SKILL.md openclaw $OPENCLAW_HOME or ~/.openclaw/skills/compartmentalize/SKILL.md @@ -36,9 +36,10 @@ BACKUP_SUFFIX = ".compartment-backup" #: Every agent known to read the Agent Skills layout, and the environment -#: variable that relocates its home. Claude Code has no such variable. +#: variable that relocates its home. Claude Code's is CLAUDE_CONFIG_DIR, +#: which moves its skills/ with everything else. SKILL_TARGETS = { - "claude": (None, Path.home() / ".claude"), + "claude": ("CLAUDE_CONFIG_DIR", Path.home() / ".claude"), "hermes": ("HERMES_HOME", Path.home() / ".hermes"), "openclaw": ("OPENCLAW_HOME", Path.home() / ".openclaw"), } diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index 2759365..bfc784a 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -25,7 +25,8 @@ from . import claude_memory, clients -SETTINGS = Path.home() / ".claude" / "settings.json" +#: Claude Code's user settings, which move with CLAUDE_CONFIG_DIR. +SETTINGS = claude_memory.claude_config_dir() / "settings.json" # Identifies OUR entries. Both tokens, not one literal string: the command # carries `--vault ...` between them when a vault is pinned, so a single # substring would stop matching and re-installing would duplicate the hook. @@ -216,12 +217,24 @@ def _file_from_payload(payload: dict) -> str | None: def is_memory_file(path: str | Path) -> bool: """True for Claude Code's own memory notes - and nothing else. The index - is a table of contents, so it is skipped like the importer skips it.""" + is a table of contents, so it is skipped like the importer skips it. + + A memory directory under the CLAUDE_CONFIG_DIR Claude Code was started + with counts as well: that directory need not be called .claude, and + when it was not, every memory write was passed over.""" p = Path(path) if p.suffix.lower() != ".md" or p.name == claude_memory.INDEX_NAME: return False parts = [x.lower() for x in p.parts] - return "memory" in parts and ".claude" in parts + if "memory" not in parts: + return False + if ".claude" in parts: + return True + try: + return Path(os.path.abspath(p)).is_relative_to( + os.path.abspath(claude_memory.claude_config_dir())) + except (OSError, ValueError): + return False def _earlier_versions(v, tag: str, ns: str, text: str) -> list[str] | None: diff --git a/src/compartment/claude_memory.py b/src/compartment/claude_memory.py index ec3eb31..15acf5c 100644 --- a/src/compartment/claude_memory.py +++ b/src/compartment/claude_memory.py @@ -27,8 +27,32 @@ import re from pathlib import Path + + +def claude_config_dir() -> Path: + """Claude Code's own directory: $CLAUDE_CONFIG_DIR when it is set, + ~/.claude otherwise. + + Claude Code keeps everything there - settings.json, the per-project + memory under projects/, skills/, the user CLAUDE.md - and moves all of + it when CLAUDE_CONFIG_DIR is set ("Respect CLAUDE_CONFIG_DIR + everywhere", Claude Code 1.0.6), which is how people run a second + account or keep it in a dotfiles tree. Compartment only ever looked in + ~/.claude, so for them the capture hook went into a settings.json + Claude Code never read, the import found no memories, and the skill + landed where no session would load it. Like Claude Code's own VS Code + extension, only an absolute path counts; an empty or relative value is + ignored.""" + raw = os.environ.get("CLAUDE_CONFIG_DIR", "").strip() + if raw: + p = Path(raw).expanduser() + if p.is_absolute(): + return p + return Path.home() / ".claude" + + # Claude Code's per-project memory lives under //memory. -DEFAULT_ROOT = Path.home() / ".claude" / "projects" +DEFAULT_ROOT = claude_config_dir() / "projects" INDEX_NAME = "MEMORY.md" # a table of contents, not a fact # Frontmatter `type` -> Compartment importance. Mirrors the vault's own tiers: @@ -199,4 +223,5 @@ def default_root_exists() -> bool: __all__ = ["DEFAULT_ROOT", "discover", "parse", "import_files", "pending", - "default_root_exists", "IMPORTANCE", "file_tag"] + "default_root_exists", "IMPORTANCE", "file_tag", + "claude_config_dir"] diff --git a/tests/test_integrations_claude_config_dir.py b/tests/test_integrations_claude_config_dir.py new file mode 100644 index 0000000..8052c3a --- /dev/null +++ b/tests/test_integrations_claude_config_dir.py @@ -0,0 +1,78 @@ +"""CLAUDE_CONFIG_DIR moves everything Claude Code keeps in ~/.claude: +settings.json, the per-project memory under projects/, skills/. People set +it to run a second account, or to keep the directory in a dotfiles tree +under a name that is not .claude.""" +import importlib + +import pytest + +from compartment import agent_skill, claude_hooks, claude_memory + + +@pytest.fixture() +def config_dir(monkeypatch, tmp_path): + """CLAUDE_CONFIG_DIR set, as it is in the environment Claude Code and + `compartment` are started with, and the modules that read it at import + imported again; put back afterwards.""" + d = tmp_path / "dotfiles" / "claude-work" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(d)) + importlib.reload(claude_memory) + importlib.reload(claude_hooks) + yield d + monkeypatch.delenv("CLAUDE_CONFIG_DIR") + importlib.reload(claude_memory) + importlib.reload(claude_hooks) + + +def test_the_hook_goes_into_the_settings_claude_code_reads(config_dir): + """The hook was installed into ~/.claude/settings.json whatever + CLAUDE_CONFIG_DIR said, a file that Claude Code then never read.""" + assert claude_hooks.SETTINGS == config_dir / "settings.json" + claude_hooks.install(compartment_bin="compartment") + assert claude_hooks.is_installed() is True + assert (config_dir / "settings.json").is_file() + + +def test_the_import_finds_the_memories_where_claude_code_keeps_them( + config_dir): + mem = config_dir / "projects" / "-home-ana-site" / "memory" + mem.mkdir(parents=True) + (mem / "likes-tea.md").write_text("---\nname: likes-tea\n---\nAna likes " + "tea.\n", encoding="utf-8") + (mem / claude_memory.INDEX_NAME).write_text("- likes-tea\n", + encoding="utf-8") + assert claude_memory.DEFAULT_ROOT == config_dir / "projects" + assert claude_memory.discover() == [mem / "likes-tea.md"] + + +def test_the_hook_captures_memory_written_under_the_config_dir(config_dir): + """is_memory_file wanted a `.claude` directory in the path, so with the + config directory named anything else every memory write was passed + over.""" + note = config_dir / "projects" / "-home-ana-site" / "memory" / "a.md" + assert claude_hooks.is_memory_file(note) is True + assert claude_hooks.is_memory_file( + config_dir / "projects" / "-home-ana-site" / "notes.md") is False + assert claude_hooks.is_memory_file( + config_dir.parent / "elsewhere" / "memory" / "a.md") is False + + +def test_the_skill_goes_where_claude_code_loads_skills(config_dir): + assert agent_skill.skill_path("claude") == ( + config_dir / "skills" / "compartmentalize" / "SKILL.md") + + +@pytest.mark.parametrize("value", ["", " ", "relative/dir"]) +def test_an_empty_or_relative_config_dir_is_ignored(value, monkeypatch, + tmp_path): + monkeypatch.setattr(claude_memory.Path, "home", + classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", value) + assert claude_memory.claude_config_dir() == tmp_path / ".claude" + + +def test_a_config_dir_under_home_is_expanded(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", "~/claude-alt") + assert claude_memory.claude_config_dir() == tmp_path / "claude-alt" From 9285cdf96136d187fc4b07a0aa830ec75752eb36 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:49:42 -0700 Subject: [PATCH 044/135] A vault config, an export or a curation plan saved by a Windows editor or PowerShell opens instead of failing on its first line Notepad's "UTF-8 with BOM" and Windows PowerShell 5.1's Set-Content and Out-File -Encoding UTF8 put a byte order mark in front of a file, and that PowerShell's `>` redirection writes UTF-16. .config.json was read as UTF-8 text, so after a user edited their access rules in the editor Windows ships, every unlock, the MCP servers and the panel failed with "JSONDecodeError: Unexpected UTF-8 BOM". `compartment import` of an export that had passed through PowerShell failed on line one the same way, and so did a curation plan. The config is now read as bytes and handed to json.loads, which detects UTF-8 with or without a BOM, UTF-16 and UTF-32 itself; the import and the plan drop a leading U+FEFF. A config that still cannot be used (not JSON, not an object, callers or settings of the wrong type, which used to crash later with AttributeError) is refused with a message naming the file and what moving it aside would grant, since the defaults let every caller read and write everywhere. The new ConfigError is an AclError and a ValueError, so callers that caught the old json error still catch it. Co-Authored-By: Claude Opus 5.5 --- src/compartment/acl.py | 39 ++++++++++++++- src/compartment/curate.py | 4 +- src/compartment/vault.py | 6 ++- tests/test_storage_windows_files.py | 73 +++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 tests/test_storage_windows_files.py diff --git a/src/compartment/acl.py b/src/compartment/acl.py index 7534f36..5f9c16b 100644 --- a/src/compartment/acl.py +++ b/src/compartment/acl.py @@ -54,6 +54,12 @@ class AclError(CryptoError): pass +class ConfigError(AclError, ValueError): + """The config file exists and cannot be used. A ValueError as well, as + the json error it replaces was, so a caller that already catches a bad + config that way (the panel's settings switch) still does.""" + + @dataclass class VaultConfig: # Deep copies: a shallow copy would hand every VaultConfig in the process @@ -73,13 +79,42 @@ def load(cls, vault_path: str) -> "VaultConfig": p = cls.path_for(vault_path) if not os.path.exists(p): return cls() - with open(p, encoding="utf-8") as f: - data = json.load(f) + # Bytes, not text: json.loads detects the encoding itself, so a file + # saved with a UTF-8 byte order mark (Notepad's "UTF-8 with BOM", + # Windows PowerShell 5.1's Set-Content -Encoding UTF8) or as UTF-16 + # (that PowerShell's `>` redirection) opens like any other. Read as + # UTF-8 text, the first of those failed every unlock with + # "Unexpected UTF-8 BOM", after a user had done nothing but edit + # their ACLs in the editor Windows ships. + with open(p, "rb") as f: + raw = f.read() + try: + data = json.loads(raw) + except (ValueError, UnicodeDecodeError) as exc: + raise ConfigError( + cls._unreadable(p, f"it is not valid JSON ({exc})")) from None + if not isinstance(data, dict) or not isinstance( + data.get("callers", {}), dict) or not isinstance( + data.get("settings", {}), dict): + raise ConfigError(cls._unreadable( + p, 'it must be a JSON object whose "callers" and "settings" ' + 'are objects')) cfg = cls() cfg.callers = data.get("callers", cfg.callers) cfg.settings = {**cfg.settings, **data.get("settings", {})} return cfg + @staticmethod + def _unreadable(path: str, why: str) -> str: + """The refusal for a config that cannot be used. It holds the access + rules, so the vault does not open on a guess, and the message says + what moving it aside would mean, since the defaults grant every + caller read-write everywhere.""" + return (f"{path} cannot be used: {why}. It holds this vault's access " + "rules and settings, so the vault is not opened without it. " + "Fix the file, or move it aside to fall back to the defaults " + "(every caller reads and writes every namespace).") + def save(self, vault_path: str) -> None: # Every unlock reads this file, so it is never rewritten in place: a # reader would meet it empty, and a write that fails (a full disk) diff --git a/src/compartment/curate.py b/src/compartment/curate.py index b3ae06a..7bea550 100644 --- a/src/compartment/curate.py +++ b/src/compartment/curate.py @@ -87,7 +87,9 @@ def parse_plan(text: str) -> list[dict]: ensure_ascii=False leaves U+2028 and U+2029 unescaped inside strings, and splitlines() would cut a valid plan line in half at them.""" plan = [] - for n, line in enumerate(text.split("\n"), 1): + # A plan saved by a Windows editor may start with a byte order mark, + # which json.loads refuses as line 1 not being JSON. + for n, line in enumerate(text.lstrip("\ufeff").split("\n"), 1): if not line.strip(): continue try: diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 298a0d5..68191b6 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -2174,7 +2174,11 @@ def export_jsonl(self, caller: str = "user") -> str: def import_jsonl(self, text: str, caller: str = "user", namespace: str | None = None) -> int: self._require_open() - records = [json.loads(l) for l in text.splitlines() if l.strip()] + # A file saved with a byte order mark (Notepad, Windows PowerShell's + # Out-File) arrives with U+FEFF in front of its first line, which + # json.loads refuses, and the whole import failed on line one. + records = [json.loads(l) for l in text.lstrip("\ufeff").splitlines() + if l.strip()] # What store() embeds: the claim without its provenance clause, every # window of it. All of them go in one batched call, and each record # is handed its own block of rows. A claim of at most CHUNK_WINDOW diff --git a/tests/test_storage_windows_files.py b/tests/test_storage_windows_files.py new file mode 100644 index 0000000..646185d --- /dev/null +++ b/tests/test_storage_windows_files.py @@ -0,0 +1,73 @@ +"""Files a Windows editor or shell wrote. + +Notepad's "UTF-8 with BOM", Windows PowerShell 5.1's Set-Content and +Out-File (-Encoding UTF8) put a byte order mark in front of a file, and +that PowerShell's `>` redirection writes UTF-16. Each used to fail on its +first line: an access-rules file the user had just edited stopped every +unlock, and an export they had passed through PowerShell would not import. +""" +import codecs +import json + +import pytest + +from compartment import curate +from compartment.acl import AclError, VaultConfig +from compartment.vault import Vault + +from conftest import PASS + +CONFIG = {"callers": {"*": {"default_namespace": "main", + "grants": {"*": "rw"}}, + "reader": {"default_namespace": "main", + "grants": {"*": "ro"}}}, + "settings": {"auto_lock_minutes": 12}} + + +@pytest.mark.parametrize("encoding,bom", [ + ("utf-8", codecs.BOM_UTF8), ("utf-16-le", codecs.BOM_UTF16_LE), + ("utf-16-be", codecs.BOM_UTF16_BE), ("utf-8", b"")]) +def test_a_config_saved_by_a_windows_editor_opens(vault, vault_path, + encoding, bom): + vault.lock() + with open(VaultConfig.path_for(vault_path), "wb") as f: + f.write(bom + json.dumps(CONFIG, indent=2).encode(encoding)) + cfg = VaultConfig.load(vault_path) + assert cfg.settings["auto_lock_minutes"] == 12 + assert cfg.grant_for("reader", "main") == "ro" + v = Vault.unlock(vault_path, passphrase=PASS) + with pytest.raises(AclError): + v.store("A reader may not write here.", caller="reader") + + +@pytest.mark.parametrize("content,why", [ + (b'{"callers": {', "not valid JSON"), + (b'["not", "an", "object"]', "must be a JSON object"), + (b'{"callers": [], "settings": {}}', "must be a JSON object"), +]) +def test_a_config_that_cannot_be_used_is_refused_with_what_to_do( + vault_path, content, why): + """Refused, not guessed at: it holds the access rules. The message + names the file and says what moving it aside would grant.""" + with open(VaultConfig.path_for(vault_path), "wb") as f: + f.write(content) + with pytest.raises(AclError) as caught: + VaultConfig.load(vault_path) + msg = str(caught.value) + assert why in msg and VaultConfig.path_for(vault_path) in msg + assert "move it aside" in msg + assert isinstance(caught.value, ValueError), \ + "still what a caller catching the old json error catches" + + +def test_an_export_passed_through_powershell_imports(vault): + vault.store("The router admin page is at 192.168.1.1.", caller="t") + exported = vault.export_jsonl() + vault.forget(vault.db.conn.execute("SELECT id FROM records") + .fetchone()["id"], caller="t") + assert vault.import_jsonl("" + exported, caller="t") == 1 + + +def test_a_curation_plan_with_a_byte_order_mark_parses(): + plan = curate.parse_plan('{"id": "abc", "pieces": [{"text": "x"}]}\n') + assert plan[0]["id"] == "abc" From 6b75e1f77fb4fc8807e595e7c9519dc0b59e03cd Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:50:04 -0700 Subject: [PATCH 045/135] An import whose memories contain a Unicode line or paragraph separator no longer fails as broken JSON import_jsonl split the file with str.splitlines(), which also breaks at U+2028, U+2029, U+0085 and a few control characters. JSONL written with ensure_ascii=False, which is how most tools other than Compartment write it, leaves those characters unescaped inside strings, so the first memory that contained one was cut in two and the whole import failed with "Unterminated string". curate.parse_plan had already been fixed for exactly this; the import now splits on newlines the same way, and a CRLF file still reads, its trailing \r being whitespace to json.loads. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 8 +++++++- tests/test_storage_windows_files.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 68191b6..a05e5a7 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -2177,7 +2177,13 @@ def import_jsonl(self, text: str, caller: str = "user", # A file saved with a byte order mark (Notepad, Windows PowerShell's # Out-File) arrives with U+FEFF in front of its first line, which # json.loads refuses, and the whole import failed on line one. - records = [json.loads(l) for l in text.lstrip("\ufeff").splitlines() + # Split on newlines only, never splitlines(), for the reason + # curate.parse_plan gives: JSONL written with ensure_ascii=False + # (most tools other than this one) carries U+2028, U+2029 and U+0085 + # unescaped inside strings, and splitlines() cut such a line in two, + # failing the whole import on a memory that merely contained one. A + # trailing \r from CRLF is whitespace to json.loads. + records = [json.loads(l) for l in text.lstrip("\ufeff").split("\n") if l.strip()] # What store() embeds: the claim without its provenance clause, every # window of it. All of them go in one batched call, and each record diff --git a/tests/test_storage_windows_files.py b/tests/test_storage_windows_files.py index 646185d..63f53a3 100644 --- a/tests/test_storage_windows_files.py +++ b/tests/test_storage_windows_files.py @@ -71,3 +71,19 @@ def test_an_export_passed_through_powershell_imports(vault): def test_a_curation_plan_with_a_byte_order_mark_parses(): plan = curate.parse_plan('{"id": "abc", "pieces": [{"text": "x"}]}\n') assert plan[0]["id"] == "abc" + + +def test_an_export_from_another_tool_with_line_separators_in_text_imports( + vault): + """JSONL written with ensure_ascii=False leaves U+2028, U+2029 and U+0085 + unescaped inside strings. splitlines() cut those lines in two and the + whole import failed on the first memory that contained one; CRLF line + ends are the Windows shape of the same file.""" + texts = ["Line one
line two of the same memory.", + "A paragraph
and the next.", + "Next line\u0085character inside."] + jsonl = "\r\n".join(json.dumps({"text": t}, ensure_ascii=False) + for t in texts) + "\r\n" + assert vault.import_jsonl(jsonl, caller="t") == 3 + stored = sorted(r["text"] for r in vault.recent("t")["results"]) + assert [t.split(" [")[0] for t in stored] == sorted(texts) From 3cdae1c319c269612568c45369e8f94d76c3ac19 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:50:28 -0700 Subject: [PATCH 046/135] `compartment serve --vault PATH` works: --vault, --caller, --keyfile and --assert-offline may follow the subcommand as well as precede it argparse takes a parent parser's options only before the subcommand, so `compartment serve --vault ~/v.vault --caller cursor` died with "unrecognized arguments: --vault". That is the order MCP client configs are commonly written in, by hand and by registries ("args": ["serve", "--vault", ...]), and the order people type for every other command; the client just showed a server that would not start. `compartment serve --help`, which the install notes point agents at, did not mention --vault at all. Every subcommand that does not define one of these options itself now accepts it after its name, with a SUPPRESS default, so it sets the value only when given there and the top-level value or default stands otherwise; each subcommand's --help lists them. The configs `integrate` writes, with the options first, are unchanged and still work. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 42 ++++++++++++++++++++++++++++++++++++++++ tests/test_cli_errors.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index a22f408..2a65108 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -2638,6 +2638,7 @@ def main(argv: list[str] | None = None) -> None: p.set_defaults(fn=cmd_setup) ps.set_defaults(fn=cmd_setup, setup_cmd=None) + _accept_global_options_anywhere(ap) args = ap.parse_args(argv) if args.assert_offline: offline_guard.activate() @@ -2669,6 +2670,47 @@ def main(argv: list[str] | None = None) -> None: _die(_os_error_message(exc, args)) +#: The options of `compartment` itself, which every subcommand also takes +#: after its own name. Help text only: the value comes from the top level. +_GLOBAL_OPTIONS = ( + (("--vault",), {"help": "vault path (also accepted before the " + "subcommand)"}), + (("--caller",), {"help": "who is asking, for the ACL and the audit log"}), + (("--keyfile",), {"help": "second-factor keyfile, if 2FA moved it"}), + (("--assert-offline",), {"action": "store_true", + "help": "abort if anything attempts network " + "access"}), +) + + +def _accept_global_options_anywhere(parser: argparse.ArgumentParser) -> None: + """Let `--vault`, `--caller`, `--keyfile` and `--assert-offline` follow + the subcommand as well as precede it. + + argparse takes a parent's options only before the subcommand, so + `compartment serve --vault ~/v.vault` died with "unrecognized arguments: + --vault". That is the order MCP client configs are written in by hand + and by registries ("command": "compartment", "args": ["serve", + "--vault", ...]), and the order people type, and `compartment serve + --help` did not mention --vault at all. Each subcommand that does not + define one of these itself gets it with a SUPPRESS default, so it sets + the value only when it is given there, and the top-level value or + default stands otherwise.""" + for action in parser._actions: + if not isinstance(action, argparse._SubParsersAction): + continue + seen = set() + for child in action.choices.values(): + if id(child) in seen: # an alias: the same parser + continue + seen.add(id(child)) + for flags, kw in _GLOBAL_OPTIONS: + if not any(f in child._option_string_actions for f in flags): + child.add_argument(*flags, default=argparse.SUPPRESS, + **kw) + _accept_global_options_anywhere(child) + + def _os_error_message(exc: OSError, args) -> str: """One line for an OSError that ended a command, naming the file.""" where = exc.filename diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 1639252..99d0cf0 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -96,3 +96,33 @@ def test_a_vault_found_but_unreadable_names_the_file(vault, vault_path, finally: os.chmod(vault_path, 0o600) assert vault_path in err and "Traceback" not in err + + +@pytest.mark.parametrize("argv", [ + ["serve", "--vault", "{vp}", "--caller", "cursor", "--assert-offline"], + ["--caller", "cursor", "serve", "--vault", "{vp}", "--assert-offline"], + ["--vault", "{vp}", "--caller", "cursor", "--assert-offline", "serve"], +]) +def test_the_vault_and_caller_may_follow_the_subcommand(tmp_path, monkeypatch, + argv): + """`compartment serve --vault PATH` is the order MCP configs are written + in by hand and by registries, and argparse refused it: "unrecognized + arguments: --vault", and the client showed a server that would not + start.""" + vp = str(tmp_path / "v.vault") + seen = [] + monkeypatch.setattr(cli, "cmd_serve", lambda args: seen.append(args)) + monkeypatch.setattr(cli.offline_guard, "activate", lambda **k: None) + cli.main([a.replace("{vp}", vp) for a in argv]) + (args,) = seen + assert (args.vault, args.caller, args.assert_offline) == (vp, "cursor", + True) + + +def test_a_subcommand_without_them_keeps_the_top_level_values(tmp_path, + monkeypatch): + seen = [] + monkeypatch.setattr(cli, "cmd_serve", lambda args: seen.append(args)) + cli.main(["--vault", str(tmp_path / "v.vault"), "serve"]) + assert seen[0].vault == str(tmp_path / "v.vault") + assert seen[0].caller == "user" and seen[0].assert_offline is False From 15919ce73f77771e92366288c901e9afd2b1d9fc Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:50:53 -0700 Subject: [PATCH 047/135] On Xfce, MATE, Cinnamon and other desktops that never start graphical-session.target, start at login keeps the autostart entry instead of a systemd unit that never runs `panel --login on` (and `init`) installs a systemd user unit wherever a user manager answers, enables and starts it, and then deletes the XDG autostart entry so there is only one starter. The unit is WantedBy graphical-session.target, and only session managers that integrate with systemd activate that target: GNOME, KDE Plasma, compositors set up for it. Xfce, MATE, Cinnamon (Linux Mint's default), LXQt and plain window managers never do. There the unit started once, at install, from `enable --now`, and never again at login, and the autostart entry, which every one of those desktops reads, had been removed. The panel came up on the day of the install and then silently stopped appearing. set_login now takes the systemd route only when graphical-session.target is active (systemctl --user is-active), which is what tells a desktop that will start the unit from one that will not. Run from SSH or a text console the target is inactive too, and the autostart entry, which works on every XDG desktop, is chosen there as well. GNOME and Plasma keep the supervised unit and its restart-on-failure. Co-Authored-By: Claude Opus 5.5 --- src/compartment/systray.py | 20 +++++++++++++++++++- tests/test_menu_bar_persistence.py | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/compartment/systray.py b/src/compartment/systray.py index 464e677..dfc81f3 100644 --- a/src/compartment/systray.py +++ b/src/compartment/systray.py @@ -376,6 +376,22 @@ def systemd_available() -> bool: or "no medium" in low or not out) +def graphical_session_active() -> bool: + """Does this login's desktop start graphical-session.target? + + The unit is WantedBy that target, so it starts at login only on a + desktop whose session manager activates it: GNOME, KDE Plasma, and + compositors set up for it. Xfce, MATE, Cinnamon (Linux Mint's default), + LXQt and a bare window manager never do, and there an enabled unit is a + unit that never starts - while the autostart entry it replaced, which + every XDG desktop reads, was deleted as the redundant second starter. + Asked while the desktop is up (install time, from its terminal); asked + from SSH or a text console it is inactive too, and the autostart entry, + which works everywhere, is the right answer there as well. + """ + return _systemctl("is-active", "graphical-session.target")[0] == 0 + + def systemd_unit_text(vault: str | None = None) -> str: return ( "[Unit]\n" @@ -742,7 +758,8 @@ def set_login(enabled: bool, vault: str | None = None) -> str: # Supervised where the machine can be. A unit that enables but never # starts is not proof of anything, so it has to be active too, and # anything short of that is undone rather than left half installed. - if systemd_available(): + # Only on a desktop that starts the unit's target at login at all. + if systemd_available() and graphical_session_active(): if install_systemd_unit(vault) == "on" and systemd_active(): remove_autostart_entry() # exactly one starter return "on" @@ -1621,6 +1638,7 @@ def quit_running(vault: str | None = None, timeout: float = 10.0) -> bool: "autostart_entry_path", "install_autostart_entry", "autostart_is_enabled", "remove_autostart_entry", "systemd_unit_path", "systemd_unit_text", "systemd_available", + "graphical_session_active", "systemd_enabled", "install_systemd_unit", "remove_systemd_unit", "scheduled_task_xml", "install_scheduled_task", "remove_scheduled_task", "scheduled_task_registered", diff --git a/tests/test_menu_bar_persistence.py b/tests/test_menu_bar_persistence.py index 82d1075..fee9589 100644 --- a/tests/test_menu_bar_persistence.py +++ b/tests/test_menu_bar_persistence.py @@ -874,6 +874,29 @@ def test_a_machine_with_no_user_manager_still_starts_at_login(as_linux, assert not systray.systemd_unit_path().exists() +def test_a_desktop_that_never_starts_the_target_keeps_its_autostart_entry( + as_linux, monkeypatch): + """Xfce, MATE, Cinnamon and LXQt never activate graphical-session.target, + which the unit is WantedBy, so an enabled unit there never starts at + login - and the autostart entry, which they all read, was removed as + the redundant second starter. The panel came up at install time and + never again.""" + monkeypatch.setattr(systray, "systemd_available", lambda: True) + base = _fake_systemctl() + + def xfce(*argv): + if argv[:2] == ("is-active", "graphical-session.target"): + return 3, "inactive" + return base(*argv) + + monkeypatch.setattr(systray, "_systemctl", xfce) + assert systray.set_login(True) == "on" + assert systray.autostart_entry_path().is_file() + assert not systray.systemd_unit_path().exists() + assert not any(c[0] == "enable" for c in base.calls), \ + "nothing enabled that will never start" + + def test_a_unit_that_will_not_run_is_undone_rather_than_left_half_done( as_linux, monkeypatch): """Enabled but not active proves nothing. Rather than leave a unit that From 1382aa4c63f3d4c8a82c6124083cea9a8e0ad25d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:50:56 -0700 Subject: [PATCH 048/135] A clock set back no longer stops the hourly expiry sweep of a vault that stays open The sweep runs at most once an hour while a vault is open, judged by time.time() minus the time of the last sweep. When the clock went back - a VM or laptop restored from a snapshot, a clock that ran ahead and was corrected by NTP - the last sweep sat in the future, the difference was negative, and every read counted it as "swept moments ago" until the clock caught up again, which in an MCP server that holds the vault for weeks can be months. Expired memories were still hidden from readers (rule 9), but never cleared. A negative interval now counts as a reason to sweep. The rule 9 test that set the last sweep to 9e18 to mean "just swept" now uses the current time, so it keeps testing the filter rather than a sweep. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 8 +++++++- tests/test_expiry.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/compartment/vault.py b/src/compartment/vault.py index a05e5a7..b94e200 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -1839,7 +1839,13 @@ def _maybe_expire(self) -> None: if not self.expiry_enabled(): return now = time.time() - if now - self._last_expiry_sweep < self.EXPIRY_SWEEP_SECONDS: + # A clock set back - a VM or a laptop restored from a snapshot, a + # clock that ran ahead and was corrected - leaves the last sweep in + # the future. Read as "swept moments ago", that held every sweep + # off until the clock caught up again, which can be months in a + # server that stays open. A negative interval says nothing about + # when the vault was last tidied, so it sweeps. + if 0 <= now - self._last_expiry_sweep < self.EXPIRY_SWEEP_SECONDS: return try: self.expire() diff --git a/tests/test_expiry.py b/tests/test_expiry.py index d09c603..85d29ea 100644 --- a/tests/test_expiry.py +++ b/tests/test_expiry.py @@ -269,7 +269,7 @@ def test_with_the_toggle_on_an_expired_memory_is_never_handed_over(v): it, so the read paths filter even inside the hour before the sweep.""" dead = _store(v, "The old wifi password is hunter2.", expires=_day(-1), _expiry_strict=False) - v._last_expiry_sweep = 9e18 # pretend a sweep just ran + v._last_expiry_sweep = __import__("time").time() # a sweep just ran hits = v.search("wifi password", caller="t")["results"] assert not any(r["id"] == dead["id"] for r in hits) assert not any(r["id"] == dead["id"] @@ -308,6 +308,18 @@ def test_a_read_sweeps_at_most_once_an_hour(v, monkeypatch): assert len(calls) == 1, "swept twice inside the hour" +def test_a_clock_set_back_does_not_hold_the_sweep_off(v, monkeypatch): + """The last sweep ran by a clock that was ahead, or before a snapshot + was restored. Counted as moments ago, no read swept again until the + clock caught up, however far that was.""" + calls = [] + monkeypatch.setattr(Vault, "expire", + lambda self, caller="expiry": calls.append(caller)) + v._last_expiry_sweep = __import__("time").time() + 90 * 86400 + v.search("anything", caller="t") + assert len(calls) == 1 + + def test_a_sweep_that_fails_does_not_take_the_search_down(v, monkeypatch): def boom(self, caller="expiry"): raise RuntimeError("disk on fire") From 5f8b45ae7f6d2d9a258392c7d7b34e18e5fc4cab Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:51:51 -0700 Subject: [PATCH 049/135] The embedder names the Visual C++ installer for the Python's own architecture, and says how to install onnxruntime or tokenizers when either is missing Windows on ARM: onnxruntime that cannot load for want of the Visual C++ runtime was always answered with vc_redist.x64.exe, which on an ARM64 Python installs a runtime that interpreter cannot load. The installer is now chosen from the architecture the interpreter was built for (arm64, x64, or x86), so an x64 Python under emulation still gets the x64 one. Anywhere: onnxruntime publishes no wheel for musl/Alpine, 32-bit ARM, the BSDs or Android, and an install made there with --no-deps or from a distribution's packages can lack it, or tokenizers. The bare ModuleNotFoundError was a traceback in the CLI and, in the MCP server, which passes the model only the text of its own errors, "Error executing tool" with no reason. It is now a ModelError that says which package is missing for which interpreter, the pip command that installs it, and what to do where pip has no build for the platform. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed.py | 49 ++++++++++++++++++++++++++++-- tests/test_server_embed_runtime.py | 46 ++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/test_server_embed_runtime.py diff --git a/src/compartment/embed.py b/src/compartment/embed.py index b82f15d..b1b4b7a 100644 --- a/src/compartment/embed.py +++ b/src/compartment/embed.py @@ -12,6 +12,8 @@ import functools import json import os +import sys +import sysconfig import threading from pathlib import Path @@ -179,11 +181,47 @@ def _missing_runtime_error(exc: ImportError) -> ModelError | None: return ModelError( "onnxruntime cannot load: this Windows machine is missing the " "Microsoft Visual C++ runtime it is built against. Install " - "https://aka.ms/vs/17/release/vc_redist.x64.exe (one small " - "installer, no restart needed), then run this command again.") + f"https://aka.ms/vs/17/release/vc_redist.{_vc_redist_arch()}.exe " + "(one small installer, no restart needed), then run this " + "command again.") return None +def _vc_redist_arch() -> str: + """Which Visual C++ runtime installer this Python needs: the one for + the architecture the interpreter was BUILT for, not the machine's. On + Windows on ARM an ARM64 Python needs vc_redist.arm64.exe, and the x64 + installer the message used to name installs a runtime it cannot load; + an x64 Python running under emulation there needs the x64 one.""" + plat = sysconfig.get_platform() + if plat == "win-arm64": + return "arm64" + if plat == "win32": + return "x86" + return "x64" + + +def _runtime_not_installed(exc: ModuleNotFoundError) -> ModelError | None: + """The actionable form of onnxruntime or tokenizers simply not being + importable, or None for any other missing module. + + pip installs both with Compartment wherever it can, but onnxruntime + publishes no wheel for some platforms (musl/Alpine, 32-bit ARM, the + BSDs, Android), where an install made with --no-deps or from a + distribution's packages can lack it. The bare error was a traceback in + the CLI, and in the MCP server, which passes the model only the text of + its own errors, "Error executing tool" with no reason at all.""" + if exc.name not in ("onnxruntime", "tokenizers"): + return None + return ModelError( + f"{exc.name} is not installed for this Python ({sys.executable}), " + "and embedding needs it. Install it with: " + f"{sys.executable} -m pip install {exc.name}. Where pip has no " + f"{exc.name} build for this platform ({sysconfig.get_platform()}), " + "install your system's own package of it, or run Compartment on a " + "glibc-based Python (a Debian or Ubuntu image rather than Alpine).") + + class Embedder: """CLS-pooled, L2-normalized sentence embeddings from a local ONNX model.""" @@ -200,12 +238,17 @@ def __init__(self, model_name: str = DEFAULT_MODEL): os.environ.setdefault("ORT_DISABLE_TELEMETRY", "1") try: import onnxruntime as ort # local import: keeps CLI startup fast + from tokenizers import Tokenizer + except ModuleNotFoundError as exc: + better = _runtime_not_installed(exc) + if better is not None: + raise better from exc + raise except ImportError as exc: better = _missing_runtime_error(exc) if better is not None: raise better from exc raise - from tokenizers import Tokenizer # Errors only, and for the DEFAULT logger, not just the session's: # device discovery runs under the default logger and its warnings go diff --git a/tests/test_server_embed_runtime.py b/tests/test_server_embed_runtime.py new file mode 100644 index 0000000..66f90cd --- /dev/null +++ b/tests/test_server_embed_runtime.py @@ -0,0 +1,46 @@ +"""What the embedder says when its runtime cannot be loaded: the remedy, in +a ModelError, which the CLI prints as one line and the MCP server passes to +the model, instead of an ImportError neither of them shows usefully.""" +import sys + +import pytest + +from compartment import embed +from compartment.crypto import CryptoError + + +@pytest.mark.parametrize("platform, installer", [ + ("win-amd64", "vc_redist.x64.exe"), + ("win-arm64", "vc_redist.arm64.exe"), + ("win32", "vc_redist.x86.exe"), +]) +def test_the_runtime_named_is_the_one_this_python_needs(monkeypatch, platform, + installer): + """Windows on ARM with an ARM64 Python was sent to the x64 installer, + which installs a runtime that Python cannot load.""" + monkeypatch.setattr(embed.os, "name", "nt") + monkeypatch.setattr(embed.sysconfig, "get_platform", lambda: platform) + err = embed._missing_runtime_error(ImportError( + "DLL load failed while importing onnxruntime_pybind11_state: The " + "specified module could not be found.")) + assert installer in str(err) + + +@pytest.mark.parametrize("module", ["onnxruntime", "tokenizers"]) +def test_a_runtime_that_is_not_installed_says_how_to_install_it(monkeypatch, + module): + """onnxruntime has no wheel for musl, 32-bit ARM, the BSDs or Android, + and an install there can lack it: the CLI printed a traceback and the + MCP server told the model "Error executing tool" and nothing else.""" + monkeypatch.setitem(sys.modules, module, None) # import now fails + with pytest.raises(embed.ModelError) as e: + embed.Embedder() + assert isinstance(e.value, CryptoError) # what the CLI and server report + assert f"-m pip install {module}" in str(e.value) + assert sys.executable in str(e.value) + + +def test_any_other_missing_module_is_left_as_it_is(monkeypatch): + err = ModuleNotFoundError("No module named 'flatbuffers'", + name="flatbuffers") + assert embed._runtime_not_installed(err) is None From 9cd15c94461ca0663b4ee1ce53e6105e2b70ae9f Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:51:53 -0700 Subject: [PATCH 050/135] Start at login refuses, and says where to run it instead, inside a Flatpak or a strictly confined Snap, where its entries could never be read Inside a Flatpak, XDG_CONFIG_HOME and XDG_DATA_HOME point into the app's own ~/.var/app// directory, and inside a strictly confined Snap so does HOME. `compartment init` typed into the terminal of a Flatpak editor (VS Code's, for one) therefore wrote the autostart entry and the applications-menu entry where no desktop reads them, naming the sandbox's own Python, which the desktop cannot run either, and reported "start at login: on". Nothing ever started. set_login now recognises the sandbox (FLATPAK_ID or /.flatpak-info; for a Snap, SNAP_NAME with HOME moved to SNAP_USER_DATA, so a classic snap such as VS Code's, which sets SNAP_NAME in every terminal and confines nothing, is not mistaken for one) and returns an error saying to run `compartment panel --login on` from a terminal outside it, writing nothing. Turning it off still removes whatever is there. Co-Authored-By: Claude Opus 5.5 --- src/compartment/systray.py | 32 ++++++++++++++++++++++ tests/test_linux_panel.py | 54 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/compartment/systray.py b/src/compartment/systray.py index dfc81f3..924951b 100644 --- a/src/compartment/systray.py +++ b/src/compartment/systray.py @@ -736,6 +736,32 @@ def _delete_run_value() -> None: pass +#: Present at the root of every Flatpak sandbox. +_FLATPAK_INFO = "/.flatpak-info" + + +def _app_sandbox() -> str | None: + """The Flatpak or Snap this process is confined in, if any. + + Inside one, XDG_CONFIG_HOME and XDG_DATA_HOME (Flatpak) or HOME itself + (Snap) point into the app's private directory, which the desktop never + reads, and the Python an entry would name is the sandbox's own, which + the desktop cannot run. `compartment init` typed into the terminal of a + Flatpak editor wrote its autostart entry there and reported start at + login as on.""" + flatpak = os.environ.get("FLATPAK_ID") + if flatpak or os.path.isfile(_FLATPAK_INFO): + return f"a Flatpak sandbox ({flatpak or 'unknown app'})" + # Strict confinement only, which is what moves HOME to the snap's own + # SNAP_USER_DATA. A classic snap - VS Code's, among others - sets + # SNAP_NAME for every terminal it opens and confines nothing. + snap = os.environ.get("SNAP_NAME") + if (snap and os.environ.get("SNAP_USER_DATA") + and os.environ.get("HOME") == os.environ.get("SNAP_USER_DATA")): + return f"a Snap sandbox ({snap})" + return None + + def set_login(enabled: bool, vault: str | None = None) -> str: """Register or drop start at sign-in. Returns what actually happened. @@ -750,6 +776,12 @@ def set_login(enabled: bool, vault: str | None = None) -> str: remove_systemd_unit() remove_autostart_entry() return "off" if remove_desktop_entry() else "error" + sandbox = _app_sandbox() + if sandbox: + return (f"error: this is running inside {sandbox}, where an " + "entry would be written somewhere your desktop never " + "reads. Run `compartment panel --login on` from a " + "terminal outside it.") # Both, and they are not the same thing: the menu entry is how the # app is found, the autostart entry is how it comes up at login. out = install_desktop_entry(vault) diff --git a/tests/test_linux_panel.py b/tests/test_linux_panel.py index b1c5edc..a51cde8 100644 --- a/tests/test_linux_panel.py +++ b/tests/test_linux_panel.py @@ -242,3 +242,57 @@ def test_it_never_signals_itself(monkeypatch, tmp_path): lambda pid, sig: killed.append(pid)) systray.quit_running(vault) assert killed == [], "an uninstall that kills its own process finishes nothing" + + +# ------------------------------------------------------- app sandboxes + +@pytest.mark.parametrize("var,value,word", [ + ("FLATPAK_ID", "com.visualstudio.code", "Flatpak"), + ("SNAP_NAME", "code", "Snap"), +]) +def test_start_at_login_from_inside_an_app_sandbox_says_it_cannot( + as_linux, monkeypatch, var, value, word): + """Inside a Flatpak the XDG directories are the app's own, and inside a + Snap so is HOME: the entries landed where no desktop reads them, naming + a Python the desktop cannot run, and start at login reported "on".""" + monkeypatch.setattr(systray, "_is_linux", lambda: True) + monkeypatch.setattr(systray, "_FLATPAK_INFO", str(as_linux / "none")) + monkeypatch.setenv(var, value) + if var == "SNAP_NAME": # strict confinement moves HOME + monkeypatch.setenv("SNAP_USER_DATA", str(as_linux / "snap" / "1")) + monkeypatch.setenv("HOME", str(as_linux / "snap" / "1")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(as_linux / "cfg")) + out = systray.set_login(True, "/home/x/memory.vault") + assert out.startswith("error") and word in out and value in out + assert "outside it" in out + assert not systray.autostart_entry_path().exists() + assert not systray.desktop_entry_path().exists() + + +def test_outside_a_sandbox_nothing_changes(as_linux, monkeypatch): + monkeypatch.setattr(systray, "_is_linux", lambda: True) + monkeypatch.setattr(systray, "_app_sandbox", lambda: None) + monkeypatch.setattr(systray, "systemd_available", lambda: False) + monkeypatch.setenv("XDG_CONFIG_HOME", str(as_linux / "cfg")) + assert systray.set_login(True, "/home/x/memory.vault") == "on" + + +def test_a_flatpak_is_recognised_by_its_info_file(as_linux, monkeypatch): + for var in ("FLATPAK_ID", "SNAP_NAME"): + monkeypatch.delenv(var, raising=False) + info = as_linux / ".flatpak-info" + monkeypatch.setattr(systray, "_FLATPAK_INFO", str(info)) + assert systray._app_sandbox() is None + info.write_text("[Application]\nname=org.example.Editor\n") + assert "Flatpak" in systray._app_sandbox() + + +def test_a_classic_snap_terminal_is_not_a_sandbox(as_linux, monkeypatch): + """VS Code's snap is classic: it sets SNAP_NAME in every terminal it + opens and confines nothing, so start at login works from there.""" + monkeypatch.delenv("FLATPAK_ID", raising=False) + monkeypatch.setattr(systray, "_FLATPAK_INFO", str(as_linux / "none")) + monkeypatch.setenv("SNAP_NAME", "code") + monkeypatch.setenv("SNAP_USER_DATA", str(as_linux / "snap" / "code" / "1")) + monkeypatch.setenv("HOME", str(as_linux / "home")) + assert systray._app_sandbox() is None From 63608c59ae6d1f64065eb8bc5333842a4db10a12 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:52:10 -0700 Subject: [PATCH 051/135] Cline and Kilo Code are wired where their current versions read MCP servers, and Cline and Roo Code are found in whichever VS Code-family editor they are installed in Cline 4 keeps one MCP settings file for its VS Code extension, CLI and JetBrains plugin, ~/.cline/data/settings/cline_mcp_settings.json (moved by CLINE_MCP_SETTINGS_PATH, CLINE_DATA_DIR or CLINE_DIR), and imports the old globalStorage file only once, the first time it runs. `integrate cline` still wrote the globalStorage file, so on a current Cline it reported the server wired and Cline never read it. The shared file is now written once ~/.cline exists; an older Cline that has never created it still gets the file in its editor's storage. Kilo Code 7 is built on OpenCode and reads MCP servers from its global config, ~/.config/kilo/kilo.jsonc (or whichever of kilo.json, opencode.jsonc, opencode.json, config.json it already has), under "mcp" with OpenCode's entry shape. The old mcp_settings.json is read only from VS Code's own storage, for migration, and never by the Kilo CLI. Kilo is now written there, as JSON with comments. Extension storage was looked for only in VS Code itself, so Cline, Roo Code or Kilo installed in Cursor, Windsurf, VSCodium, VS Code Insiders or the Flathub builds was reported as not installed, and wiring it wrote into a VS Code that might not exist. The first of those editors that has run the extension is now used, and VS Code when none has. An extension whose storage exists but has no settings directory yet counts as installed. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 143 ++++++++++++++++++++++++++--- tests/test_integrations_clients.py | 134 +++++++++++++++++++++++++++ 2 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 tests/test_integrations_clients.py diff --git a/src/compartment/clients.py b/src/compartment/clients.py index f169a0d..dade5cd 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -82,16 +82,113 @@ def _app_support(name: str) -> Path: def _vscode_user(app: str = "Code") -> Path: """VS Code's User directory, which several extensions nest inside. - Cline, Roo Code and Kilo Code are VS Code extensions, so their settings - live under VS Code's storage rather than anywhere of their own. + Roo Code, and older Cline and Kilo Code, are VS Code extensions whose + settings live under the editor's storage rather than anywhere of their + own. """ return _app_support(app) / "User" +#: The editors an extension may be installed in, by the name of their user +#: data folder: VS Code, its Insiders build, VSCodium, and the forks that +#: run VS Code extensions (vscode-docs, VSCodium's prepare_vscode.sh). +_EDITORS = ("Code", "Code - Insiders", "VSCodium", "Cursor", "Windsurf") + + +def _editor_users() -> list[Path]: + """Every editor User directory an extension could keep storage in, + the Flathub builds of VS Code and VSCodium included (Flatpak sets their + XDG_CONFIG_HOME to ~/.var/app//config).""" + users = [_vscode_user(app) for app in _EDITORS] + if sys.platform != "darwin" and os.name != "nt": + flat = Path.home() / ".var" / "app" + users += [flat / "com.visualstudio.code" / "config" / "Code" / "User", + flat / "com.vscodium.codium" / "config" / "VSCodium" + / "User"] + return users + + +def _ext_storage(ext: str) -> Path | None: + """The globalStorage directory of extension `ext` in the first editor + that has one, or None when no editor has run it.""" + for user in _editor_users(): + d = user / "globalStorage" / ext + try: + if d.is_dir(): + return d + except OSError: + pass + return None + + def _ext_settings(ext: str, filename: str) -> Callable[[], Path]: - """A VS Code extension's own settings file, by extension id.""" - return lambda: (_vscode_user() / "globalStorage" / ext / "settings" - / filename) + """A VS Code extension's own settings file, by extension id, in the + editor it is installed in. + + Only VS Code's own storage used to be looked at, so the extension in + Cursor, Windsurf, VSCodium or VS Code Insiders was reported as not + installed, and wiring it wrote a file in a VS Code that might not even + be there. VS Code is still the answer when no editor has run it.""" + def path() -> Path: + storage = _ext_storage(ext) or (_vscode_user() / "globalStorage" + / ext) + return storage / "settings" / filename + return path + + +def _env_dir(var: str) -> Path | None: + """The directory an environment variable names, or None when it is + unset or blank.""" + raw = os.environ.get(var, "").strip() + return Path(raw).expanduser() if raw else None + + +def _cline_dir() -> Path: + return _env_dir("CLINE_DIR") or Path.home() / ".cline" + + +def _cline_settings() -> Path: + """cline_mcp_settings.json, where the Cline that is installed reads it. + + Cline 4 keeps one file for the VS Code extension, the CLI and the + JetBrains plugin, ~/.cline/data/settings/cline_mcp_settings.json + (CLINE_MCP_SETTINGS_PATH, CLINE_DATA_DIR and CLINE_DIR move it; + sdk/packages/shared/src/storage/paths.ts), and imports the old + globalStorage file once, the first time it runs. A server written to + the old file after that is never read. An older Cline, one that has + not created ~/.cline, still reads the old file in its editor.""" + explicit = _env_dir("CLINE_MCP_SETTINGS_PATH") + if explicit: + return explicit + data = _env_dir("CLINE_DATA_DIR") + current = ((data or _cline_dir() / "data") / "settings" + / "cline_mcp_settings.json") + try: + if data or current.exists() or _cline_dir().is_dir(): + return current + except OSError: + return current + legacy = _ext_storage("saoudrizwan.claude-dev") + if legacy is not None: + return legacy / "settings" / "cline_mcp_settings.json" + return current + + +def _kilo_config() -> Path: + """Kilo Code's global config. + + Kilo Code 7 is built on OpenCode and reads MCP servers from + ~/.config/kilo (XDG_CONFIG_HOME, on every platform): the first of + kilo.jsonc, kilo.json, opencode.jsonc, opencode.json and config.json + that exists, kilo.jsonc when none does (packages/opencode/src/config/ + config.ts). The extension's old mcp_settings.json is read only from VS + Code's own storage, only for migration, and never by the Kilo CLI.""" + d = _xdg() / "kilo" + for name in ("kilo.jsonc", "kilo.json", "opencode.jsonc", + "opencode.json", "config.json"): + if (d / name).exists(): + return d / name + return d / "kilo.jsonc" def _home(*parts: str) -> Callable[[], Path]: @@ -133,6 +230,9 @@ class Client: #: Whether the client reads its file as JSON with comments, so a file #: holding comments or trailing commas is edited as text, not refused. jsonc: bool = False + #: Whether the client is installed, when the config file's directory is + #: not the proof (it may not exist until the first server is added). + installed: Callable[[], bool] | None = None def config_path(self) -> Path: return self.path() @@ -148,7 +248,14 @@ def present(self) -> bool: p = self.config_path() if not p.is_absolute(): return p.exists() - return p.exists() or p.parent.is_dir() + if p.exists() or p.parent.is_dir(): + return True + if self.installed is not None: + try: + return bool(self.installed()) + except OSError: + return False + return False # --- the table ------------------------------------------------------------- @@ -164,17 +271,25 @@ def present(self) -> bool: Client("cursor", "Cursor", _home(".cursor", "mcp.json")), Client("windsurf", "Windsurf", _home(".codeium", "windsurf", "mcp_config.json")), - Client("cline", "Cline", - _ext_settings("saoudrizwan.claude-dev", "cline_mcp_settings.json"), - note="reload the VS Code window to pick it up"), + Client("cline", "Cline", _cline_settings, + installed=lambda: (_cline_dir().is_dir() or _ext_storage( + "saoudrizwan.claude-dev") is not None), + note="reload the editor window, or restart the Cline CLI, to " + "pick it up"), + # Roo Code's repository was archived in May 2026; the extension still + # reads globalStorage/rooveterinaryinc.roo-cline/settings. Client("roo", "Roo Code", _ext_settings("rooveterinaryinc.roo-cline", "mcp_settings.json"), aliases=("roo-code",), - note="reload the VS Code window to pick it up"), - Client("kilo", "Kilo Code", - _ext_settings("kilocode.kilo-code", "mcp_settings.json"), - aliases=("kilo-code",), - note="reload the VS Code window to pick it up"), + installed=lambda: _ext_storage( + "rooveterinaryinc.roo-cline") is not None, + note="reload the editor window to pick it up"), + Client("kilo", "Kilo Code", _kilo_config, root="mcp", + entry=_entry_opencode, jsonc=True, aliases=("kilo-code",), + installed=lambda: ((_xdg() / "kilo").is_dir() or _ext_storage( + "kilocode.kilo-code") is not None), + note="reload the editor window, or restart the Kilo CLI, to " + "pick it up"), Client("gemini", "Gemini CLI", _home(".gemini", "settings.json"), aliases=("gemini-cli",)), Client("qwen", "Qwen Code CLI", _home(".qwen", "settings.json"), diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py new file mode 100644 index 0000000..fd17b04 --- /dev/null +++ b/tests/test_integrations_clients.py @@ -0,0 +1,134 @@ +"""Where each client reads its MCP servers today. + +Every case puts a fake home directory under tmp_path and lays out only what +the client itself would have created, then checks that `integrate` finds +the client and writes the file that client reads.""" +import json +import os +import sys + +import pytest + +from compartment import clients + +VAULT = "/v/memory.vault" + + +@pytest.fixture() +def home(tmp_path, monkeypatch): + h = tmp_path / "home" + h.mkdir() + monkeypatch.setattr(clients.Path, "home", classmethod(lambda cls: h)) + monkeypatch.setenv("APPDATA", str(h / "AppData" / "Roaming")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(h / ".config")) + for var in ("CLINE_DIR", "CLINE_DATA_DIR", "CLINE_MCP_SETTINGS_PATH", + "CODEX_HOME", "GEMINI_CLI_HOME", "QWEN_HOME", "COPILOT_HOME", + "OPENCODE_CONFIG_DIR", "CRUSH_GLOBAL_CONFIG", "KIRO_HOME", + "GOOSE_PATH_ROOT", "CONTINUE_GLOBAL_DIR"): + monkeypatch.delenv(var, raising=False) + return h + + +def _editor_storage(editor, ext): + """An extension's globalStorage directory in `editor`, as the editor + creates it the first time the extension runs.""" + d = clients._vscode_user(editor) / "globalStorage" / ext + d.mkdir(parents=True) + return d + + +def _servers(p, root="mcpServers"): + return clients._plain(clients._jsonc_parse( + p.read_text(encoding="utf-8")))[root] + + +# ------------------------------------------------------------------ Cline + +def test_cline_4_is_written_in_its_shared_settings_file(home): + """Cline 4 reads ~/.cline/data/settings/cline_mcp_settings.json and + imports the old globalStorage file once, so a server written there + afterwards was never seen: `integrate cline` said it was wired.""" + (home / ".cline").mkdir() + _editor_storage("Code", "saoudrizwan.claude-dev") + cline = clients.CLIENTS["cline"] + + assert cline.present() is True + want = home / ".cline" / "data" / "settings" / "cline_mcp_settings.json" + assert cline.config_path() == want + assert clients.register(cline, VAULT)["written"] is True + assert "compartment" in _servers(want) + + +def test_cline_honours_its_directory_variables(home, monkeypatch, tmp_path): + cline = clients.CLIENTS["cline"] + monkeypatch.setenv("CLINE_DIR", str(tmp_path / "c")) + assert cline.config_path() == (tmp_path / "c" / "data" / "settings" + / "cline_mcp_settings.json") + monkeypatch.setenv("CLINE_DATA_DIR", str(tmp_path / "d")) + assert cline.config_path() == (tmp_path / "d" / "settings" + / "cline_mcp_settings.json") + monkeypatch.setenv("CLINE_MCP_SETTINGS_PATH", str(tmp_path / "m.json")) + assert cline.config_path() == tmp_path / "m.json" + + +def test_an_older_cline_in_cursor_is_found_and_written_there(home): + """Only VS Code's own storage was looked at, so Cline installed in + Cursor (or Windsurf, VSCodium, Insiders) was reported as not installed + and wiring it wrote into a VS Code that was not there.""" + storage = _editor_storage("Cursor", "saoudrizwan.claude-dev") + cline = clients.CLIENTS["cline"] + assert cline.present() is True + assert cline.config_path() == (storage / "settings" + / "cline_mcp_settings.json") + + +def test_roo_code_in_windsurf_is_found(home): + storage = _editor_storage("Windsurf", "rooveterinaryinc.roo-cline") + roo = clients.CLIENTS["roo"] + assert roo.present() is True + assert roo.config_path() == storage / "settings" / "mcp_settings.json" + + +@pytest.mark.skipif(sys.platform == "darwin" or os.name == "nt", + reason="Flatpak is Linux") +def test_an_extension_in_flatpak_vs_code_is_found(home): + storage = (home / ".var" / "app" / "com.visualstudio.code" / "config" + / "Code" / "User" / "globalStorage" + / "rooveterinaryinc.roo-cline") + storage.mkdir(parents=True) + assert clients.CLIENTS["roo"].config_path().parent.parent == storage + + +# ------------------------------------------------------------------- Kilo + +def test_kilo_code_7_gets_an_opencode_style_entry_in_its_global_config(home): + """Kilo Code 7 reads MCP servers from ~/.config/kilo/kilo.jsonc under + "mcp"; the old mcp_settings.json in VS Code's storage is read only for + migration, and never by the Kilo CLI.""" + kilo_dir = home / ".config" / "kilo" + kilo_dir.mkdir(parents=True) + kilo = clients.CLIENTS["kilo"] + assert kilo.present() is True + assert kilo.config_path() == kilo_dir / "kilo.jsonc" + + clients.register(kilo, VAULT) + entry = _servers(kilo_dir / "kilo.jsonc", "mcp")["compartment"] + assert entry["type"] == "local" and entry["enabled"] is True + assert entry["command"][1:3] == ["--vault", VAULT] + + +def test_kilo_code_writes_the_config_file_it_already_has(home): + kilo_dir = home / ".config" / "kilo" + kilo_dir.mkdir(parents=True) + (kilo_dir / "opencode.json").write_text( + '{\n // mine\n "model": "x",\n}\n', encoding="utf-8") + kilo = clients.CLIENTS["kilo"] + assert kilo.config_path() == kilo_dir / "opencode.json" + assert clients.register(kilo, VAULT)["written"] is True + text = (kilo_dir / "opencode.json").read_text(encoding="utf-8") + assert "// mine" in text and '"compartment"' in text + + +def test_kilo_code_installed_in_an_editor_but_never_run_is_present(home): + _editor_storage("VSCodium", "kilocode.kilo-code") + assert clients.CLIENTS["kilo"].present() is True From bf36b7223491b8f597e18a6266f2677a23c930fc Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:53:00 -0700 Subject: [PATCH 052/135] Codex, Gemini CLI, Qwen Code, Copilot CLI, OpenCode, Zed and LM Studio are found and written where they read their config when it has been moved Each of these lets the user move its configuration, and Compartment wrote the default location whatever it said: a file the client never read, and in `integrate --list` a client whose only directory was the moved one reported as not installed. Now, as each client's own source resolves it: - Codex: $CODEX_HOME/config.toml (codex-rs/utils/home-dir). - Gemini CLI: $GEMINI_CLI_HOME/.gemini/settings.json, the variable standing in for the home directory (packages/core/src/utils/paths.ts). - Qwen Code: $QWEN_HOME/settings.json. Copilot CLI: $COPILOT_HOME/mcp-config.json. - OpenCode: $OPENCODE_CONFIG_DIR, and in it whichever of opencode.json and opencode.jsonc exists, since it loads both; a new opencode.json beside an opencode.jsonc split the user's config in two. - Zed on macOS: ~/.config/zed whatever XDG_CONFIG_HOME says, since Zed does not consult it there (crates/paths/src/paths.rs). - LM Studio: the home named in ~/.lmstudio-home-pointer, which LM Studio writes and which is how a home moved to another disk is found, then ~/.cache/lm-studio, then ~/.lmstudio (lmstudio-js findLMStudioHome). Gemini CLI and Qwen Code strip comments from settings.json and OpenCode reads JSON with comments, so a file of theirs with comments in it is now edited keeping them instead of refused. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 80 ++++++++++++++++++++++++------ tests/test_integrations_clients.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 14 deletions(-) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index dade5cd..cfd7ddb 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -196,6 +196,53 @@ def _home(*parts: str) -> Callable[[], Path]: return lambda: Path.home().joinpath(*parts) +def _home_from(var: str, *default: str) -> Path: + """The directory `var` names, or ~/ when it is unset.""" + return _env_dir(var) or Path.home().joinpath(*default) + + +def _zed_settings() -> Path: + """Zed's settings.json (crates/paths/src/paths.rs): %APPDATA%\\Zed on + Windows, the XDG config directory on Linux, and ~/.config/zed on macOS + whatever XDG_CONFIG_HOME says, since Zed does not consult it there. + Taking XDG_CONFIG_HOME on a Mac that sets it wrote a file Zed never + read.""" + if os.name == "nt": + return _appdata() / "Zed" / "settings.json" + if sys.platform == "darwin": + return Path.home() / ".config" / "zed" / "settings.json" + return _xdg() / "zed" / "settings.json" + + +def _opencode_config() -> Path: + """OpenCode's global config: OPENCODE_CONFIG_DIR, or opencode under the + XDG config directory on every platform, and in it opencode.json or + opencode.jsonc, both of which it loads. The one that exists is + written; a new opencode.json beside an opencode.jsonc would split the + user's config in two.""" + d = _env_dir("OPENCODE_CONFIG_DIR") or _xdg() / "opencode" + if not (d / "opencode.json").exists() and (d / "opencode.jsonc").exists(): + return d / "opencode.jsonc" + return d / "opencode.json" + + +def _lmstudio_home() -> Path: + """LM Studio's home, as LM Studio finds it (lmstudio-js, + findLMStudioHome.ts): the path in ~/.lmstudio-home-pointer, which LM + Studio writes and which is how a home moved in its settings is found; + else ~/.cache/lm-studio, where older versions lived; else ~/.lmstudio.""" + h = Path.home() + try: + pointer = (h / ".lmstudio-home-pointer").read_text( + encoding="utf-8").strip() + if pointer: + return Path(pointer).expanduser() + except (OSError, UnicodeDecodeError): + pass + legacy = h / ".cache" / "lm-studio" + return legacy if legacy.is_dir() else h / ".lmstudio" + + # --- the shape each client wants an entry in ------------------------------- def _entry_plain(command: str, args: list[str]) -> dict: @@ -290,12 +337,17 @@ def present(self) -> bool: "kilocode.kilo-code") is not None), note="reload the editor window, or restart the Kilo CLI, to " "pick it up"), - Client("gemini", "Gemini CLI", _home(".gemini", "settings.json"), - aliases=("gemini-cli",)), - Client("qwen", "Qwen Code CLI", _home(".qwen", "settings.json"), - aliases=("qwen-code",)), + # GEMINI_CLI_HOME stands in for the home directory, .gemini under it; + # QWEN_HOME and COPILOT_HOME are the directory itself. Both CLIs strip + # comments before parsing settings.json. + Client("gemini", "Gemini CLI", + lambda: (_env_dir("GEMINI_CLI_HOME") or Path.home()) / ".gemini" + / "settings.json", jsonc=True, aliases=("gemini-cli",)), + Client("qwen", "Qwen Code CLI", + lambda: _home_from("QWEN_HOME", ".qwen") / "settings.json", + jsonc=True, aliases=("qwen-code",)), Client("copilot-cli", "GitHub Copilot CLI", - _home(".copilot", "mcp-config.json")), + lambda: _home_from("COPILOT_HOME", ".copilot") / "mcp-config.json"), # Oh My Pi reads a project-level .omp/mcp.json as well, but that one is # relative to whatever repository you happen to be in, so the user-level # file is the only one an integrate command can honestly write. @@ -311,20 +363,20 @@ def present(self) -> bool: root="servers", entry=_entry_vscode, jsonc=True), # Zed writes its settings.json with a block of // comments on first # run, so it is edited as JSON with comments: comments kept. - Client("zed", "Zed", - lambda: (_appdata() / "Zed" / "settings.json" if os.name == "nt" - else _xdg() / "zed" / "settings.json"), - root="context_servers", jsonc=True), - Client("opencode", "OpenCode", - lambda: _xdg() / "opencode" / "opencode.json", - root="mcp", entry=_entry_opencode), + Client("zed", "Zed", _zed_settings, root="context_servers", jsonc=True), + # The legacy "mcp": {name: entry} shape, which every release reads and + # which current ones merge with the newer "mcp": {"servers": ...}. + Client("opencode", "OpenCode", _opencode_config, + root="mcp", entry=_entry_opencode, jsonc=True), # -- Codex keeps TOML -------------------------------------------------- - Client("codex", "Codex CLI", _home(".codex", "config.toml"), + # CODEX_HOME moves the whole directory (codex-rs/utils/home-dir). + Client("codex", "Codex CLI", + lambda: _home_from("CODEX_HOME", ".codex") / "config.toml", root="mcp_servers", kind="toml", aliases=("codex-cli",)), # -- desktop and self-hosted chat apps --------------------------------- - Client("lmstudio", "LM Studio", _home(".lmstudio", "mcp.json"), + Client("lmstudio", "LM Studio", lambda: _lmstudio_home() / "mcp.json", aliases=("lm-studio",), note="LM Studio reloads mcp.json on save, no restart needed"), Client("anythingllm", "AnythingLLM", diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py index fd17b04..8d96ba3 100644 --- a/tests/test_integrations_clients.py +++ b/tests/test_integrations_clients.py @@ -132,3 +132,79 @@ def test_kilo_code_writes_the_config_file_it_already_has(home): def test_kilo_code_installed_in_an_editor_but_never_run_is_present(home): _editor_storage("VSCodium", "kilocode.kilo-code") assert clients.CLIENTS["kilo"].present() is True + + +# ------------------------------------------ homes moved by an environment + +@pytest.mark.parametrize("key, var, expect", [ + ("codex", "CODEX_HOME", "{d}/config.toml"), + ("gemini", "GEMINI_CLI_HOME", "{d}/.gemini/settings.json"), + ("qwen", "QWEN_HOME", "{d}/settings.json"), + ("copilot-cli", "COPILOT_HOME", "{d}/mcp-config.json"), + ("opencode", "OPENCODE_CONFIG_DIR", "{d}/opencode.json"), +]) +def test_a_home_moved_by_its_variable_is_followed(key, var, expect, home, + monkeypatch, tmp_path): + """Each of these moves its whole configuration directory with one + variable; the default was written whatever it said, a file the client + never read, and `--list` called a client with only the moved directory + not installed.""" + d = tmp_path / "moved" + d.mkdir() + monkeypatch.setenv(var, str(d)) + c = clients.CLIENTS[key] + assert str(c.config_path()) == expect.format(d=d).replace("/", os.sep) + if key == "gemini": + (d / ".gemini").mkdir() + assert c.present() is True + assert clients.register(c, VAULT)["written"] is True + assert clients.is_registered(c) is True + + +def test_an_opencode_jsonc_is_written_rather_than_split_in_two(home): + d = home / ".config" / "opencode" + d.mkdir(parents=True) + (d / "opencode.jsonc").write_text('{\n // theme\n "theme": "x"\n}\n', + encoding="utf-8") + oc = clients.CLIENTS["opencode"] + assert oc.config_path() == d / "opencode.jsonc" + assert clients.register(oc, VAULT)["written"] is True + assert not (d / "opencode.json").exists() + assert "// theme" in (d / "opencode.jsonc").read_text(encoding="utf-8") + + +def test_gemini_settings_with_comments_are_written(home): + """Gemini CLI and Qwen Code strip comments before parsing settings.""" + p = home / ".gemini" / "settings.json" + p.parent.mkdir() + p.write_text('{\n // dark mode\n "theme": "Dracula"\n}\n', + encoding="utf-8") + assert clients.register(clients.CLIENTS["gemini"], VAULT)["written"] + assert "// dark mode" in p.read_text(encoding="utf-8") + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS rule") +def test_zed_on_macos_ignores_xdg_config_home(home, monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg")) + assert clients.CLIENTS["zed"].config_path() == ( + home / ".config" / "zed" / "settings.json") + + +def test_lm_studio_is_found_where_its_home_pointer_says(home, tmp_path): + """LM Studio records its home in ~/.lmstudio-home-pointer, which is how + a home moved in its settings (to another disk, say) is found.""" + moved = tmp_path / "Volumes" / "Big" / "lmstudio" + moved.mkdir(parents=True) + (home / ".lmstudio-home-pointer").write_text(f"{moved}\n", + encoding="utf-8") + lm = clients.CLIENTS["lmstudio"] + assert lm.config_path() == moved / "mcp.json" + assert lm.present() is True + + +def test_lm_studio_without_a_pointer_uses_its_default_home(home): + assert clients.CLIENTS["lmstudio"].config_path() == ( + home / ".lmstudio" / "mcp.json") + (home / ".cache" / "lm-studio").mkdir(parents=True) + assert clients.CLIENTS["lmstudio"].config_path() == ( + home / ".cache" / "lm-studio" / "mcp.json") From 6bdfe2e9734dcf0a839e475a6afb082a6ea8bd0f Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:53:04 -0700 Subject: [PATCH 053/135] In a container with a CPU quota, or under a narrowed CPU set, the embedder runs as many inference threads as the process may use onnxruntime sizes its thread pool from the host's cores and does not see a cgroup CPU quota. Under `docker run --cpus 2` (or a Kubernetes CPU limit) on a 64-core host it ran 32 threads that the kernel let use two CPUs between them: the quota was spent in the first milliseconds of each period, and every search and store then waited, throttled, for the next. A CPU set narrowed with taskset or cpuset was likewise ignored. The pool is now sized to the CPUs this process may actually use, read from the cgroup v2 cpu.max on the way up from the process's own group (or the v1 cfs quota) and from its CPU affinity, whenever that is at most half the machine. Above half, onnxruntime's own choice, the physical cores, is left alone, so an unconstrained machine runs exactly as before. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed.py | 82 ++++++++++++++++++++++++++++++ tests/test_server_embed_runtime.py | 63 +++++++++++++++++++++++ 2 files changed, 145 insertions(+) diff --git a/src/compartment/embed.py b/src/compartment/embed.py index b1b4b7a..71ebb4b 100644 --- a/src/compartment/embed.py +++ b/src/compartment/embed.py @@ -222,6 +222,85 @@ def _runtime_not_installed(exc: ModuleNotFoundError) -> ModelError | None: "glibc-based Python (a Debian or Ubuntu image rather than Alpine).") +def _cgroup_cpu_limit(root: str = "/sys/fs/cgroup", + own: str = "/proc/self/cgroup") -> float | None: + """The CPUs a cgroup quota grants this process (2.0 for `docker run + --cpus 2`), or None where no quota applies. + + cgroup v2 keeps " " in cpu.max, at every level of the + hierarchy, and the tightest one on the way up from this process's own + group is the one that binds; inside a container the namespace makes + that group the root. cgroup v1 keeps the two numbers in separate files + of the cpu controller's mount.""" + limits: list[float] = [] + rel = None + try: + with open(own, encoding="utf-8") as f: + for line in f: + if line.startswith("0::"): + rel = line[3:].strip() + except OSError: + pass + if rel is not None: + top = Path(root) + here = top / rel.lstrip("/") + while here == top or top in here.parents: + try: + quota, period = (here / "cpu.max").read_text( + encoding="ascii").split()[:2] + if quota != "max" and int(period) > 0: + limits.append(int(quota) / int(period)) + except (OSError, ValueError): + pass + if here == top: + break + here = here.parent + for d in ("cpu", "cpu,cpuacct", "cpuacct,cpu"): + try: + quota = int((Path(root) / d / "cpu.cfs_quota_us").read_text( + encoding="ascii")) + period = int((Path(root) / d / "cpu.cfs_period_us").read_text( + encoding="ascii")) + except (OSError, ValueError): + continue + if quota > 0 and period > 0: + limits.append(quota / period) + break + return min(limits) if limits else None + + +def intra_op_threads() -> int: + """Threads for one inference, or 0 for onnxruntime's own choice. + + onnxruntime sizes its pool from the machine's cores. It does not see a + container's CPU quota, so under `docker run --cpus 2` on a 64-core host + it ran 32 threads that the kernel let use two CPUs between them: the + quota was spent in the first few milliseconds of each period and every + search then waited, throttled, for the next one. Nor, before Python + 3.13 made it easy to ask, a CPU set narrowed with taskset or cpuset. + So when either leaves this process at most half of the machine, the + pool is sized to what it may use. Half, because onnxruntime's own + choice is the PHYSICAL cores, half the logical ones where each core + runs two threads: above half, its default already fits on such a + machine, and setting more threads than that would only add contention. + """ + total = os.cpu_count() or 1 + usable = total + count = getattr(os, "process_cpu_count", None) + try: + if count is not None: + usable = count() or total + elif hasattr(os, "sched_getaffinity"): + usable = len(os.sched_getaffinity(0)) or total + except OSError: + pass + if sys.platform.startswith("linux"): + limit = _cgroup_cpu_limit() + if limit is not None: + usable = min(usable, max(1, int(limit))) + return usable if usable <= total // 2 else 0 + + class Embedder: """CLS-pooled, L2-normalized sentence embeddings from a local ONNX model.""" @@ -286,6 +365,9 @@ def __init__(self, model_name: str = DEFAULT_MODEL): # the arena the same work returns to under 90 MB when it is done, at # the same speed for the batch sizes BATCH_TOKENS allows. so.enable_cpu_mem_arena = False + threads = intra_op_threads() + if threads: + so.intra_op_num_threads = threads self.sess = ort.InferenceSession(str(info.onnx), so, providers=["CPUExecutionProvider"]) self._needs_token_type = any( diff --git a/tests/test_server_embed_runtime.py b/tests/test_server_embed_runtime.py index 66f90cd..84c4b7f 100644 --- a/tests/test_server_embed_runtime.py +++ b/tests/test_server_embed_runtime.py @@ -44,3 +44,66 @@ def test_any_other_missing_module_is_left_as_it_is(monkeypatch): err = ModuleNotFoundError("No module named 'flatbuffers'", name="flatbuffers") assert embed._runtime_not_installed(err) is None + + +# ------------------------------------------------- CPUs in a container -- +def _cgroup(tmp_path, own, files): + root = tmp_path / "sys" / "fs" / "cgroup" + for rel, text in files.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + proc = tmp_path / "proc_self_cgroup" + proc.write_text(own) + return str(root), str(proc) + + +def test_a_cgroup_v2_quota_is_read_from_the_tightest_level(tmp_path): + root, own = _cgroup(tmp_path, "0::/user.slice/agent.scope\n", { + "user.slice/cpu.max": "150000 100000\n", + "user.slice/agent.scope/cpu.max": "max 100000\n", + }) + assert embed._cgroup_cpu_limit(root, own) == 1.5 + + +def test_a_container_sees_its_own_quota_at_the_root(tmp_path): + root, own = _cgroup(tmp_path, "0::/\n", {"cpu.max": "200000 100000\n"}) + assert embed._cgroup_cpu_limit(root, own) == 2.0 + + +def test_a_cgroup_v1_quota_and_none(tmp_path): + root, own = _cgroup(tmp_path, "4:cpu,cpuacct:/docker/abc\n", { + "cpu,cpuacct/cpu.cfs_quota_us": "50000\n", + "cpu,cpuacct/cpu.cfs_period_us": "100000\n", + }) + assert embed._cgroup_cpu_limit(root, own) == 0.5 + root, own = _cgroup(tmp_path / "unlimited", "0::/\n", + {"cpu.max": "max 100000\n"}) + assert embed._cgroup_cpu_limit(root, own) is None + + +@pytest.mark.parametrize("limit, affinity, threads", [ + (2.0, 64, 2), # docker run --cpus 2 on a 64-core host + (0.5, 64, 1), # never fewer than one + (None, 64, 0), # nothing limits it: onnxruntime's own choice + (48.0, 64, 0), # above half: its own choice already fits + (None, 4, 4), # taskset -c 0-3 +]) +def test_the_inference_pool_fits_the_cpus_this_process_may_use( + monkeypatch, limit, affinity, threads): + """onnxruntime sizes its pool from the host's cores and does not see a + container's quota: 32 threads sharing two CPUs' worth of time, every + search throttled until the next period.""" + monkeypatch.setattr(embed.os, "cpu_count", lambda: 64) + monkeypatch.setattr(embed.os, "process_cpu_count", lambda: affinity, + raising=False) + monkeypatch.setattr(embed.sys, "platform", "linux") + monkeypatch.setattr(embed, "_cgroup_cpu_limit", lambda: limit) + assert embed.intra_op_threads() == threads + + +def test_the_embedder_runs_with_that_many_threads(monkeypatch): + monkeypatch.setattr(embed, "intra_op_threads", lambda: 2) + e = embed.Embedder() + assert e.sess.get_session_options().intra_op_num_threads == 2 + assert e.embed_query("router passphrase").shape == (e.dim,) From 73f588fec5e798716b82d31bfed47142c4b8de07 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:53:17 -0700 Subject: [PATCH 054/135] `pip install compartment` works on Windows on ARM and Intel Macs again, and the mcp floor is one that actually starts the server Two indirect dependencies stopped shipping wheels for those machines - cryptography (via mcp -> pyjwt[crypto]) after 46.0.3 on Windows ARM64 and after 48 on macOS x86_64, argon2-cffi-bindings after 25 on macOS x86_64 - and pip compiles the newest release regardless, which needs a Rust toolchain for cryptography. Platform-scoped caps hold each at its last release with a wheel there; nothing changes anywhere else. Compartment never imports cryptography itself. The mcp floor moves from 1.10 to 1.21.2: 1.10 through 1.21.1 break against the pydantic 2.12+ pip installs beside them (an ImportError inside mcp, or a TypeError on postponed annotations), measured with the suite at each version. CI's lowest-dependencies job now runs at these floors unpinned. tools/wheel_coverage.py resolves the dependencies for every supported platform and Python, as pip would and wheels-only, and fails when they differ; CI runs it on every push and weekly. The clock-fallback tests no longer assume time.clock_gettime exists, which Windows CI showed it does not there. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 35 ++++++++++--- pyproject.toml | 31 +++++++++--- tests/test_platforms.py | 7 ++- tools/wheel_coverage.py | 104 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 tools/wheel_coverage.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71f6e5f..b49b9f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,11 @@ name: CI on: push: pull_request: + # Weekly, because a dependency's new release can break an install on some + # platform without any commit here (see wheel-coverage). + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: jobs: test-offline: @@ -111,13 +116,30 @@ jobs: shell: bash run: python tools/mcp_smoke.py --command compartment + wheel-coverage: + # pip installs the newest release of every dependency and compiles it when + # that release has no wheel for the machine. That is how a plain + # `pip install compartment` broke on Windows on ARM and Intel Macs: an + # indirect dependency stopped shipping wheels there. This resolves the + # dependencies for every supported platform and Python, as pip would and + # wheels-only, and fails when they differ. A dependency release can break + # this without a commit here, so it also runs weekly. + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: python -m pip install uv + - run: python tools/wheel_coverage.py + lowest-dependencies: # pyproject.toml's lower bounds are a promise to everyone whose distro or # lock file holds an older library. It used to promise mcp>=1.0, and no - # release before 1.10 can even start the server (FastMCP arrived in 1.2, - # tool titles in 1.10). This installs the lowest version of every direct - # dependency the bounds allow and runs the suite, so a bound is a tested - # fact rather than a guess. + # release before 1.21.2 runs the server beside today's pydantic. This + # installs the lowest version of every direct dependency the bounds + # allow and runs the suite, so a bound is a tested fact, not a guess. runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -126,12 +148,9 @@ jobs: with: python-version: "3.11" - name: Install at the lowest versions the bounds allow - # mcp 1.10-1.12 import pydantic internals that pydantic 2.12 moved, a - # pairing their own metadata does not exclude; the constraint keeps - # the transitive pydantic to what those releases were built against. run: | python -m pip install uv - uv pip install --system --resolution lowest-direct -e ".[dev]" "pydantic<2.12" + uv pip install --system --resolution lowest-direct -e ".[dev]" python -m pip list - name: Test suite with offline guard active env: diff --git a/pyproject.toml b/pyproject.toml index d7109e3..690d8ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,12 +42,16 @@ dependencies = [ # Both majors. mcp 2.0 (2026-07-28) removed mcp.server.fastmcp in favour # of mcp.server.mcpserver; server.py imports whichever is present, so the # same wheel runs on 1.x and 2.x and single-version distros are not - # stranded on either side of the rename. The floor is 1.10, not 1.0: - # FastMCP only arrived in 1.2 and the tool titles every tool is - # registered with in 1.10, so anything older crashed on import while - # pip called the install satisfied. CI's lowest-dependencies job runs - # the suite at exactly this floor. - "mcp>=1.10,<3", + # stranded on either side of the rename. The floor is 1.21.2, not 1.0: + # nothing before 1.10 can register the tools at all (FastMCP arrived in + # 1.2, tool titles in 1.10), and 1.10 through 1.21.1 break against the + # pydantic 2.12+ that pip installs beside them - an import error inside + # mcp, or a TypeError on this module's postponed annotations - so an + # older mcp already in the environment meant a server that never + # started while pip called the install satisfied. With this floor pip + # upgrades it instead. CI's lowest-dependencies job runs the suite at + # exactly these floors. + "mcp>=1.21.2,<3", # The app ships with every install. It is not an add-on: unlocking, # locking and changing the passphrase are things people do through a # panel, not through a terminal, so an install that only lays down a CLI @@ -58,6 +62,21 @@ dependencies = [ "pyobjc-framework-Cocoa>=10.0; sys_platform == 'darwin'", "pystray>=0.19; sys_platform == 'win32'", "pillow>=10; sys_platform == 'win32'", + # Two platforms the newest releases of two indirect dependencies left + # without wheels. pip takes the newest version whether or not it has a + # wheel for the machine, and then tries to compile it: cryptography (via + # mcp -> pyjwt[crypto]) needs a Rust toolchain, so `pip install + # compartment` simply failed on a Snapdragon Windows laptop and on an + # Intel Mac, and the release build of the Intel app failed with it. + # These caps hold each one at its last release with a wheel there, on + # that platform only; everywhere else nothing changes. Compartment + # itself never imports cryptography - mcp needs it only for its HTTP + # transports' auth, which a stdio server does not use. Lift a cap when + # the project ships those wheels again; CI's windows-11-arm and + # macos-15-intel jobs show it. + "cryptography<46.0.4; sys_platform == 'win32' and platform_machine == 'ARM64'", + "cryptography<49; sys_platform == 'darwin' and platform_machine == 'x86_64'", + "argon2-cffi-bindings<26; sys_platform == 'darwin' and platform_machine == 'x86_64'", ] # PyPI is the repo's declared homepage, so it is a real front door - and diff --git a/tests/test_platforms.py b/tests/test_platforms.py index 8f634a2..b179234 100644 --- a/tests/test_platforms.py +++ b/tests/test_platforms.py @@ -154,8 +154,9 @@ def test_a_sandbox_that_hides_the_boot_record_still_gets_a_boot_token( _no_kernel_boot_record(monkeypatch, system) clock = "CLOCK_MONOTONIC" if system == "Darwin" else "CLOCK_BOOTTIME" monkeypatch.setattr(platforms.time, clock, 777, raising=False) + # raising=False: Windows' time module has no clock_gettime at all. monkeypatch.setattr(platforms.time, "clock_gettime", - lambda c: 500.0 if c == 777 else 1.0) + lambda c: 500.0 if c == 777 else 1.0, raising=False) monkeypatch.setattr(platforms.time, "time", lambda: 10_500.0) assert platforms.boot_time() == platforms.boot_time() == "10000" @@ -171,7 +172,9 @@ def test_the_clock_fallback_agrees_with_the_pinned_kernel_reading( lambda: (10_000.0, 500.0)) assert platforms.boot_time() == "10000" monkeypatch.setattr(platforms, "_sysctl_boot_estimate", lambda: None) - monkeypatch.setattr(platforms.time, "clock_gettime", lambda c: 501.0) + monkeypatch.setattr(platforms.time, "CLOCK_MONOTONIC", 1, raising=False) + monkeypatch.setattr(platforms.time, "clock_gettime", lambda c: 501.0, + raising=False) monkeypatch.setattr(platforms.time, "time", lambda: 10_501.7) assert platforms.boot_time() == "10000" diff --git a/tools/wheel_coverage.py b/tools/wheel_coverage.py new file mode 100644 index 0000000..cd9e4aa --- /dev/null +++ b/tools/wheel_coverage.py @@ -0,0 +1,104 @@ +"""Fail when a plain install would have to compile something on a supported platform. + + python tools/wheel_coverage.py # needs `uv` on PATH + +pip installs the newest release of every dependency, direct or not, whether +or not that release has a wheel for the machine, and then tries to build it +from source. A user without a compiler (or a Rust toolchain, for +cryptography) gets a wall of build errors from a package they never asked +for. That is how `pip install compartment` broke on Windows on ARM and on +Intel Macs in 2026: cryptography and argon2-cffi-bindings stopped shipping +wheels there and nothing here noticed. + +For every platform and Python Compartment supports, this resolves the +dependencies twice with uv - as pip would, and with wheels only - and fails +if the two disagree, naming the package. pyproject.toml's platform caps are +the fix it points at. +""" +from __future__ import annotations + +import os +import pathlib +import subprocess +import sys +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +#: (uv --python-platform, Pythons, macOS deployment target or None). Kept in +#: step with README's "Where it runs": onnxruntime decides most of it. +SUPPORTED = [ + ("x86_64-pc-windows-msvc", ["3.11", "3.12", "3.13", "3.14"], None), + # 3.11 on Windows on ARM has no pyyaml wheel (via tokenizers -> + # huggingface-hub), so ARM64 Windows starts at 3.12. + ("aarch64-pc-windows-msvc", ["3.12", "3.13", "3.14"], None), + ("x86_64-manylinux_2_28", ["3.11", "3.12", "3.13", "3.14"], None), + ("aarch64-manylinux_2_28", ["3.11", "3.12", "3.13", "3.14"], None), + # onnxruntime's arm64 wheels need macOS 14 from 1.24 on, and 1.24 is the + # first with 3.14 wheels. + ("aarch64-apple-darwin", ["3.11", "3.12", "3.13", "3.14"], "14.0"), + # Intel Macs stop at onnxruntime 1.23.2, which stops at Python 3.13. + ("x86_64-apple-darwin", ["3.11", "3.12", "3.13"], "13.0"), +] + + +def _compile(platform: str, python: str, target: str | None, + binary_only: bool, extra: str | None = None) -> tuple[int, str]: + env = dict(os.environ) + if target: + env["MACOSX_DEPLOYMENT_TARGET"] = target + with tempfile.TemporaryDirectory() as td: + out = pathlib.Path(td) / "req.txt" + cmd = ["uv", "pip", "compile", str(ROOT / "pyproject.toml"), + *(["--extra", extra] if extra else []), + "--python-platform", platform, + "--python-version", python, "--no-header", "--no-annotate", + "-q", "-o", str(out)] + if binary_only: + cmd += ["--only-binary", ":all:"] + p = subprocess.run(cmd, capture_output=True, text=True, env=env) + text = out.read_text() if out.exists() else "" + return p.returncode, text if p.returncode == 0 else p.stderr + + +def _check(platform: str, py: str, target: str | None, + extra: str | None) -> str | None: + """None when a plain install needs no compiler; otherwise why.""" + rc_any, any_ = _compile(platform, py, target, False, extra) + rc_bin, bin_ = _compile(platform, py, target, True, extra) + if rc_any != 0: + return f"does not resolve at all\n{any_}" + if rc_bin != 0: + return f"no wheel-only solution exists\n{bin_}" + if any_ != bin_: + a, b = set(any_.splitlines()), set(bin_.splitlines()) + return "pip would build from source:\n" + "\n".join( + [f" {ln} (newest; no wheel here)" for ln in sorted(a - b)] + + [f" {ln} (newest with a wheel)" for ln in sorted(b - a)]) + return None + + +def main() -> int: + bad = 0 + for platform, pythons, target in SUPPORTED: + for py in pythons: + label = f"{platform} / Python {py}" + why = _check(platform, py, target, None) + if why: + print(f"FAIL {label}: {why}") + bad += 1 + continue + # The optional [hnsw] extra is reported, not enforced: it is an + # opt-in for vaults past 20k records, and its own dependency + # (numkong) has shipped partial wheel sets that fix themselves + # within days. + why = _check(platform, py, target, "hnsw") + print(f"ok {label}" + (f" (note: [hnsw] {why})" if why else "")) + if bad: + print(f"\n{bad} platform(s) would compile a dependency on install. " + "Cap the package for that platform in pyproject.toml, with the " + "reason, or drop the platform from SUPPORTED and the README.") + return 1 if bad else 0 + +if __name__ == "__main__": + sys.exit(main()) From 4363ee5089771571993685b1a0491d0213ec6f6b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:53:49 -0700 Subject: [PATCH 055/135] `integrate continue` connects Continue instead of printing a block, through a server file of its own in Continue's mcpServers folder Continue was print-only, because its config.yaml is YAML with comments that a rewrite would lose. But Continue also loads every JSON file under its global mcpServers folder (~/.continue/mcpServers, or under CONTINUE_GLOBAL_DIR) in the Claude Desktop shape (core/context/mcp/json/loadJsonMcpConfigs.ts), for config.yaml and config.json setups alike. `integrate continue` and `--all` now write compartment.json there: a file of Compartment's own, so nothing the user wrote is edited, and config.yaml is never touched. Continue counts as installed when its global folder exists, which it creates on first run. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 16 ++++++++++++--- tests/test_integrations_clients.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index cfd7ddb..b7307d5 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -392,15 +392,25 @@ def present(self) -> bool: if sys.platform == "darwin" or os.name == "nt" else _xdg() / "trae" / "mcp.json")), + # Continue loads every *.json under its global mcpServers folder, in the + # Claude Desktop shape (core/context/mcp/json/loadJsonMcpConfigs.ts), so + # it gets a file of its own: nothing of the user's is edited, and its + # config.yaml, whose comments a rewrite would lose, is never touched. + Client("continue", "Continue", + lambda: (_home_from("CONTINUE_GLOBAL_DIR", ".continue") + / "mcpServers" / "compartment.json"), + installed=lambda: _home_from("CONTINUE_GLOBAL_DIR", + ".continue").is_dir(), + jsonc=True, + note="Continue loads it from its mcpServers folder; reload the " + "editor window to pick it up"), + # -- known, documented, but not ours to write -------------------------- # # Each of these keeps its configuration somewhere a merge cannot be made # safely: a YAML document whose comments and anchors we would flatten, or # an application database, or a settings blob with no stable on-disk # shape. Printing the exact block is honest; writing a guess is not. - Client("continue", "Continue", _home(".continue", "config.yaml"), - kind="manual", writes=False, - note="paste under `mcpServers:` in config.yaml"), Client("goose", "Goose", lambda: _xdg() / "goose" / "config.yaml", kind="manual", writes=False, note="or run `goose configure` and add it as an extension"), diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py index 8d96ba3..f717576 100644 --- a/tests/test_integrations_clients.py +++ b/tests/test_integrations_clients.py @@ -208,3 +208,34 @@ def test_lm_studio_without_a_pointer_uses_its_default_home(home): (home / ".cache" / "lm-studio").mkdir(parents=True) assert clients.CLIENTS["lmstudio"].config_path() == ( home / ".cache" / "lm-studio" / "mcp.json") + + +# --------------------------------------------------------------- Continue + +def test_continue_is_wired_with_a_file_of_its_own(home): + """Continue was print-only, since its config.yaml is YAML with comments. + It also loads every JSON file in ~/.continue/mcpServers in the Claude + Desktop shape, so it gets a file of its own and nothing of the user's + is edited.""" + cont = home / ".continue" + cont.mkdir() + (cont / "config.yaml").write_text("# mine\nname: x\n", encoding="utf-8") + c = clients.CLIENTS["continue"] + assert c.writes is True and c.present() is True + + res = clients.register(c, VAULT) + assert res["written"] is True + p = cont / "mcpServers" / "compartment.json" + assert json.loads(p.read_text(encoding="utf-8"))["mcpServers"][ + "compartment"]["args"][:2] == ["--vault", VAULT] + assert (cont / "config.yaml").read_text(encoding="utf-8") == ( + "# mine\nname: x\n") + assert clients.unregister(c) is True + + +def test_continue_follows_its_global_dir_variable(home, monkeypatch, + tmp_path): + monkeypatch.setenv("CONTINUE_GLOBAL_DIR", str(tmp_path / "cont")) + assert clients.CLIENTS["continue"].config_path() == ( + tmp_path / "cont" / "mcpServers" / "compartment.json") + assert clients.CLIENTS["continue"].present() is False From 531f438fe7831def7201267461b4d897007329cf Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:54:15 -0700 Subject: [PATCH 056/135] A server whose terminal closes saves the reads it audited before it ends, as one stopped with SIGTERM does Claude Code, Codex and Gemini CLI run in a terminal and start their MCP servers in their own process group, so closing that terminal window sends every `compartment serve` SIGHUP. Only SIGTERM had a handler: SIGHUP's default action ended the server on the spot, and with it every search and get of the session that only RAM had audited, a loss SECURITY.md attributes to kill -9 alone. SIGHUP now takes the same path as SIGTERM: one save, then the end the signal always meant. Windows has no SIGHUP and is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/compartment/server.py | 20 ++++++++++++++------ tests/test_server.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/compartment/server.py b/src/compartment/server.py index 4cec57f..75a89c4 100644 --- a/src/compartment/server.py +++ b/src/compartment/server.py @@ -959,11 +959,11 @@ def _save_on_exit() -> None: def _stop(signum, frame) -> None: - """SIGTERM: the save a closed stdin gets, then the end SIGTERM always - meant. Not SystemExit raised through the event loop: anyio tells its - idle worker threads to stop only when the server's task ends, and the - interpreter waits for them on the way out, so a server ended that way - never exited. A second SIGTERM during the save is ignored: the process + """SIGTERM or SIGHUP: the save a closed stdin gets, then the end the + signal always meant. Not SystemExit raised through the event loop: + anyio tells its idle worker threads to stop only when the server's task + ends, and the interpreter waits for them on the way out, so a server + ended that way never exited. A second signal during the save is ignored: the process ends as soon as the save is done.""" signal.signal(signum, signal.SIG_IGN) try: @@ -995,7 +995,15 @@ def main(argv: list[str] | None = None) -> None: _state["vault"] = None # start locked; tools will say so threading.Thread(target=_autolock_loop, daemon=True).start() threading.Thread(target=_retag_loop, daemon=True).start() - signal.signal(signal.SIGTERM, _stop) + # SIGHUP too, where there is one: a client running in a terminal, as + # Claude Code, Codex and Gemini CLI do, starts its servers in its own + # process group, and closing that terminal hangs them all up. Its + # default action ended the server on the spot, and the reads it had + # audited only in RAM with it. + for name in ("SIGTERM", "SIGHUP"): + sig = getattr(signal, name, None) + if sig is not None: + signal.signal(sig, _stop) try: mcp.run() # stdio transport finally: diff --git a/tests/test_server.py b/tests/test_server.py index bb1a84b..eb26cd2 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -533,8 +533,8 @@ def _serve(monkeypatch, vault_path, session, passphrase="CorrectHorse"): """server.main() with `session` in place of the stdio transport, its two background loops not started, and the passphrase in the environment as the only credential (no keychain lookup; None for no credential at - all). main() handles SIGTERM for the whole process, so the handler it - found is put back.""" + all). main() handles SIGTERM and SIGHUP for the whole process, so the + handlers it found are put back.""" from compartment import vault as vault_module monkeypatch.setattr(vault_module, "keychain_get", lambda path: None) monkeypatch.delenv("ENGRAM_PASSPHRASE", raising=False) @@ -545,11 +545,14 @@ def _serve(monkeypatch, vault_path, session, passphrase="CorrectHorse"): monkeypatch.setattr(server, "_autolock_loop", lambda: None) monkeypatch.setattr(server, "_retag_loop", lambda: None) monkeypatch.setattr(server.mcp, "run", lambda *a, **k: session()) - before = signal.getsignal(signal.SIGTERM) + handled = [getattr(signal, n) for n in ("SIGTERM", "SIGHUP") + if hasattr(signal, n)] + before = {sig: signal.getsignal(sig) for sig in handled} try: server.main(["--vault", vault_path, "--caller", "agent"]) finally: - signal.signal(signal.SIGTERM, before) + for sig, handler in before.items(): + signal.signal(sig, handler) def _audited_on_disk(vault_path): @@ -597,6 +600,28 @@ def session(): assert ended == [(os.getpid(), signal.SIGTERM, signal.SIG_DFL, "search")] +@pytest.mark.skipif(not hasattr(signal, "SIGHUP"), + reason="Windows has no SIGHUP") +def test_a_server_whose_terminal_closes_saves_them_too( + vault, vault_path, own_state, monkeypatch): + """A client running in a terminal starts its servers in its process + group, and closing the terminal sends them all SIGHUP, whose default + action ended the server with the session's audited reads unsaved.""" + vault.lock() + ended = [] + monkeypatch.setattr(os, "kill", lambda pid, sig: ended.append( + (pid, sig, signal.getsignal(sig), _audited_on_disk(vault_path)[-1]))) + + def session(): + _call(server.memory_search, "which Postgres version does staging run") + if not callable(signal.getsignal(signal.SIGHUP)): + pytest.fail("compartment serve leaves SIGHUP to end it on the " + "spot, with nothing saved") + signal.raise_signal(signal.SIGHUP) + _serve(monkeypatch, vault_path, session) + assert ended == [(os.getpid(), signal.SIGHUP, signal.SIG_DFL, "search")] + + def _set(vault_path, **settings): from compartment.acl import VaultConfig cfg = VaultConfig.load(vault_path) From f7b8394ba44d956589e41647b6f57b8c32f82877 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:54:15 -0700 Subject: [PATCH 057/135] On Windows, the check for a running tray app counts only this user's session, so with fast user switching another person's copy is not taken for yours running_pids lists every process through Win32_Process and matches the tray app's command line. With fast user switching a second person signed in on the same PC runs a tray app of their own, and from an elevated prompt (where other users' command lines are readable) theirs matched too. `compartment init` then took the other user's copy as its own app coming up and started none in this session, and update and uninstall waited on, or counted, a copy that was not theirs. quit_running was already safe, since it acts only on the pid this vault's lock names. The query now asks for SessionId as a fourth column, and rows from another session are left out when this process's session can be read (ProcessIdToSessionId). If it cannot, or a row has no session column, nothing is filtered, as before. Co-Authored-By: Claude Opus 5.5 --- src/compartment/systray.py | 30 +++++++++++++++++++++++++++--- tests/test_menu_bar_persistence.py | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/compartment/systray.py b/src/compartment/systray.py index 924951b..40a0ba5 100644 --- a/src/compartment/systray.py +++ b/src/compartment/systray.py @@ -1570,11 +1570,28 @@ def from_tray(fn): return 0 -#: Every process on Windows with the command line that started it, as CSV. +#: Every process on Windows with the command line that started it and the +#: sign-in session it runs in, as CSV. SessionId is last, so the first three +#: columns are where they always were. _WINDOWS_PROCESSES = ("Get-CimInstance Win32_Process | Select-Object " - "ProcessId, ParentProcessId, CommandLine | " + "ProcessId, ParentProcessId, CommandLine, SessionId | " "ConvertTo-Csv -NoTypeInformation") + +def _windows_session() -> str | None: + """The Windows session this process runs in (one per signed-in user, + fast user switching included), or None if it cannot be asked.""" + try: + import ctypes + from ctypes import wintypes + sid = wintypes.DWORD() + if ctypes.windll.kernel32.ProcessIdToSessionId(os.getpid(), + ctypes.byref(sid)): + return str(sid.value) + except Exception: # noqa: BLE001 + pass + return None + #: A Windows command line that is the panel: the CLI, then its subcommand. _WINDOWS_APP = re.compile(r"compartment.*\s(menubar|panel|tray)(\s|$)", re.IGNORECASE) @@ -1603,12 +1620,19 @@ def running_pids() -> list[int]: except (OSError, subprocess.SubprocessError): return [] mine, pids = {os.getpid(), os.getppid()}, [] + # Only this session's. With fast user switching, a second person + # signed in on the same PC has a tray app of their own, and from an + # elevated prompt their command lines are readable too: `init` then + # took theirs as this user's app coming up, and started none. + session = _windows_session() # Anything that is not a row - the header, a message instead of a # table - has no pid in its first field to misread. for row in csv.reader((out or "").splitlines()): if (len(row) > 2 and row[0].isdigit() and int(row[0]) not in mine - and _WINDOWS_APP.search(row[2])): + and _WINDOWS_APP.search(row[2]) + and (session is None or len(row) < 4 + or row[3] == session)): pids.append(int(row[0])) return pids # The app's own command line and this user's processes, as on macOS: diff --git a/tests/test_menu_bar_persistence.py b/tests/test_menu_bar_persistence.py index fee9589..875a6f7 100644 --- a/tests/test_menu_bar_persistence.py +++ b/tests/test_menu_bar_persistence.py @@ -1053,6 +1053,33 @@ def fake_run(argv, *a, **k): assert not [c for c in ran if c[0] == "taskkill"] +def test_windows_counts_only_the_tray_app_of_this_session(monkeypatch): + """Fast user switching: a second person signed in on the same PC runs a + tray app of their own, and from an elevated prompt its command line is + readable. `init` took it as this user's app coming up and started none; + an update counted it as the copy to wait for.""" + _as_windows(monkeypatch) + exe = WINDOWS_EXE + mine = f'"{exe}" --vault "C:\\Users\\ana\\.compartment\\memory.vault" tray' + theirs = f'"{exe}" --vault "C:\\Users\\bo\\.compartment\\memory.vault" tray' + table = ('"ProcessId","ParentProcessId","CommandLine","SessionId"\r\n' + + f'"4100","900","{mine.replace(chr(34), chr(34) * 2)}","1"\r\n' + + f'"6100","950","{theirs.replace(chr(34), chr(34) * 2)}","2"\r\n') + asked = [] + + def fake_run(argv, *a, **k): + asked.append(argv[-1]) + return subprocess.CompletedProcess(argv, 0, table, "") + monkeypatch.setattr(systray.subprocess, "run", fake_run) + monkeypatch.setattr(systray, "_windows_session", lambda: "1", + raising=False) + assert systray.running_pids() == [4100] + assert "SessionId" in asked[-1], "the session has to be asked for" + # Where the session cannot be told, nothing is left out. + monkeypatch.setattr(systray, "_windows_session", lambda: None) + assert systray.running_pids() == [4100, 6100] + + def test_the_task_restarts_a_panel_that_failed(monkeypatch): _as_windows(monkeypatch) xml = systray.scheduled_task_xml("C:\\v\\memory.vault") From 4b8c5f6b6b5cdadb879cb76c67854ecd0e5ba5b9 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:55:02 -0700 Subject: [PATCH 058/135] `integrate goose` prints an extension block Goose reads, for the config.yaml Goose uses on each platform Goose is print-only, since its config.yaml is YAML a rewrite would flatten, and what it printed could not be used: `{"mcpServers": ...}` JSON, a shape Goose's config does not have. Its servers are extensions, under `extensions:`, with `cmd` rather than `command` and a `type` (ExtensionConfig::Stdio in crates/goose/src/agents/extension.rs). The block is now that YAML, with the command and arguments as double-quoted scalars, which are JSON strings and so escape any path. The file it named was the XDG one on every platform. On Windows Goose keeps it in %APPDATA%\Block\goose\config\config.yaml, and GOOSE_PATH_ROOT (when absolute) moves it to /config/config.yaml on all of them (crates/goose/src/config/paths.rs); both are now what is shown and what detection looks for. LibreChat's block, which its note says to merge into `mcpServers:` in librechat.yaml, is printed as YAML as well. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 66 +++++++++++++++++++++++++++--- tests/test_clients.py | 2 +- tests/test_integrations_clients.py | 46 +++++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index b7307d5..1837524 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -243,6 +243,55 @@ def _lmstudio_home() -> Path: return legacy if legacy.is_dir() else h / ".lmstudio" +def _goose_config() -> Path: + """Goose's config.yaml (crates/goose/src/config/paths.rs): + $GOOSE_PATH_ROOT/config when that is an absolute path; otherwise the + XDG config directory on macOS and Linux, and on Windows + %APPDATA%\\Block\\goose\\config, where the XDG path that was printed + named a file Goose never reads.""" + root = _env_dir("GOOSE_PATH_ROOT") + if root is not None and root.is_absolute(): + return root / "config" / "config.yaml" + if os.name == "nt": + return _appdata() / "Block" / "goose" / "config" / "config.yaml" + return _xdg() / "goose" / "config.yaml" + + +def _yaml_str(v: str) -> str: + """A YAML double-quoted scalar. YAML 1.2 is a superset of JSON, so a + JSON string is one, escapes and all.""" + return json.dumps(v, ensure_ascii=False) + + +def _goose_block(entry: dict) -> str: + """The extension as Goose's config.yaml spells it: under `extensions:`, + with `cmd` rather than `command` and a `type` (ExtensionConfig::Stdio in + crates/goose/src/agents/extension.rs). The `mcpServers` JSON that used + to be printed for Goose is not a shape Goose reads at all.""" + return "\n".join([ + "extensions:", + f" {NAME}:", + " enabled: true", + " type: stdio", + f" name: {NAME}", + " description: Encrypted offline memory", + f" cmd: {_yaml_str(entry['command'])}", + " args: [" + ", ".join(_yaml_str(a) for a in entry["args"]) + "]", + " timeout: 300", + ]) + + +def _librechat_block(entry: dict) -> str: + """The server under librechat.yaml's `mcpServers:` mapping, as YAML.""" + return "\n".join([ + "mcpServers:", + f" {NAME}:", + " type: stdio", + f" command: {_yaml_str(entry['command'])}", + " args: [" + ", ".join(_yaml_str(a) for a in entry["args"]) + "]", + ]) + + # --- the shape each client wants an entry in ------------------------------- def _entry_plain(command: str, args: list[str]) -> dict: @@ -280,6 +329,9 @@ class Client: #: Whether the client is installed, when the config file's directory is #: not the proof (it may not exist until the first server is added). installed: Callable[[], bool] | None = None + #: The block to print, from the entry, for a client whose file is not + #: JSON (Goose's YAML); JSON under `root` when None. + block: Callable[[dict], str] | None = None def config_path(self) -> Path: return self.path() @@ -411,12 +463,14 @@ def present(self) -> bool: # safely: a YAML document whose comments and anchors we would flatten, or # an application database, or a settings blob with no stable on-disk # shape. Printing the exact block is honest; writing a guess is not. - Client("goose", "Goose", lambda: _xdg() / "goose" / "config.yaml", - kind="manual", writes=False, - note="or run `goose configure` and add it as an extension"), + Client("goose", "Goose", _goose_config, kind="manual", writes=False, + block=_goose_block, + note="add this to config.yaml (merge it into `extensions:` if " + "you have one), or run `goose configure` and add it as a " + "command-line extension"), Client("librechat", "LibreChat", lambda: Path("librechat.yaml"), - kind="manual", writes=False, - note="paste under `mcpServers:` in your librechat.yaml, then " + kind="manual", writes=False, block=_librechat_block, + note="merge this into `mcpServers:` in your librechat.yaml, then " "restart LibreChat"), Client("jan", "Jan", lambda: _app_support("Jan") / "data" / "mcp_config.json", kind="manual", writes=False, @@ -618,6 +672,8 @@ def snippet(client: Client, vault: str, caller: str | None = None) -> str: """What to print when we are not writing the file ourselves.""" if client.kind == "toml": return _toml_block(entry_for(client, vault, caller)) + if client.block is not None: + return client.block(entry_for(client, vault, caller)) return json.dumps({client.root: {NAME: entry_for(client, vault, caller)}}, indent=2) diff --git a/tests/test_clients.py b/tests/test_clients.py index ec9a2fe..07f6a1c 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -438,7 +438,7 @@ def test_a_client_known_by_a_bare_filename_is_not_found_everywhere( assert clients.CLIENTS["librechat"].present() is True assert clients.CLIENTS["open-webui"].present() is False cli.main(["--vault", VAULT, "integrate", "librechat"]) - assert '"mcpServers"' in capsys.readouterr().out + assert "mcpServers:" in capsys.readouterr().out def test_one_client_that_fails_does_not_stop_integrate_all(tmp_path, diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py index f717576..f1f73ea 100644 --- a/tests/test_integrations_clients.py +++ b/tests/test_integrations_clients.py @@ -239,3 +239,49 @@ def test_continue_follows_its_global_dir_variable(home, monkeypatch, assert clients.CLIENTS["continue"].config_path() == ( tmp_path / "cont" / "mcpServers" / "compartment.json") assert clients.CLIENTS["continue"].present() is False + + +# ------------------------------------------------------------------ Goose + +def test_the_goose_block_is_an_extension_goose_reads(home): + """Goose was handed `{"mcpServers": ...}` JSON, a shape Goose's + config.yaml does not have: its servers are extensions, with `cmd`, not + `command`, and a `type`.""" + goose = clients.CLIENTS["goose"] + block = clients.snippet(goose, "/Users/ana/My Vault/mémoire.vault") + lines = block.splitlines() + assert lines[:3] == ["extensions:", " compartment:", " enabled: true"] + assert " type: stdio" in lines and " name: compartment" in lines + cmd = next(x for x in lines if x.startswith(" cmd: ")) + args = next(x for x in lines if x.startswith(" args: ")) + # Double-quoted YAML scalars are JSON strings. + assert json.loads(cmd.split(": ", 1)[1]) == clients.executable() + assert json.loads(args.split(": ", 1)[1])[:2] == [ + "--vault", "/Users/ana/My Vault/mémoire.vault"] + assert "mcpServers" not in block + + +class _WindowsOs: + """`os` as clients.py sees it on Windows, without telling pathlib.""" + name = "nt" + + def __getattr__(self, attr): + return getattr(os, attr) + + +def test_goose_on_windows_is_under_appdata_block(home, monkeypatch): + """The XDG path was printed on Windows too, a file Goose never reads: + there its config is %APPDATA%\\Block\\goose\\config\\config.yaml.""" + monkeypatch.setattr(clients, "os", _WindowsOs()) + p = clients.CLIENTS["goose"].config_path() + assert p == (home / "AppData" / "Roaming" / "Block" / "goose" / "config" + / "config.yaml") + + +def test_goose_path_root_moves_its_config(home, monkeypatch, tmp_path): + monkeypatch.setenv("GOOSE_PATH_ROOT", str(tmp_path / "g")) + assert clients.CLIENTS["goose"].config_path() == ( + tmp_path / "g" / "config" / "config.yaml") + monkeypatch.setenv("GOOSE_PATH_ROOT", "relative") # ignored by Goose + assert clients.CLIENTS["goose"].config_path() == ( + home / ".config" / "goose" / "config.yaml") From 0c7b51fda27c8f161beac388ef6ef9d011b4a3be Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:56:04 -0700 Subject: [PATCH 059/135] `integrate` knows Kiro, Amazon Q Developer CLI, Junie, Factory Droid and Crush, so `--all` wires them too Five MCP clients in wide use had no row in the table, so `integrate --list` never mentioned them, `--all` passed them by, and naming one was "unknown agent". Each now has its user-level file, from its own source or its maker's documentation: - Kiro (IDE and CLI): ~/.kiro/settings/mcp.json, moved by KIRO_HOME. - Amazon Q Developer CLI: ~/.aws/amazonq/mcp.json. - Junie: ~/.junie/mcp/mcp.json; present once ~/.junie exists. - Factory Droid: ~/.factory/mcp.json (alias `droid`). - Crush: crush/crush.json under XDG_CONFIG_HOME or ~/.config on every platform, or $CRUSH_GLOBAL_CONFIG/crush.json; servers under "mcp" with the required "type": "stdio". All five take the usual merge: other servers kept, backup first, the block printed when the file cannot be read. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 21 +++++++++++++++ tests/test_integrations_clients.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 1837524..953acd6 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -400,6 +400,21 @@ def present(self) -> bool: jsonc=True, aliases=("qwen-code",)), Client("copilot-cli", "GitHub Copilot CLI", lambda: _home_from("COPILOT_HOME", ".copilot") / "mcp-config.json"), + # Kiro (IDE and CLI) and the Amazon Q Developer CLI it replaced; Kiro's + # directory moves with KIRO_HOME (kirodotdev/KiroCrew, config/paths.py), + # Amazon Q's is fixed (crates/chat-cli/src/util/paths.rs). + Client("kiro", "Kiro", lambda: _home_from("KIRO_HOME", ".kiro") + / "settings" / "mcp.json", aliases=("kiro-cli",)), + Client("amazon-q", "Amazon Q Developer CLI", + _home(".aws", "amazonq", "mcp.json"), aliases=("amazonq",)), + # Junie's user-level file, from JetBrains' own skill for driving it + # (JetBrains/junie, hermes-plugin/skills/junie/SKILL.md). + Client("junie", "Junie", _home(".junie", "mcp", "mcp.json"), + installed=lambda: (Path.home() / ".junie").is_dir()), + # Factory's droid; stdio is the default type (docs/cli/configuration/ + # mcp.mdx in Factory-AI/factory). + Client("factory", "Factory Droid", _home(".factory", "mcp.json"), + aliases=("droid",)), # Oh My Pi reads a project-level .omp/mcp.json as well, but that one is # relative to whatever repository you happen to be in, so the user-level # file is the only one an integrate command can honestly write. @@ -416,6 +431,12 @@ def present(self) -> bool: # Zed writes its settings.json with a block of // comments on first # run, so it is edited as JSON with comments: comments kept. Client("zed", "Zed", _zed_settings, root="context_servers", jsonc=True), + # Crush: $CRUSH_GLOBAL_CONFIG/crush.json, else crush/crush.json under + # XDG_CONFIG_HOME or ~/.config on every platform, servers under "mcp" + # with a required type (internal/config/load.go, config.go). + Client("crush", "Crush", + lambda: (_env_dir("CRUSH_GLOBAL_CONFIG") or _xdg() / "crush") + / "crush.json", root="mcp", entry=_entry_vscode), # The legacy "mcp": {name: entry} shape, which every release reads and # which current ones merge with the newer "mcp": {"servers": ...}. Client("opencode", "OpenCode", _opencode_config, diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py index f1f73ea..f405809 100644 --- a/tests/test_integrations_clients.py +++ b/tests/test_integrations_clients.py @@ -285,3 +285,44 @@ def test_goose_path_root_moves_its_config(home, monkeypatch, tmp_path): monkeypatch.setenv("GOOSE_PATH_ROOT", "relative") # ignored by Goose assert clients.CLIENTS["goose"].config_path() == ( home / ".config" / "goose" / "config.yaml") + + +# ---------------------------------------------------------- newer clients + +@pytest.mark.parametrize("key, rel, root, typed", [ + ("kiro", ".kiro/settings/mcp.json", "mcpServers", False), + ("amazon-q", ".aws/amazonq/mcp.json", "mcpServers", False), + ("junie", ".junie/mcp/mcp.json", "mcpServers", False), + ("factory", ".factory/mcp.json", "mcpServers", False), + ("crush", ".config/crush/crush.json", "mcp", True), +]) +def test_newer_clients_are_known_and_wired(key, rel, root, typed, home): + c = clients.CLIENTS[key] + p = home.joinpath(*rel.split("/")) + assert c.config_path() == p + assert c.present() is False + p.parent.mkdir(parents=True) + p.write_text(json.dumps({root: {"theirs": {"command": "x"}}}), + encoding="utf-8") + assert c.present() is True + assert clients.register(c, VAULT)["written"] is True + servers = json.loads(p.read_text(encoding="utf-8"))[root] + assert servers["theirs"] == {"command": "x"} + assert servers["compartment"]["args"][:2] == ["--vault", VAULT] + assert ("type" in servers["compartment"]) is typed + + +def test_kiro_and_crush_follow_their_variables(home, monkeypatch, tmp_path): + monkeypatch.setenv("KIRO_HOME", str(tmp_path / "k")) + monkeypatch.setenv("CRUSH_GLOBAL_CONFIG", str(tmp_path / "c")) + assert clients.CLIENTS["kiro"].config_path() == ( + tmp_path / "k" / "settings" / "mcp.json") + assert clients.CLIENTS["crush"].config_path() == ( + tmp_path / "c" / "crush.json") + + +def test_junie_is_present_before_its_first_server(home): + (home / ".junie").mkdir() + assert clients.CLIENTS["junie"].present() is True + assert clients.resolve("droid").key == "factory" + assert clients.resolve("amazonq").key == "amazon-q" From 6a42253aa83e0a69bf5df76a168cc195ef98e7ec Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:56:34 -0700 Subject: [PATCH 060/135] The Claude Code plugin declares its version, and every manifest in the tree that carries one is gated against the package .claude-plugin/plugin.json had no version. Claude Code compares that field to decide whether an installed plugin is out of date, and a plugin without one is restored at whatever commit it was installed from, so users of `/plugin install compartment@maxfreedompollard` had no version to see and no clean update signal. Codex and OpenClaw read the same file when they install from this marketplace. It now carries the package version (and the SchemaStore $schema Claude Code accepts), and tests/test_version_lockstep.py checks it and any version a marketplace entry repeats. The lockstep test only knew the files someone had remembered to list, which is how the Hermes plugin.yaml sat at 1.7.0 for months. It now walks the tree for every JSON, YAML, TOML and SKILL.md that carries a version and fails on any it neither checks nor names as having a version of its own (the engram tombstone, the skill's revision). The one stale copy it finds, integrations/hermes/upstream/plugin.yaml at 1.7.0, is outside this change and is pinned as a strict xfail so its fix has to retire the marker. RELEASING.md lists the new file. Before the fix: test_claude_code_plugin_json_matches_the_package E KeyError: 'version' Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/plugin.json | 2 + RELEASING.md | 7 +- tests/test_version_lockstep.py | 116 +++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index aad7aa1..8fe41a8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,8 @@ { + "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "compartment", "displayName": "Compartment", + "version": "4.12.0", "description": "Encrypted, fully offline memory for Claude Code: one vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client. Adds 14 memory tools and the /compartmentalize skill. Requires `pip install compartment && compartment init` first (the vault passphrase is yours to choose).", "author": { "name": "Max Freedom Pollard", diff --git a/RELEASING.md b/RELEASING.md index 14c6ea8..3d6fea3 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -33,6 +33,8 @@ server without the app. # server.json version = "2.1.0" the SEMVER form # plugin.json version = "2.1.0" Agent Plugins manifest # gemini-extension.json version = "2.1.0" Gemini CLI extension manifest +# .claude-plugin/plugin.json version = "2.1.0" Claude Code plugin (Codex, +# OpenClaw read it too) # # The two server.json fields are deliberately NOT the same value. Step 5 # explains why; setting them both to "2.1" is what published a registry @@ -49,7 +51,10 @@ server without the app. # plugin.yaml pair sat at 1.7.0 while the package shipped 4.9.2. # # tests/test_version_lockstep.py now fails when any of them disagree, so -# this list is a gate rather than a thing to remember. +# this list is a gate rather than a thing to remember. It also fails when +# a file anywhere in the tree carries a version it does not check, so a +# new manifest cannot join the list silently. docs/PLUGINS.md says what +# each store needs after the bump. python -m pytest -q # must be green diff --git a/tests/test_version_lockstep.py b/tests/test_version_lockstep.py index 1f3a8e4..3ee612a 100644 --- a/tests/test_version_lockstep.py +++ b/tests/test_version_lockstep.py @@ -75,6 +75,122 @@ def test_plugin_json_matches_the_package(): assert d["version"] == _semver(_version()) +def test_claude_code_plugin_json_matches_the_package(): + """Claude Code decides whether an installed plugin is out of date by this + field, and a plugin that declares none is pinned to whatever commit it + was installed from, so it has to exist and has to move with every + release. Codex and OpenClaw read the same file when they install from + this marketplace.""" + d = json.loads((ROOT / ".claude-plugin" / "plugin.json").read_text( + encoding="utf-8")) + assert d["version"] == _semver(_version()) + + +def test_marketplace_entries_that_carry_a_version_match_the_package(): + """The entry may repeat the plugin's version; Claude Code shows the + entry's metadata before plugin.json's, so a stale copy there would be + the number users see. None is set today; this stops one from rotting.""" + d = json.loads((ROOT / ".claude-plugin" / "marketplace.json").read_text( + encoding="utf-8")) + for entry in d["plugins"]: + if "version" in entry: + assert entry["version"] == _semver(_version()), entry["name"] + + +# ----------------------------------------------------------- coverage ------ +#: Every file above, and nothing else, may carry the package's version. +GATED = { + "pyproject.toml", + "server.json", + "plugin.json", + "gemini-extension.json", + ".claude-plugin/plugin.json", + ".claude-plugin/marketplace.json", + "integrations/hermes/compartment/plugin.yaml", + "src/compartment/data/hermes-plugin/plugin.yaml", +} + +#: Files whose version field is their own, not the package's. +OWN_VERSION = { + # The tombstone for the pre-rename PyPI name. It only depends on + # compartment; republishing it under every Compartment version would + # upload an identical empty package each release. + "packaging/engram-memory-vault/pyproject.toml", + # The skill's own revision, which Hermes shows; it changes when the + # skill's text does, not when the package does. + "skills/compartmentalize/SKILL.md", + "src/compartment/data/agent-skill/SKILL.md", +} + +#: Known drift in a file this test does not own the fix for. Strict, so the +#: day it is fixed the marker has to go and the file joins GATED. +KNOWN_STALE = { + # The plugin.yaml prepared for a PR to hermes-agent still says 1.7.0 and + # compartment>=2.3. It is not shipped by anything in this repository, + # which is why nothing caught it. + "integrations/hermes/upstream/plugin.yaml", +} + +_SKIP_DIRS = {".git", "tests", "models", "build", "dist", ".venv", "venv", + "node_modules", "__pycache__"} + + +def _carries_a_version(p: pathlib.Path) -> bool: + """A top-level version field, in whatever syntax the file is written in, + or a pin on the compartment package itself.""" + text = p.read_text(encoding="utf-8", errors="replace") + if p.suffix == ".json": + try: + d = json.loads(text) + except ValueError: + return False + if not isinstance(d, dict): + return False + return "version" in d or any( + isinstance(e, dict) and "version" in e + for e in d.get("plugins", []) or []) + if p.name == "SKILL.md": + head = text.split("---", 2)[1] if text.startswith("---") else "" + return bool(re.search(r"^\s*version:", head, re.M)) + if p.suffix in (".yaml", ".yml", ".toml"): + return bool(re.search(r"^version\s*[:=]", text, re.M) + or re.search(r"\bcompartment\s*==\s*[0-9]", text)) + return False + + +def _manifests_with_a_version() -> set[str]: + found = set() + for p in ROOT.rglob("*"): + rel = p.relative_to(ROOT) + if any(part in _SKIP_DIRS or part.endswith(".egg-info") + for part in rel.parts[:-1]): + continue + if not p.is_file(): + continue + if p.suffix in (".json", ".yaml", ".yml", ".toml") or p.name == "SKILL.md": + if _carries_a_version(p): + found.add(rel.as_posix()) + return found + + +def test_every_file_that_carries_a_version_is_gated(): + """A new manifest nobody added to this file is exactly how plugin.yaml + drifted. Anything in the tree with a version field must be checked + above, or say here why its version is its own.""" + unaccounted = _manifests_with_a_version() - GATED - OWN_VERSION - KNOWN_STALE + assert not unaccounted, ( + f"{sorted(unaccounted)} carry a version this test does not check: " + "gate it against the package version above, or add it to " + "OWN_VERSION with the reason it differs") + + +@pytest.mark.parametrize("rel", sorted(KNOWN_STALE)) +@pytest.mark.xfail(strict=True, reason="stale copy in a file this test does " + "not own; see KNOWN_STALE") +def test_known_stale_copies_match_the_package(rel): + assert _yaml_version(ROOT / rel) == _version() + + @pytest.mark.parametrize("path", PLUGIN_YAMLS, ids=lambda p: p.parent.name) def test_hermes_plugin_yaml_matches_the_package(path): """`hermes plugins list` prints this in its Version column, so a stale From 48b87f929bdf6fdde44302d5284ecbfe372f6aa3 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:56:44 -0700 Subject: [PATCH 061/135] An interrupted LongMemEval download leaves no truncated dataset behind for the next benchmark run to trust `compartment setup download-longmemeval` wrote straight to the dataset's final path. A download cut short (Ctrl-C, a dropped connection, a full disk; the m variant is gigabytes) left a truncated file there, `bench longmemeval` trusted it because it existed, and every run after failed parsing it with "Unterminated string" and nothing to say that the download was the problem or that deleting the file was the fix. The body now goes to a .part file beside it, renamed into place only once it has all arrived and removed if it has not. Co-Authored-By: Claude Opus 5.5 --- src/compartment/longmemeval.py | 34 +++++++++++++----- tests/test_storage_bench_download.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 tests/test_storage_bench_download.py diff --git a/src/compartment/longmemeval.py b/src/compartment/longmemeval.py index 8ff4bb2..ae00526 100644 --- a/src/compartment/longmemeval.py +++ b/src/compartment/longmemeval.py @@ -42,6 +42,7 @@ from .home import env, home import hashlib import json +import os import sqlite3 import time from pathlib import Path @@ -92,15 +93,30 @@ def download(variant: str = "s") -> Path: print(f"downloading {url}") h = hashlib.sha256() done = 0 - with urllib.request.urlopen(url) as r, open(dest, "wb") as f: - while True: - chunk = r.read(1 << 22) - if not chunk: - break - f.write(chunk) - h.update(chunk) - done += len(chunk) - print(f"\r {done // (1 << 20)} MB", end="", flush=True) + # Into a file beside it, renamed into place only once the whole body has + # arrived. Written straight to `dest`, a download cut short (Ctrl-C, a + # dropped connection, a full disk; the m variant is gigabytes) left a + # truncated file that run() trusted because it existed, and every run + # after failed parsing it as "Unterminated string" with nothing to say + # the download was the problem or that deleting it was the fix. + part = dest.with_name(dest.name + ".part") + try: + with urllib.request.urlopen(url) as r, open(part, "wb") as f: + while True: + chunk = r.read(1 << 22) + if not chunk: + break + f.write(chunk) + h.update(chunk) + done += len(chunk) + print(f"\r {done // (1 << 20)} MB", end="", flush=True) + os.replace(part, dest) + except BaseException: + try: + part.unlink() + except OSError: + pass + raise print(f"\n sha256 {h.hexdigest()}") print(f" → {dest}") return dest diff --git a/tests/test_storage_bench_download.py b/tests/test_storage_bench_download.py new file mode 100644 index 0000000..1b6bd4f --- /dev/null +++ b/tests/test_storage_bench_download.py @@ -0,0 +1,54 @@ +"""The benchmark dataset download, cut short. + +The LongMemEval files run to gigabytes. A download interrupted part way +used to leave a truncated file where the dataset belongs, and every run +after trusted it because it existed. +""" +import io +import urllib.request + +import pytest + +from compartment import longmemeval as L + + +class _Response(io.BytesIO): + """A response body that drops after `cut` bytes, as a connection does.""" + + def __init__(self, body: bytes, cut: int | None = None): + super().__init__(body) + self._cut = cut + + def read(self, n=-1): + if self._cut is not None and self.tell() >= self._cut: + raise ConnectionResetError(54, "Connection reset by peer") + if self._cut is not None and n > 0: + n = min(n, self._cut - self.tell()) + return super().read(n) + + +@pytest.fixture() +def bench_home(tmp_path, monkeypatch): + monkeypatch.setattr(L, "data_dir", lambda: tmp_path / "longmemeval") + return tmp_path / "longmemeval" + + +def test_a_download_cut_short_leaves_nothing_that_looks_like_the_dataset( + bench_home, monkeypatch): + body = b'[{"question_id": "q1", "haystack_sessions": []}]' * 1000 + monkeypatch.setattr(urllib.request, "urlopen", + lambda url: _Response(body, cut=len(body) // 3)) + with pytest.raises(ConnectionResetError): + L.download("s") + assert not L.dataset_path("s").exists() + assert list(bench_home.iterdir()) == [], "and no partial file beside it" + with pytest.raises(L.CryptoError, match="download-longmemeval"): + L.run("s") # says to fetch it, not "bad JSON" + + +def test_a_whole_download_lands_where_run_looks(bench_home, monkeypatch): + body = b'[{"question_id": "q1", "haystack_sessions": []}]' + monkeypatch.setattr(urllib.request, "urlopen", lambda url: _Response(body)) + dest = L.download("s") + assert dest == L.dataset_path("s") and dest.read_bytes() == body + assert [p.name for p in bench_home.iterdir()] == [dest.name] From e148a18e576a913e14377ab53b4b895089b491b4 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:30 -0700 Subject: [PATCH 062/135] The MCP Registry entry names the project's website and icon, and the registry page no longer states a client count server.json had neither websiteUrl nor icons, both fields of the current (2025-12-11) registry schema, so every client that renders the registry - VS Code's MCP gallery, GitHub's MCP registry, the directory sites that mirror it - showed Compartment as a blank tile with no link. It now points at the GitHub Pages site and at docs/images/logo-400.png on main (PNG, 400x400, under the schema's 255-character limit). The two version fields are untouched. Validated against the 2025-12-11 schema from modelcontextprotocol/registry with jsonschema; the new tests check the parts the registry enforces without fetching anything: the schema URL, the mcp-name line PyPI ownership depends on, the 100-character description, the pypi.org base URL, and that the icon named is the file in the repository at the size claimed. registry.html said "any of the 28 clients that `compartment integrate --list` names"; the list grows, so the page now says no number, and a test holds this set of pages to the real count if one ever comes back. Before the fix: test_server_json_gives_registry_clients_a_website_and_an_icon E KeyError: 'websiteUrl' Co-Authored-By: Claude Opus 5.5 --- registry.html | 2 +- server.json | 10 ++++ tests/test_plugin_manifests.py | 85 ++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 tests/test_plugin_manifests.py diff --git a/registry.html b/registry.html index 5f106d1..536fda8 100644 --- a/registry.html +++ b/registry.html @@ -15,7 +15,7 @@

Compartment in the official MCP Registry

Compartment is published in the Model Context Protocol registry, the shared catalogue that MCP clients read, under the name io.github.MaxFreedomPollard/compartment. Each release is published there by the same workflow that publishes to PyPI, so the registry's latest version is always the one pip install compartment serves.

Install

pip install compartment && compartment init
-

Then wire your client: compartment integrate claude, hermes, openclaw, cursor, or any of the 28 clients that compartment integrate --list names. Clients that install straight from the registry run it as uvx compartment serve.

+

Then wire your client: compartment integrate claude, hermes, openclaw, cursor, or any other client that compartment integrate --list names. Clients that install straight from the registry run it as uvx compartment serve.

Verify it yourself

The registry has no web page of its own; it is an API. The current entry, as JSON: every published version, or a search for "compartment". The version flagged isLatest names the PyPI release it installs.

Compartment on GitHub · on PyPI

diff --git a/server.json b/server.json index 0baeef5..aaa6143 100644 --- a/server.json +++ b/server.json @@ -3,6 +3,16 @@ "name": "io.github.MaxFreedomPollard/compartment", "title": "Compartment", "description": "Durable agentic memory, encrypted at rest. Fully offline: no network, no API key, no cloud.", + "websiteUrl": "https://maxfreedompollard.github.io/Compartment/", + "icons": [ + { + "src": "https://raw.githubusercontent.com/MaxFreedomPollard/Compartment/main/docs/images/logo-400.png", + "mimeType": "image/png", + "sizes": [ + "400x400" + ] + } + ], "repository": { "url": "https://github.com/MaxFreedomPollard/Compartment", "source": "github", diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py new file mode 100644 index 0000000..2715310 --- /dev/null +++ b/tests/test_plugin_manifests.py @@ -0,0 +1,85 @@ +"""Every store listing in the repository parses, says what its host needs, +and launches the same server. + +Compartment is listed in a dozen places from one repository: the root is +at once a Claude Code plugin, a Codex plugin (through the Agent Plugins +manifest), an OpenClaw bundle, a Gemini CLI extension and a Hermes +package, and packaging/ holds the manifests other catalogues want. None of +those hosts runs in CI, so a manifest that drifts from its host's format, +or from the server it describes, fails on a user's machine and nowhere +else. These tests check each one against the parts of its host's current +format that matter (docs/PLUGINS.md names the source of each), and check +them against each other: one command, the same subcommand, a caller each +host can be told apart by, and no claim about the server that the server +does not make. + +Offline, like the rest of the suite: nothing here fetches a schema. +""" +import json +import pathlib +import re + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def _json(rel: str) -> dict: + return json.loads((ROOT / rel).read_text(encoding="utf-8")) + + +# ------------------------------------------------------------ MCP Registry -- +#: The newest released server.json schema in modelcontextprotocol/registry +#: (docs/reference/server-json/CHANGELOG.md), still current in October 2026. +REGISTRY_SCHEMA = ("https://static.modelcontextprotocol.io/schemas/" + "2025-12-11/server.schema.json") + + +def test_server_json_targets_the_current_registry_schema(): + assert _json("server.json")["$schema"] == REGISTRY_SCHEMA + + +def test_server_json_name_is_the_one_pypi_proves_ownership_of(): + """The registry checks a PyPI package by finding `mcp-name: ` in + its README, so the two must be the same string or every publish fails.""" + name = _json("server.json")["name"] + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert f"mcp-name: {name}" in readme + + +def test_server_json_meets_the_registry_limits(): + d = _json("server.json") + assert 1 <= len(d["description"]) <= 100 # schema maxLength + assert d["repository"]["url"] == "https://github.com/MaxFreedomPollard/Compartment" + pkg = d["packages"][0] + assert pkg["registryType"] == "pypi" + assert pkg["registryBaseUrl"] == "https://pypi.org" # the only one allowed + assert pkg["identifier"] == "compartment" + assert pkg["transport"] == {"type": "stdio"} + assert [a["value"] for a in pkg["packageArguments"]][-1] == "serve" + + +def test_server_json_gives_registry_clients_a_website_and_an_icon(): + """Clients that render the registry (VS Code's MCP gallery, GitHub's MCP + registry, directory sites) show a blank tile and no link without these.""" + d = _json("server.json") + assert d["websiteUrl"].startswith("https://") + icon = d["icons"][0] + assert icon["src"].startswith("https://") and len(icon["src"]) <= 255 + assert icon["mimeType"] == "image/png" + assert all(re.fullmatch(r"\d+x\d+|any", s) for s in icon["sizes"]) + # the icon it names is the one in this repository, at the size it says + local = ROOT / icon["src"].split("/main/", 1)[1] + head = local.read_bytes()[:24] + w, h = int.from_bytes(head[16:20], "big"), int.from_bytes(head[20:24], "big") + assert icon["sizes"] == [f"{w}x{h}"] + + +def test_listing_pages_never_state_a_client_count_that_is_wrong(): + """`integrate --list` grows; a page that says "28 clients" is wrong the + day it does. Pages in this set either say no number or the right one.""" + from compartment import clients + for rel in ("registry.html", "llms-install.md", "docs/PLUGINS.md"): + p = ROOT / rel + if not p.exists(): + continue + for n in re.findall(r"\b(\d+) (?:MCP )?clients\b", p.read_text(encoding="utf-8")): + assert int(n) == len(clients.CLIENTS), (rel, n) From db52e318971a24f08a31e6619c9bde5996d3cc40 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:40 -0700 Subject: [PATCH 063/135] A mistyped index_precision setting no longer stops a large vault from opening index_precision lives in .config.json, which people edit by hand, and it was passed to usearch as written. usearch raises KeyError for any spelling it does not know ("int-8", a number, a stray space), and above the 20,000-record brute-force limit the HNSW index is built on every open, so one typo meant the vault could not be opened by the CLI, any MCP server or the panel. Its bit and unsigned kinds were accepted and quietly destroyed recall under the cosine metric. build_index now maps the setting onto the float widths and int8 under every spelling usearch accepts, case and whitespace aside, and anything else builds an f32 index with a one-time notice naming the setting and the command that sets it properly. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vindex.py | 32 +++++++++++++++++++++++++++++++- tests/test_vindex.py | 20 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/compartment/vindex.py b/src/compartment/vindex.py index dc0be08..f00e68b 100644 --- a/src/compartment/vindex.py +++ b/src/compartment/vindex.py @@ -117,6 +117,35 @@ def __len__(self) -> int: _HNSW_WARNED = False +_PRECISION_WARNED: set[str] = set() + +#: What `index_precision` may name: the float widths and int8, each under the +#: spellings usearch itself accepts. The setting lives in a JSON file people +#: edit by hand, and usearch raised KeyError for anything else ("int-8", 8, +#: "INT8 " with a space): above BRUTE_FORCE_LIMIT records the index is built +#: on every open, so one typo meant the vault could not be opened at all. +#: Its bit and unsigned kinds ("b1", "u8") are refused too: under the cosine +#: metric they do not approximate the vectors, they discard them. +PRECISIONS = { + "f32": "f32", "float32": "f32", "f16": "f16", "float16": "f16", + "bf16": "bf16", "bfloat16": "bf16", "f64": "f64", "float64": "f64", + "i8": "i8", "int8": "i8", +} + + +def _precision(value) -> str: + """The usearch dtype `value` names, or f32 with a notice once.""" + name = PRECISIONS.get(str(value).strip().lower()) + if name is not None: + return name + if repr(value) not in _PRECISION_WARNED: + import sys + print(f"notice: index_precision {value!r} is not one of " + f"{', '.join(sorted(set(PRECISIONS.values())))}; using f32. " + "Set it with `compartment reindex --int8` or `--f32`.", + file=sys.stderr) + _PRECISION_WARNED.add(repr(value)) + return "f32" def build_index(dim: int, ikeys: list[int], mat: np.ndarray, @@ -130,7 +159,8 @@ def build_index(dim: int, ikeys: list[int], mat: np.ndarray, if force == "brute" or (force is None and len(ikeys) < BRUTE_FORCE_LIMIT): return BruteForceIndex.build(dim, ikeys, mat) try: - return UsearchIndex.build(dim, ikeys, mat, precision=precision) + return UsearchIndex.build(dim, ikeys, mat, + precision=_precision(precision)) except ImportError: global _HNSW_WARNED if not _HNSW_WARNED: diff --git a/tests/test_vindex.py b/tests/test_vindex.py index 4336b1e..22a138a 100644 --- a/tests/test_vindex.py +++ b/tests/test_vindex.py @@ -167,3 +167,23 @@ def test_an_add_writes_one_row_instead_of_copying_the_index(): moves += 1 before = built._mat assert moves <= 2, f"the rows moved on {moves} of 1,000 adds" + + +def test_a_mistyped_precision_setting_still_builds_an_index(capsys): + """index_precision is hand-edited JSON, and usearch raised KeyError for + anything it does not spell. Above the brute-force limit the index is + built on every open, so one typo meant the vault would not open.""" + import pytest + pytest.importorskip("usearch") + from compartment.vindex import build_index + rng = np.random.default_rng(3) + mat = rng.standard_normal((50, 16)).astype(np.float32) + for setting in ("int-8", 8, None, "b1", ["f32"]): + idx = build_index(16, list(range(1, 51)), mat, precision=setting, + force="hnsw") + assert idx.kind == "hnsw/usearch (f32)" + assert idx.search(mat[7], 1)[0][0] == 8 + assert "index_precision" in capsys.readouterr().err + for spelled, kind in (("INT8", "i8"), (" i8", "i8"), ("float16", "f16")): + assert build_index(16, list(range(1, 51)), mat, precision=spelled, + force="hnsw").kind == f"hnsw/usearch ({kind})" From f7d394c0411ea3f0e5bde4cc41839f4659aa6a2b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:43 -0700 Subject: [PATCH 064/135] The Linux and macOS installers work with externally managed Pythons, pick a Python the dependencies have wheels for, and survive being re-run or piped into bash Run on the systems most people have, both scripts failed before installing anything: - PEP 668. `python3 -m pip install --user` is refused with "externally-managed-environment" by Homebrew's Python and by Debian 12+, Ubuntu 23.04+, Arch and Crostini, and `set -e` ended the script there. Inside an activated virtualenv pip refuses --user as well, and a python3 without pip (minimal containers) died on "No module named pip". The scripts now install into an active virtualenv directly, keep `pip install --user` where the interpreter allows it (so earlier installs upgrade in place), and otherwise use pipx, then uv, then a private venv in ~/.local/share/compartment/venv linked into ~/.local/bin. A Python without the venv module is told to add python3-venv or pipx. - The wrong Python. Each script took `python3` as it found it. They now probe candidates (python3, python3.14 ... python3.11, and on macOS the Homebrew and python.org locations) and take the first inside the matrix tools/wheel_coverage.py checks: Python 3.11-3.14 on x86_64/aarch64 with glibc 2.28+; on macOS 13+, 3.11-3.13 on Intel (and Rosetta) Pythons and on macOS 13, 3.14 too on Apple silicon with macOS 14+. Outside it, pip failed after a long resolve with "No matching distribution found for onnxruntime"; now the script says why and what to install: the Python version that fits (brew install python@3.13), or, where onnxruntime has no wheels at all, the OS package that provides it - Alpine `apk add python3 py3-onnxruntime`, FreeBSD `pkg install pyXY-onnxruntime` and the other compiled deps, Termux `pkg install python onnxruntime python-numpy rust` (package names checked in aports, freebsd-ports and termux-packages on GitHub) - and then installs into a venv with --system-site-packages so pip uses that copy. /usr/bin/python3 is only probed where the Command Line Tools exist, since otherwise it opens an installer dialog. - `curl ... | bash`. With no script file, $0 is "bash", so `dirname "$0"` was the current directory and a pyproject.toml one level up - whatever project the user stood in - was installed instead of Compartment. Only a checkout found through BASH_SOURCE whose pyproject names compartment is installed from now; otherwise the PyPI release. `init` gets /dev/tty when stdin is the piped script, since it prompts for the passphrase. - Re-running. `init` refuses an existing vault, so a second run ended in an error; an existing vault (COMPARTMENT_VAULT, ENGRAM_VAULT, ~/.compartment or legacy ~/.engram, as home.py resolves it) is now left alone, and installs use --upgrade/--force so a re-run upgrades. - Termux, and Linux without /dev/shm: there is nowhere to keep the per-boot secret, so `init` created the vault and then failed. The vault is now created with --no-session and the script says how to open it. The closing text no longer tells people to pip-install into Hermes and copy plugin folders by hand: `init` already connects Claude Code, Hermes Agent and OpenClaw, and `compartment integrate` does any other client. Everything runs under macOS's bash 3.2. tests/test_platform_installers.py runs both scripts with bash against stubs for python (which runs the scripts' own probes with the platform facts each test sets), pip, venv, pipx, uv and compartment. Against the old scripts 22 of its first 24 tests failed; the two that passed are the syntax check and the arm64/macOS 15/Python 3.14 case. Co-Authored-By: Claude Opus 5.5 --- install/install-linux.sh | 328 ++++++++++++++++++--- install/install-macos.sh | 281 ++++++++++++++++-- tests/test_platform_installers.py | 460 ++++++++++++++++++++++++++++++ 3 files changed, 1006 insertions(+), 63 deletions(-) create mode 100644 tests/test_platform_installers.py diff --git a/install/install-linux.sh b/install/install-linux.sh index fecb2f1..184a777 100755 --- a/install/install-linux.sh +++ b/install/install-linux.sh @@ -1,41 +1,250 @@ #!/usr/bin/env bash -# Compartment installer - Linux -# Installs the package, creates + unlocks a vault, -# and prints the Hermes selection step. Fully offline after this script. +# Compartment installer - Linux, WSL and ChromeOS (Crostini). On Alpine, +# Termux and the BSDs it says what to install first. +# +# Installs the package, creates and unlocks a vault, and connects the agents +# it finds. Fully offline after this script. +# +# bash install/install-linux.sh from a checkout +# PYTHON=python3.12 bash install-linux.sh to choose the interpreter +# +# Running it again upgrades Compartment and leaves an existing vault alone. +# Written for bash 3.2 and up, so the same constructs run on macOS's bash. set -euo pipefail echo "Compartment installer (Linux)" -PY="${PYTHON:-python3}" -command -v "$PY" >/dev/null || { echo "python3 not found. Install via your package manager (apt/dnf/pacman)."; exit 1; } -if [ -f "$(dirname "$0")/../pyproject.toml" ]; then - "$PY" -m pip install --user "$(cd "$(dirname "$0")/.." && pwd)" +# --- 1. A Python that can run Compartment ------------------------------------ +# +# Compartment runs on Python 3.11 to 3.14 with x86_64 or aarch64 and glibc +# 2.28 or newer: that is where every compiled dependency publishes wheels, +# onnxruntime above all, which Compartment cannot embed anything without +# (tools/wheel_coverage.py checks the matrix in CI). Elsewhere - musl +# (Alpine), Android (Termux), the BSDs, 32-bit, armv7, riscv64, ppc64le, +# s390x, an older glibc, a Python newer than its wheels - pip fails with a +# bare "No matching distribution found for onnxruntime". So this is checked +# first, and where the operating system packages onnxruntime instead, that +# copy is used. +# +# The probe runs inside each candidate. First line: "ok", or why this +# platform has no onnxruntime wheel. Second line: whether onnxruntime is +# importable already. Exit 0: usable. Exit 1: not Python 3.11 to 3.14. +# Exit 2: no wheel and no onnxruntime. +PROBE=' +import os, platform, sys +if not (3, 11) <= sys.version_info[:2] <= (3, 14): + print("Python %d.%d" % sys.version_info[:2]) + sys.exit(1) +reason = "ok" +machine = platform.machine().lower() +if hasattr(sys, "getandroidapilevel") or sys.platform == "android": + reason = "android" +elif platform.system() != "Linux": + reason = platform.system() or "unknown" +elif machine not in ("x86_64", "amd64", "aarch64", "arm64"): + reason = "arch " + machine +else: + try: + libc = os.confstr("CS_GNU_LIBC_VERSION") or "" + except (ValueError, OSError): + libc = "" + if not libc.startswith("glibc "): + reason = "musl" + else: + parts = libc.split()[1].split(".") + if (int(parts[0]), int(parts[1])) < (2, 28): + reason = "glibc " + libc.split()[1] +try: + import importlib.util + present = importlib.util.find_spec("onnxruntime") is not None +except Exception: + present = False +print(reason) +print("present" if present else "absent") +sys.exit(0 if reason == "ok" or present else 2) +' + +PY="" +PLATFORM="" +NO_WHEEL="" +NO_WHEEL_PY="" +TOO_OLD="" +if [ -n "${PYTHON:-}" ]; then + set -- "$PYTHON" +else + set -- python3 python3.14 python3.13 python3.12 python3.11 +fi +for cand in "$@"; do + command -v "$cand" >/dev/null 2>&1 || continue + rc=0 + out="$("$cand" -c "$PROBE" 2>/dev/null)" || rc=$? + first="$(printf '%s\n' "$out" | sed -n 1p)" + case "$rc" in + 0) PY="$(command -v "$cand")"; PLATFORM="$first"; break ;; + 2) if [ -z "$NO_WHEEL" ]; then NO_WHEEL="$first"; NO_WHEEL_PY="$cand"; fi ;; + *) TOO_OLD="${TOO_OLD:+$TOO_OLD, }$cand is ${first:-not runnable}" ;; + esac +done + +if [ -z "$PY" ] && [ -n "$NO_WHEEL" ]; then + pyxy="$("$NO_WHEEL_PY" -c 'import sys; print("%d%d" % sys.version_info[:2])')" + echo + echo "onnxruntime, which Compartment needs, publishes no build for this system ($NO_WHEEL)." + case "$NO_WHEEL" in + android) + echo "On Termux, install it from Termux's own packages, then run this script again:" + echo " pkg install python onnxruntime python-numpy rust" + echo "(rust lets pip build tokenizers and pydantic-core, which have no Android wheels.)" ;; + musl) + echo "On Alpine, install it from the community repository, then run this script again:" + echo " apk add python3 py3-onnxruntime" + echo "Every other dependency has musllinux wheels." ;; + FreeBSD) + echo "On FreeBSD, install it and the other compiled dependencies from packages," + echo "then run this script again:" + echo " pkg install py${pyxy}-onnxruntime py${pyxy}-tokenizers py${pyxy}-pydantic-core \\" + echo " py${pyxy}-pynacl py${pyxy}-argon2-cffi py${pyxy}-numpy" ;; + arch\ *) + echo "onnxruntime ships wheels for x86_64 and aarch64 only. On a Raspberry Pi," + echo "install the 64-bit OS. Elsewhere, install onnxruntime for $NO_WHEEL_PY from" + echo "your distribution if it packages it, then run this script again." ;; + glibc\ *) + echo "Its wheels need glibc 2.28 or newer (Debian 10, Ubuntu 20.04, RHEL 8" + echo "and later). Use a newer distribution, or a container of one." + exit 1 ;; + *) + echo "Build onnxruntime's Python package for $NO_WHEEL_PY from source" + echo "(https://github.com/microsoft/onnxruntime), then run this script again." ;; + esac + echo "This script then installs Compartment in its own environment that uses" + echo "that onnxruntime." + exit 1 +fi + +if [ -z "$PY" ]; then + echo + if [ -n "$TOO_OLD" ]; then + echo "Compartment needs Python 3.11 to 3.14; found: $TOO_OLD." + else + echo "Compartment needs Python 3.11 to 3.14, and none was found." + fi + echo "Install one with your package manager, for example" + echo " sudo apt install python3.12 python3.12-venv (Ubuntu 22.04: python3.11)" + echo " sudo dnf install python3.12" + echo "then run this script again, or name it: PYTHON=python3.12 bash ${BASH_SOURCE[0]:-install-linux.sh}" + exit 1 +fi +echo "Using $PY" + +# --- 2. Install --------------------------------------------------------------- +# +# This checkout when the script is run from one, otherwise the release on +# PyPI. Only a checkout whose pyproject names compartment counts: run as +# `curl ... | bash` there is no script file, and `dirname "$0"` named the +# directory the user happened to be in, so the pyproject one level up was +# somebody else's project, which got installed instead. +SRC="compartment" +SCRIPT="${BASH_SOURCE[0]:-}" +if [ -n "$SCRIPT" ] && [ -f "$SCRIPT" ]; then + ROOT="$(cd "$(dirname "$SCRIPT")/.." && pwd)" + if [ -f "$ROOT/pyproject.toml" ] && grep -q '^name = "compartment"$' "$ROOT/pyproject.toml"; then + SRC="$ROOT" + fi +fi + +# Where to put it. `pip install --user` into the system Python fails on +# Debian 12+, Ubuntu 23.04+, Arch, Crostini and Homebrew with +# "externally-managed-environment" (PEP 668), which under `set -e` ended the +# script before anything was installed. So: +# - inside an activated virtualenv: into it (--user is refused there); +# - onnxruntime from the OS (Alpine, Termux, FreeBSD): a private venv that +# can see the system site-packages, so pip uses that copy; +# - a Python that allows it: `pip install --user`, as before; +# - an externally managed one, or one without pip (minimal containers): +# pipx, then uv, then a private venv, which brings its own pip. +KIND="$("$PY" -c ' +import importlib.util, os, sys, sysconfig +if sys.prefix != sys.base_prefix: + print("venv") +elif os.path.isfile(os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")): + print("managed") +elif importlib.util.find_spec("pip") is None: + print("nopip") +else: + print("plain") +')" +if [ "$KIND" = venv ]; then + METHOD=venv-active +elif [ "$PLATFORM" != ok ]; then + METHOD=private +elif [ "$KIND" = plain ]; then + METHOD=user +elif command -v pipx >/dev/null 2>&1; then + METHOD=pipx +elif command -v uv >/dev/null 2>&1; then + METHOD=uv else - "$PY" -m pip install --user compartment + METHOD=private fi -# `pip install --user` puts the console script in the interpreter's own user -# bin directory (usually ~/.local/bin), which is not on PATH on a clean -# machine. Ask the interpreter where that is and use it, so the rest of this -# script runs instead of dying on "compartment: command not found". -USER_BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" +LOCAL_BIN="$HOME/.local/bin" +case "$METHOD" in + venv-active) + "$PY" -m pip install --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" ;; + user) + # `pip install --user` puts the console script in the interpreter's own + # user bin directory (usually ~/.local/bin), which is not on PATH on a + # clean machine, so the interpreter is asked where that is. + "$PY" -m pip install --user --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" ;; + pipx) + pipx install --force --python "$PY" "$SRC" + BIN="$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + uv) + uv tool install --force --upgrade --python "$PY" "$SRC" + BIN="$(uv tool dir --bin 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + private) + VENV="${XDG_DATA_HOME:-$HOME/.local/share}/compartment/venv" + mkdir -p "$(dirname "$VENV")" + SITE_FLAG="" + if [ "$PLATFORM" != ok ]; then SITE_FLAG="--system-site-packages"; fi + # No --clear: a re-run upgrades in place, under a status bar app that + # may be running from this very environment. + if ! "$PY" -m venv $SITE_FLAG "$VENV"; then + echo + echo "Could not create a virtual environment with $PY." + echo "On Debian, Ubuntu and Crostini install the venv module first:" + echo " sudo apt install python3-venv (or python3.X-venv for $PY)" + echo "or install pipx (sudo apt install pipx), then run this script again." + exit 1 + fi + "$VENV/bin/python" -m pip install --upgrade "$SRC" + mkdir -p "$LOCAL_BIN" + for cmd in compartment cmpt engram; do + if [ -x "$VENV/bin/$cmd" ]; then ln -sf "$VENV/bin/$cmd" "$LOCAL_BIN/$cmd"; fi + done + BIN="$LOCAL_BIN" ;; +esac + ON_PATH_ALREADY=no -if command -v compartment >/dev/null 2>&1; then ON_PATH_ALREADY=yes; fi -if [ -d "$USER_BIN" ]; then - PATH="$USER_BIN:$PATH" +case ":$PATH:" in *":$BIN:"*) ON_PATH_ALREADY=yes ;; esac +if [ -d "$BIN" ]; then + PATH="$BIN:$PATH" export PATH fi -if [ -x "$USER_BIN/compartment" ]; then - COMPARTMENT="$USER_BIN/compartment" +if [ -x "$BIN/compartment" ]; then + COMPARTMENT="$BIN/compartment" elif command -v compartment >/dev/null 2>&1; then COMPARTMENT="$(command -v compartment)" elif "$PY" -c 'import compartment.cli' >/dev/null 2>&1; then COMPARTMENT="" # no console script on PATH, but the module is importable else echo "compartment was installed but cannot be found." - echo "Expected the command in: $USER_BIN" - echo "Try: \"$PY\" -m pip install --user --force-reinstall compartment" + echo "Expected the command in: $BIN" exit 1 fi @@ -47,26 +256,81 @@ run_compartment() { fi } +# --- 3. The vault --------------------------------------------------------------- +# +# Where `compartment` looks by default (home.py): COMPARTMENT_VAULT, then +# the legacy ENGRAM_VAULT, then memory.vault in ~/.compartment, or in +# ~/.engram when only that exists. A vault that is there is never touched: +# `init` refuses to overwrite one, and a re-run should not fail on that. +if [ -n "${COMPARTMENT_VAULT:-}" ]; then + VAULT="$COMPARTMENT_VAULT" +elif [ -n "${ENGRAM_VAULT:-}" ]; then + VAULT="$ENGRAM_VAULT" +elif [ ! -e "$HOME/.compartment" ] && [ -e "$HOME/.engram" ]; then + VAULT="$HOME/.engram/memory.vault" +else + VAULT="$HOME/.compartment/memory.vault" +fi + +# Staying unlocked until the next restart keeps a per-boot secret in POSIX +# shared memory. Termux has none, and glibc keeps it in /dev/shm, which some +# containers and chroots lack; there `init` would create the vault and then +# fail. Such a vault is created locked and opened with the passphrase. +NO_SESSION="" +if [ -n "${TERMUX_VERSION:-}" ] || case "${PREFIX:-}" in *com.termux*) true ;; *) false ;; esac \ + || [ ! -d /dev/shm ] || [ ! -w /dev/shm ]; then + NO_SESSION="--no-session" +fi + echo -echo "Creating your encrypted vault..." -run_compartment init +INIT_RAN=no +if [ -e "$VAULT" ]; then + echo "Your vault at $VAULT is kept as it is." + echo "Open it with: compartment unlock" +else + INIT_RAN=yes + echo "Creating your encrypted vault..." + # `init` asks for a passphrase on the terminal. Run as `curl ... | bash`, + # stdin is the script itself, so the terminal is handed over explicitly. + if [ -t 0 ]; then + run_compartment init $NO_SESSION + elif (: /dev/null; then + run_compartment init $NO_SESSION /dev/null || { echo "python3 not found. Install from python.org or 'brew install python'."; exit 1; } -# From a source checkout, install this repo; otherwise install from PyPI. -if [ -f "$(dirname "$0")/../pyproject.toml" ]; then - "$PY" -m pip install --user "$(cd "$(dirname "$0")/.." && pwd)" +# --- 1. A Python that can run Compartment ------------------------------------ +# +# Compartment runs on macOS 13 or newer, with a Python its compiled +# dependencies publish wheels for (tools/wheel_coverage.py checks the +# matrix in CI). onnxruntime, which it cannot embed anything without, +# decides it: +# - Apple silicon, macOS 14 or newer: Python 3.11 to 3.14. +# - Apple silicon on macOS 13: Python 3.11 to 3.13 - onnxruntime's +# builds for 3.14 need macOS 14. +# - Intel Macs (and an x86_64 Python under Rosetta): Python 3.11 to 3.13 - +# onnxruntime stopped building for Intel after 1.23.2, which stops at +# 3.13. +# Any other Python fails in pip with a bare "No matching distribution found +# for onnxruntime", so each candidate is checked first and the first one +# that fits is used. +# +# The probe runs inside each candidate. Exit 0: usable. Exit 1: older than +# Python 3.11. Exit 2: Python 3.11+, but this Mac has no wheels for it; its +# output names the newest Python that would do, or "none" for a macOS +# older than 13. +MACOS_VERSION="$(sw_vers -productVersion 2>/dev/null || true)" +export CMPT_MACOS_VERSION="$MACOS_VERSION" +PROBE=' +import os, platform, sys +v = sys.version_info[:2] +if v < (3, 11): + print("Python %d.%d" % v) + sys.exit(1) +text = os.environ.get("CMPT_MACOS_VERSION") or platform.mac_ver()[0] or "0" +parts = [int(x) for x in text.split(".")[:2] if x.isdigit()] + [0, 0] +mac = (parts[0], parts[1]) +if mac == (10, 16): + mac = (11, 0) +if mac < (13, 0): + print("none") + sys.exit(2) +intel = platform.machine().lower() not in ("arm64", "aarch64") +newest = (3, 13) if (intel or mac < (14, 0)) else (3, 14) +if v > newest: + print("%d.%d %s" % (newest + (platform.machine(),))) + sys.exit(2) +' + +# /usr/bin/python3 on a Mac without the Command Line Tools is a stub that +# opens an installer dialog instead of running, so it is only tried where +# the tools are installed. +usable_name() { + case "$(command -v "$1" 2>/dev/null)" in + "") return 1 ;; + /usr/bin/python3) xcode-select -p >/dev/null 2>&1 ;; + *) return 0 ;; + esac +} + +PY="" +NEWEST="" +PYARCH="" +TOO_OLD="" +if [ -n "${PYTHON:-}" ]; then + set -- "$PYTHON" +else + set -- python3 python3.13 python3.12 python3.11 python3.14 \ + /opt/homebrew/bin/python3.13 /opt/homebrew/bin/python3.12 /opt/homebrew/bin/python3.11 \ + /usr/local/bin/python3.13 /usr/local/bin/python3.12 /usr/local/bin/python3.11 \ + /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 \ + /Library/Frameworks/Python.framework/Versions/3.12/bin/python3 \ + /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 +fi +for cand in "$@"; do + usable_name "$cand" || continue + rc=0 + out="$("$cand" -c "$PROBE" 2>/dev/null)" || rc=$? + case "$rc" in + 0) PY="$(command -v "$cand")"; break ;; + 2) if [ -z "$NEWEST" ]; then NEWEST="${out%% *}"; PYARCH="${out#* }"; fi + TOO_OLD="${TOO_OLD:+$TOO_OLD, }$cand has no onnxruntime build here" ;; + *) TOO_OLD="${TOO_OLD:+$TOO_OLD, }$cand is ${out:-not runnable}" ;; + esac +done + +if [ -z "$PY" ]; then + echo + if [ "$NEWEST" = none ]; then + echo "Compartment needs macOS 13 or newer; this Mac has macOS $MACOS_VERSION," + echo "for which onnxruntime and the menu bar app's libraries publish no builds." + exit 1 + elif [ -n "$NEWEST" ]; then + echo "For an $PYARCH Python on macOS $MACOS_VERSION, onnxruntime, which" + echo "Compartment needs, publishes builds only for Python 3.11 to $NEWEST." + echo "Found: $TOO_OLD." + echo "Install Python $NEWEST, then run this script again:" + echo " brew install python@$NEWEST (or the installer from python.org)" + echo " PYTHON=python$NEWEST bash ${BASH_SOURCE[0]:-install-macos.sh}" + else + if [ -n "$TOO_OLD" ]; then + echo "Compartment needs Python 3.11 or newer; found: $TOO_OLD." + else + echo "Compartment needs Python 3.11 or newer, and none was found." + fi + echo "Install it from python.org or with Homebrew (brew install python@3.13)," + echo "then run this script again." + fi + exit 1 +fi +echo "Using $PY" + +# --- 2. Install --------------------------------------------------------------- +# +# This checkout when the script is run from one, otherwise the release on +# PyPI. Only a checkout whose pyproject names compartment counts: run as +# `curl ... | bash` there is no script file, and `dirname "$0"` named the +# directory the user happened to be in, so the pyproject one level up was +# somebody else's project, which got installed instead. +SRC="compartment" +SCRIPT="${BASH_SOURCE[0]:-}" +if [ -n "$SCRIPT" ] && [ -f "$SCRIPT" ]; then + ROOT="$(cd "$(dirname "$SCRIPT")/.." && pwd)" + if [ -f "$ROOT/pyproject.toml" ] && grep -q '^name = "compartment"$' "$ROOT/pyproject.toml"; then + SRC="$ROOT" + fi +fi + +# Where to put it. Homebrew's Python is externally managed (PEP 668), so +# `pip install --user` there fails with "externally-managed-environment", +# which under `set -e` ended the script before anything was installed. So: +# - inside an activated virtualenv: into it (--user is refused there); +# - a Python that allows it (python.org's): `pip install --user`, as before; +# - an externally managed one, or one without pip (minimal containers): +# pipx, then uv, then a private venv, which brings its own pip. +KIND="$("$PY" -c ' +import importlib.util, os, sys, sysconfig +if sys.prefix != sys.base_prefix: + print("venv") +elif os.path.isfile(os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")): + print("managed") +elif importlib.util.find_spec("pip") is None: + print("nopip") +else: + print("plain") +')" +if [ "$KIND" = venv ]; then + METHOD=venv-active +elif [ "$KIND" = plain ]; then + METHOD=user +elif command -v pipx >/dev/null 2>&1; then + METHOD=pipx +elif command -v uv >/dev/null 2>&1; then + METHOD=uv else - "$PY" -m pip install --user compartment + METHOD=private fi -# `pip install --user` puts the console script in the interpreter's own user -# bin directory (on macOS ~/Library/Python/3.X/bin), which is not on PATH on a -# clean machine. Ask the interpreter where that is and use it, so the rest of -# this script runs instead of dying on "compartment: command not found". -USER_BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" +LOCAL_BIN="$HOME/.local/bin" +case "$METHOD" in + venv-active) + "$PY" -m pip install --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" ;; + user) + # `pip install --user` puts the console script in the interpreter's own + # user bin directory (~/Library/Python/3.X/bin), which is not on PATH on + # a clean machine, so the interpreter is asked where that is. + "$PY" -m pip install --user --upgrade "$SRC" + BIN="$("$PY" -c 'import sysconfig; print(sysconfig.get_path("scripts", scheme="posix_user"))')" ;; + pipx) + pipx install --force --python "$PY" "$SRC" + BIN="$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + uv) + uv tool install --force --upgrade --python "$PY" "$SRC" + BIN="$(uv tool dir --bin 2>/dev/null || true)" + BIN="${BIN:-$LOCAL_BIN}" ;; + private) + VENV="${XDG_DATA_HOME:-$HOME/.local/share}/compartment/venv" + mkdir -p "$(dirname "$VENV")" + # No --clear: a re-run upgrades in place, under a menu bar app that may + # be running from this very environment. + if ! "$PY" -m venv "$VENV"; then + echo + echo "Could not create a virtual environment with $PY." + echo "Install pipx (brew install pipx) and run this script again." + exit 1 + fi + "$VENV/bin/python" -m pip install --upgrade "$SRC" + mkdir -p "$LOCAL_BIN" + for cmd in compartment cmpt engram; do + if [ -x "$VENV/bin/$cmd" ]; then ln -sf "$VENV/bin/$cmd" "$LOCAL_BIN/$cmd"; fi + done + BIN="$LOCAL_BIN" ;; +esac + ON_PATH_ALREADY=no -if command -v compartment >/dev/null 2>&1; then ON_PATH_ALREADY=yes; fi -if [ -d "$USER_BIN" ]; then - PATH="$USER_BIN:$PATH" +case ":$PATH:" in *":$BIN:"*) ON_PATH_ALREADY=yes ;; esac +if [ -d "$BIN" ]; then + PATH="$BIN:$PATH" export PATH fi -if [ -x "$USER_BIN/compartment" ]; then - COMPARTMENT="$USER_BIN/compartment" +if [ -x "$BIN/compartment" ]; then + COMPARTMENT="$BIN/compartment" elif command -v compartment >/dev/null 2>&1; then COMPARTMENT="$(command -v compartment)" elif "$PY" -c 'import compartment.cli' >/dev/null 2>&1; then COMPARTMENT="" # no console script on PATH, but the module is importable else echo "compartment was installed but cannot be found." - echo "Expected the command in: $USER_BIN" - echo "Try: \"$PY\" -m pip install --user --force-reinstall compartment" + echo "Expected the command in: $BIN" exit 1 fi @@ -48,26 +231,62 @@ run_compartment() { fi } +# --- 3. The vault --------------------------------------------------------------- +# +# Where `compartment` looks by default (home.py): COMPARTMENT_VAULT, then +# the legacy ENGRAM_VAULT, then memory.vault in ~/.compartment, or in +# ~/.engram when only that exists. A vault that is there is never touched: +# `init` refuses to overwrite one, and a re-run should not fail on that. +if [ -n "${COMPARTMENT_VAULT:-}" ]; then + VAULT="$COMPARTMENT_VAULT" +elif [ -n "${ENGRAM_VAULT:-}" ]; then + VAULT="$ENGRAM_VAULT" +elif [ ! -e "$HOME/.compartment" ] && [ -e "$HOME/.engram" ]; then + VAULT="$HOME/.engram/memory.vault" +else + VAULT="$HOME/.compartment/memory.vault" +fi + echo -echo "Creating your encrypted vault..." -run_compartment init +INIT_RAN=no +if [ -e "$VAULT" ]; then + echo "Your vault at $VAULT is kept as it is." + echo "Open it with: compartment unlock (or from the menu bar)" +else + INIT_RAN=yes + echo "Creating your encrypted vault..." + # `init` asks for a passphrase on the terminal. Run as `curl ... | bash`, + # stdin is the script itself, so the terminal is handed over explicitly. + if [ -t 0 ]; then + run_compartment init + elif (: /dev/null; then + run_compartment init ", "exec"), {{"__name__": "__main__"}}) + sys.exit(0) +if args[:2] == ["-m", "pip"]: + log("pip", *args[2:]) + if kind == "managed" and "--user" in args: + print("error: externally-managed-environment", file=sys.stderr) + sys.exit(1) + console_script(os.path.dirname(me) if in_venv + else os.environ["STUB_USER_BIN"] if "--user" in args + else os.environ["STUB_VENV_BIN"]) + sys.exit(0) +if args[:2] == ["-m", "venv"]: + log("venv", *args[2:]) + if f.get("no_venv"): + print("ensurepip is not available", file=sys.stderr) + sys.exit(1) + d = args[-1] + os.makedirs(os.path.join(d, "bin"), exist_ok=True) + open(os.path.join(d, "pyvenv.cfg"), "w").close() + shutil.copy(me, os.path.join(d, "bin", "python")) + sys.exit(0) +log("unexpected", *args) +sys.exit(3) +''' + +_TOOL_STUB = r'''#!{python} +import json, os, shutil, sys +args = sys.argv[1:] +tool = os.path.basename(sys.argv[0]) +with open(os.environ["STUB_LOG"], "a") as fh: + fh.write(json.dumps([tool, *args]) + "\n") +if tool == "compartment": + sys.exit(0) +if tool == "sw_vers": + print(os.environ.get("STUB_MACOS", "15.1")) + sys.exit(0) +bindir = os.environ["STUB_TOOL_BIN"] +if args[:1] in (["environment"], ["tool"]) and "install" not in args: + print(bindir) + sys.exit(0) +os.makedirs(bindir, exist_ok=True) +shutil.copy(os.environ["STUB_COMPARTMENT"], os.path.join(bindir, "compartment")) +os.chmod(os.path.join(bindir, "compartment"), 0o755) +''' + + +class Run: + def __init__(self, proc, log): + self.rc = proc.returncode + self.out = proc.stdout + proc.stderr + self.calls = [json.loads(line) for line in log.read_text().splitlines()] + + def of(self, what): + """The argument lists of every call to `what` (pip, venv, pipx, + uv, compartment).""" + found = [] + for call in self.calls: + head = call[0] if call[0] in ("pipx", "uv", "compartment", "sw_vers") else call[1] + if head == what: + found.append(call[1:] if call[0] in ("pipx", "uv", "compartment", "sw_vers") else call[2:]) + return found + + def interpreter_of(self, what): + return [c[0] for c in self.calls if len(c) > 1 and c[1] == what] + + def init_args(self): + """How `init` was run, or how the script told the user to run it + when there was no terminal to ask for a passphrase on (no pty in + this sandbox, say). None when it was neither.""" + inits = [c for c in self.of("compartment") if c[:1] == ["init"]] + if inits: + return inits[0] + m = re.search(r"Create the vault with:\n\s+compartment (init[^\n]*)", + self.out) + return m.group(1).split() if m else None + + +@pytest.fixture() +def installer(tmp_path): + """Run an installer with the stand-ins. `pythons` names the python + stubs to put on PATH; `facts` is per-stub (or "*") platform facts; + `tools` adds pipx and/or uv.""" + def run(script, pythons=("python3",), facts=None, tools=(), env=None, + tty=True, stdin_script=False, cwd=None): + root = tmp_path / "stubs" + bin_ = root / "bin" + bin_.mkdir(parents=True, exist_ok=True) + home = tmp_path / "home" + home.mkdir(exist_ok=True) + log = root / "log" + log.write_text("") + py = _PYTHON_STUB.format(python=sys.executable) + tool = _TOOL_STUB.format(python=sys.executable) + comp = root / "compartment-stub" + comp.write_text(tool) + comp.chmod(0o755) + for name in pythons: + (bin_ / name).write_text(py) + (bin_ / name).chmod(0o755) + for name in (*tools, "sw_vers"): + (bin_ / name).write_text(tool) + (bin_ / name).chmod(0o755) + for name in ("sed", "grep", "dirname", "mkdir", "ln", "cat", "uname"): + real = shutil.which(name, path="/usr/bin:/bin") + if real: + (bin_ / name).symlink_to(real) + e = {"PATH": str(bin_), "HOME": str(home), + "STUB_LOG": str(log), "STUB_ROOT": str(root), + "STUB_FACTS": json.dumps(facts or {}), + "STUB_COMPARTMENT": str(comp), + "STUB_USER_BIN": str(home / ".local" / "bin"), + "STUB_VENV_BIN": str(root / "active-venv" / "bin"), + "STUB_TOOL_BIN": str(home / ".local" / "bin")} + e.update(env or {}) + master = slave = None + if stdin_script: + argv, stdin = [BASH, "-s"], open(script, "rb") + else: + argv, stdin = [BASH, str(script)], subprocess.DEVNULL + if tty: + try: + master, slave = pty.openpty() + stdin = slave + except OSError: + pass # no ptys here: the no-terminal branch + try: + # A session of its own, so /dev/tty is never the terminal of + # whoever runs the suite. + proc = subprocess.run(argv, env=e, stdin=stdin, cwd=cwd or tmp_path, + capture_output=True, text=True, timeout=120, + start_new_session=True) + finally: + if stdin_script: + stdin.close() + for fd in (master, slave): + if fd is not None: + os.close(fd) + return Run(proc, log) + return run + + +# --- Linux: PEP 668 ----------------------------------------------------------- + +def test_an_externally_managed_python_is_installed_with_pipx(installer): + """Debian 12+, Ubuntu 23.04+, Arch, Crostini: `pip install --user` fails + with externally-managed-environment, and under `set -e` the script + ended there with nothing installed.""" + r = installer(LINUX, facts={"*": {"kind": "managed"}}, tools=("pipx",)) + assert r.rc == 0, r.out + assert not [a for a in r.of("pip") if "--user" in a] + install = [a for a in r.of("pipx") if a[:1] == ["install"]] + assert install and str(ROOT) in install[0] + assert r.init_args()[0] == "init" + + +def test_uv_is_used_where_there_is_no_pipx(installer): + r = installer(LINUX, facts={"*": {"kind": "managed"}}, tools=("uv",)) + assert r.rc == 0, r.out + assert [a for a in r.of("uv") if a[:2] == ["tool", "install"]] + + +def test_with_neither_a_private_environment_is_made_and_linked(installer, + tmp_path): + r = installer(LINUX, facts={"*": {"kind": "managed"}}) + assert r.rc == 0, r.out + venv = tmp_path / "home" / ".local" / "share" / "compartment" / "venv" + assert r.of("venv") == [[str(venv)]] + assert r.interpreter_of("pip") == [str(venv / "bin" / "python")] + link = tmp_path / "home" / ".local" / "bin" / "compartment" + assert link.is_symlink() and link.resolve() == (venv / "bin" / "compartment").resolve() + assert r.init_args()[0] == "init" + + +def test_a_python_without_the_venv_module_says_which_package_to_add(installer): + """Debian splits ensurepip into python3-venv, and Crostini has none.""" + r = installer(LINUX, facts={"*": {"kind": "managed", "no_venv": True}}) + assert r.rc == 1 + assert "python3-venv" in r.out and "pipx" in r.out + + +def test_a_python_that_allows_user_installs_keeps_the_old_route(installer): + r = installer(LINUX) + assert r.rc == 0, r.out + assert r.of("pip") == [["install", "--user", "--upgrade", str(ROOT)]] + + +def test_a_python_without_pip_gets_a_private_environment(installer, tmp_path): + """Minimal containers and some distributions ship python3 without pip: + `python3 -m pip` died with "No module named pip".""" + r = installer(LINUX, facts={"*": {"no_pip": True}}) + assert r.rc == 0, r.out + venv = tmp_path / "home" / ".local" / "share" / "compartment" / "venv" + assert r.of("venv") == [[str(venv)]] + assert r.interpreter_of("pip") == [str(venv / "bin" / "python")] + + +def test_an_activated_virtualenv_is_installed_into_without_user(installer): + """pip refuses --user inside a virtualenv.""" + r = installer(LINUX, facts={"*": {"kind": "venv"}}) + assert r.rc == 0, r.out + assert r.of("pip") == [["install", "--upgrade", str(ROOT)]] + + +# --- Linux: which Python ------------------------------------------------------ + +def test_a_too_old_python3_is_passed_over_for_a_versioned_one(installer): + """Ubuntu 22.04's python3 is 3.10; python3.11 is a separate package.""" + r = installer(LINUX, pythons=("python3", "python3.11"), + facts={"python3": {"version": [3, 10]}, + "python3.11": {"version": [3, 11]}}) + assert r.rc == 0, r.out + assert r.interpreter_of("pip")[0].endswith("python3.11") + + +def test_a_python_newer_than_the_wheels_is_passed_over_too(installer): + """Python 3.15 is out before onnxruntime builds for it; the supported + range is 3.11 to 3.14 (tools/wheel_coverage.py).""" + r = installer(LINUX, pythons=("python3", "python3.14"), + facts={"python3": {"version": [3, 15]}, + "python3.14": {"version": [3, 14]}}) + assert r.rc == 0, r.out + assert r.interpreter_of("pip")[0].endswith("python3.14") + + +def test_no_python_in_range_is_explained(installer): + r = installer(LINUX, facts={"*": {"version": [3, 9]}}) + assert r.rc == 1 + assert "3.11 to 3.14" in r.out and "python3 is Python 3.9" in r.out + assert not r.of("pip") + + +# --- Linux: platforms onnxruntime has no wheel for ---------------------------- + +@pytest.mark.parametrize("facts, says", [ + ({"libc": "musl"}, "apk add python3 py3-onnxruntime"), + ({"android": True}, "pkg install python onnxruntime"), + ({"system": "FreeBSD", "version": [3, 12]}, "pkg install py312-onnxruntime"), + ({"machine": "armv7l"}, "64-bit OS"), + ({"libc": "glibc 2.17"}, "glibc 2.28 or newer"), + ({"libc": "glibc 2.27"}, "glibc 2.28 or newer"), +]) +def test_a_platform_without_onnxruntime_wheels_is_told_what_to_install( + installer, facts, says): + """pip used to fail there with a bare "No matching distribution found + for onnxruntime", after a long resolve.""" + r = installer(LINUX, facts={"*": facts}) + assert r.rc == 1 + assert says in r.out + assert not r.of("pip") and not r.of("venv") + + +def test_onnxruntime_from_the_os_is_used_through_system_site_packages( + installer, tmp_path): + """Alpine's py3-onnxruntime, Termux's onnxruntime, FreeBSD's port: pip + must see that copy, or it goes looking for a wheel that does not + exist.""" + r = installer(LINUX, facts={"*": {"libc": "musl", "ort": True, + "kind": "managed"}}, tools=("pipx",)) + assert r.rc == 0, r.out + venv = tmp_path / "home" / ".local" / "share" / "compartment" / "venv" + assert r.of("venv") == [["--system-site-packages", str(venv)]] + assert not r.of("pipx") + + +def test_termux_creates_the_vault_locked_because_it_has_no_shared_memory( + installer): + """Termux has no shm_open, so `init` would create the vault and then + fail storing the session credential.""" + r = installer(LINUX, facts={"*": {"android": True, "ort": True}}, + env={"TERMUX_VERSION": "0.119"}) + assert r.rc == 0, r.out + assert r.init_args() == ["init", "--no-session"] + assert "COMPARTMENT_PASSPHRASE" in r.out + + +# --- Linux: what gets installed, and re-running ------------------------------ + +def test_piped_into_bash_it_installs_compartment_not_the_project_next_door( + installer, tmp_path): + """`curl ... | bash` has no script file: $0 is "bash", so `dirname $0` + was the current directory, and a pyproject.toml one level up - any + other Python project the user was standing in - was installed instead + of Compartment.""" + other = tmp_path / "someone-elses-project" + (other / "docs").mkdir(parents=True) + (other / "pyproject.toml").write_text('[project]\nname = "other"\n') + vault = tmp_path / "home" / "existing.vault" + vault.parent.mkdir(parents=True, exist_ok=True) + vault.write_text("") + r = installer(LINUX, stdin_script=True, cwd=other / "docs", + env={"COMPARTMENT_VAULT": str(vault)}) + assert r.rc == 0, r.out + assert r.of("pip") == [["install", "--user", "--upgrade", "compartment"]] + + +def test_running_it_again_leaves_the_vault_alone_and_succeeds(installer, + tmp_path): + """`init` refuses to overwrite a vault, so a second run used to end in + an error after upgrading.""" + (tmp_path / "home" / ".compartment").mkdir(parents=True) + (tmp_path / "home" / ".compartment" / "memory.vault").write_text("") + r = installer(LINUX) + assert r.rc == 0, r.out + assert r.init_args() is None + assert "kept as it is" in r.out + + +def test_a_legacy_engram_vault_counts_as_existing(installer, tmp_path): + (tmp_path / "home" / ".engram").mkdir(parents=True) + (tmp_path / "home" / ".engram" / "memory.vault").write_text("") + r = installer(LINUX) + assert r.rc == 0, r.out + assert r.init_args() is None + + +# --- macOS -------------------------------------------------------------------- + +def test_homebrew_python_is_installed_with_pipx_on_macos(installer): + r = installer(MACOS, facts={"*": {"kind": "managed", "machine": "arm64"}}, + tools=("pipx",)) + assert r.rc == 0, r.out + assert not [a for a in r.of("pip") if "--user" in a] + assert [a for a in r.of("pipx") if a[:1] == ["install"]] + + +@pytest.mark.parametrize("machine, macos", [("x86_64", "15.1"), + ("arm64", "13.6")]) +def test_python_3_14_is_passed_over_where_onnxruntime_stops_at_3_13( + installer, machine, macos): + """onnxruntime has no Intel Mac build after 1.23.2 and no macOS 13 build + after it either; 1.23.2 covers Python 3.11-3.13. With Homebrew's + python3 at 3.14, pip failed on onnxruntime.""" + r = installer(MACOS, pythons=("python3", "python3.13"), + facts={"*": {"machine": machine}, + "python3": {"version": [3, 14]}, + "python3.13": {"version": [3, 13]}}, + env={"STUB_MACOS": macos}) + assert r.rc == 0, r.out + assert r.interpreter_of("pip")[0].endswith("python3.13") + + +def test_only_python_3_14_on_an_intel_mac_names_the_python_to_install( + installer): + r = installer(MACOS, facts={"*": {"machine": "x86_64", + "version": [3, 14]}}) + assert r.rc == 1 + assert "brew install python@3.13" in r.out + assert "x86_64 Python" in r.out + assert not r.of("pip") + + +def test_a_mac_older_than_macos_13_is_told_it_is_unsupported(installer): + r = installer(MACOS, facts={"*": {"machine": "arm64"}}, + env={"STUB_MACOS": "12.7"}) + assert r.rc == 1 + assert "macOS 13 or newer" in r.out + assert not r.of("pip") + + +def test_apple_silicon_on_a_current_macos_takes_python_3_14(installer): + r = installer(MACOS, facts={"*": {"machine": "arm64", "version": [3, 14]}}, + env={"STUB_MACOS": "15.1"}) + assert r.rc == 0, r.out + assert r.init_args() == ["init"] + + +def test_both_scripts_parse(): + for script in (LINUX, MACOS): + subprocess.run([BASH, "-n", str(script)], check=True) From 58c5a215eb3358e83f295dd13101d8ea8730c1f5 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:46 -0700 Subject: [PATCH 065/135] The /compartmentalize skill for OpenClaw goes into the skills directory OpenClaw loads when OPENCLAW_HOME, OPENCLAW_STATE_DIR or OPENCLAW_CONFIG_PATH is set OpenClaw loads managed skills from /skills, and works out its config directory from three variables (src/infra/config-dir.ts): OPENCLAW_STATE_DIR is the directory; otherwise the directory holding OPENCLAW_CONFIG_PATH; otherwise OPENCLAW_HOME, which stands in for the home directory, with .openclaw under it. The skill installer read only OPENCLAW_HOME, and as the directory itself, so with it set the skill landed one level above the skills/ OpenClaw reads, and with either of the other two set it went to ~/.openclaw, which OpenClaw was not using. The skill's home is now resolved as OpenClaw resolves it. Without any of the variables it is ~/.openclaw, as before. A SKILL_TARGETS entry may now name a function instead of a variable for this. Co-Authored-By: Claude Opus 5.5 --- src/compartment/agent_skill.py | 33 +++++++++++++++++++---- tests/test_integrations_skills.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 tests/test_integrations_skills.py diff --git a/src/compartment/agent_skill.py b/src/compartment/agent_skill.py index 748fc30..1d1fa52 100644 --- a/src/compartment/agent_skill.py +++ b/src/compartment/agent_skill.py @@ -17,7 +17,7 @@ claude $CLAUDE_CONFIG_DIR or ~/.claude/skills/compartmentalize/SKILL.md hermes $HERMES_HOME or ~/.hermes/skills/compartmentalize/SKILL.md - openclaw $OPENCLAW_HOME or ~/.openclaw/skills/compartmentalize/SKILL.md + openclaw /skills/compartmentalize/SKILL.md Writing into somebody's skills directory is a larger claim on their machine than registering an MCP server, so this module never does it silently and never @@ -35,13 +35,34 @@ #: Written next to a skill we are about to replace, when it is not ours. BACKUP_SUFFIX = ".compartment-backup" -#: Every agent known to read the Agent Skills layout, and the environment -#: variable that relocates its home. Claude Code's is CLAUDE_CONFIG_DIR, -#: which moves its skills/ with everything else. +def _openclaw_dir() -> Path | None: + """OpenClaw's config directory when the environment moves it, resolved + as OpenClaw resolves it (src/infra/config-dir.ts): OPENCLAW_STATE_DIR is + the directory; else the one holding OPENCLAW_CONFIG_PATH; else + OPENCLAW_HOME, which stands in for the home directory, with .openclaw + under it. OPENCLAW_HOME used to be taken for the directory itself, so + with it set the skill went one level above the skills/ OpenClaw loads + managed skills from.""" + state = os.environ.get("OPENCLAW_STATE_DIR", "").strip() + if state: + return Path(state).expanduser() + config = os.environ.get("OPENCLAW_CONFIG_PATH", "").strip() + if config: + return Path(config).expanduser().parent + home = os.environ.get("OPENCLAW_HOME", "").strip() + if home: + return Path(home).expanduser() / ".openclaw" + return None + + +#: Every agent known to read the Agent Skills layout, and what relocates +#: its home: the environment variable naming it, or a function working it +#: out. Claude Code's is CLAUDE_CONFIG_DIR, which moves its skills/ with +#: everything else. SKILL_TARGETS = { "claude": ("CLAUDE_CONFIG_DIR", Path.home() / ".claude"), "hermes": ("HERMES_HOME", Path.home() / ".hermes"), - "openclaw": ("OPENCLAW_HOME", Path.home() / ".openclaw"), + "openclaw": (_openclaw_dir, Path.home() / ".openclaw"), } @@ -56,6 +77,8 @@ def agent_home(target: str) -> Path | None: if entry is None: return None var, default = entry + if callable(var): + return var() or default if var and os.environ.get(var): return Path(os.environ[var]).expanduser() return default diff --git a/tests/test_integrations_skills.py b/tests/test_integrations_skills.py new file mode 100644 index 0000000..b3a809a --- /dev/null +++ b/tests/test_integrations_skills.py @@ -0,0 +1,44 @@ +"""Where each agent loads the /compartmentalize skill from.""" +import pytest + +from compartment import agent_skill + + +@pytest.fixture() +def no_openclaw_env(monkeypatch): + for var in ("OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH", "OPENCLAW_HOME"): + monkeypatch.delenv(var, raising=False) + + +def test_openclaw_home_stands_in_for_the_home_directory(no_openclaw_env, + monkeypatch, + tmp_path): + """OPENCLAW_HOME replaces the home directory and OpenClaw keeps + .openclaw under it (src/infra/config-dir.ts). It was taken for the + .openclaw directory itself, so the skill landed one level above the + skills/ OpenClaw loads.""" + monkeypatch.setenv("OPENCLAW_HOME", str(tmp_path / "h")) + assert agent_skill.skill_path("openclaw") == ( + tmp_path / "h" / ".openclaw" / "skills" / "compartmentalize" + / "SKILL.md") + + +def test_openclaw_state_dir_and_config_path_move_it(no_openclaw_env, + monkeypatch, tmp_path): + monkeypatch.setenv("OPENCLAW_HOME", str(tmp_path / "h")) + monkeypatch.setenv("OPENCLAW_CONFIG_PATH", + str(tmp_path / "cfg" / "openclaw.json")) + assert agent_skill.agent_home("openclaw") == tmp_path / "cfg" + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(tmp_path / "state")) + assert agent_skill.agent_home("openclaw") == tmp_path / "state" + + +def test_openclaw_without_any_variable_is_in_the_home_directory( + no_openclaw_env, monkeypatch, tmp_path): + monkeypatch.setitem(agent_skill.SKILL_TARGETS, "openclaw", + (agent_skill._openclaw_dir, tmp_path / ".openclaw")) + assert agent_skill.agent_home("openclaw") == tmp_path / ".openclaw" + r = agent_skill.install("openclaw") + assert r["action"] == "written" + assert agent_skill.is_current("openclaw") + assert agent_skill.remove("openclaw") is True From 924a2f3199102f1434635df75375692dde4df94b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:51 -0700 Subject: [PATCH 066/135] `compartment import` reads a file saved by PowerShell or Notepad, and refuses one that is not records by its line number instead of a traceback PowerShell 5.1, the one Windows ships, writes UTF-8 with a byte-order mark for `Out-File -Encoding utf8` and UTF-16 for a plain `>` redirect, and Notepad wrote a BOM for years. Import read the file as bare UTF-8: the first ended in JSONDecodeError "Unexpected UTF-8 BOM", the second in UnicodeDecodeError, both as tracebacks. A line that was not JSON, or a record with no "text", ended the same way, naming neither the line nor the field. The file is now decoded by its BOM (UTF-16 or UTF-8 with or without one), anything else is refused as not UTF-8, and every line is checked before the vault is opened: "FILE, line 2: not JSON (...)" or "FILE, line 1: a record needs a \"text\" string", exit 1, nothing imported. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 40 +++++++++++++++++++++++++++++++++++++++- tests/test_cli_errors.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 2a65108..ea61108 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -1179,9 +1179,47 @@ def cmd_export(args) -> None: _save_reads(v) +def _read_text_file(path: str) -> str: + """A text file the user handed the CLI, as text, whatever Windows tool + saved it. + + PowerShell 5.1 writes UTF-8 with a byte-order mark (`Out-File -Encoding + utf8`) and UTF-16 for a plain `>` redirect; Notepad wrote a BOM for + years. Read as bare UTF-8, the first gave JSONDecodeError "Unexpected + UTF-8 BOM" and the second UnicodeDecodeError, both as tracebacks. The + BOM says which it is; with none, the file is UTF-8, as `export` writes + it, and anything else is refused by name.""" + raw = Path(path).read_bytes() + try: + if raw.startswith((b"\xff\xfe", b"\xfe\xff")): + return raw.decode("utf-16") + return raw.decode("utf-8-sig") + except UnicodeDecodeError as exc: + _die(f"{path} is not UTF-8 text (byte {exc.start}): save it as " + "UTF-8 and import it again") + + +def _check_jsonl(path: str, text: str) -> None: + """Refuse a JSONL file by line number before anything is imported: a + line that is not JSON, or a record with no text, ended the import in a + traceback that named neither the file's line nor the field.""" + for no, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + rec = json.loads(line) + except ValueError as exc: + _die(f"{path}, line {no}: not JSON ({exc.msg}). Each line must " + "be one record, as `compartment export` writes them") + if not isinstance(rec, dict) or not isinstance(rec.get("text"), str): + _die(f"{path}, line {no}: a record needs a \"text\" string") + + def cmd_import(args) -> None: + text = _read_text_file(args.file) + _check_jsonl(args.file, text) v = _open_vault(args) - n = v.import_jsonl(Path(args.file).read_text(encoding="utf-8"), namespace=args.namespace) + n = v.import_jsonl(text, namespace=args.namespace) print(f"imported {n} {'record' if n == 1 else 'records'}") diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 99d0cf0..bb952e1 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -126,3 +126,37 @@ def test_a_subcommand_without_them_keeps_the_top_level_values(tmp_path, cli.main(["--vault", str(tmp_path / "v.vault"), "serve"]) assert seen[0].vault == str(tmp_path / "v.vault") assert seen[0].caller == "user" and seen[0].assert_offline is False + + +# ------------------------------------------------------- import and export -- +@pytest.fixture() +def open_vault(vault, vault_path, monkeypatch): + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + vault.lock() + return vault_path + + +@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16"]) +def test_an_import_saved_by_a_windows_tool_is_read(open_vault, tmp_path, + capsys, encoding): + """PowerShell 5.1 writes UTF-8 with a BOM, or UTF-16 for `>`: the first + ended in "Unexpected UTF-8 BOM", the second in UnicodeDecodeError.""" + f = tmp_path / "memories.jsonl" + f.write_text('{"text": "The staging database runs Postgres 16"}\n', + encoding=encoding) + cli.main(["--vault", open_vault, "import", str(f)]) + assert "imported 1 record" in capsys.readouterr().out + + +@pytest.mark.parametrize("body, said", [ + ('{"text": "fine"}\nnot json\n', "line 2: not JSON"), + ('{"txt": "misspelt"}\n', 'line 1: a record needs a "text" string'), + ("[1, 2]\n", 'line 1: a record needs a "text" string'), +]) +def test_an_import_that_is_not_records_is_refused_by_line(open_vault, + tmp_path, capsys, + body, said): + f = tmp_path / "memories.jsonl" + f.write_text(body, encoding="utf-8") + err = _ends(["--vault", open_vault, "import", str(f)], capsys) + assert said in err and "Traceback" not in err From e56f9462a32ac9dce669c3f98d53e78db7eb4a64 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:58 -0700 Subject: [PATCH 067/135] A plaintext export is written readable by its owner alone `compartment export --plaintext` writes every memory in the vault in the clear, the credentials the agent was told to keep there included, and it wrote them with Path.write_text, so with the umask's mode: 0644 on most systems. An export made into /tmp, or into a home directory other accounts may enter (Debian's default until recently), could be read by any local user. The file is now created 0600, the mode of the vault file itself, and an existing file it overwrites is set to 0600 before the data goes in. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 15 ++++++++++++++- tests/test_cli_errors.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index ea61108..d236b41 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -1168,7 +1168,20 @@ def cmd_export(args) -> None: data = v.export_jsonl() if args.plaintext: print("WARNING: exporting PLAINTEXT memories to disk", file=sys.stderr) - Path(args.out).write_text(data, encoding="utf-8") + # Readable by this user alone, as the vault file is (VAULT_MODE): + # this is every memory the vault holds, credentials included, in + # the clear, and write_text left it to the umask, 0644 on most + # systems, so any local account could read an export made into + # /tmp or a shared home. fchmod for a file that already existed, + # whose mode O_CREAT leaves as it was. + fd = os.open(args.out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as f: + if hasattr(os, "fchmod"): + try: + os.fchmod(fd, 0o600) + except OSError: + pass + f.write(data) # Count the lines actually written rather than newline characters: # export_jsonl ends with a trailing newline only when it is non-empty, # so a newline count is one off the moment that changes. diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index bb952e1..8bf8397 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -160,3 +160,17 @@ def test_an_import_that_is_not_records_is_refused_by_line(open_vault, f.write_text(body, encoding="utf-8") err = _ends(["--vault", open_vault, "import", str(f)], capsys) assert said in err and "Traceback" not in err + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +def test_a_plaintext_export_is_readable_by_its_owner_alone(open_vault, + tmp_path, capsys): + """Every memory in the clear, credentials included, was written with the + umask's 0644: any account on the machine could read an export left in + /tmp or in a home other users can enter.""" + out = tmp_path / "export.jsonl" + out.write_text("an older export\n") + os.chmod(out, 0o644) + cli.main(["--vault", open_vault, "export", str(out), "--plaintext"]) + assert os.stat(out).st_mode & 0o777 == 0o600 + assert "an older export" not in out.read_text(encoding="utf-8") From 0c1f72b0eb151cdf6b1d253792f1140823dd031e Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:58:18 -0700 Subject: [PATCH 068/135] Codex lists the plugin as Compartment, under Productivity, with starter prompts and a logo `codex plugin marketplace add MaxFreedomPollard/Compartment` does read .claude-plugin/marketplace.json, as the README says (Codex looks for .agents/plugins, then .claude-plugin, then .cursor-plugin). But for a plugin at the repository root Codex loads the root plugin.json first whenever it declares the Agent Plugins 1.0.0 schema, and that manifest has no interface fields, so Codex derived one: display name "compartment", category "Other", the one-line description twice, no prompts and no logo. Agent Plugins reserves `extensions` for exactly this, and Codex reads `extensions["com.openai"]` with the same interface block as its own .codex-plugin/plugin.json. The manifest now carries one: display name, short and long descriptions, developer, Productivity, the Read and Write capabilities, the website and privacy policy pages, two starter prompts and docs/images/logo-400.png. Hermes and OpenClaw ignore namespaces they do not own, as the specification requires, and the file still validates against the published 1.0.0 schema. The new tests hold plugin.json and mcp.json inside the closed 1.0.0 schemas (a host must reject the plugin on any other field), keep the server command a single bare executable as Codex, Hermes and OpenClaw require, and check the Codex interface against Codex's own limits. Before the fix: test_codex_gets_a_listing_of_its_own_from_the_agent_plugins_manifest E KeyError: 'extensions' Co-Authored-By: Claude Opus 5.5 --- plugin.json | 24 +++++++++++++++- tests/test_plugin_manifests.py | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/plugin.json b/plugin.json index ea770df..e877fd5 100644 --- a/plugin.json +++ b/plugin.json @@ -19,5 +19,27 @@ "vector-search", "privacy", "mcp" - ] + ], + "extensions": { + "com.openai": { + "interface": { + "displayName": "Compartment", + "shortDescription": "Encrypted, offline memory for agents", + "longDescription": "One encrypted memory vault on your own machine, shared by Codex, Claude, Gemini CLI, Hermes, OpenClaw and every other MCP client. Recall before answering, store what is worth keeping; no network, no API key, no account. Requires `pip install compartment && compartment init` first.", + "developerName": "Max Freedom Pollard", + "category": "Productivity", + "capabilities": [ + "Read", + "Write" + ], + "websiteURL": "https://maxfreedompollard.github.io/Compartment/", + "privacyPolicyURL": "https://maxfreedompollard.github.io/Compartment/privacy", + "defaultPrompt": [ + "What do you remember about this project?", + "Remember the decisions we made today." + ], + "logo": "./docs/images/logo-400.png" + } + } + } } diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index 2715310..19e6df0 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -83,3 +83,55 @@ def test_listing_pages_never_state_a_client_count_that_is_wrong(): continue for n in re.findall(r"\b(\d+) (?:MCP )?clients\b", p.read_text(encoding="utf-8")): assert int(n) == len(clients.CLIENTS), (rel, n) + + +# ------------------------------------------- Agent Plugins (Hermes, Codex) -- +#: Agent Plugins 1.0.0 is still the published release (1.1.0 is a working +#: draft that Codex, Hermes and OpenClaw do not accept yet), and both of its +#: schemas are closed: a host must reject the plugin on any field outside +#: these, so this is the whole vocabulary. +AGENT_PLUGIN_FIELDS = {"$schema", "name", "version", "description", "author", + "homepage", "repository", "license", "keywords", + "extensions"} +AGENT_PLUGIN_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json" +AGENT_MCP_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json" + + +def test_the_agent_plugins_manifest_stays_inside_the_closed_schema(): + d = _json("plugin.json") + assert set(d) <= AGENT_PLUGIN_FIELDS, set(d) - AGENT_PLUGIN_FIELDS + assert d["$schema"] == AGENT_PLUGIN_SCHEMA + assert re.fullmatch(r"(?!.*(?:--|\.\.))[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?", + d["name"]) and len(d["name"]) <= 64 + assert set(d["author"]) <= {"name", "email", "url"} + assert all(isinstance(v, dict) for v in d.get("extensions", {}).values()) + + +def test_the_agent_plugins_mcp_json_stays_inside_the_closed_schema(): + d = _json("mcp.json") + assert set(d) == {"$schema", "mcpServers"} + assert d["$schema"] == AGENT_MCP_SCHEMA + for server in d["mcpServers"].values(): + assert set(server) <= {"type", "command", "args", "env", "cwd"} + assert server["type"] == "stdio" + # one executable token, never a path or a shell line: Codex, Hermes + # and OpenClaw all refuse anything else for this format + assert re.fullmatch(r"[A-Za-z0-9._-]+", server["command"]) + cwd = server.get("cwd", "./") + assert re.match(r"(?:\./|\$\{PLUGIN_ROOT\}(?:/|$)|\$\{PLUGIN_DATA\}(?:/|$))", cwd) + + +def test_codex_gets_a_listing_of_its_own_from_the_agent_plugins_manifest(): + """Codex installs from .claude-plugin/marketplace.json but loads the root + plugin.json first when it declares the Agent Plugins schema, and without + a com.openai extension it lists the plugin as "compartment", category + Other, with no prompt or logo. The extension is where Codex looks.""" + ui = _json("plugin.json")["extensions"]["com.openai"]["interface"] + for field in ("displayName", "shortDescription", "longDescription", + "developerName", "category", "capabilities", "defaultPrompt"): + assert ui[field], field + assert 1 <= len(ui["defaultPrompt"]) <= 3 + assert all(len(p) <= 128 for p in ui["defaultPrompt"]) + for field in ("websiteURL", "privacyPolicyURL"): + assert ui[field].startswith("https://") + assert ui["logo"].startswith("./") and (ROOT / ui["logo"]).is_file() From 67ba6112268a9d08af44815f5ee773a3f551cac5 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:59:07 -0700 Subject: [PATCH 069/135] `atomize --out` writes its decrypted listing readable by its owner alone, as export now does `compartment atomize --out FILE --plaintext` writes the over-limit memories decrypted, behind the same confirmation export asks for, and like export it left the file to the umask, 0644 on most systems. Both now go through one helper that creates the file 0600 and sets an existing one to 0600 before writing. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 37 +++++++++++++++++++++---------------- tests/test_cli_errors.py | 11 +++++++++++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index d236b41..0eb8e2a 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -800,6 +800,25 @@ def cmd_status(args) -> None: _print(v.status()) +def _write_private(path: str, text: str) -> None: + """Write decrypted memory text where only this user can read it. + + Readable by its owner alone, as the vault file is (VAULT_MODE): an + export is every memory the vault holds, credentials included, in the + clear, and a plain write left it to the umask, 0644 on most systems, so + any local account could read one made into /tmp or a shared home. + fchmod for a file that already existed, whose mode O_CREAT leaves as it + was.""" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as f: + if hasattr(os, "fchmod"): + try: + os.fchmod(fd, 0o600) + except OSError: + pass + f.write(text) + + def cmd_store(args) -> None: v = _open_vault(args) supersedes = getattr(args, "supersedes", None) @@ -852,8 +871,7 @@ def cmd_atomize(args) -> None: lines = "".join(json.dumps(r, sort_keys=True) + "\n" for r in rows) if args.out: try: - with open(args.out, "w", encoding="utf-8") as f: - f.write(lines) + _write_private(args.out, lines) except OSError as exc: _die(f"cannot write {args.out}: {exc}") else: @@ -1168,20 +1186,7 @@ def cmd_export(args) -> None: data = v.export_jsonl() if args.plaintext: print("WARNING: exporting PLAINTEXT memories to disk", file=sys.stderr) - # Readable by this user alone, as the vault file is (VAULT_MODE): - # this is every memory the vault holds, credentials included, in - # the clear, and write_text left it to the umask, 0644 on most - # systems, so any local account could read an export made into - # /tmp or a shared home. fchmod for a file that already existed, - # whose mode O_CREAT leaves as it was. - fd = os.open(args.out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - with os.fdopen(fd, "w", encoding="utf-8") as f: - if hasattr(os, "fchmod"): - try: - os.fchmod(fd, 0o600) - except OSError: - pass - f.write(data) + _write_private(args.out, data) # Count the lines actually written rather than newline characters: # export_jsonl ends with a trailing newline only when it is non-empty, # so a newline count is one off the moment that changes. diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 8bf8397..8e2b826 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -174,3 +174,14 @@ def test_a_plaintext_export_is_readable_by_its_owner_alone(open_vault, cli.main(["--vault", open_vault, "export", str(out), "--plaintext"]) assert os.stat(out).st_mode & 0o777 == 0o600 assert "an older export" not in out.read_text(encoding="utf-8") + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +def test_an_atomize_listing_is_readable_by_its_owner_alone(open_vault, + tmp_path, capsys): + """`atomize --out` writes the over-limit memories decrypted, behind the + same --plaintext confirmation as export, and with the umask's mode.""" + out = tmp_path / "blobs.jsonl" + cli.main(["--vault", open_vault, "atomize", "--out", str(out), + "--plaintext"]) + assert os.stat(out).st_mode & 0o777 == 0o600 From f8e1c86a8b78a991fe9e8683479b86ec8deeb837 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:59:19 -0700 Subject: [PATCH 070/135] The Add to LM Studio button registers the server under the same caller `integrate lmstudio` writes The deeplink on add/lmstudio.html, and the manual block beside it, started the server with `--caller lm-studio`, while `compartment integrate lmstudio` (and the Connect button behind it) writes `--caller lmstudio`. The caller is the name the audit log records and the key an ACL entry in the vault config is written against, so a vault configured for `lmstudio` treated a button-installed LM Studio as an unknown client, and the same app showed up under two names. Both now say `lmstudio`. The base64 padding in the link is percent-encoded the way LM Studio's own link generator writes it. The new tests decode both one-click pages: the meta refresh and the fallback link must be the same URL, the LM Studio entry must match the manual block on the page and use integrate's caller, and the goose link must use uvx, which is one of the six launchers goose accepts from a deeplink (a bare `compartment` would be refused), with goose's caller. Before the fix: test_the_lm_studio_button_registers_what_integrate_registers E AssertionError: assert 'lm-studio' == 'lmstudio' Co-Authored-By: Claude Opus 5.5 --- add/lmstudio.html | 6 ++-- tests/test_plugin_manifests.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/add/lmstudio.html b/add/lmstudio.html index 8d8a493..153965e 100644 --- a/add/lmstudio.html +++ b/add/lmstudio.html @@ -7,20 +7,20 @@ - + Add Compartment to LM Studio

Add Compartment to LM Studio

-

The app should open now with Compartment ready to add. If it did not, open it by hand.

+

The app should open now with Compartment ready to add. If it did not, open it by hand.

Before the first run, install the package and create the vault once: pip install compartment && compartment init. The vault is encrypted under a passphrase you choose, and nothing leaves your machine.

Manual setup: in LM Studio open the MCP configuration (Program, then Install, then Edit mcp.json) and add:

{
   "mcpServers": {
     "compartment": {
       "command": "compartment",
-      "args": ["--caller", "lm-studio", "serve"]
+      "args": ["--caller", "lmstudio", "serve"]
     }
   }
 }
diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index 19e6df0..359e3e5 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -135,3 +135,53 @@ def test_codex_gets_a_listing_of_its_own_from_the_agent_plugins_manifest(): for field in ("websiteURL", "privacyPolicyURL"): assert ui[field].startswith("https://") assert ui["logo"].startswith("./") and (ROOT / ui["logo"]).is_file() + + +# ------------------------------------------------------ one-click pages ----- +def _deeplinks(rel: str) -> list[str]: + """The meta refresh and the fallback anchor, which must be one link.""" + import html + s = (ROOT / rel).read_text(encoding="utf-8") + refresh = re.search(r'http-equiv="refresh" content="0; url=([^"]+)"', s).group(1) + anchors = re.findall(r' str: + return args[args.index("--caller") + 1] + + +def test_the_lm_studio_button_registers_what_integrate_registers(): + """The caller names the client in the audit log and is the key an ACL + entry is written against; `integrate lmstudio` writes `lmstudio`, so a + vault configured for that caller must see the same name from the + button. The manual block on the page is the same entry.""" + import base64 + from urllib.parse import parse_qs, urlsplit + from compartment import clients + links = _deeplinks("add/lmstudio.html") + assert len(set(links)) == 1, links + q = parse_qs(urlsplit(links[0]).query) + assert q["name"] == ["compartment"] + entry = json.loads(base64.b64decode(q["config"][0])) + assert entry["command"] == "compartment" and entry["args"][-1] == "serve" + assert _caller(entry["args"]) == clients.CLIENTS["lmstudio"].key + page = (ROOT / "add/lmstudio.html").read_text(encoding="utf-8") + manual = json.loads(re.search(r"
(.*?)
", page, re.S).group(1)) + assert manual["mcpServers"]["compartment"] == entry + + +def test_the_goose_button_uses_a_command_goose_accepts(): + """goose refuses a deeplink whose command is not one of its launchers + (cu, docker, jbang, npx, uvx, goose), so the bare `compartment` the + other manifests use would be rejected; uvx runs the PyPI package.""" + from urllib.parse import parse_qs, urlsplit + from compartment import clients + links = _deeplinks("add/goose.html") + assert len(set(links)) == 1, links + q = parse_qs(urlsplit(links[0]).query) + assert q["cmd"] == ["uvx"] and q["name"] == ["Compartment"] + assert q["arg"][0] == "compartment" and q["arg"][-1] == "serve" + assert _caller(q["arg"]) == clients.CLIENTS["goose"].key + page = (ROOT / "add/goose.html").read_text(encoding="utf-8") + assert f"
uvx {' '.join(q['arg'])}
" in page From 412e16f70c786eba781e706cae603fdfe6fa30e0 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:59:41 -0700 Subject: [PATCH 071/135] Memory Claude Code keeps in the directory its autoMemoryDirectory setting names is imported and captured like the rest Since Claude Code 2.1.74 the autoMemoryDirectory setting moves auto memory out of projects//memory to any directory the user names. `import-claude`, the import `integrate claude` runs, and the capture hook looked only under projects/, and the hook also wanted `memory` and `.claude` in the path, so every note written to a moved memory directory was invisible to all three. claude_memory.auto_memory_dir() reads the setting from the user settings.json beside the projects directory (Claude Code ignores it in a project's checked-in settings), expanding ~/ and ignoring a relative value. discover() includes that directory's notes when no root is given, and the hook takes a note there for a memory file. The MEMORY.md index is still skipped. Without the setting nothing changes. Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_hooks.py | 7 ++- src/compartment/claude_memory.py | 39 +++++++++++++-- tests/test_integrations_claude_memory.py | 64 ++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/test_integrations_claude_memory.py diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index bfc784a..4ea5a92 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -221,10 +221,15 @@ def is_memory_file(path: str | Path) -> bool: A memory directory under the CLAUDE_CONFIG_DIR Claude Code was started with counts as well: that directory need not be called .claude, and - when it was not, every memory write was passed over.""" + when it was not, every memory write was passed over. So does the + directory the autoMemoryDirectory setting names.""" p = Path(path) if p.suffix.lower() != ".md" or p.name == claude_memory.INDEX_NAME: return False + custom = claude_memory.auto_memory_dir() + if custom is not None and os.path.abspath(p.parent) == os.path.abspath( + custom): + return True # autoMemoryDirectory's notes parts = [x.lower() for x in p.parts] if "memory" not in parts: return False diff --git a/src/compartment/claude_memory.py b/src/compartment/claude_memory.py index 15acf5c..15dc1d1 100644 --- a/src/compartment/claude_memory.py +++ b/src/compartment/claude_memory.py @@ -68,9 +68,42 @@ def claude_config_dir() -> Path: _FRONTMATTER = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", re.DOTALL) +def auto_memory_dir() -> Path | None: + """The directory Claude Code's `autoMemoryDirectory` setting moves + auto memory to, from the user settings.json beside DEFAULT_ROOT, or + None when it is not set. + + Since Claude Code 2.1.74 that one setting replaces + projects//memory/ with any directory the user names (~/ + expanded; it is ignored in a project's checked-in settings, so only + the user's own file is read). Memory written there was invisible to + both the import and the capture hook, which looked only under + projects/. A relative value is ignored: there is no project here to + resolve it against.""" + try: + from .clients import _read_json + value = _read_json(DEFAULT_ROOT.parent / "settings.json").get( + "autoMemoryDirectory") + except (OSError, ValueError): + return None + if not isinstance(value, str) or not value.strip(): + return None + d = Path(value.strip()).expanduser() + return d if d.is_absolute() else None + + def discover(root: Path | str | None = None) -> list[Path]: - """Every Claude Code memory file, sorted, index files excluded.""" - root = Path(root) if root else DEFAULT_ROOT + """Every Claude Code memory file, sorted, index files excluded: under + `root`, or by default under DEFAULT_ROOT and in the directory + autoMemoryDirectory names.""" + if root is None: + out = discover(DEFAULT_ROOT) + extra = auto_memory_dir() + if extra is not None and extra.is_dir(): + out += [p for p in sorted(extra.glob("*.md")) + if p.name != INDEX_NAME and p.is_file() and p not in out] + return out + root = Path(root) if not root.exists(): return [] if root.name == "memory" and root.is_dir(): # a memory dir directly @@ -224,4 +257,4 @@ def default_root_exists() -> bool: __all__ = ["DEFAULT_ROOT", "discover", "parse", "import_files", "pending", "default_root_exists", "IMPORTANCE", "file_tag", - "claude_config_dir"] + "claude_config_dir", "auto_memory_dir"] diff --git a/tests/test_integrations_claude_memory.py b/tests/test_integrations_claude_memory.py new file mode 100644 index 0000000..81d00d4 --- /dev/null +++ b/tests/test_integrations_claude_memory.py @@ -0,0 +1,64 @@ +"""Claude Code's auto memory, wherever Claude Code keeps it.""" +import json + +import pytest + +from compartment import claude_hooks, claude_memory + + +@pytest.fixture() +def claude_home(tmp_path, monkeypatch): + root = tmp_path / ".claude" / "projects" + root.mkdir(parents=True) + monkeypatch.setattr(claude_memory, "DEFAULT_ROOT", root) + return tmp_path / ".claude" + + +def _settings(claude_home, **values): + (claude_home / "settings.json").write_text(json.dumps(values), + encoding="utf-8") + + +def _note(d, name, body="Ana prefers tea."): + d.mkdir(parents=True, exist_ok=True) + p = d / name + p.write_text(f"---\nname: {p.stem}\ndescription: x\nmetadata:\n type: " + f"user\n---\n{body}\n", encoding="utf-8") + return p + + +def test_memory_in_the_auto_memory_directory_is_found_and_captured( + claude_home, tmp_path): + """autoMemoryDirectory (Claude Code 2.1.74) moves auto memory out of + projects//memory to any directory. Notes written there were + invisible to `import-claude` and to the capture hook.""" + custom = tmp_path / "notes" / "claude" + _settings(claude_home, autoMemoryDirectory=str(custom)) + moved = _note(custom, "tea.md") + (custom / claude_memory.INDEX_NAME).write_text("- tea\n", + encoding="utf-8") + usual = _note(claude_home / "projects" / "p" / "memory", "old.md") + + assert claude_memory.auto_memory_dir() == custom + assert claude_memory.discover() == [usual, moved] + assert claude_hooks.is_memory_file(moved) is True + assert claude_hooks.is_memory_file(custom / claude_memory.INDEX_NAME) is ( + False) + assert claude_hooks.is_memory_file(tmp_path / "notes" / "x.md") is False + + +def test_an_auto_memory_directory_under_home_is_expanded(claude_home, + tmp_path, + monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + _settings(claude_home, autoMemoryDirectory="~/mem") + assert claude_memory.auto_memory_dir() == tmp_path / "mem" + + +@pytest.mark.parametrize("value", [None, "", "relative/mem", 3]) +def test_no_or_unusable_auto_memory_directory_changes_nothing(claude_home, + value): + _settings(claude_home, **({} if value is None + else {"autoMemoryDirectory": value})) + assert claude_memory.auto_memory_dir() is None From 2134d60ef964cc27ac72424adc5a2bd49aaefa80 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:59:45 -0700 Subject: [PATCH 072/135] A Claude Code memory note saved with a byte order mark is imported with its frontmatter instead of storing the frontmatter as the fact A note edited in a Windows editor that writes a UTF-8 byte order mark began with the mark, so the opening `---` was not at the start of the text, the frontmatter went unrecognised, and the import and the capture hook stored the whole block, name, description and type lines included, as the fact, with none of its tags or its importance tier. Notes are now read as utf-8-sig, which drops a leading mark and changes nothing else. Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_memory.py | 5 ++++- tests/test_integrations_claude_memory.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/compartment/claude_memory.py b/src/compartment/claude_memory.py index 15dc1d1..ca3cefb 100644 --- a/src/compartment/claude_memory.py +++ b/src/compartment/claude_memory.py @@ -145,7 +145,10 @@ def _parse_frontmatter(raw: str) -> tuple[dict, str]: def parse(path: Path) -> dict: """One memory file -> the record Compartment should store.""" - raw = path.read_text(encoding="utf-8", errors="replace") + # utf-8-sig: a note saved by a Windows editor with a byte order mark + # still opens with its frontmatter, instead of the mark hiding it and + # the whole block being stored as the fact. + raw = path.read_text(encoding="utf-8-sig", errors="replace") fields, body = _parse_frontmatter(raw) name = fields.get("name") or path.stem description = fields.get("description", "") diff --git a/tests/test_integrations_claude_memory.py b/tests/test_integrations_claude_memory.py index 81d00d4..713abf4 100644 --- a/tests/test_integrations_claude_memory.py +++ b/tests/test_integrations_claude_memory.py @@ -62,3 +62,16 @@ def test_no_or_unusable_auto_memory_directory_changes_nothing(claude_home, _settings(claude_home, **({} if value is None else {"autoMemoryDirectory": value})) assert claude_memory.auto_memory_dir() is None + + +def test_a_note_saved_with_a_byte_order_mark_keeps_its_frontmatter( + tmp_path): + """The mark hid the opening ---, so the frontmatter was not recognised + and the whole block was stored as the fact, tagged with nothing.""" + p = tmp_path / "memory" / "tea.md" + p.parent.mkdir() + p.write_bytes(b"\xef\xbb\xbf---\nname: tea\ndescription: Ana likes tea" + b"\nmetadata:\n type: user\n---\nGreen, no sugar.\n") + rec = claude_memory.parse(p) + assert rec["name"] == "tea" and "user" in rec["tags"] + assert "---" not in rec["text"] and "" not in rec["text"] From 50b25d8ae4172dc31105ae57ac50af1682851135 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:00:16 -0700 Subject: [PATCH 073/135] On Windows the Claude Code capture hook stores notes kept under a home directory with a non-ASCII name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `compartment hook capture` read Claude Code's PostToolUse payload through sys.stdin, which Python decodes with the locale's encoding: on Windows the ANSI code page, not UTF-8. JSON.stringify leaves non-ASCII characters as they are, so for a user named José the note path C:\Users\José\.claude\... arrived as C:\Users\José\.claude\..., which does not exist, and every capture on such a machine was dropped silently as "file no longer exists". The payload is now read as bytes and decoded as UTF-8 (a BOM tolerated). Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 21 +++++++++++++++++- tests/test_cli_hook_stdin.py | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/test_cli_hook_stdin.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 0eb8e2a..f4ad18d 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -10,6 +10,7 @@ import datetime import getpass import hashlib +import io import json import os import shutil @@ -1046,11 +1047,29 @@ def cmd_menubar(args) -> None: sys.exit(app.run(args.vault, show=args.show, render_to=args.render)) +def _hook_payload() -> io.StringIO: + """The hook's JSON from stdin, decoded as the UTF-8 Claude Code writes. + + Python decodes a piped stdin with the locale's encoding, which on + Windows is the ANSI code page: JSON.stringify leaves non-ASCII as it + is, so the path of a memory note under C:\\Users\\José\\.claude arrived + as C:\\Users\\José\\.claude, which does not exist, and every capture on + such a machine was dropped as "file no longer exists".""" + buf = getattr(sys.stdin, "buffer", None) + try: + raw = buf.read() if buf is not None else sys.stdin.read() + except (OSError, ValueError): + raw = b"" + if isinstance(raw, bytes): + raw = raw.decode("utf-8-sig", errors="replace") + return io.StringIO(raw) + + def cmd_hook(args) -> None: """Deterministic capture, so remembering does not depend on the model choosing to call a tool.""" if args.hook_cmd == "capture": - res = claude_hooks.capture(vault_path=args.vault) + res = claude_hooks.capture(_hook_payload(), vault_path=args.vault) if args.json: _print(res) sys.exit(0) # never fail a user's edit over a memory write diff --git a/tests/test_cli_hook_stdin.py b/tests/test_cli_hook_stdin.py new file mode 100644 index 0000000..cb76ed7 --- /dev/null +++ b/tests/test_cli_hook_stdin.py @@ -0,0 +1,41 @@ +"""`compartment hook capture` reads Claude Code's hook payload as the UTF-8 +it is written in, whatever the locale would decode stdin as.""" +import io +import json + +import pytest + +from compartment import cli, session +from compartment.vault import Vault + +from conftest import PASS + + +def test_a_memory_note_under_a_non_ascii_home_is_captured(tmp_path, + vault_path, + monkeypatch, + capsys): + """On Windows Python decodes a piped stdin with the ANSI code page, so + the path of a note under C:\\Users\\José\\.claude arrived as + C:\\Users\\José\\.claude and every capture there was dropped.""" + note = (tmp_path / "José" / ".claude" / "projects" / "app" / "memory" + / "deploy.md") + note.parent.mkdir(parents=True) + note.write_text("The deploy key lives in 1Password under 'prod'.\n", + encoding="utf-8") + payload = json.dumps({"hook_event_name": "PostToolUse", + "tool_name": "Write", + "tool_input": {"file_path": str(note)}}, + ensure_ascii=False).encode("utf-8") + # stdin as a Windows pipe presents it: the bytes, decoded as cp1252. + monkeypatch.setattr("sys.stdin", + io.TextIOWrapper(io.BytesIO(payload), + encoding="cp1252")) + v = Vault.create(vault_path, PASS, creator="t") + session.store(vault_path, v._master) + v.lock() + with pytest.raises(SystemExit) as e: + cli.main(["--vault", vault_path, "hook", "capture", "--json"]) + assert e.value.code == 0 + res = json.loads(capsys.readouterr().out) + assert res["stored"] is True, res From 5b5c83efe26df2691ead309cfd1e4a84f2923019 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:00:39 -0700 Subject: [PATCH 074/135] The Windows installer picks a 64-bit Python its dependencies have wheels for, stops on a failed pip install, and runs when piped into PowerShell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The script ran whatever `py` started, else `python`: the default Python, which may be 32-bit (no onnxruntime build at all), 3.10, a 3.15 newer than the wheels, or 3.11 on Windows on ARM, which has no pyyaml wheel. pip then failed on a dependency with no wheel, and because a native command's exit code does not stop PowerShell, the script carried on to "compartment was installed but cannot be found". - It now probes `py -3`, `py -3.14` ... `py -3.11` (and the -arm64 tags on ARM64), `python` and `python3`, or $env:PYTHON, and takes the first inside the matrix tools/wheel_coverage.py checks: 64-bit Python 3.11-3.14 on x64, 3.12-3.14 on ARM64 (an x64 Python under emulation counts as x64). Outside it, it says which Python to install. The probe runs from a file, since Windows PowerShell 5.1 mangles a -c argument with double quotes, and with stderr contained, since the Store's python.exe stand-in and a missing launcher write there and 5.1 makes that terminating under "Stop". - pip's exit code is checked, and so is init's. - Run as `irm ... | iex`, $PSScriptRoot is empty and Split-Path refused it, ending the script; it now installs from PyPI then, and from a checkout only when its pyproject names compartment. A failure there no longer calls `exit`, which would have closed the user's PowerShell window. - An activated virtualenv is installed into without --user, which pip refuses there. An existing vault is left alone, so a re-run upgrades instead of ending in init's "already exists"; --upgrade makes it one. - Paths are read from Python as UTF-8, so a profile under a non-ASCII user name (C:\Users\José) is found rather than mis-decoded through the console's OEM code page, and the PATH command it prints escapes a quote in the user name. - The header says how to run it past the default execution policy for this one run. PowerShell is not available here, so the script itself was not run; its probe, which makes the decision, is tested under Python with the interpreter facts stood in for, and failed before (there was none): test_the_windows_installer_picks_a_python_inside_the_wheel_matrix E AssertionError: install-windows.ps1 has no interpreter probe Co-Authored-By: Claude Opus 5.5 --- install/install-windows.ps1 | 214 ++++++++++++++++++++++++++---- tests/test_platform_installers.py | 52 +++++++- 2 files changed, 236 insertions(+), 30 deletions(-) diff --git a/install/install-windows.ps1 b/install/install-windows.ps1 index 51a1224..118b82f 100644 --- a/install/install-windows.ps1 +++ b/install/install-windows.ps1 @@ -1,18 +1,143 @@ -# Compartment installer - Windows (PowerShell) -# Installs the package, creates + unlocks a vault, -# and prints the Hermes selection step. Fully offline after this script. +# Compartment installer - Windows (Windows PowerShell 5.1 or PowerShell 7) +# +# Installs the package, creates and unlocks a vault, connects the agents it +# finds, and puts Compartment in the tray. Fully offline after this script. +# +# powershell -ExecutionPolicy Bypass -File install\install-windows.ps1 +# +# -ExecutionPolicy Bypass applies to this one run: Windows' default policy +# refuses unsigned scripts, and changing it for good is the user's call. +# Set $env:PYTHON to a python.exe to choose the interpreter. Running it again +# upgrades Compartment and leaves an existing vault alone. $ErrorActionPreference = "Stop" Write-Host "Compartment installer (Windows)" -$py = if (Get-Command py -ErrorAction SilentlyContinue) { "py" } else { "python" } -& $py --version *> $null -if ($LASTEXITCODE -ne 0) { Write-Error "Python not found. Install from python.org (check 'Add to PATH')."; exit 1 } -$repoRoot = Split-Path -Parent $PSScriptRoot -if (Test-Path (Join-Path $repoRoot "pyproject.toml")) { - & $py -m pip install --user $repoRoot +function Stop-Install([string]$message) { + Write-Host "" + Write-Host $message + # `exit` would close the PowerShell window of someone who ran this as + # `irm ... | iex`, where there is no script file to exit from. + if ($PSCommandPath) { exit 1 } else { throw "Compartment was not installed." } +} + +# --- 1. A Python that can run Compartment ------------------------------------ +# +# Compartment's compiled dependencies publish Windows wheels for 64-bit +# Python 3.11 to 3.14 on x64, and 3.12 to 3.14 on ARM64 (no pyyaml wheel for +# 3.11 there); tools/wheel_coverage.py checks that matrix in CI. 32-bit +# Python has no onnxruntime build at all. An x64 Python on an ARM64 machine +# runs under emulation and takes the x64 wheels. Anything else fails in pip +# with a bare "No matching distribution found", so each candidate is probed +# first and the first one inside the matrix is used. +# +# The probe is a file rather than `-c`: Windows PowerShell 5.1 passes an +# argument with double quotes in it to a native program mangled. +$probe = @' +import sys, sysconfig +v = sys.version_info[:2] +plat = sysconfig.get_platform() +if plat == 'win32': + print('32-bit Python %d.%d' % v) + sys.exit(2) +low = (3, 12) if plat == 'win-arm64' else (3, 11) +if not (low <= v <= (3, 14)): + print('Python %d.%d (%s)' % (v[0], v[1], plat)) + sys.exit(1) +print('ok') +'@ +$probeFile = Join-Path ([System.IO.Path]::GetTempPath()) "compartment-probe-$PID.py" +Set-Content -LiteralPath $probeFile -Value $probe -Encoding ASCII + +function Invoke-Probe($cand) { + # Local to this function: a candidate that is missing, or the Microsoft + # Store's python.exe stand-in, writes to stderr, and Windows PowerShell + # turns redirected stderr into a terminating error under "Stop". + $ErrorActionPreference = "Continue" + if (-not (Get-Command $cand.Exe -ErrorAction SilentlyContinue)) { return $null } + try { + $a = $cand.Args + $out = & $cand.Exe @a $probeFile 2>$null + return @{ Code = $LASTEXITCODE; Out = ((@($out) -join " ").Trim()) } + } catch { + return $null + } +} + +$os = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE } +$candidates = @() +if ($env:PYTHON) { + $candidates += @{ Exe = $env:PYTHON; Args = @() } } else { - & $py -m pip install --user compartment + if ($os -eq "ARM64") { + foreach ($v in "3.14", "3.13", "3.12") { $candidates += @{ Exe = "py"; Args = @("-$v-arm64") } } + } + foreach ($v in "3", "3.14", "3.13", "3.12", "3.11") { $candidates += @{ Exe = "py"; Args = @("-$v") } } + $candidates += @{ Exe = "python"; Args = @() } + $candidates += @{ Exe = "python3"; Args = @() } +} + +$exe = $null +$pyArgs = @() +$found = @() +try { + foreach ($cand in $candidates) { + $r = Invoke-Probe $cand + if ($null -eq $r) { continue } + $label = (@($cand.Exe) + $cand.Args) -join " " + if ($r.Code -eq 0) { $exe = $cand.Exe; $pyArgs = $cand.Args; break } + $found += "$label is $(if ($r.Out) { $r.Out } else { 'not runnable' })" + } +} finally { + Remove-Item -LiteralPath $probeFile -ErrorAction SilentlyContinue +} + +if (-not $exe) { + $range = if ($os -eq "ARM64") { "3.12 to 3.14 (ARM64), or 3.11 to 3.14 (x64)" } else { "3.11 to 3.14" } + $what = if ($found.Count) { "Found: " + ($found -join "; ") + "." } else { "None was found." } + Stop-Install ("Compartment needs 64-bit Python $range. $what`n" + + "Install it from python.org, or: winget install Python.Python.3.13`n" + + "then run this script again, or name it with `$env:PYTHON.") +} +Write-Host "Using $((@($exe) + $pyArgs) -join ' ')" + +function Get-PyText([string]$code) { + # UTF-8 both ways, so a profile path with a non-ASCII user name (C:\Users\ + # José) comes back as itself rather than in the console's OEM code page. + $saved = $null + try { $saved = [Console]::OutputEncoding; [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch { } + try { + $out = & $exe @pyArgs -X utf8 -c $code + } finally { + if ($saved) { try { [Console]::OutputEncoding = $saved } catch { } } + } + return ((@($out) -join "`n").Trim()) +} + +# --- 2. Install --------------------------------------------------------------- +# +# This checkout when the script is run from one, otherwise the release on +# PyPI. Run as `irm ... | iex` there is no script file and $PSScriptRoot is +# empty, which Split-Path refused, ending the script under "Stop". +$src = "compartment" +if ($PSScriptRoot) { + $repoRoot = Split-Path -Parent $PSScriptRoot + $pyproject = Join-Path $repoRoot "pyproject.toml" + if ((Test-Path -LiteralPath $pyproject) -and + (Select-String -LiteralPath $pyproject -Pattern '^name = "compartment"$' -Quiet)) { + $src = $repoRoot + } +} + +# pip refuses --user inside a virtualenv, so an activated one is installed +# into directly; otherwise --user, as before. Windows Pythons are not +# externally managed. +$kind = Get-PyText "import sys; print('venv' if sys.prefix != sys.base_prefix else 'plain')" +$pipArgs = @("-m", "pip", "install", "--upgrade") +if ($kind -ne "venv") { $pipArgs += "--user" } +& $exe @pyArgs @pipArgs $src +if ($LASTEXITCODE -ne 0) { + Stop-Install "pip could not install Compartment (exit code $LASTEXITCODE); its reason is above." } # `pip install --user` puts compartment.exe in the interpreter's own user @@ -20,8 +145,11 @@ if (Test-Path (Join-Path $repoRoot "pyproject.toml")) { # on a clean machine. Ask the interpreter where that is and use it, so the # rest of this script runs instead of dying on "compartment is not # recognized". -$userBin = & $py -c "import sysconfig; print(sysconfig.get_path('scripts', scheme='nt_user'))" -$userBin = "$userBin".Trim() +if ($kind -eq "venv") { + $userBin = Get-PyText "import sysconfig; print(sysconfig.get_path('scripts'))" +} else { + $userBin = Get-PyText "import sysconfig; print(sysconfig.get_path('scripts', scheme='nt_user'))" +} $onPathAlready = [bool](Get-Command compartment -ErrorAction SilentlyContinue) if ($userBin -and (Test-Path -LiteralPath $userBin)) { $env:Path = "$userBin;$env:Path" @@ -37,43 +165,73 @@ if ($userBin -and (Test-Path -LiteralPath (Join-Path $userBin "compartment.exe") # PowerShell turns a native command's redirected stderr into an error # under "Stop", which here means only that the import failed. try { - & $py -c "import compartment.cli" *> $null + & $exe @pyArgs -c "import compartment.cli" *> $null $importable = ($LASTEXITCODE -eq 0) } catch { $importable = $false } if (-not $importable) { - Write-Host "compartment was installed but cannot be found." - Write-Host "Expected the command in: $userBin" - Write-Host "Try: $py -m pip install --user --force-reinstall compartment" - exit 1 + Stop-Install "compartment was installed but cannot be found. Expected the command in: $userBin" } } function Invoke-Compartment { - if ($compartment) { & $compartment @args } else { & $py -m compartment.cli @args } + if ($compartment) { & $compartment @args } else { & $exe @pyArgs -m compartment.cli @args } +} + +# --- 3. The vault --------------------------------------------------------------- +# +# Where `compartment` looks by default (home.py): COMPARTMENT_VAULT, then +# the legacy ENGRAM_VAULT, then memory.vault in %USERPROFILE%\.compartment, +# or in .engram when only that exists. A vault that is there is never +# touched: `init` refuses to overwrite one, and a re-run should not fail on +# that. +$profileDir = $env:USERPROFILE +if ($env:COMPARTMENT_VAULT) { + $vault = $env:COMPARTMENT_VAULT +} elseif ($env:ENGRAM_VAULT) { + $vault = $env:ENGRAM_VAULT +} elseif (-not (Test-Path -LiteralPath (Join-Path $profileDir ".compartment")) -and + (Test-Path -LiteralPath (Join-Path $profileDir ".engram"))) { + $vault = Join-Path $profileDir ".engram\memory.vault" +} else { + $vault = Join-Path $profileDir ".compartment\memory.vault" } Write-Host "" -Write-Host "Creating your encrypted vault..." -Invoke-Compartment init +$initRan = $false +if (Test-Path -LiteralPath $vault) { + Write-Host "Your vault at $vault is kept as it is." + Write-Host "Open it with: compartment unlock (or from the tray icon)" +} else { + Write-Host "Creating your encrypted vault..." + Invoke-Compartment init + if ($LASTEXITCODE -ne 0) { + Stop-Install "compartment init stopped (exit code $LASTEXITCODE); its reason is above." + } + $initRan = $true +} if (-not $onPathAlready) { + $quoted = $userBin -replace "'", "''" Write-Host "" Write-Host "Note: $userBin is not on your PATH." Write-Host "Run this once in PowerShell so 'compartment' works in new terminals:" - Write-Host " [Environment]::SetEnvironmentVariable('Path', '$userBin;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User')" + Write-Host " [Environment]::SetEnvironmentVariable('Path', '$quoted;' + [Environment]::GetEnvironmentVariable('Path', 'User'), 'User')" } +Write-Host "" +Write-Host "Done. Compartment is installed." +if ($initRan) { + Write-Host "``init`` connected the agents it found (Claude Code, Hermes Agent," + Write-Host "OpenClaw); restart them to load it." +} Write-Host @" -Done. Compartment is installed and your vault is unlocked (it stays unlocked -through logins until the next restart, then asks for your passphrase once). - -To use it with Hermes: - 1. python -m pip install --user compartment # into the Hermes venv if separate - 2. Copy-Item -Recurse integrations\hermes\compartment `$env:USERPROFILE\.hermes\plugins\compartment - 3. hermes memory setup # pick "compartment" in the list +To connect an agent: + compartment integrate claude # Claude Code + Claude Desktop + compartment integrate hermes # Hermes Agent + compartment integrate --list # every MCP client it can wire Verify anytime: compartment selftest "@ diff --git a/tests/test_platform_installers.py b/tests/test_platform_installers.py index 0457550..3209114 100644 --- a/tests/test_platform_installers.py +++ b/tests/test_platform_installers.py @@ -11,7 +11,6 @@ import json import os import pathlib -import pty import re import shutil import subprocess @@ -24,7 +23,7 @@ MACOS = ROOT / "install" / "install-macos.sh" BASH = shutil.which("bash") -pytestmark = pytest.mark.skipif(os.name == "nt" or BASH is None, +needs_bash = pytest.mark.skipif(os.name == "nt" or BASH is None, reason="the shell installers need bash") _PYTHON_STUB = r'''#!{python} @@ -171,6 +170,9 @@ def installer(tmp_path): """Run an installer with the stand-ins. `pythons` names the python stubs to put on PATH; `facts` is per-stub (or "*") platform facts; `tools` adds pipx and/or uv.""" + if os.name == "nt" or BASH is None: + pytest.skip("the shell installers need bash") + def run(script, pythons=("python3",), facts=None, tools=(), env=None, tty=True, stdin_script=False, cwd=None): root = tmp_path / "stubs" @@ -210,6 +212,7 @@ def run(script, pythons=("python3",), facts=None, tools=(), env=None, argv, stdin = [BASH, str(script)], subprocess.DEVNULL if tty: try: + import pty # POSIX only, like these tests master, slave = pty.openpty() stdin = slave except OSError: @@ -455,6 +458,51 @@ def test_apple_silicon_on_a_current_macos_takes_python_3_14(installer): assert r.init_args() == ["init"] +@needs_bash def test_both_scripts_parse(): for script in (LINUX, MACOS): subprocess.run([BASH, "-n", str(script)], check=True) + + +# --- Windows: the interpreter probe ------------------------------------------ +# +# PowerShell is not here to run the script itself, so its probe - the part +# that decides which Python is used - runs under this Python with the +# interpreter facts it reads stood in for. + +WINDOWS = ROOT / "install" / "install-windows.ps1" + + +def _windows_probe(version, plat, monkeypatch, capsys): + text = WINDOWS.read_text(encoding="utf-8") + m = re.search(r"\$probe = @'\r?\n(.*?)\r?\n'@", text, re.S) + assert m, "install-windows.ps1 has no interpreter probe" + import sysconfig + monkeypatch.setattr(sys, "version_info", tuple(version) + (0, "final", 0)) + monkeypatch.setattr(sysconfig, "get_platform", lambda: plat) + try: + exec(compile(m.group(1), "", "exec"), {"__name__": "__main__"}) + code = 0 + except SystemExit as exc: + code = exc.code or 0 + return code, capsys.readouterr().out.strip() + + +@pytest.mark.parametrize("version, plat, usable", [ + ((3, 13), "win-amd64", True), + ((3, 11), "win-amd64", True), + ((3, 14), "win-arm64", True), + ((3, 11), "win-arm64", False), # no pyyaml wheel for 3.11 on ARM64 + ((3, 12), "win32", False), # no 32-bit onnxruntime at all + ((3, 15), "win-amd64", False), # newer than the wheels + ((3, 10), "win-amd64", False), +]) +def test_the_windows_installer_picks_a_python_inside_the_wheel_matrix( + version, plat, usable, monkeypatch, capsys): + """The old script took whatever `py` started: the default Python, which + may be 32-bit, 3.10, 3.15, or 3.11 on ARM64, where pip then failed on a + dependency with no wheel.""" + code, out = _windows_probe(version, plat, monkeypatch, capsys) + assert (code == 0) is usable, out + if not usable: + assert out From 3c206407fa0c0c633a6973f92a148006f40fb417 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:00:41 -0700 Subject: [PATCH 075/135] Docs: INTEGRATIONS.md says where `integrate` writes for every client on each platform, and why the command it writes is an absolute path The Claude Desktop example named a bare `compartment`, which an app started from the Dock or the Start menu cannot find; it now shows an absolute path and says why, and what `integrate` writes instead. A table gives Claude Desktop's config file on macOS, on Windows for the MSIX builds and the older installer, and on Linux, with the warning that a running Claude Desktop can write its old config back over a new entry (anthropics/claude-ai-mcp#1150). The capture hook paragraph names the settings file on Windows and under CLAUDE_CONFIG_DIR, the autoMemoryDirectory setting, and Git Bash. A new section lists every client `integrate` knows with the file it writes, the variables that move it, and which are edited keeping comments, written as a file of their own, or printed to paste. Co-Authored-By: Claude Opus 5.5 --- docs/INTEGRATIONS.md | 82 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 186f3a8..650976a 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -28,11 +28,37 @@ Claude Desktop (`claude_desktop_config.json`): ```json { "mcpServers": { "compartment": { - "command": "compartment", + "command": "/Users/you/Library/Python/3.12/bin/compartment", "args": ["--vault", "/Users/you/.compartment/memory.vault", "--caller", "claude-desktop", "serve"] } } } ``` +Use the absolute path of the `compartment` command (`which compartment`, +or `where compartment` on Windows), not the bare name: an app started from +the Dock or the Start menu does not get your shell's PATH, and a bare +`compartment` fails there with "spawn compartment ENOENT". `integrate` +writes the absolute path for you: the script pip installed (including the +`pip install --user` directory the install scripts use), or the link a +package manager keeps for it (Homebrew, pipx, uv), which survives +upgrades. + +Where Claude Desktop keeps that file: + +| Platform | File | +|---|---| +| macOS | `~/Library/Application Support/Claude/claude_desktop_config.json` | +| Windows (MSIX: the Store build and the current direct download) | `%LOCALAPPDATA%\Packages\Claude_pzs8sxrjxfjjc\LocalCache\Roaming\Claude\claude_desktop_config.json` (Store: `AnthropicPBC.Claude_fnn82j28hfe8t`) | +| Windows (older installer) | `%APPDATA%\Claude\claude_desktop_config.json` | +| Linux (official beta, and community builds) | `~/.config/Claude/claude_desktop_config.json` | + +An MSIX app reads its private copy under `Packages` once it has saved its +settings, and ignores `%APPDATA%\Claude` from then on; `integrate claude` +writes whichever one the installed app is reading. Quit Claude Desktop +completely before you run it (on Windows, from the tray icon), or quit +and reopen it straight after: a running Claude Desktop can write the +config it loaded at startup back over the file, dropping a server added +while it ran (anthropics/claude-ai-mcp#1150). + ### Claude Code already has a memory - Compartment takes it over Claude Code keeps its own memory as Markdown files in a per-project @@ -51,9 +77,14 @@ and neither is complete. `compartment integrate claude` resolves it in three way in the system prompt, which otherwise outranks anything a tool says. 3. **It stops relying on the model.** `integrate claude` installs a - `PostToolUse` hook into `~/.claude/settings.json`. When Claude Code writes - a file into its memory directory, the hook stores it in the vault - no - tool call required, so capture is deterministic instead of discretionary. + `PostToolUse` hook into `~/.claude/settings.json` (`%USERPROFILE%\.claude` + on Windows, or `$CLAUDE_CONFIG_DIR/settings.json` when that is set). When + Claude Code writes a file into its memory directory (under `projects/`, or + wherever its `autoMemoryDirectory` setting points), the hook stores it in + the vault - no tool call required, so capture is deterministic instead of + discretionary. Claude Code runs hooks through bash, Git Bash on Windows, + so the hook names `compartment` by an absolute path written with forward + slashes there. Your existing hooks are preserved, settings.json is backed up first, and re-installing updates in place rather than duplicating. The hook always exits 0 (a memory tool must never break an edit) and does nothing when the @@ -212,11 +243,52 @@ shutdown and before compaction, and recalled memory is injected into the compaction context so summaries do not drop prior decisions. `memory_search` and `memory_store` are also registered as tools, independent of MCP wiring. +## Every other MCP client - `compartment integrate ` + +`compartment integrate --list` shows every client below and whether it is +installed and connected here; `compartment integrate --all` wires every one +it finds. Each write keeps the client's other servers, takes a byte-exact +backup first (`.compartment-backup`), follows a symlinked config to +the file it points at, and refuses - printing the block to paste instead - +anything it cannot read or edit safely. Files saved with a byte order mark +or as UTF-16 (Notepad, PowerShell 5) are read and written back as UTF-8. + +| `integrate …` | File it writes (macOS / Linux; Windows where it differs) | Notes | +|---|---|---| +| `cursor` | `~/.cursor/mcp.json` | | +| `windsurf` | `~/.codeium/windsurf/mcp_config.json` | | +| `vscode`, `vscode-insiders` | `/mcp.json`, key `servers` | comments kept | +| `zed` | `~/.config/zed/settings.json` (Linux: `$XDG_CONFIG_HOME/zed`; Windows: `%APPDATA%\Zed`), key `context_servers` | comments kept | +| `cline` | `~/.cline/data/settings/cline_mcp_settings.json` (`CLINE_DIR`, `CLINE_DATA_DIR`, `CLINE_MCP_SETTINGS_PATH`); older Cline: its editor's globalStorage | found in VS Code, Insiders, VSCodium, Cursor, Windsurf | +| `roo` | `/User/globalStorage/rooveterinaryinc.roo-cline/settings/mcp_settings.json` | | +| `kilo` | `~/.config/kilo/kilo.jsonc` (or the kilo/opencode file already there), key `mcp` | comments kept | +| `continue` | `~/.continue/mcpServers/compartment.json` (`CONTINUE_GLOBAL_DIR`) | a file of its own; config.yaml untouched | +| `codex` | `~/.codex/config.toml` (`CODEX_HOME`), `[mcp_servers.compartment]` | re-running refreshes a stale entry | +| `gemini` | `~/.gemini/settings.json` (`$GEMINI_CLI_HOME/.gemini`) | comments kept | +| `qwen` | `~/.qwen/settings.json` (`QWEN_HOME`) | comments kept | +| `copilot-cli` | `~/.copilot/mcp-config.json` (`COPILOT_HOME`) | | +| `opencode` | `~/.config/opencode/opencode.json` or `.jsonc` (`OPENCODE_CONFIG_DIR`), key `mcp` | comments kept | +| `crush` | `~/.config/crush/crush.json` (`CRUSH_GLOBAL_CONFIG`), key `mcp` | | +| `kiro` | `~/.kiro/settings/mcp.json` (`KIRO_HOME`) | | +| `amazon-q` | `~/.aws/amazonq/mcp.json` | | +| `junie` | `~/.junie/mcp/mcp.json` | | +| `factory` | `~/.factory/mcp.json` | | +| `omp` | `~/.omp/agent/mcp.json` | | +| `lmstudio` | `/mcp.json`, from `~/.lmstudio-home-pointer` (default `~/.lmstudio`) | | +| `anythingllm`, `boltai`, `5ire`, `trae` | each app's own `mcp.json` | | +| `goose` | prints an `extensions:` block for `~/.config/goose/config.yaml` (Windows: `%APPDATA%\Block\goose\config\config.yaml`; `GOOSE_PATH_ROOT`) | paste, not written | +| `librechat`, `jan`, `cherry-studio`, `chatbox`, `msty`, `witsy`, `open-webui` | prints the block and where it goes | paste, not written | + +"Comments kept" means the client reads its file as JSON with comments, so a +file holding `//` comments or trailing commas is edited as text: the entry +is spliced in, the result is read back and must hold exactly the old +settings plus the entry, and every comment stays where it was. + ## Everything else | Agent kind | Mechanism | What to do | |---|---|---| -| Any MCP-capable host (Cursor, Windsurf, custom SDK agents, …) | MCP stdio | the same one-block config as Claude, with its own `--caller` name | +| Any other MCP-capable host (custom SDK agents, …) | MCP stdio | the same one-block config as Claude, with its own `--caller` name | | Python frameworks (LangChain/LlamaIndex-style) | vector-store adapter | import `compartment.vault.Vault` directly, or use the CLI; a drop-in `VectorStore` adapter class is on the roadmap | | Anything that can run a subprocess | CLI/JSON | `compartment search "query" --json`, `compartment store "text" --source "how you know"` | From 021b3524f4f91e3d9d01730be9e8ce041b65ad28 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:01:35 -0700 Subject: [PATCH 076/135] A new signing identity is written readable by its owner alone `compartment lock --sign` and `compartment pack build` create identity.json on first use, and that file holds the Ed25519 seed itself. Both wrote it with Path.write_text, so with the umask's mode, 0644 on most systems: any local account that could reach it could sign packs that this user's trusted keys accept, or seal a vault that `compartment verify` reports as signed by them. Both now go through one helper that creates the file 0600 (and its folder, which `pack build` did not make, so a first build into a new folder no longer fails). Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 35 ++++++++++++++++++++--------------- tests/test_cli_errors.py | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index f4ad18d..83adc75 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -754,17 +754,28 @@ def cmd_unlock(args) -> None: v.save() +def _signing_identity(path: str, creator: str) -> dict: + """The Ed25519 identity that signs a sealed vault or a pack, made on + first use. + + The file holds the private seed, so it is written readable by its owner + alone: written plainly it took the umask's 0644, and any account on the + machine that could read it could sign a pack this user's keys trust, or + seal a vault `compartment verify` reports as signed by them.""" + ident_path = Path(path) + if ident_path.exists(): + return json.loads(ident_path.read_text(encoding="utf-8")) + identity = packs.new_identity(creator) + ident_path.parent.mkdir(parents=True, exist_ok=True) + _write_private(str(ident_path), json.dumps(identity, indent=2)) + print(f"generated signing identity → {ident_path} (keep it private)") + return identity + + def cmd_lock(args) -> None: if args.sign: v = _open_vault(args) - ident_path = Path(args.identity) - if ident_path.exists(): - identity = json.loads(ident_path.read_text(encoding="utf-8")) - else: - identity = packs.new_identity(args.creator) - ident_path.parent.mkdir(parents=True, exist_ok=True) - ident_path.write_text(json.dumps(identity, indent=2), encoding="utf-8") - print(f"generated signing identity → {ident_path} (keep it private)") + identity = _signing_identity(args.identity, args.creator) v.lock(signing_key=packs.load_signing_key(identity)) print(f"vault sealed + signed by {identity['signer']} " f"(pub {identity['pub_hex'][:16]}…); verify with `compartment verify`") @@ -1549,13 +1560,7 @@ def cmd_pack_build(args) -> None: _die("source must be a .jsonl, .csv, or a directory of .md files") if not records: _die("no records found in source") - ident_path = Path(args.identity) - if ident_path.exists(): - identity = json.loads(ident_path.read_text(encoding="utf-8")) - else: - identity = packs.new_identity(args.creator) - ident_path.write_text(json.dumps(identity, indent=2), encoding="utf-8") - print(f"generated new signing identity → {ident_path} (keep it private)") + identity = _signing_identity(args.identity, args.creator) emb = get_embedder(DEFAULT_MODEL) vectors = emb.embed_passages([r["text"] for r in records]) pw = None diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 8e2b826..ffa4c16 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -185,3 +185,25 @@ def test_an_atomize_listing_is_readable_by_its_owner_alone(open_vault, cli.main(["--vault", open_vault, "atomize", "--out", str(out), "--plaintext"]) assert os.stat(out).st_mode & 0o777 == 0o600 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes") +@pytest.mark.parametrize("command", [ + ["lock", "--sign", "--identity", "{identity}"], + ["pack", "build", "{source}", "--name", "notes", "--identity", + "{identity}", "--out", "{out}"], +]) +def test_a_new_signing_identity_is_readable_by_its_owner_alone( + open_vault, tmp_path, capsys, command): + """identity.json holds the Ed25519 seed that signs packs and sealed + vaults, and it was written with the umask's 0644: any local account + could have signed packs this user trusts.""" + identity = tmp_path / "keys" / "identity.json" + source = tmp_path / "notes.jsonl" + source.write_text('{"text": "The staging database runs Postgres 16"}\n', + encoding="utf-8") + cli.main(["--vault", open_vault, *[ + a.format(identity=identity, source=source, + out=tmp_path / "notes.mpack") for a in command]]) + assert os.stat(identity).st_mode & 0o777 == 0o600 + assert "generated signing identity" in capsys.readouterr().out From 5420bf771837b230034b398a4b70f1652d5e8643 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:02:54 -0700 Subject: [PATCH 077/135] The macOS app is built on every push as well as at release, for Apple silicon and Intel, and the release uploads the files the build names for itself Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 38 +++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 13 ++++++------ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b49b9f3..0bc8aff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,44 @@ jobs: shell: bash run: python tools/mcp_smoke.py --command compartment + macos-app: + # The same build the Release workflow runs at tag time, so a broken + # installer shows up on the pull request that broke it rather than on + # release day. The .pkg and .dmg are kept for a week for anyone who + # wants to try a change before it ships. + if: github.event_name != 'schedule' + strategy: + fail-fast: false + matrix: + include: + - runner: macos-latest + arch: apple-silicon + - runner: macos-15-intel + arch: intel + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10 + - name: A standalone, non-framework Python to embed + shell: bash + run: | + uv python install 3.12 + echo "EMBED_PY=$(uv python find --managed-python 3.12)" >> "$GITHUB_ENV" + - name: Build Compartment.app, the .pkg and the .dmg + shell: bash + run: | + "$EMBED_PY" tools/build_macos_app.py --python "$EMBED_PY" --pkg --dmg + ls -l build + - uses: actions/upload-artifact@v7 + with: + name: compartment-macos-${{ matrix.arch }} + path: | + build/Compartment-*.pkg + build/Compartment-*.dmg + retention-days: 7 + if-no-files-found: error + wheel-coverage: # pip installs the newest release of every dependency and compiles it when # that release has no wheel for the machine. That is how a plain diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5831383..4e96804 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,12 +101,10 @@ jobs: include: - runner: macos-latest arch: apple-silicon - suffix: "" # onnxruntime's last macOS x86_64 wheels are 1.23.2 for Python up - # to 3.13; pip resolves to it here on its own. + # to 3.13; the build selects it on its own. - runner: macos-15-intel arch: intel - suffix: "-intel" runs-on: ${{ matrix.runner }} timeout-minutes: 60 steps: @@ -122,20 +120,21 @@ jobs: uv python install 3.12 echo "EMBED_PY=$(uv python find --managed-python 3.12)" >> "$GITHUB_ENV" - name: Build Compartment.app, the .pkg and the .dmg + # On an Intel runner the script names them Compartment--intel.* + # itself, and the installer refuses to run on the other architecture. shell: bash run: | "$EMBED_PY" tools/build_macos_app.py --python "$EMBED_PY" --pkg --dmg ls -l build - - name: Name them for their architecture + - name: Exactly one .pkg and one .dmg of this version shell: bash env: V: ${{ needs.check.outputs.version }} - SUFFIX: ${{ matrix.suffix }} run: | mkdir -p out - cp "build/Compartment-$V.pkg" "out/Compartment-$V$SUFFIX.pkg" - cp "build/Compartment-$V.dmg" "out/Compartment-$V$SUFFIX.dmg" + cp build/Compartment-"$V"*.pkg build/Compartment-"$V"*.dmg out/ ls -l out + test "$(ls out | wc -l)" -eq 2 - uses: actions/upload-artifact@v7 with: name: macos-${{ matrix.arch }} From 468a0f89517c9ebcd19ae3887020efc91ed2fa0a Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:03:19 -0700 Subject: [PATCH 078/135] The Docker image reads a host user's vault, keeps itself offline, and builds its wheel from the package alone - The documented `docker run -v "$HOME/.compartment:/data"` could not open the vault on a Linux host: the vault is 0600 and owned by the host user, and the image runs as uid 10001. The run line now passes `--user "$(id -u):$(id -g)"` and the header says why (Docker Desktop maps ownership itself). Such a uid has HOME=/; with the passphrase in the environment that now works, because looking up a session credential no longer creates a session directory (24dc490). Checked by hand outside Docker: HOME unwritable, COMPARTMENT_PASSPHRASE set, the guard on - `search` answers; with the old session.py it died with PermissionError making the home directory. - The header promised no outbound connection at runtime and nothing held the container to it. COMPARTMENT_ASSERT_OFFLINE=1 is now set in the image, which seals the offline guard on in every process, and the run line adds `--network none`. COMPARTMENT_EMBED_DAEMON=0 too: embed_daemon.enabled() names containers as wanting it off, and a container runs one server. - `COPY . /src` built the wheel from the whole checkout, including a local build/ directory, whose stale build/lib setuptools folds into the wheel. Only pyproject.toml, README.md, LICENSE and src/ are copied now; the wheel built that way lists the same 52 files as one built from `git archive HEAD`. - python:3.13-slim instead of 3.11-slim, inside the supported 3.11-3.14 range, and the header names linux/amd64 and linux/arm64 with the buildx line, since onnxruntime has Linux wheels for those two only. HEALTHCHECK NONE says outright that a stdio server has nothing to probe. Docker is not available here, so the image was not built. The contract is pinned in tests/test_platform_docker.py, all five of which fail on the old Dockerfile. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 42 +++++++++++++++++++++----- tests/test_platform_docker.py | 55 +++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 tests/test_platform_docker.py diff --git a/Dockerfile b/Dockerfile index b2dd465..1599326 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,13 @@ # # The default embedding model ships inside the wheel, so the network is needed # only while this image is being built. At runtime the container makes no -# outbound connection. +# outbound connection, and it is held to that twice: the offline guard is on +# (COMPARTMENT_ASSERT_OFFLINE=1, below), and the run line gives it no network. # # Build: docker build -t compartment . -# Run: docker run -i --rm \ +# docker buildx build --platform linux/amd64,linux/arm64 -t compartment . +# Run: docker run -i --rm --network none \ +# --user "$(id -u):$(id -g)" \ # -v "$HOME/.compartment:/data" \ # -e COMPARTMENT_PASSPHRASE \ # compartment @@ -18,26 +21,47 @@ # /data. Create it once on the host with `compartment init` before the first # container run, because init prompts for a passphrase and that passphrase is # the only key to the vault. +# +# --user: the vault is mode 0600 and owned by you, so on a Linux host the +# image's own unprivileged user (uid 10001) cannot read a bind-mounted one; +# running as your own uid can. (Docker Desktop on macOS and Windows maps +# ownership itself and needs no --user.) A uid the image has never heard of +# gets HOME=/, which is fine: with the passphrase in the environment nothing +# is written outside /data. +# +# linux/amd64 and linux/arm64 only: onnxruntime, which Compartment cannot run +# without, publishes Linux wheels for x86_64 and aarch64 and nothing else. -FROM python:3.11-slim AS build +FROM python:3.13-slim AS build WORKDIR /src -COPY . /src +# Only what the wheel is built from. `COPY .` also brought in whatever the +# checkout had lying about - a local build/ directory, whose stale build/lib +# setuptools folds into the wheel, a virtualenv, .git - into the build. +COPY pyproject.toml README.md LICENSE /src/ +COPY src /src/src RUN pip install --no-cache-dir build \ && python -m build --wheel --outdir /dist -FROM python:3.11-slim +FROM python:3.13-slim # Unbuffered so the stdio transport is not held up by Python's block buffering. +# The offline guard is sealed on for every process in the container. The +# shared embedding daemon is for a machine full of agents sharing one model; +# a container runs one server, and the daemon would be one more process +# wanting a writable session directory. ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 + PYTHONDONTWRITEBYTECODE=1 \ + COMPARTMENT_ASSERT_OFFLINE=1 \ + COMPARTMENT_EMBED_DAEMON=0 COPY --from=build /dist/*.whl /tmp/ RUN pip install --no-cache-dir /tmp/*.whl \ && rm -rf /tmp/*.whl # Runs unprivileged. /data is the mount point for the host's vault, so the uid -# here has to be able to read and write whatever is mounted there. +# here has to be able to read and write whatever is mounted there (see --user +# above). RUN useradd --create-home --uid 10001 compartment \ && mkdir -p /data \ && chown compartment:compartment /data @@ -45,5 +69,9 @@ USER compartment VOLUME ["/data"] +# A stdio server has no listener for a probe to reach, and its health is the +# client's pipe: the MCP client sees at once when the server stops answering. +HEALTHCHECK NONE + # No EXPOSE and no port: stdio only, zero listeners. ENTRYPOINT ["compartment", "--vault", "/data/memory.vault", "--caller", "docker", "serve"] diff --git a/tests/test_platform_docker.py b/tests/test_platform_docker.py new file mode 100644 index 0000000..8733240 --- /dev/null +++ b/tests/test_platform_docker.py @@ -0,0 +1,55 @@ +"""The Dockerfile's contract, read from the file: Docker is not available to +the suite, so what the image promises is pinned here instead. + +The behaviour behind the run line - a uid with no home, HOME=/, the +passphrase in the environment - is exercised for real in +test_platform_session_store.py.""" +import pathlib +import re + +DOCKERFILE = (pathlib.Path(__file__).resolve().parents[1] / "Dockerfile" + ).read_text(encoding="utf-8") + + +def _env(): + """Every ENV key=value the final stage sets.""" + final = DOCKERFILE.split("\nFROM ")[-1] + pairs = {} + for block in re.findall(r"^ENV ((?:.*\\\n)*.*)$", final, re.M): + for key, value in re.findall(r"(\w+)=(\S+)", block): + pairs[key] = value + return pairs + + +def test_the_container_is_held_offline_at_runtime(): + """The header promised no outbound connection and nothing enforced it.""" + assert _env().get("COMPARTMENT_ASSERT_OFFLINE") == "1" + assert "--network none" in DOCKERFILE + + +def test_one_server_per_container_runs_no_shared_embedding_daemon(): + """embed_daemon.enabled() names containers as wanting it off; the daemon + also needs a writable session directory a --user run does not have.""" + assert _env().get("COMPARTMENT_EMBED_DAEMON") == "0" + + +def test_the_run_line_can_read_a_vault_owned_by_the_host_user(): + """The vault is 0600 and owned by the host user, so the image's uid + 10001 could not open a bind-mounted one on a Linux host.""" + assert '--user "$(id -u):$(id -g)"' in DOCKERFILE + + +def test_the_wheel_is_built_from_the_package_alone(): + """`COPY . /src` brought a checkout's stale build/ directory, which + setuptools folds into the wheel, into the image's build.""" + build = DOCKERFILE.split("\nFROM ")[1] + assert not re.search(r"^COPY \. ", build, re.M) + assert re.search(r"^COPY src /src/src$", build, re.M) + + +def test_a_supported_python_and_both_architectures(): + bases = re.findall(r"^FROM python:(\d+\.\d+)-slim", DOCKERFILE, re.M) + assert bases and all((3, 11) <= tuple(map(int, b.split("."))) <= (3, 14) + for b in bases) + assert len(set(bases)) == 1 + assert "linux/amd64,linux/arm64" in DOCKERFILE From b0fbe77ff3b85f38fbdfdbfbdc7649bdd247465c Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:03:29 -0700 Subject: [PATCH 079/135] A vault past 20,000 vectors opens in milliseconds where usearch has no SIMD, including every agent-run command on a Mac, instead of taking minutes to build its HNSW index Above BRUTE_FORCE_LIMIT the vault built a usearch HNSW graph from scratch on every open. Where usearch finds no SIMD kernels it can use it computes distances one element at a time and builds at a few hundred vectors a second: measured 50 s for 25,000 vectors and 321 s for 100,000 (f32; int8 18 s and 114 s), on every unlock, every `compartment` command and every MCP server reopen after another agent wrote - far past the clients' tool-call timeouts. The exact index answers a search over 100,000 vectors in about 9 ms and costs nothing to build. This is not a rare machine. SimSIMD's dispatch asks the kernel what the CPU supports (hw.optional.neon on Apple Silicon), macOS Seatbelt refuses that query, and Claude Code and Codex run an agent's shell commands under Seatbelt, so usearch reports "serial" there on every Apple Silicon Mac. The same holds on CPUs usearch ships no kernels for. build_index now creates the usearch index first and, when its hardware_acceleration is "serial" and the choice is automatic, returns the exact index without building a graph. An explicit force is honoured, and an older usearch that does not report acceleration keeps the old behaviour. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vindex.py | 35 ++++++++++++++++++++++---- tests/test_vindex.py | 52 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/src/compartment/vindex.py b/src/compartment/vindex.py index f00e68b..f4904df 100644 --- a/src/compartment/vindex.py +++ b/src/compartment/vindex.py @@ -97,11 +97,21 @@ def __init__(self, dim: int, precision: str = "f32"): def build(cls, dim: int, ikeys: list[int], mat: np.ndarray, precision: str = "f32") -> "UsearchIndex": idx = cls(dim, precision) - if len(ikeys): - idx._index.add(np.array(ikeys, dtype=np.uint64), - np.ascontiguousarray(mat, dtype=np.float32)) + idx.add_all(ikeys, mat) return idx + def add_all(self, ikeys: list[int], mat: np.ndarray) -> None: + if len(ikeys): + self._index.add(np.array(ikeys, dtype=np.uint64), + np.ascontiguousarray(mat, dtype=np.float32)) + + @property + def serial(self) -> bool: + """True when usearch found no SIMD kernels it can use on this CPU + and computes every distance one element at a time. An older usearch + that does not say is taken to have them, as it always was.""" + return getattr(self._index, "hardware_acceleration", None) == "serial" + def add(self, ikey: int, vec: np.ndarray) -> None: self._index.add(np.uint64(ikey), vec.astype(np.float32)) @@ -159,8 +169,23 @@ def build_index(dim: int, ikeys: list[int], mat: np.ndarray, if force == "brute" or (force is None and len(ikeys) < BRUTE_FORCE_LIMIT): return BruteForceIndex.build(dim, ikeys, mat) try: - return UsearchIndex.build(dim, ikeys, mat, - precision=_precision(precision)) + idx = UsearchIndex(dim, _precision(precision)) + # The HNSW graph is built from scratch on every open, and without + # SIMD usearch builds it at a few hundred vectors a second: measured + # 50 s for 25,000 vectors and over five minutes for 100,000, per + # open, where the exact index answers a search over 100,000 in about + # 9 ms and costs nothing to build. That is not a rare machine: SIMD + # dispatch asks the kernel what the CPU supports, and inside macOS + # Seatbelt - where Claude Code and Codex run an agent's commands - + # that query is refused, so usearch reports "serial" on every Apple + # Silicon Mac; likewise on CPUs it has no kernels for. A vault past + # the limit there took minutes to open in every CLI call and every + # MCP server reopen, long past the clients' tool-call timeouts. + # Chosen automatically only; an explicit force is honoured. + if force is None and idx.serial: + return BruteForceIndex.build(dim, ikeys, mat) + idx.add_all(ikeys, mat) + return idx except ImportError: global _HNSW_WARNED if not _HNSW_WARNED: diff --git a/tests/test_vindex.py b/tests/test_vindex.py index 22a138a..6e3630f 100644 --- a/tests/test_vindex.py +++ b/tests/test_vindex.py @@ -187,3 +187,55 @@ def test_a_mistyped_precision_setting_still_builds_an_index(capsys): for spelled, kind in (("INT8", "i8"), (" i8", "i8"), ("float16", "f16")): assert build_index(16, list(range(1, 51)), mat, precision=spelled, force="hnsw").kind == f"hnsw/usearch ({kind})" + + +def test_a_large_vault_is_not_given_an_hnsw_index_usearch_would_build_serially( + monkeypatch): + """Without SIMD - which is what usearch reports inside macOS Seatbelt, + where it cannot ask the kernel what the CPU supports - the graph is + built at a few hundred vectors a second, on every open: minutes for a + vault past the limit, against milliseconds per exact search.""" + import pytest + pytest.importorskip("usearch") + from usearch import index as usearch_index + + from compartment import vindex + + built = [] + + class _Serial(usearch_index.Index): + hardware_acceleration = "serial" + + def add(self, *args, **kwargs): + built.append(args) + return super().add(*args, **kwargs) + + monkeypatch.setattr(usearch_index, "Index", _Serial) + monkeypatch.setattr(vindex, "BRUTE_FORCE_LIMIT", 10) + rng = np.random.default_rng(5) + mat = rng.standard_normal((40, 16)).astype(np.float32) + idx = vindex.build_index(16, list(range(1, 41)), mat) + assert isinstance(idx, BruteForceIndex) + assert not built, "no graph was built only to be thrown away" + assert idx.search(mat[3], 1)[0][0] == 4 + + forced = vindex.build_index(16, list(range(1, 41)), mat, force="hnsw") + assert forced.kind.startswith("hnsw"), "an explicit force is honoured" + + +def test_with_simd_a_large_vault_still_gets_hnsw(monkeypatch): + import pytest + pytest.importorskip("usearch") + from usearch import index as usearch_index + + from compartment import vindex + + class _Simd(usearch_index.Index): + hardware_acceleration = "neon" + + monkeypatch.setattr(usearch_index, "Index", _Simd) + monkeypatch.setattr(vindex, "BRUTE_FORCE_LIMIT", 10) + mat = np.random.default_rng(6).standard_normal((40, 16)).astype(np.float32) + idx = vindex.build_index(16, list(range(1, 41)), mat) + assert idx.kind.startswith("hnsw") + assert idx.search(mat[3], 1)[0][0] == 4 From 94ec5a2e69d1cd1dec5a9d39d497385bccaf444d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:03:44 -0700 Subject: [PATCH 080/135] Claude Desktop gets a one-click extension: an MCP Bundle that runs the pinned release in its own environment There was no Claude Desktop extension. The routes into Claude Desktop all wrote a config naming the `compartment` command, which a Dock-launched app often cannot find, and none of them could appear in Claude Desktop's extension list or directory. packaging/mcpb is now an MCP Bundle (modelcontextprotocol/mcpb, manifest_version 0.4) of server type "uv": a pyproject.toml pinning compartment==, a .python-version, a three-line entry point that runs `compartment --caller claude-desktop serve` (the caller `integrate claude` writes for Desktop), the icon and the licence. Claude Desktop has uv build the environment from PyPI once, at install; the server then makes no connection. A bundle of type "python" cannot work here: onnxruntime, PyNaCl, argon2-cffi, numpy and tokenizers are compiled, so a bundle built on one platform is dead on the rest. uv uses the newest interpreter it can get unless told otherwise, and it picked CPython 3.15 for the first trial build, for which onnxruntime 1.31 has no wheels, so the environment could not have been built. The bundle pins 3.12 and bounds requires-python below 3.15. tools/build_mcpb.py builds dist/compartment-.mcpb with the standard library, byte for byte reproducibly, refuses a manifest or pin that is not the package's version, and with uv on the PATH adds a uv.lock resolved at build time so every user installs what the release was built against. Release step (not yet in the workflows): after the version is on PyPI, `pip install uv && python tools/build_mcpb.py`, then attach the .mcpb to the GitHub release. Checked end to end with uv 0.13.2: the bundle, repinned to 4.11.0 (4.12.0 is not on PyPI yet), locked 54 packages, built its environment, and `uv run --directory server/main.py` under COMPARTMENT_ASSERT_OFFLINE=1 and UV_OFFLINE=1 answered initialize as compartment, listed the 14 tools and answered memory_status. The manifest validates against the v0.4 schema. The tests check the bundle's shape, that its tool list is exactly what server.py registers, the caller, the interpreter pin, a reproducible complete build, and the version gate; tests/test_version_lockstep.py gates both new files. Co-Authored-By: Claude Opus 5.5 --- packaging/mcpb/.python-version | 1 + packaging/mcpb/manifest.json | 70 ++++++++++++++++ packaging/mcpb/pyproject.toml | 31 +++++++ packaging/mcpb/server/main.py | 19 +++++ tests/test_plugin_manifests.py | 81 ++++++++++++++++++ tests/test_version_lockstep.py | 15 ++++ tools/build_mcpb.py | 148 +++++++++++++++++++++++++++++++++ 7 files changed, 365 insertions(+) create mode 100644 packaging/mcpb/.python-version create mode 100644 packaging/mcpb/manifest.json create mode 100644 packaging/mcpb/pyproject.toml create mode 100644 packaging/mcpb/server/main.py create mode 100644 tools/build_mcpb.py diff --git a/packaging/mcpb/.python-version b/packaging/mcpb/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/packaging/mcpb/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/packaging/mcpb/manifest.json b/packaging/mcpb/manifest.json new file mode 100644 index 0000000..33d3364 --- /dev/null +++ b/packaging/mcpb/manifest.json @@ -0,0 +1,70 @@ +{ + "manifest_version": "0.4", + "name": "compartment", + "display_name": "Compartment", + "version": "4.12.0", + "description": "Encrypted, fully offline memory for Claude: one vault on your machine, shared with every MCP client. No network, no API key.", + "long_description": "Compartment is persistent memory for AI agents, stored on your own computer and encrypted at rest. What Claude learns in one conversation is available in every later one, and to Claude Code, Cursor, Codex, Gemini CLI, Hermes Agent, OpenClaw and any other MCP client on the same machine. Nothing leaves it: the embedding model runs locally, the server speaks stdio only and opens no ports, and there is no account.\n\n**Before the first run**, create the vault once in a terminal: `pip install compartment && compartment init` (or `pipx install compartment`, or `uv tool install compartment`). It asks for a passphrase you choose; that passphrase is the only key and Compartment cannot recover it. If the vault is locked after a restart, run `compartment unlock`.\n\nIf you already connected Claude Desktop with `compartment integrate claude` or the app's Connect button, you do not need this extension as well: both would start the same server and Claude would see every tool twice.", + "author": { + "name": "Max Freedom Pollard", + "url": "https://github.com/MaxFreedomPollard" + }, + "repository": { + "type": "git", + "url": "https://github.com/MaxFreedomPollard/Compartment" + }, + "homepage": "https://maxfreedompollard.github.io/Compartment/", + "documentation": "https://github.com/MaxFreedomPollard/Compartment#readme", + "support": "https://github.com/MaxFreedomPollard/Compartment/issues", + "icon": "icon.png", + "server": { + "type": "uv", + "entry_point": "server/main.py", + "mcp_config": { + "command": "uv", + "args": [ + "run", + "--directory", + "${__dirname}", + "server/main.py" + ] + } + }, + "tools": [ + {"name": "memory_store", "description": "Save one claim to the encrypted memory, with its source and date."}, + {"name": "memory_store_many", "description": "Save several separate facts in one call, one memory each."}, + {"name": "memory_search", "description": "Recall from memory before answering: hybrid vector and keyword search."}, + {"name": "memory_link", "description": "Record a relationship between two entities in the memory graph."}, + {"name": "memory_relations", "description": "Query the memory graph, optionally as of a date."}, + {"name": "memory_unlink", "description": "Remove one relation from the memory graph."}, + {"name": "memory_get", "description": "Fetch one memory by id."}, + {"name": "memory_forget", "description": "Delete a memory, optionally crypto-shredding it."}, + {"name": "memory_list_namespaces", "description": "List namespaces and their record counts."}, + {"name": "memory_recent", "description": "The most recently stored memories."}, + {"name": "memory_status", "description": "Vault status: lock state, counts, model, index."}, + {"name": "memory_selftest", "description": "Health check against the built-in reference facts."}, + {"name": "memory_lock", "description": "Panic lock: seal the vault and drop the key now."}, + {"name": "memory_unlock", "description": "Unlock from the agent; disabled unless the vault config enables it."} + ], + "tools_generated": false, + "keywords": [ + "memory", + "agent-memory", + "encryption", + "offline", + "local-first", + "privacy", + "vector-search" + ], + "license": "Apache-2.0", + "compatibility": { + "platforms": [ + "darwin", + "win32", + "linux" + ], + "runtimes": { + "python": ">=3.11" + } + } +} diff --git a/packaging/mcpb/pyproject.toml b/packaging/mcpb/pyproject.toml new file mode 100644 index 0000000..82c4f94 --- /dev/null +++ b/packaging/mcpb/pyproject.toml @@ -0,0 +1,31 @@ +# The environment Claude Desktop builds for the Compartment extension. +# +# The bundle is an MCP Bundle of server type "uv" (manifest_version 0.4): +# it ships no Python packages of its own. Claude Desktop reads this file and +# has uv install the pinned release from PyPI, with the right interpreter and +# the right compiled wheels for the machine, once, when the extension is +# installed. That is the same network use as `pip install compartment`, and +# the only one: the server it then starts makes no connection at all. +# +# Not a package. It has a different name from its one dependency because a +# project may not depend on itself, and uv is told not to build it. +# +# The pin moves with every release (tests/test_version_lockstep.py), so the +# bundle attached to a release installs exactly that release. + +[project] +name = "compartment-claude-desktop" +version = "4.12.0" +description = "Compartment as a Claude Desktop extension (MCP Bundle)" +# uv runs the newest interpreter it can find or download unless told +# otherwise, and the compiled dependencies trail new Python releases: +# onnxruntime 1.31 has wheels for 3.11 to 3.14 and none for 3.15, so an +# unpinned bundle installed after 3.15 shipped would fail to build its +# environment. .python-version picks 3.12 (what CI runs); the upper bound +# here makes a host that overrides it fail with a clear message instead of +# a missing-wheel error. Raise both when onnxruntime ships wheels for more. +requires-python = ">=3.11,<3.15" +dependencies = ["compartment==4.12.0"] + +[tool.uv] +package = false diff --git a/packaging/mcpb/server/main.py b/packaging/mcpb/server/main.py new file mode 100644 index 0000000..9f78ddb --- /dev/null +++ b/packaging/mcpb/server/main.py @@ -0,0 +1,19 @@ +"""Compartment's MCP server, as the Claude Desktop extension starts it. + +Claude Desktop runs this file with `uv run` inside the environment it built +from the bundle's pyproject.toml, so `compartment` here is the pinned +release in that environment, not whatever may or may not be on the PATH a +GUI application was started with. That is the reason the bundle exists: +every other Claude Desktop route needs the `compartment` command findable, +and a Dock-launched app does not inherit the shell's PATH. + +The caller is `claude-desktop`, the same name `compartment integrate +claude` registers, so the audit log and any per-caller ACL treat the two +routes as one client. The vault is Compartment's default (COMPARTMENT_HOME +or ~/.compartment, or the ~/.engram an older install left), which is where +`compartment init` creates it. +""" +from compartment.cli import main + +if __name__ == "__main__": + main(["--caller", "claude-desktop", "serve"]) diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index 359e3e5..2115d6f 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -185,3 +185,84 @@ def test_the_goose_button_uses_a_command_goose_accepts(): assert _caller(q["arg"]) == clients.CLIENTS["goose"].key page = (ROOT / "add/goose.html").read_text(encoding="utf-8") assert f"
uvx {' '.join(q['arg'])}
" in page + + +# --------------------------------------------- Claude Desktop (MCP Bundle) -- +def _registered_tools() -> list[str]: + import asyncio + from compartment import server + return [t.name for t in asyncio.run(server.mcp.list_tools())] + + +def _build_mcpb(): + import importlib.util + spec = importlib.util.spec_from_file_location( + "build_mcpb", ROOT / "tools" / "build_mcpb.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_the_bundle_is_a_uv_bundle_claude_desktop_can_run(): + """manifest_version 0.4 is the MCPB release that added the uv server + type, the only type that can carry compiled dependencies (onnxruntime, + PyNaCl, argon2-cffi, numpy) to every platform from one bundle.""" + m = _json("packaging/mcpb/manifest.json") + assert m["manifest_version"] == "0.4" + assert m["name"] == "compartment" and m["author"]["name"] + server = m["server"] + assert server["type"] == "uv" + assert (ROOT / "packaging/mcpb" / server["entry_point"]).is_file() + cfg = server["mcp_config"] + assert cfg["command"] == "uv" + assert cfg["args"][:3] == ["run", "--directory", "${__dirname}"] + assert cfg["args"][-1] == server["entry_point"] + + +def test_the_bundle_lists_exactly_the_tools_the_server_registers(): + """Claude Desktop shows this list before the server ever runs.""" + m = _json("packaging/mcpb/manifest.json") + assert [t["name"] for t in m["tools"]] == _registered_tools() + assert m["tools_generated"] is False + + +def test_the_bundle_starts_the_server_as_claude_desktop(): + """The same caller `integrate claude` writes into Claude Desktop's own + config, so the two routes are one client to the audit log and ACL.""" + entry = (ROOT / "packaging/mcpb/server/main.py").read_text(encoding="utf-8") + assert 'main(["--caller", "claude-desktop", "serve"])' in entry + assert '"--caller", "claude-desktop", "serve"' in ( + ROOT / "src/compartment/cli.py").read_text(encoding="utf-8") + + +def test_the_bundle_pins_an_interpreter_its_dependencies_have_wheels_for(): + """uv otherwise picks the newest Python it can get, and onnxruntime + trails Python releases: a bundle without a pin, installed the week a + new Python ships, cannot build its environment.""" + pinned = (ROOT / "packaging/mcpb/.python-version").read_text().strip() + text = (ROOT / "packaging/mcpb/pyproject.toml").read_text(encoding="utf-8") + lo, hi = re.search(r'requires-python = ">=(\d+\.\d+),<(\d+\.\d+)"', text).groups() + as_tuple = lambda v: tuple(int(x) for x in v.split(".")) # noqa: E731 + assert as_tuple(lo) <= as_tuple(pinned) < as_tuple(hi) + + +def test_the_bundle_build_is_complete_and_reproducible(tmp_path): + import zipfile + mod = _build_mcpb() + a = mod.build(tmp_path / "a", with_lock=False) + b = mod.build(tmp_path / "b", with_lock=False) + assert a.name == f"compartment-{mod.package_version()}.mcpb" + assert a.read_bytes() == b.read_bytes() + with zipfile.ZipFile(a) as z: + names = set(z.namelist()) + manifest = json.loads(z.read("manifest.json")) + assert names == set(mod.CONTENTS) + assert manifest == _json("packaging/mcpb/manifest.json") + assert {manifest["icon"], manifest["server"]["entry_point"]} <= names + + +def test_the_bundle_build_refuses_a_bundle_for_another_release(): + import pytest + mod = _build_mcpb() + with pytest.raises(SystemExit, match="not the package's"): + mod.check("0.0.1") diff --git a/tests/test_version_lockstep.py b/tests/test_version_lockstep.py index 3ee612a..8a537fe 100644 --- a/tests/test_version_lockstep.py +++ b/tests/test_version_lockstep.py @@ -97,6 +97,19 @@ def test_marketplace_entries_that_carry_a_version_match_the_package(): assert entry["version"] == _semver(_version()), entry["name"] +def test_the_claude_desktop_bundle_installs_this_release(): + """The .mcpb attached to a release must install that release: its + manifest is the version Claude Desktop shows and compares on update, and + the pin is what uv installs.""" + d = json.loads((ROOT / "packaging" / "mcpb" / "manifest.json").read_text( + encoding="utf-8")) + assert d["version"] == _semver(_version()) + text = (ROOT / "packaging" / "mcpb" / "pyproject.toml").read_text( + encoding="utf-8") + assert re.search(r'^version\s*=\s*"([^"]+)"', text, re.M).group(1) == _version() + assert re.search(r'"compartment==([^"]+)"', text).group(1) == _version() + + # ----------------------------------------------------------- coverage ------ #: Every file above, and nothing else, may carry the package's version. GATED = { @@ -108,6 +121,8 @@ def test_marketplace_entries_that_carry_a_version_match_the_package(): ".claude-plugin/marketplace.json", "integrations/hermes/compartment/plugin.yaml", "src/compartment/data/hermes-plugin/plugin.yaml", + "packaging/mcpb/manifest.json", + "packaging/mcpb/pyproject.toml", } #: Files whose version field is their own, not the package's. diff --git a/tools/build_mcpb.py b/tools/build_mcpb.py new file mode 100644 index 0000000..9bd2dad --- /dev/null +++ b/tools/build_mcpb.py @@ -0,0 +1,148 @@ +"""Build dist/compartment-.mcpb, the Claude Desktop extension. + +An MCP Bundle is a zip with manifest.json at its root (the format lives in +github.com/modelcontextprotocol/mcpb, formerly anthropics/dxt). This one is +server type "uv", manifest_version 0.4: it carries a pyproject.toml that +pins the release, a three-line entry point and the icon, and Claude Desktop +has uv install compartment and its compiled dependencies for the user's own +platform when the extension is installed. Bundling the dependencies instead +("python" type) cannot work for Compartment: onnxruntime, PyNaCl, +argon2-cffi, numpy and tokenizers are compiled, so a bundle built on one +platform is dead on the others, and the MCPB README says as much. + + python tools/build_mcpb.py # what the release runs; needs uv + python tools/build_mcpb.py --no-lock # no uv.lock (development, tests) + +With uv on the PATH the bundle also carries a uv.lock, resolved here, with +hashes, for every platform: Claude Desktop then installs exactly what the +release was built against instead of resolving again on each user's +machine, and a launch whose environment is already in place has nothing to +look up. Resolving is the one step that uses the network, and it happens at +build time, on the release runner. The built bundle and the server in it +make no connection. + +Standard library only, and deterministic: the same inputs give the same +bytes, so a rebuilt bundle can be compared with the one attached to a +release. +""" +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SRC = ROOT / "packaging" / "mcpb" + +#: Archive path -> source file. The manifest's entry point and icon must be +#: in here, and nothing that is not declared ships. +CONTENTS = { + "manifest.json": SRC / "manifest.json", + "pyproject.toml": SRC / "pyproject.toml", + ".python-version": SRC / ".python-version", + "server/main.py": SRC / "server" / "main.py", + "icon.png": ROOT / "docs" / "images" / "logo-400.png", + "LICENSE": ROOT / "LICENSE", +} + +#: 1980-01-01, the earliest time a zip can record: every entry gets it, so +#: the archive does not change with the checkout's file times. +_EPOCH = (1980, 1, 1, 0, 0, 0) + + +def package_version() -> str: + text = (ROOT / "src" / "compartment" / "__init__.py").read_text(encoding="utf-8") + m = re.search(r'^__version__\s*=\s*"([^"]+)"', text, re.M) + if not m: + raise SystemExit("cannot find __version__ in src/compartment/__init__.py") + return m.group(1) + + +def semver(v: str) -> str: + parts = v.split(".") + return ".".join(parts + ["0"] * (3 - len(parts))) + + +def check(version: str) -> dict: + """Refuse to build a bundle that would install the wrong release or + that Claude Desktop would reject. Returns the manifest.""" + manifest = json.loads((SRC / "manifest.json").read_text(encoding="utf-8")) + problems = [] + for field in ("manifest_version", "name", "version", "description", + "author", "server"): + if field not in manifest: + problems.append(f"manifest.json has no {field}") + if manifest.get("version") != semver(version): + problems.append(f"manifest.json version {manifest.get('version')!r} is " + f"not the package's {semver(version)!r}") + pin = re.search(r'"compartment==([^"]+)"', + (SRC / "pyproject.toml").read_text(encoding="utf-8")) + if not pin or pin.group(1) != version: + problems.append(f"pyproject.toml must pin compartment=={version}") + server = manifest.get("server", {}) + if server.get("type") != "uv": + problems.append("server.type must be uv: the dependencies are compiled") + for declared in (server.get("entry_point"), manifest.get("icon")): + if declared and declared not in CONTENTS: + problems.append(f"manifest.json names {declared}, which the bundle " + "does not contain") + for name, source in CONTENTS.items(): + if not source.is_file(): + problems.append(f"{name}: {source} is missing") + if problems: + raise SystemExit("cannot build the bundle:\n " + "\n ".join(problems)) + return manifest + + +def lock(staging: pathlib.Path) -> pathlib.Path: + """Resolve uv.lock for the pinned release. Network: PyPI, build time only.""" + uv = shutil.which("uv") + if not uv: + raise SystemExit("uv is not on PATH, so the bundle cannot carry a lock " + "file. Install it (`pip install uv`) and run again, or " + "pass --no-lock for a development build.") + for name in ("pyproject.toml", ".python-version"): + shutil.copy2(SRC / name, staging / name) + subprocess.run([uv, "lock", "--directory", str(staging)], check=True) + return staging / "uv.lock" + + +def build(out_dir: pathlib.Path, with_lock: bool = True) -> pathlib.Path: + version = package_version() + check(version) + files = dict(CONTENTS) + with tempfile.TemporaryDirectory() as tmp: + if with_lock: + files["uv.lock"] = lock(pathlib.Path(tmp)) + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / f"compartment-{version}.mcpb" + partial = out.with_suffix(".mcpb.partial") + with zipfile.ZipFile(partial, "w", zipfile.ZIP_DEFLATED) as z: + for name in sorted(files): + info = zipfile.ZipInfo(name, date_time=_EPOCH) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + z.writestr(info, files[name].read_bytes()) + partial.replace(out) + return out + + +def main(argv: list[str] | None = None) -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--out", default=str(ROOT / "dist"), + help="directory to write the bundle into (default dist/)") + ap.add_argument("--no-lock", action="store_true", + help="leave out uv.lock (no uv, no network); not for release") + args = ap.parse_args(argv) + out = build(pathlib.Path(args.out), with_lock=not args.no_lock) + print(f"built {out} ({out.stat().st_size / 1024:.0f} KB)") + + +if __name__ == "__main__": + sys.exit(main()) From 4ac6f9aa40f4730e7c5f0660c141145f36645d99 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:04:22 -0700 Subject: [PATCH 081/135] A Windows machine up for more than 24.9 days no longer relocks the vault and deletes the unlock credential _windows_boot_token called kernel32.GetTickCount64 through ctypes without a restype, and ctypes reads an undescribed function's result as a C int. The 64-bit tick count was cut to 32 signed bits, so past 2**31 ms of uptime - 24.9 days, routine for a Windows machine that only sleeps or shuts down with Fast Startup, which resumes the same kernel - the uptime went negative, the derived boot instant jumped 49.7 days into the future, _pinned_boot_token took that for a reboot and pinned a new token, and session.get() deleted the credential it could no longer open. Every process saw the vault locked until the passphrase was typed again, and it happened again every 49.7 days. The function now gets restype c_uint64 (and no arguments). A machine up less than 24.9 days computes exactly what it did before, so its token and credential are unchanged; one already past it re-pins once. The docstrings said GetTickCount64 excludes sleep. It includes it (PEP 418; QueryUnbiasedInterruptTime is the one that skips sleep), and the 300 s tolerance in the pin depends on that, so they now say so. Before the fix (30 days of uptime): test_windows_uptime_past_24_days_is_not_cut_to_32_bits E AssertionError: assert '1801702967' == '1797408000' Co-Authored-By: Claude Opus 5.5 --- src/compartment/platforms.py | 28 +++++++++++++++++++++------- tests/test_platform_boot.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index fa9db98..0b18c4c 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -283,8 +283,8 @@ def _pinned_boot_token(computed: float, uptime_s: float, marker: Path) -> str: mid-session in three ways this avoids: 1. sub-second int() truncation flapping between two processes; 2. wall-clock steps and NTP slew moving the derived boot instant; - 3. a jump of the whole sleep duration where the uptime source - excludes sleep (Windows GetTickCount64). + 3. a jump of the whole sleep duration, where an uptime source + excludes sleep (none of those used here does). A real reboot resets uptime and moves the boot instant far beyond the tolerance below, so it is detected and the token changes, as intended.""" try: @@ -325,13 +325,27 @@ def _windows_boot_token() -> str: credential and relocks the vault mid-session in three ways this avoids: 1. sub-second int() truncation flapping between two processes; 2. wall-clock drift after NTP slew accumulating past a second boundary; - 3. a jump of the whole sleep duration after every sleep/resume - (GetTickCount64 excludes sleep; time.time() does not). + 3. a jump of the whole sleep duration after every sleep/resume, were + the uptime source to skip sleep. GetTickCount64 does not (it + includes time suspended; PEP 418), and the tolerance relies on + that: QueryUnbiasedInterruptTime, which skips sleep, would read + every sleep longer than five minutes as a reboot. A real reboot resets uptime and moves the boot instant far beyond the - tolerance below, so it is detected and the token changes, as intended.""" + tolerance below, so it is detected and the token changes, as intended. + + GetTickCount64 returns a 64-bit count, and ctypes assumes a C int for + any function it is not told about. Called bare, the count was cut to 32 + signed bits: past 2**31 ms of uptime - 24.9 days, which a Windows + machine that only ever sleeps, or shuts down with Fast Startup, reaches + easily - it went negative, the derived boot instant jumped 49.7 days, + the pin read that as a reboot, and the session credential was deleted; + then again every 49.7 days.""" import ctypes - uptime_s = ctypes.windll.kernel32.GetTickCount64() / 1000.0 # excludes sleep - computed = time.time() - uptime_s # ~boot instant + get_tick_count = ctypes.windll.kernel32.GetTickCount64 + get_tick_count.restype = ctypes.c_uint64 + get_tick_count.argtypes = [] + uptime_s = get_tick_count() / 1000.0 # counts through sleep + computed = time.time() - uptime_s # ~boot instant return _pinned_boot_token(computed, uptime_s, _win_marker()) diff --git a/tests/test_platform_boot.py b/tests/test_platform_boot.py index f740f06..a9520d0 100644 --- a/tests/test_platform_boot.py +++ b/tests/test_platform_boot.py @@ -92,3 +92,35 @@ class NoSysctlbyname: _record_subprocesses(monkeypatch, stdout=stdout) est = platforms._sysctl_boot_estimate() assert est is not None and est[0] == 1_700_000_000.0 + + +# --- Windows: the tick count is 64 bits wide --------------------------------- + +class _CtypesFunction: + """A foreign function as ctypes models one: its result is read as the + restype says, and a function nobody described returns a C int.""" + + def __init__(self, value): + self.value = value + self.restype = ctypes.c_int + self.argtypes = None + + def __call__(self): + return self.restype(self.value).value + + +def test_windows_uptime_past_24_days_is_not_cut_to_32_bits(tmp_path, + monkeypatch): + """Bare ctypes reads GetTickCount64's 64-bit result as a 32-bit int. + Past 2**31 ms (24.9 days, routine for a Windows machine that sleeps or + uses Fast Startup) the uptime went negative, the boot instant jumped by + 49.7 days, the pin took it for a reboot, and the stored unlock + credential was deleted - and again every 49.7 days.""" + import types + thirty_days_ms = 30 * 86_400 * 1000 + kernel32 = types.SimpleNamespace( + GetTickCount64=_CtypesFunction(thirty_days_ms)) + monkeypatch.setattr(ctypes, "windll", + types.SimpleNamespace(kernel32=kernel32), raising=False) + monkeypatch.setattr(platforms.time, "time", lambda: 1_800_000_000.0) + assert platforms._windows_boot_token() == str(1_800_000_000 - 30 * 86_400) From dfb2ca28889e3013744f64cca7ab816fc65c651b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:05:38 -0700 Subject: [PATCH 082/135] A save by another process is noticed even where the filesystem reports the same inode, size and time, so it is never silently overwritten A vault handle knew another process had rewritten the file by comparing (inode, size, mtime) with what it had read. On the disks people carry vaults on those three miss a rewrite: FAT and exFAT keep times to two seconds or ten milliseconds and derive inode numbers from where a file's directory entry or first cluster landed, which a later save reuses; a compaction often leaves the same size, the payload being whole SQLite pages; and a clock stepped back can stamp a newer file with an older time on any filesystem. A miss is a lost update: the handle's next save wrote its stale image over the other process's work, or its next append was sealed to a journal position the new file does not have, and the next open refused the vault as tampered. The disk state now carries an identity as well: a hash of the header and the payload's nonce, which every save draws fresh, read from the same open file as the stat (a few kilobytes). Appends leave it alone, any rewrite changes it. The read-only panel helper already told rewrites apart this way. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 12 ++-- src/compartment/vaultfile.py | 59 ++++++++++++++++-- tests/test_storage_staleness.py | 102 ++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 10 deletions(-) create mode 100644 tests/test_storage_staleness.py diff --git a/src/compartment/vault.py b/src/compartment/vault.py index b94e200..6a815d7 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -388,7 +388,7 @@ def __init__(self, path: str, header: vaultfile.VaultHeader, store: Store, #: When the expiry sweep last ran. Zero means never, so the first read #: after opening always sweeps. self._last_expiry_sweep = 0.0 - self._disk_state: tuple[int, int, int] | None = None + self._disk_state: tuple[int, int, int, str] | None = None if os.path.exists(path): self._disk_state = self._stat_disk() self.index = None @@ -397,12 +397,14 @@ def __init__(self, path: str, header: vaultfile.VaultHeader, store: Store, # -------------------------------------------------- multi-process safety - def _stat_disk(self) -> tuple[int, int, int]: + def _stat_disk(self) -> tuple[int, int, int, str]: # The inode as well, as in platforms.file_signature: another # process's compaction can leave a file of the same length with, on a - # coarse clock, the same time, and only the inode shows it is new. - st = os.stat(self.path) - return (st.st_ino, st.st_size, st.st_mtime_ns) + # coarse clock, the same time, and only the inode shows it is new - + # where the filesystem does not hand the new file an old number. The + # identity read with them is what shows it everywhere else; see + # vaultfile.disk_state. + return vaultfile.disk_state(self.path) def is_stale(self) -> bool: """True if the vault file changed under us since we read it. diff --git a/src/compartment/vaultfile.py b/src/compartment/vaultfile.py index 39211b3..b0a9489 100644 --- a/src/compartment/vaultfile.py +++ b/src/compartment/vaultfile.py @@ -265,10 +265,11 @@ class LoadedVaultFile: #: decrypting the whole payload again. journal_start: int = 0 journal_end: int = 0 - #: (inode, size, mtime in ns) of the file as it was read - taken before the - #: read, from the same open file. What an opener compares against to learn - #: that another process wrote while it was reading. See Vault.unlock. - disk_state: tuple[int, int, int] | None = None + #: (inode, size, mtime in ns, identity) of the file as it was read - taken + #: before the read, from the same open file. What an opener compares + #: against to learn that another process wrote while it was reading. See + #: Vault.unlock and disk_state below. + disk_state: tuple[int, int, int, str] | None = None def parse_journal(raw: bytes, pos: int) -> tuple[list[bytes], int, bool, int]: @@ -324,6 +325,53 @@ def parse_journal(raw: bytes, pos: int) -> tuple[list[bytes], int, bool, int]: return journal_cts, pos, truncated_tail, tail_bytes +#: The most header an identity reads. A real header is a few kilobytes; this +#: only bounds the read on a file whose length field is garbage. +_IDENTITY_HEADER_CAP = 1 << 20 + + +def _identity_span(head: bytes) -> int: + """How many leading bytes identify one write of the file: the header and + the payload's nonce, given at least the first ten bytes.""" + if len(head) >= 10 and head[:4] == MAGIC: + (hlen,) = struct.unpack(">I", head[6:10]) + return 10 + min(hlen, _IDENTITY_HEADER_CAP) + crypto.NONCE_LEN + return min(len(head), 10) + + +def _identity(prefix: bytes) -> str: + return crypto.sha256(prefix[:_identity_span(prefix)]) + + +def disk_state(path: str) -> tuple[int, int, int, str]: + """(inode, size, mtime in ns, identity) of the vault file as it is now, + all read from one open file. + + The first three are what a stat can say, and between them they miss a + rewrite on exactly the disks people carry vaults on. A compaction leaves + a file of the same size often - the payload is a whole number of SQLite + pages - and FAT and exFAT keep modification times to two seconds or ten + milliseconds, hand a new file an inode number derived from where its + directory entry or first cluster landed (which a later save reuses), and + a clock stepped back can stamp a newer file with an older time anywhere. + Missed, another process's save is overwritten by this one's, or this + process appends entries sealed to a journal position the new file does + not have, and the next open refuses the vault as tampered. + + The identity closes that: every save reseals the payload under a fresh + random nonce at its front, so the header plus those bytes differ between + any two writes of the file, whatever the filesystem reports. It costs + one read of a few kilobytes. The read-only panel helper (vaultview) has + told rewrites apart the same way.""" + with open(path, "rb") as f: + st = os.fstat(f.fileno()) + head = f.read(10) + span = _identity_span(head) + if span > len(head): + head += f.read(span - len(head)) + return (st.st_ino, st.st_size, st.st_mtime_ns, _identity(head)) + + def read_vault_file(path: str) -> LoadedVaultFile: with open(path, "rb") as f: # Before the read, so an append that lands during it makes the file @@ -332,7 +380,8 @@ def read_vault_file(path: str) -> LoadedVaultFile: st = os.fstat(f.fileno()) raw = f.read() loaded = parse_vault_bytes(raw, path) - loaded.disk_state = (st.st_ino, st.st_size, st.st_mtime_ns) + loaded.disk_state = (st.st_ino, st.st_size, st.st_mtime_ns, + _identity(raw)) return loaded diff --git a/tests/test_storage_staleness.py b/tests/test_storage_staleness.py new file mode 100644 index 0000000..3352d49 --- /dev/null +++ b/tests/test_storage_staleness.py @@ -0,0 +1,102 @@ +"""Telling that another process rewrote the vault, on any filesystem. + +A handle knows the vault changed under it by comparing what it read with +what is on disk now. Inode, size and modification time miss a rewrite on +the disks people carry vaults on: FAT and exFAT keep times to two seconds or +ten milliseconds and derive inode numbers from where a file landed, which a +later save reuses; a compaction often leaves the same size, since the +payload is whole SQLite pages; and a clock stepped back stamps a newer file +with an older time anywhere. A missed rewrite is a lost update: this +handle's next save writes its stale image over the other process's work. +""" +import os + +import pytest + +from compartment import vault as vault_mod, vaultfile +from compartment.vault import Vault, VaultStaleError + +from conftest import PASS + + +class _With: + def __init__(self, module, **replaced): + self._module = module + self.__dict__.update(replaced) + + def __getattr__(self, name): + return getattr(self._module, name) + + +def _fat_like_stat(monkeypatch): + """Every stat of the vault reports one inode number and one time, as a + FAT stick can for two writes landing in the same slot within its clock + tick. The size stays true.""" + def fake(result): + fields = list(result) + fields[1] = 4242 # st_ino + fields[8] = 1_700_000_000 # st_mtime + st = os.stat_result(fields, {"st_mtime_ns": 1_700_000_000 * 10**9}) + return st + + def stat(path, *args, **kwargs): + return fake(os.stat(path, *args, **kwargs)) + + def fstat(fd): + return fake(os.fstat(fd)) + + for module in (vaultfile, vault_mod): + monkeypatch.setattr(module, "os", _With(os, stat=stat, fstat=fstat)) + + +def test_a_rewrite_the_stat_cannot_see_is_still_seen(vault_path, monkeypatch): + first = Vault.create(vault_path, PASS, creator="t") + rid = first.store("The build server is called ostrich.", caller="t", + tags=["aaaa"])["id"] + first.lock() + _fat_like_stat(monkeypatch) + + writer = Vault.unlock(vault_path, passphrase=PASS) + stale = Vault.unlock(vault_path, passphrase=PASS) + size = os.path.getsize(vault_path) + writer.db.set_tags(rid, ["bbbb"]) # same length: same pages + writer.save() + assert os.path.getsize(vault_path) == size, \ + "the premise: a rewrite the same size as what it replaced" + + assert stale.is_stale(), "the other handle's save is noticed" + with pytest.raises(VaultStaleError): + stale.save() + assert Vault.unlock(vault_path, passphrase=PASS).get( + rid, caller="t")["tags"] == ["bbbb"], "and its change survives" + + +def test_an_untouched_vault_is_not_called_stale(vault_path): + v = Vault.create(vault_path, PASS, creator="t") + v.store("A memory nobody else touches.", caller="t") + assert not v.is_stale() + again = Vault.unlock(vault_path, passphrase=PASS) + assert not again.is_stale() + again.store("Another, from the same handle.", caller="t") + assert not again.is_stale(), "its own writes are not someone else's" + + +def test_the_identity_is_the_header_and_the_payload_nonce(vault_path): + """What changes on every save and on nothing else: an append keeps it, + a save replaces it.""" + v = Vault.create(vault_path, PASS, creator="t") + before = vaultfile.disk_state(vault_path)[3] + v.store("Appended to the journal only.", caller="t") + assert vaultfile.disk_state(vault_path)[3] == before + v.save() + assert vaultfile.disk_state(vault_path)[3] != before + assert vaultfile.read_vault_file(vault_path).disk_state == \ + vaultfile.disk_state(vault_path) + + +def test_a_file_that_is_not_a_vault_still_has_a_state(tmp_path): + p = tmp_path / "junk.vault" + p.write_bytes(b"NUC") + assert vaultfile.disk_state(str(p))[1] == 3 + p.write_bytes(b"NUCV\x00\x01\xff\xff\xff\xff" + b"x" * 100) + assert vaultfile.disk_state(str(p))[1] == 110, "a garbage length is capped" From a8d931ca75ab6e2c9b667e3b1994def441d07d53 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:06:34 -0700 Subject: [PATCH 083/135] An import or a memory pack whose text holds half an emoji is installed with U+FFFD in its place instead of failing The earlier fix cleaned lone surrogates where store() receives text, but two paths compute vectors before store() is reached. import_jsonl embeds every line in one batch first, so a JSONL line carrying a "\ud83d" escape still failed the whole import in the tokenizer ("TextEncodeInput must be Union[...]"). Pack records are JSON lines as well, and a pack whose model differs from the vault's has its vectors recomputed from that text the same way. Both now pass their text through clean_text as it is read, so the vector and the stored record are computed from the same valid UTF-8. Co-Authored-By: Claude Opus 5.5 --- src/compartment/packs.py | 7 +++++++ src/compartment/vault.py | 5 +++++ tests/test_storage_text.py | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/compartment/packs.py b/src/compartment/packs.py index 287a88a..df17bc2 100644 --- a/src/compartment/packs.py +++ b/src/compartment/packs.py @@ -207,6 +207,13 @@ def read_pack(blob: bytes, passphrase: str | None = None, *, sections = unpack_sections(body) records = [json.loads(l) for l in sections["records"].decode("utf-8").splitlines() if l.strip()] + # A "\ud83d" escape in a record decodes to a lone surrogate, which the + # encoder refuses when the vectors have to be recomputed and SQLite + # refuses on insert; see vault.clean_text. + from .vault import clean_text + for r in records: + if isinstance(r.get("text"), str): + r["text"] = clean_text(r["text"]) dim = int(header["model"]["dim"]) vectors = np.frombuffer(sections["vectors"], dtype=np.float32).reshape(-1, dim) if vectors.shape[0] != len(records) or vectors.shape[0] != header["records"]: diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 6a815d7..2284f16 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -2193,6 +2193,11 @@ def import_jsonl(self, text: str, caller: str = "user", # trailing \r from CRLF is whitespace to json.loads. records = [json.loads(l) for l in text.lstrip("\ufeff").split("\n") if l.strip()] + # Cleaned here, not only in store(): the vectors below are computed + # before store() sees the text, and the tokenizer refuses a lone + # surrogate exactly as it does there. + for r in records: + r["text"] = clean_text(r["text"]) # What store() embeds: the claim without its provenance clause, every # window of it. All of them go in one batched call, and each record # is handed its own block of rows. A claim of at most CHUNK_WINDOW diff --git a/tests/test_storage_text.py b/tests/test_storage_text.py index 8a00e78..9223fcd 100644 --- a/tests/test_storage_text.py +++ b/tests/test_storage_text.py @@ -50,3 +50,29 @@ def test_the_graph_takes_and_finds_the_same_text(vault): rels = vault.relations("t", entity="Ana " + HALF_EMOJI)["relations"] assert [(r["subject"], r["object"]) for r in rels] == [ ("Ana cut here \ufffd", "the \ufffdrelease")] + + +def test_an_import_line_with_half_an_emoji_is_imported(vault): + """The import computes its vectors before store() sees the text, so the + tokenizer refused it there and the whole import failed.""" + line = '{"text": "Imported with half an emoji \\ud83d at the end."}\n' + assert vault.import_jsonl(line, caller="t") == 1 + hit = vault.search("imported half an emoji \ud83d", caller="t") + assert hit["results"][0]["text"].startswith( + "Imported with half an emoji � at the end.") + + +def test_a_pack_record_with_half_an_emoji_reads_clean(): + """Pack records are JSON lines too; a recomputed vector or the insert + refused the lone surrogate the escape decodes to.""" + import numpy as np + + from compartment import packs + ident = packs.new_identity("tester") + blob = packs.build_pack( + name="t", version="1.0.0", description="", identity=ident, + records=[{"id": "r1", "text": "Pack fact with half an emoji \ud83d"}], + vectors=np.zeros((1, 4), dtype=np.float32), + model={"name": "m", "sha256": "0", "dim": 4}) + _, records, _ = packs.read_pack(blob, trusted_keys=[ident["pub_hex"]]) + assert records[0]["text"] == "Pack fact with half an emoji �" From 8cc3da801a363facbc90d08f0a8facb7df8c2de1 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:06:51 -0700 Subject: [PATCH 084/135] Docker's MCP Catalog and Continue's hub get entries prepared in the repository, the container one running with no network Neither catalogue had anything to submit. packaging/docker-mcp/server.yaml is the entry docker/mcp-registry wants as servers/compartment/server.yaml: Docker builds and signs mcp/compartment from the existing Dockerfile at a pinned commit (current main), mounts the user's vault folder at /data where the image's entrypoint looks, passes the passphrase as COMPARTMENT_PASSPHRASE from Docker Desktop's secret store (the route the Dockerfile already documents, since a container has no Keychain or boot session), and sets disableNetwork, so the offline guarantee is enforced by Docker for that route. It follows the rules in the registry's cmd/validate: a capitalised title without "MCP" or "Server", a 40-hex commit, a prefix.name secret, a PNG icon under 512x512. No tools.json: the server lists its 14 tools without a vault, so Docker's build can ask it, and a static list would go stale. packaging/continue/compartment.yaml is a Continue block (schema v1) for hub.continue.dev, and a file Continue loads as it is from .continue/mcpServers/. It uses the caller `integrate continue` prints, and its version is gated with the package. The tests tie the Docker entry to the Dockerfile (the mounted folder is the entrypoint vault's folder, the volume's parameter is declared and required, the only secret is the passphrase, no EXPOSE), and check the Continue block's shape and caller. Neither file is read by Compartment; submitting them is a step for the maintainer, described in docs/PLUGINS.md. Co-Authored-By: Claude Opus 5.5 --- packaging/continue/compartment.yaml | 23 ++++++++++ packaging/docker-mcp/server.yaml | 67 +++++++++++++++++++++++++++++ tests/test_plugin_manifests.py | 55 +++++++++++++++++++++++ tests/test_version_lockstep.py | 7 +++ 4 files changed, 152 insertions(+) create mode 100644 packaging/continue/compartment.yaml create mode 100644 packaging/docker-mcp/server.yaml diff --git a/packaging/continue/compartment.yaml b/packaging/continue/compartment.yaml new file mode 100644 index 0000000..5d62fd9 --- /dev/null +++ b/packaging/continue/compartment.yaml @@ -0,0 +1,23 @@ +# Compartment as a Continue block: the MCP server entry Continue's hub +# (hub.continue.dev) publishes, and a file Continue reads as it is. +# +# Without the hub, copy this file into a workspace's .continue/mcpServers/ +# folder (or ~/.continue/mcpServers/ for every workspace) and Continue picks +# it up; `compartment integrate continue` prints the same server for pasting +# under `mcpServers:` in config.yaml instead. Either way the vault must exist +# first: `pip install compartment && compartment init`. +# +# The caller is `continue`, the name `integrate continue` registers. The +# version moves with the package (tests/test_version_lockstep.py), so the +# hub shows the release the block was published from. +name: Compartment +version: 4.12.0 +schema: v1 +mcpServers: + - name: Compartment + type: stdio + command: compartment + args: + - --caller + - continue + - serve diff --git a/packaging/docker-mcp/server.yaml b/packaging/docker-mcp/server.yaml new file mode 100644 index 0000000..5f2645f --- /dev/null +++ b/packaging/docker-mcp/server.yaml @@ -0,0 +1,67 @@ +# Compartment's entry for the Docker MCP Catalog (Docker Desktop's MCP +# Toolkit, hub.docker.com/mcp). It belongs in github.com/docker/mcp-registry +# as servers/compartment/server.yaml; it is kept here so it is reviewed and +# tested with the code it describes. docs/PLUGINS.md has the submission +# steps. This repository does not read the file itself. +# +# Docker builds the image from the Dockerfile at `source.commit` and signs it +# under mcp/compartment. That Dockerfile runs the server as an unprivileged +# user over stdio, with the vault at /data/memory.vault and no port. The +# catalog adds the one thing a container cannot do for itself: mount the +# user's vault folder. disableNetwork runs the container with no network at +# all, which is the offline guarantee enforced by Docker rather than +# promised by us. +# +# The vault has to exist before the first run (`compartment init` on the +# host), because creating it asks for the passphrase and a catalog server +# has no terminal. Inside the container there is no Keychain and no boot +# session to inherit, so the passphrase arrives the way the Dockerfile +# documents, as COMPARTMENT_PASSPHRASE, kept in Docker Desktop's secret store. +# +# source.commit is the current main; Docker's pin-update workflow moves it +# with each release once the entry is merged. +name: compartment +image: mcp/compartment +type: server +meta: + category: productivity + tags: + - memory + - agent-memory + - encryption + - offline + - local-first + - privacy +about: + title: Compartment + description: | + Encrypted, fully offline long-term memory for AI agents. One vault on + your own machine, AEAD-encrypted at rest including the vector index, + shared by every MCP client: what one agent stores, the others recall. + Embedding runs locally inside the container, which runs with no + network. Create the vault once on the host with + `pip install compartment && compartment init`, then mount its folder. + icon: https://raw.githubusercontent.com/MaxFreedomPollard/Compartment/main/docs/images/logo-400.png +source: + project: https://github.com/MaxFreedomPollard/Compartment + commit: be47c40ce1e7fc62ff213a926fe5405e571d34a8 +run: + volumes: + - "{{compartment.vault_dir}}:/data" + disableNetwork: true +config: + description: Mount the folder that holds your vault and give its passphrase + secrets: + - name: compartment.passphrase + env: COMPARTMENT_PASSPHRASE + example: + description: The passphrase you chose at `compartment init` + parameters: + type: object + properties: + vault_dir: + type: string + description: Folder on this machine that holds memory.vault (created by `compartment init`, usually ~/.compartment) + default: /Users/local-test/.compartment + required: + - vault_dir diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index 2115d6f..1e87aa2 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -266,3 +266,58 @@ def test_the_bundle_build_refuses_a_bundle_for_another_release(): mod = _build_mcpb() with pytest.raises(SystemExit, match="not the package's"): mod.check("0.0.1") + + +# ---------------------------------------------------- Docker MCP Catalog ---- +def _yaml(rel: str): + import pytest + yaml = pytest.importorskip("yaml") + return yaml.safe_load((ROOT / rel).read_text(encoding="utf-8")) + + +def _dockerfile_entrypoint() -> list[str]: + text = (ROOT / "Dockerfile").read_text(encoding="utf-8") + return json.loads(re.search(r"^ENTRYPOINT (\[.*\])$", text, re.M).group(1)) + + +def test_the_docker_catalog_entry_builds_from_this_repository(): + """docker/mcp-registry builds mcp/compartment from the Dockerfile at a + pinned commit; the entry is useless if either is wrong.""" + d = _yaml("packaging/docker-mcp/server.yaml") + assert d["name"] == "compartment" and d["type"] == "server" + assert d["image"] == "mcp/compartment" + assert d["source"]["project"] == "https://github.com/MaxFreedomPollard/Compartment" + assert re.fullmatch(r"[0-9a-f]{40}", d["source"]["commit"]) + assert d["about"]["title"] and d["about"]["icon"].startswith("https://") + assert d["meta"]["category"] + + +def test_the_docker_catalog_runs_the_container_offline_on_the_users_vault(): + """No network at all, enforced by Docker; the user's vault folder mounted + where the image's entrypoint looks for it; the passphrase from Docker + Desktop's secret store, as the Dockerfile documents.""" + d = _yaml("packaging/docker-mcp/server.yaml") + assert d["run"]["disableNetwork"] is True + entry = _dockerfile_entrypoint() + vault = entry[entry.index("--vault") + 1] + (volume,) = d["run"]["volumes"] + param, target = re.fullmatch(r"\{\{compartment\.(\w+)\}\}:(/\S+)", volume).groups() + assert pathlib.PurePosixPath(vault).parent == pathlib.PurePosixPath(target) + assert param in d["config"]["parameters"]["properties"] + assert param in d["config"]["parameters"]["required"] + assert [s["env"] for s in d["config"]["secrets"]] == ["COMPARTMENT_PASSPHRASE"] + assert all(s["name"].startswith("compartment.") for s in d["config"]["secrets"]) + assert entry[-1] == "serve" + assert not re.search(r"^EXPOSE\b", (ROOT / "Dockerfile").read_text( + encoding="utf-8"), re.M) + + +# -------------------------------------------------------------- Continue ---- +def test_the_continue_block_is_a_block_continue_loads(): + from compartment import clients + d = _yaml("packaging/continue/compartment.yaml") + assert d["schema"] == "v1" and d["name"] and d["version"] + (server,) = d["mcpServers"] + assert server["type"] == "stdio" and server["command"] == "compartment" + assert server["args"][-1] == "serve" + assert _caller(server["args"]) == clients.CLIENTS["continue"].key diff --git a/tests/test_version_lockstep.py b/tests/test_version_lockstep.py index 8a537fe..c727e29 100644 --- a/tests/test_version_lockstep.py +++ b/tests/test_version_lockstep.py @@ -110,6 +110,12 @@ def test_the_claude_desktop_bundle_installs_this_release(): assert re.search(r'"compartment==([^"]+)"', text).group(1) == _version() +def test_the_continue_block_carries_this_release(): + """Continue's hub shows a block's version; it is published from here.""" + assert _yaml_version(ROOT / "packaging" / "continue" / "compartment.yaml") \ + == _version() + + # ----------------------------------------------------------- coverage ------ #: Every file above, and nothing else, may carry the package's version. GATED = { @@ -123,6 +129,7 @@ def test_the_claude_desktop_bundle_installs_this_release(): "src/compartment/data/hermes-plugin/plugin.yaml", "packaging/mcpb/manifest.json", "packaging/mcpb/pyproject.toml", + "packaging/continue/compartment.yaml", } #: Files whose version field is their own, not the package's. From a5c7796235c24c11fa75769d8d8f66fb4d8dd00d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:07:33 -0700 Subject: [PATCH 085/135] The Gemini CLI extension identifies itself with the caller `integrate gemini` writes, and every listing is checked to start the same server gemini-extension.json started the server with `--caller gemini-cli`; `compartment integrate gemini` (where `gemini-cli` is only an alias the user may type) writes `--caller gemini`. The caller is the name the audit log records and the key per-caller ACL entries use, so the extension and integrate showed one Gemini CLI as two clients, and an ACL entry for `gemini` did not apply to the extension. It now says `gemini`, the same fix the LM Studio button got. New tests read every place the repository tells a host how to start the server - the Claude Code plugin, the Agent Plugins mcp.json, the Gemini extension, the LM Studio and goose buttons, the Dockerfile, the Claude Desktop bundle and the Continue block - and require the same shape (`compartment [--vault V] [--caller C] serve`), a different caller per host, and integrate's caller wherever integrate wires the same host. Another checks that no listing claims a number of tools the server does not register (14 today). Before the fix: test_every_listing_names_its_host_as_integrate_does E AssertionError: ('gemini', 'gemini-cli') Co-Authored-By: Claude Opus 5.5 --- gemini-extension.json | 2 +- tests/test_plugin_manifests.py | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/gemini-extension.json b/gemini-extension.json index 3f0986a..c8d88c4 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -8,7 +8,7 @@ "command": "compartment", "args": [ "--caller", - "gemini-cli", + "gemini", "serve" ] } diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index 1e87aa2..bc3c65f 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -321,3 +321,76 @@ def test_the_continue_block_is_a_block_continue_loads(): assert server["type"] == "stdio" and server["command"] == "compartment" assert server["args"][-1] == "serve" assert _caller(server["args"]) == clients.CLIENTS["continue"].key + + +# ------------------------------------------------- across every listing ----- +def _listings() -> dict[str, tuple[str, list[str]]]: + """host -> (command, compartment arguments) for every place this + repository tells a host how to start the server.""" + import base64 + from urllib.parse import parse_qs, urlsplit + out = {} + out["claude-code"] = tuple(_json(".claude-plugin/plugin.json")["mcpServers"] + ["compartment"][k] for k in ("command", "args")) + out["agent-plugins"] = tuple(_json("mcp.json")["mcpServers"]["compartment"][k] + for k in ("command", "args")) + out["gemini"] = tuple(_json("gemini-extension.json")["mcpServers"] + ["compartment"][k] for k in ("command", "args")) + q = parse_qs(urlsplit(_deeplinks("add/lmstudio.html")[0]).query) + lm = json.loads(base64.b64decode(q["config"][0])) + out["lmstudio"] = (lm["command"], lm["args"]) + q = parse_qs(urlsplit(_deeplinks("add/goose.html")[0]).query) + out["goose"] = (q["arg"][0], q["arg"][1:]) # uvx + entry = _dockerfile_entrypoint() + out["docker"] = (entry[0], entry[1:]) + main_py = (ROOT / "packaging/mcpb/server/main.py").read_text(encoding="utf-8") + out["claude-desktop"] = ("compartment", json.loads( + re.search(r"main\((\[.*?\])\)", main_py).group(1))) + try: + cont = _yaml("packaging/continue/compartment.yaml")["mcpServers"][0] + out["continue"] = (cont["command"], cont["args"]) + except Exception: # noqa: BLE001 + pass # no PyYAML here + return out + + +def test_every_listing_starts_the_same_server(): + """One executable, global options only before the subcommand, and the + subcommand last: `compartment [--vault V] [--caller C] serve`.""" + for host, (command, args) in _listings().items(): + assert command == "compartment", host + assert args[-1] == "serve", host + opts = args[:-1] + assert len(opts) % 2 == 0, (host, args) + assert set(opts[::2]) <= {"--vault", "--caller"}, (host, args) + + +def test_every_listing_names_its_host_as_integrate_does(): + """The caller is the audit log's name for a client and the key ACL + entries are written against. Where `compartment integrate` wires the + same host, a listing must use the caller integrate writes, or one app + shows up as two clients; and no two hosts may share a caller.""" + from compartment import clients + listings = _listings() + callers = {host: _caller(args) for host, (_, args) in listings.items()} + assert len(set(callers.values())) == len(callers), callers + for host, caller in callers.items(): + if host in clients.CLIENTS: + assert caller == clients.CLIENTS[host].key, (host, caller) + assert callers["claude-code"] == "claude-code" # integrate claude + assert callers["claude-desktop"] == "claude-desktop" + + +def test_no_listing_claims_a_tool_count_the_server_does_not_have(): + count = len(_registered_tools()) + for rel in (".claude-plugin/plugin.json", ".claude-plugin/marketplace.json", + "plugin.json", "gemini-extension.json", "server.json", + "packaging/mcpb/manifest.json", "packaging/docker-mcp/server.yaml", + "packaging/continue/compartment.yaml", "llms-install.md", + "docs/PLUGINS.md", "registry.html"): + p = ROOT / rel + if not p.exists(): + continue + for n in re.findall(r"\b(\d+) (?:memory |MCP )?tools\b", + p.read_text(encoding="utf-8")): + assert int(n) == count, (rel, n, count) From ca280a104677ac60c0dd0073dd8e87114b1b9801 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:08:01 -0700 Subject: [PATCH 086/135] Claude Code settings with `"hooks": null` get the capture hook instead of a TypeError `"hooks": null` in settings.json means no hooks, and the shape check reads it that way, but install() then took the null from setdefault and indexed it: a TypeError that `integrate claude` and `hook install` do not catch. A null hooks key is now replaced by an empty object first. (Also drops a variable the JSONC inserter assigned and never read.) Co-Authored-By: Claude Opus 5.5 --- src/compartment/claude_hooks.py | 4 +++- src/compartment/clients.py | 2 +- tests/test_integrations_hooks.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index 4ea5a92..66b0515 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -154,7 +154,9 @@ def install(compartment_bin: str | None = None, vault: str | None = None, data = _read(path) # raises on malformed groups = _groups(data, path) # and on a bad shape - hooks = data.setdefault("hooks", {}) + if data.get("hooks") is None: # absent, or null + data["hooks"] = {} + hooks = data["hooks"] # drop only our own previous entries, so re-installing updates in place kept = [] for group in groups: diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 953acd6..364967a 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -965,7 +965,7 @@ def _jsonc_insert(text: str, obj: _JsoncObject, key: str, value, + text[obj.open + 1:]) first_key = min(k for k, _, _ in obj.spans.values()) - last_key, _, last_end = max(obj.spans.values()) + _, _, last_end = max(obj.spans.values()) member_indent = _own_line_indent(text, first_key) if member_indent is None: # a one-line object compact = json.dumps(value) diff --git a/tests/test_integrations_hooks.py b/tests/test_integrations_hooks.py index b9dbc20..01e6a60 100644 --- a/tests/test_integrations_hooks.py +++ b/tests/test_integrations_hooks.py @@ -106,3 +106,13 @@ def test_settings_whose_hooks_are_the_wrong_shape_are_refused_untouched( with pytest.raises(ValueError, match="refusing to touch it"): call() assert p.read_text(encoding="utf-8") == settings + + +def test_settings_whose_hooks_are_null_get_the_hook(tmp_path): + """`"hooks": null` is no hooks at all; install died with TypeError on + it, which nothing catches.""" + p = tmp_path / "settings.json" + p.write_text('{"hooks": null, "model": "opus"}', encoding="utf-8") + claude_hooks.install("compartment", settings=p) + assert claude_hooks.is_installed(settings=p) is True + assert claude_hooks.uninstall(settings=p) is True From 54e2d8731bede8f90ef1d4e8aa07d721ea5f8b30 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:08:10 -0700 Subject: [PATCH 087/135] The Oh My Pi extension's memory tools declare their approval tier and stay in the model's tool list omp's ToolDefinition grew `approval` and `loadMode`. A tool without an approval tier is treated as "exec", running code, so approval modes that pass reads and writes still stopped to ask before every memory_search; and an extension tool without a loadMode is "discoverable", taken out of the top-level tool list and reachable only through omp's tool search, which buries the one tool the description tells the model to call before answering. memory_search now declares "read", memory_store "write", and both load as "essential". Older omp releases load the module without type checking and ignore the two fields. Field names and semantics are from can1357/oh-my-pi packages/agent/src/types.ts and the extension types as of 2026-10-19; the other events and return shapes the extension uses are unchanged there. Before the fix: test_the_omp_extension_declares_what_each_tool_does E {'memory_search': {}} != {'memory_search': {'approval': 'read', 'loadMode': 'essential'}} Co-Authored-By: Claude Opus 5.5 --- integrations/omp/README.md | 4 ++++ integrations/omp/extension.ts | 10 ++++++++++ tests/test_plugin_manifests.py | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/integrations/omp/README.md b/integrations/omp/README.md index 5a4466b..d48a5c1 100644 --- a/integrations/omp/README.md +++ b/integrations/omp/README.md @@ -62,5 +62,9 @@ The extension works with or without the MCP entry; both share the same vault. state or user instructions. - User-turn collection deduplicates by text; flushes are capped (20 turns, 4000 chars) per store call. +- The two tools declare their omp approval tier, `memory_search` a read and + `memory_store` a write, so an approval mode that passes reads does not + stop every recall as if it ran code, and both load as `essential`, in the + model's top-level tool list rather than behind tool search. - Vault locked? The CLI exits non-zero: recall adds nothing, a flush logs one line saying the turns were not stored, and nothing blocks the session. diff --git a/integrations/omp/extension.ts b/integrations/omp/extension.ts index b74a0c0..226534c 100644 --- a/integrations/omp/extension.ts +++ b/integrations/omp/extension.ts @@ -126,11 +126,19 @@ export default function compartmentOmp(pi: ExtensionAPI) { }); // --- Explicit tools (model-driven, MCP-independent) --------------------- + // omp treats a tool without an `approval` tier as "exec" (running code) + // and an extension tool without a `loadMode` as "discoverable", hidden + // behind tool search. Recall is a read and a store is a write, so approval + // modes that pass reads stop asking before every search; and a tool the + // model must call BEFORE answering cannot sit where it has to be found + // first, so both stay top-level. const z = pi.zod; pi.registerTool({ name: "memory_search", label: "Memory Search", + approval: "read", + loadMode: "essential", description: "Recall from the user's persistent encrypted memory vault BEFORE answering " + "anything that may depend on past work, decisions, preferences, or project " @@ -148,6 +156,8 @@ export default function compartmentOmp(pi: ExtensionAPI) { pi.registerTool({ name: "memory_store", label: "Memory Store", + approval: "write", + loadMode: "essential", description: "Store one claim into the encrypted memory vault, one per call, dated " + "automatically. " diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index bc3c65f..76287a1 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -394,3 +394,22 @@ def test_no_listing_claims_a_tool_count_the_server_does_not_have(): for n in re.findall(r"\b(\d+) (?:memory |MCP )?tools\b", p.read_text(encoding="utf-8")): assert int(n) == count, (rel, n, count) + + +# ------------------------------------------------------------- Oh My Pi ----- +def test_the_omp_extension_declares_what_each_tool_does(): + """omp's ToolDefinition treats a missing `approval` as "exec" and a + missing `loadMode` on an extension tool as "discoverable" (removed from + the top-level tool list, found by search). Recall must not be gated + like running code, and a tool the model is told to call before + answering must be in front of it.""" + src = (ROOT / "integrations/omp/extension.ts").read_text(encoding="utf-8") + tools = {} + for block in src.split("pi.registerTool({")[1:]: + name = re.search(r'name:\s*"([^"]+)"', block).group(1) + head = block.split("async execute")[0] + tools[name] = dict(re.findall(r'\b(approval|loadMode):\s*"([^"]+)"', head)) + assert tools == { + "memory_search": {"approval": "read", "loadMode": "essential"}, + "memory_store": {"approval": "write", "loadMode": "essential"}, + } From f24c4e81849197edf7e0f1c92dab1f2982cd2ac4 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:08:25 -0700 Subject: [PATCH 088/135] `integrate vscode` finds and writes the Flathub VS Code on Linux The Flathub build of VS Code runs with XDG_CONFIG_HOME set to its sandbox, so its User directory, and its mcp.json, are under ~/.var/app/com.visualstudio.code/config/Code/User. Only the native ~/.config/Code/User was looked at, so on a machine with just the Flathub build VS Code was reported as not installed, and `integrate vscode` wrote a file it never reads. The sandboxed User directory is now used when it is the only one; a native install, where there is one, is still the one written. Insiders likewise. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 24 ++++++++++++++++++++++-- tests/test_integrations_clients.py | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 364967a..9afef8b 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -108,6 +108,25 @@ def _editor_users() -> list[Path]: return users +def _vscode_mcp(app: str, flatpak_id: str) -> Callable[[], Path]: + """VS Code's user-level mcp.json, in the Flathub build's sandboxed + config (~/.var/app//config//User) when that is the only one + on this Linux machine. The native path alone used to be written, which + the Flathub VS Code never reads.""" + def path() -> Path: + native = _vscode_user(app) + if sys.platform != "darwin" and os.name != "nt": + flat = (Path.home() / ".var" / "app" / flatpak_id / "config" + / app / "User") + try: + if not native.is_dir() and flat.is_dir(): + return flat / "mcp.json" + except OSError: + pass + return native / "mcp.json" + return path + + def _ext_storage(ext: str) -> Path | None: """The globalStorage directory of extension `ext` in the first editor that has one, or None when no editor has run it.""" @@ -423,10 +442,11 @@ def present(self) -> bool: note="run `/mcp reload` in omp to pick it up without restarting"), # -- coding agents that spell the key differently ---------------------- - Client("vscode", "VS Code", lambda: _vscode_user() / "mcp.json", + Client("vscode", "VS Code", + _vscode_mcp("Code", "com.visualstudio.code"), root="servers", entry=_entry_vscode, jsonc=True), Client("vscode-insiders", "VS Code Insiders", - lambda: _vscode_user("Code - Insiders") / "mcp.json", + _vscode_mcp("Code - Insiders", "com.visualstudio.code.insiders"), root="servers", entry=_entry_vscode, jsonc=True), # Zed writes its settings.json with a block of // comments on first # run, so it is edited as JSON with comments: comments kept. diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py index f405809..498e31f 100644 --- a/tests/test_integrations_clients.py +++ b/tests/test_integrations_clients.py @@ -326,3 +326,22 @@ def test_junie_is_present_before_its_first_server(home): assert clients.CLIENTS["junie"].present() is True assert clients.resolve("droid").key == "factory" assert clients.resolve("amazonq").key == "amazon-q" + + +def test_flathub_vs_code_on_linux_is_found_and_written(home, monkeypatch): + """The Flathub VS Code keeps its User directory in its sandbox, + ~/.var/app/com.visualstudio.code/config/Code/User; only the native + path was looked at, so it was not found and never written.""" + monkeypatch.setattr(clients.sys, "platform", "linux") + if os.name == "nt": + pytest.skip("POSIX layout") + user = (home / ".var" / "app" / "com.visualstudio.code" / "config" + / "Code" / "User") + user.mkdir(parents=True) + vscode = clients.CLIENTS["vscode"] + assert vscode.config_path() == user / "mcp.json" + assert vscode.present() is True + # A native install, where there is one, is still the one written. + (home / ".config" / "Code" / "User").mkdir(parents=True) + assert vscode.config_path() == (home / ".config" / "Code" / "User" + / "mcp.json") From 77fb2a9046223a293d976478e20ef83598c43a61 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:08:26 -0700 Subject: [PATCH 089/135] A text holding half of a cut emoji is embedded, as U+FFFD, instead of failing every store and search with a TypeError An MCP client may legally send a lone surrogate as a JSON "\ud83d" escape, and a string sliced in the middle of a pair holds one too. tokenizers 0.16 and later refuse such a str ("TextInputSequence must be str"), so memory_store, memory_search and `compartment store` with one in the text failed outright, in the shared daemon and in-process alike, while tokenizers 0.15, the floor, let it through, which is why the lowest-dependencies CI job disagreed with the rest about the daemon's error-reply test. Every code point in U+D800..U+DFFF (in a Python str always a lone surrogate) is now replaced by U+FFFD on the way into the tokenizer, in the two places every embedding path goes through (the batch encoder and the chunker), so the daemon and this process embed such a text identically on every tokenizers version. The daemon test that relied on the tokenizer refusing it now checks that the text embeds through the daemon with the daemon kept, and the error-reply path is tested with a stand-in model that raises, over a socketpair, so it no longer depends on the tokenizer's version or on bind() being allowed. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed.py | 25 ++++++++++++ tests/test_embed_batching.py | 18 +++++++++ tests/test_embed_daemon.py | 74 ++++++++++++++++++++++++++++++++---- 3 files changed, 109 insertions(+), 8 deletions(-) diff --git a/src/compartment/embed.py b/src/compartment/embed.py index 71ebb4b..dd3b51f 100644 --- a/src/compartment/embed.py +++ b/src/compartment/embed.py @@ -12,6 +12,7 @@ import functools import json import os +import re import sys import sysconfig import threading @@ -88,6 +89,26 @@ class ModelError(CryptoError): pass +#: Every code point the tokenizer cannot take. In a Python str a code point +#: in U+D800..U+DFFF is always a lone surrogate - a pair is one code point +#: above U+FFFF - and it gets there as half of a cut emoji: a JSON "\\ud83d" +#: escape an MCP client may legally send, or a string sliced mid-pair. +_LONE_SURROGATE = re.compile("[\ud800-\udfff]") + + +def well_formed(text: str) -> str: + """`text` with each lone surrogate replaced by U+FFFD. + + tokenizers 0.16 and later refuse a str holding one with TypeError + ("TextInputSequence must be str"), from every embedding call, so a + store or a search with half an emoji in it failed outright, in the + shared daemon and in-process alike, while 0.15 let it through. Every + path into the tokenizer comes through here, so the daemon and this + process embed such a text identically, and as the replacement + character any UTF-8 decoder would have put in its place.""" + return _LONE_SURROGATE.sub("\ufffd", text) if text else text + + def bundled_model_dir() -> Path: return Path(__file__).resolve().parent / "models" / DEFAULT_MODEL @@ -375,6 +396,7 @@ def __init__(self, model_name: str = DEFAULT_MODEL): def _encode(self, texts: list[str]) -> list[list[int]]: """Token ids per text, truncated to the model's 512, never padded.""" + texts = [well_formed(t) for t in texts] with self._tok_lock: return [e.ids for e in self.tok.encode_batch(texts)] @@ -444,6 +466,9 @@ def chunk(self, text: str, window: int = CHUNK_WINDOW, """ if not text: return [text] + # The windows are cut from the text the tokenizer saw: replacing a + # surrogate is one code point for one, so its offsets index this. + text = well_formed(text) with self._tok_lock: self.tok.no_truncation() try: diff --git a/tests/test_embed_batching.py b/tests/test_embed_batching.py index c4cfa52..e403236 100644 --- a/tests/test_embed_batching.py +++ b/tests/test_embed_batching.py @@ -141,3 +141,21 @@ def test_a_vault_pinned_to_another_model_is_still_refused(vault, vault_path): vault.lock() with pytest.raises(Exception, match="does not match the model"): Vault.unlock(vault_path, passphrase=PASS) + + +def test_a_lone_surrogate_embeds_as_the_replacement_character(emb): + """Half of a cut emoji (a JSON "\\ud83d" escape from an MCP client, or a + string sliced in the middle of a pair) made tokenizers 0.16 and later + raise TypeError from every embedding call, so a store or a search with + one in it failed with a traceback. In a Python str every code point in + U+D800..U+DFFF is a lone surrogate, and it now embeds as U+FFFD.""" + cut, whole = "router \ud83d passphrase", "router � passphrase" + assert np.array_equal(emb.embed_query(cut), emb.embed_query(whole)) + assert np.array_equal(emb.embed_passages([cut, "x \udc00"]), + emb.embed_passages([whole, "x �"])) + long = "a sentence about vaults \udfff and agents. " * 400 + assert emb.chunk(long) == emb.chunk(long.replace("\udfff", "�")) + assert np.array_equal(emb.embed_record(long), + emb.embed_record(long.replace("\udfff", "�"))) + # A real pair, written as one code point, is untouched. + assert emb.chunk("a \U0001F600 b") == ["a \U0001F600 b"] diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index d2b371c..20f8b5c 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -246,21 +246,79 @@ def test_a_daemon_that_stops_answering_does_not_hang_its_clients( @needs_bind -def test_a_text_the_model_rejects_is_an_error_not_a_lost_daemon(daemon_env): - """A text the tokenizer cannot encode, such as one with a lone surrogate - from a cut emoji, comes back from the daemon as an error reply. The - client took that for a daemon that had gone: it dropped the connection, - loaded a model of its own for the rest of the process, ran the text - again there, and the caller saw the tokenizer's bare TypeError.""" +def test_a_text_with_a_lone_surrogate_embeds_through_the_daemon(daemon_env, + local): + """Half of a cut emoji, which an MCP client may send as a JSON \\ud83d + escape, made tokenizers 0.16 and later raise TypeError, in the daemon + and in-process alike (0.15 let it through). It now embeds as U+FFFD + would, through the same daemon, which stays the one serving.""" r = RemoteEmbedder.connect() r.embed_query("warm up") pid = r.daemon_pid - with pytest.raises(ValueError, match="TypeError"): - r.embed_query("where is the router \ud83d passphrase") + vec = r.embed_query("where is the router \ud83d passphrase") assert r.shared and r.daemon_pid == pid assert embed_daemon.status(daemon_env)["clients"] == 1 + assert np.array_equal( + vec, local.embed_query("where is the router \ufffd passphrase")) + + +def _served(monkeypatch, model=None): + """RemoteEmbedder.connect() wired to a daemon's own connection handler + over a socketpair, which needs no bind(), so this runs everywhere, + sandboxes included. `model` stands in for Embedder in the daemon.""" + if model is not None: + monkeypatch.setattr(embed_daemon, "Embedder", model) + daemon = embed_daemon._Daemon(Path("unused.sock"), idle=60) + + def pair(*a, **k): + mine, theirs = socket.socketpair(socket.AF_UNIX) + threading.Thread(target=daemon._serve, args=(theirs,), + daemon=True).start() + return mine + monkeypatch.setattr(embed_daemon, "connect_or_spawn", pair) + r = RemoteEmbedder.connect(path=Path("unused.sock"), spawn=False) + assert r is not None and r.shared + return r, daemon + + +def test_a_text_the_model_rejects_is_an_error_not_a_lost_daemon(monkeypatch): + """A request the daemon's model fails on comes back as an error reply. + The client took that for a daemon that had gone: it dropped the + connection, loaded a model of its own for the rest of the process, ran + the text again there, and the caller saw the model's bare error.""" + class Refusing: + def __init__(self, name): + self.dim = model_info(name).dim + + def embed_query(self, text): + if "refuse" in text: + raise TypeError("TextInputSequence must be str") + return np.ones(self.dim, np.float32) + + r, daemon = _served(monkeypatch, Refusing) + pid = r.daemon_pid + with pytest.raises(ValueError, match="TypeError"): + r.embed_query("please refuse this one") + assert r.shared and r.daemon_pid == pid + assert daemon._status()["clients"] == 1 assert r.embed_query("an ordinary query afterwards").shape == (r.dim,) assert r.shared + r.close() + + +def test_the_daemon_and_this_process_agree_on_a_lone_surrogate(monkeypatch, + local): + """The daemon gets the text through JSON, where a lone surrogate is a + \\ud83d escape; it must embed what this process's own model embeds.""" + r, _ = _served(monkeypatch) + text = "where is the router \ud83d passphrase" + assert np.array_equal(r.embed_query(text), local.embed_query(text)) + assert np.array_equal(r.embed_passages([text, "plain"]), + local.embed_passages([text, "plain"])) + long = ("sentence about vaults \udc00 and agents. " * 400) + assert r.chunk(long) == local.chunk(long) + assert r.shared + r.close() @needs_bind From 9ee627c4d431f6cf0a5e00be5732f91f030fd908 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:08:39 -0700 Subject: [PATCH 090/135] The Claude Code skill follows CLAUDE_CONFIG_DIR by the same rule as the hook and the import, ignoring a relative value The skill's home took CLAUDE_CONFIG_DIR as given, while the capture hook and the memory import ignore a value that is not an absolute path, as Claude Code's own VS Code extension does. With a relative value the skill was written relative to whatever directory `integrate` ran in. It now goes through claude_memory.claude_config_dir(), like the other two. Co-Authored-By: Claude Opus 5.5 --- src/compartment/agent_skill.py | 11 ++++++++++- tests/test_integrations_skills.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/compartment/agent_skill.py b/src/compartment/agent_skill.py index 1d1fa52..56ffd87 100644 --- a/src/compartment/agent_skill.py +++ b/src/compartment/agent_skill.py @@ -55,12 +55,21 @@ def _openclaw_dir() -> Path | None: return None +def _claude_dir() -> Path | None: + """Claude Code's directory when CLAUDE_CONFIG_DIR moves it, by the same + rule the hook and the import use (an absolute path only).""" + if not os.environ.get("CLAUDE_CONFIG_DIR", "").strip(): + return None + from .claude_memory import claude_config_dir + return claude_config_dir() + + #: Every agent known to read the Agent Skills layout, and what relocates #: its home: the environment variable naming it, or a function working it #: out. Claude Code's is CLAUDE_CONFIG_DIR, which moves its skills/ with #: everything else. SKILL_TARGETS = { - "claude": ("CLAUDE_CONFIG_DIR", Path.home() / ".claude"), + "claude": (_claude_dir, Path.home() / ".claude"), "hermes": ("HERMES_HOME", Path.home() / ".hermes"), "openclaw": (_openclaw_dir, Path.home() / ".openclaw"), } diff --git a/tests/test_integrations_skills.py b/tests/test_integrations_skills.py index b3a809a..5aa3b3e 100644 --- a/tests/test_integrations_skills.py +++ b/tests/test_integrations_skills.py @@ -42,3 +42,18 @@ def test_openclaw_without_any_variable_is_in_the_home_directory( assert r["action"] == "written" assert agent_skill.is_current("openclaw") assert agent_skill.remove("openclaw") is True + + +def test_a_relative_claude_config_dir_does_not_scatter_the_skill( + monkeypatch, tmp_path): + """The hook and the import ignore a CLAUDE_CONFIG_DIR that is not + absolute, as Claude Code's own VS Code extension does; the skill must + not land relative to whatever directory the command ran in.""" + monkeypatch.setitem(agent_skill.SKILL_TARGETS, "claude", + (agent_skill._claude_dir, tmp_path / ".claude")) + monkeypatch.setattr(agent_skill.Path, "home", + classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", "relative/claude") + assert agent_skill.agent_home("claude") == tmp_path / ".claude" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "abs")) + assert agent_skill.agent_home("claude") == tmp_path / "abs" From fea9c1e1ed0b390b86b34010de9467598b584c9f Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:09:02 -0700 Subject: [PATCH 091/135] The AI install guide for Cline gets an agent past PEP 668, Windows and a client that cannot see the PATH Cline's marketplace installs a server by handing llms-install.md to the agent, and the guide's one install line was `pip install compartment`. On Debian 12+, Ubuntu 23.04+, Homebrew's Python, Arch and others pip refuses with "externally-managed-environment", and an agent left without guidance reaches for --break-system-packages. The guide now names the error and sends the agent to `pipx install compartment` or `uv tool install compartment`, and says not to break the system Python. It gives the Windows forms (`py --version`, `py -m pip install`, `where`), how to put the install directory on PATH (`pipx ensurepath`, `uv tool update-shell`), and that a client started from the Dock or Start menu does not see the shell's PATH: a hand-written block should use the absolute path, which `compartment integrate` already writes. The step that said Cline launches "compartment serve" now says it launches the path integrate wrote. Before the fix: test_llms_install_gets_an_agent_past_the_common_install_failures E assert 'externally-managed-environment' in '# Installing ...' Co-Authored-By: Claude Opus 5.5 --- llms-install.md | 45 ++++++++++++++++++++++++++-------- tests/test_plugin_manifests.py | 19 ++++++++++++++ 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/llms-install.md b/llms-install.md index 5f38818..385af05 100644 --- a/llms-install.md +++ b/llms-install.md @@ -16,8 +16,9 @@ Compartment needs **Python 3.11 or newer**. python3 --version ``` -If that reports anything below 3.11, stop and tell the user to install a newer -Python first. Do not try to work around it. +On Windows the command is `py --version` (or `python --version`). If it +reports anything below 3.11, stop and tell the user to install a newer Python +first. Do not try to work around it. ## 2. Install the package @@ -25,16 +26,30 @@ Python first. Do not try to work around it. pip install compartment ``` -`pipx install compartment` and `uv tool install compartment` both work too, and -are better if the user keeps their global Python clean. All three put a -`compartment` executable on the PATH. Confirm it landed: +On Windows: `py -m pip install compartment`. + +If pip refuses with `error: externally-managed-environment` (Debian 12 and +newer, Ubuntu 23.04 and newer, Homebrew's Python, Arch and others mark the +system Python as managed by the OS), do not pass `--break-system-packages`. +Use one of these instead, which install Compartment into its own environment: + +```bash +pipx install compartment +uv tool install compartment +``` + +They are also the better choice when the user keeps their global Python +clean. All three routes put a `compartment` executable on the PATH. Confirm it +landed: ```bash compartment --version ``` If the shell cannot find `compartment`, the install directory is not on PATH. -On most systems that directory is `~/.local/bin`. +On most systems that directory is `~/.local/bin` (`pipx ensurepath` or +`uv tool update-shell` adds it); on Windows it is the `Scripts` folder of the +Python installation, which `py -m pip show compartment` reveals. ## 3. Create the vault (the user must do this step, not you) @@ -100,7 +115,8 @@ Then confirm the MCP server itself starts: compartment serve --help ``` -Cline launches it as `compartment serve` over stdio. You do not need to run +Cline launches the server itself over stdio, using the full path to the +`compartment` executable that `integrate` wrote. You do not need to run `compartment serve` yourself, and you should not leave a copy running. ## Manual configuration (any MCP client) @@ -121,14 +137,23 @@ If a client is not in `--list`, add this block to its MCP config by hand: VS Code uses the key `servers` and wants `"type": "stdio"`. Zed uses `context_servers`. Codex uses TOML under `[mcp_servers.compartment]`. +A client started from the Dock, the Start menu or a desktop launcher does not +see the PATH the user's shell sets up, so a bare `compartment` that works in +the terminal can fail inside it. When writing the block by hand, prefer the +absolute path: `which compartment` prints it on macOS and Linux, +`where compartment` on Windows. `compartment integrate` always writes the +absolute path. + ## Troubleshooting **Tools return a locked error.** The vault is locked. `compartment unlock`, or `compartment unlock --keychain` on macOS so it survives the next reboot. -**`compartment: command not found` inside the client but not in the shell.** -The client is not inheriting the user's PATH. Put the absolute path to the -executable in the `command` field instead. `which compartment` prints it. +**`compartment: command not found` (or `spawn compartment ENOENT`) inside the +client but not in the shell.** The client is not inheriting the user's PATH. +Put the absolute path to the executable in the `command` field instead, or run +`compartment integrate `, which does. `which compartment` prints the +path on macOS and Linux, `where compartment` on Windows. **The client shows the server but lists no tools.** Reload the client window. Most MCP clients only read their server list at startup. diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index 76287a1..dd59c49 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -413,3 +413,22 @@ def test_the_omp_extension_declares_what_each_tool_does(): "memory_search": {"approval": "read", "loadMode": "essential"}, "memory_store": {"approval": "write", "loadMode": "essential"}, } + + +# ---------------------------------------------------- Cline (llms-install) -- +def test_llms_install_gets_an_agent_past_the_common_install_failures(): + """Cline's marketplace installs by handing this file to the agent, which + then meets what users meet: pip refusing on an OS-managed Python + (PEP 668), a Windows shell, a GUI client that cannot see the PATH.""" + from compartment import clients + text = (ROOT / "llms-install.md").read_text(encoding="utf-8") + assert "externally-managed-environment" in text + assert "pipx install compartment" in text and "uv tool install compartment" in text + # --break-system-packages appears only as something not to do + for line in text.splitlines(): + if "--break-system-packages" in line: + assert "do not" in line.lower(), line + assert "py -m pip install compartment" in text + assert "where compartment" in text and "which compartment" in text + for client in re.findall(r"compartment integrate ([a-z0-9][a-z0-9-]*)", text): + assert client in clients.CLIENTS, client From 63eec0c2fe1afe4dd8a639f14aa6f9801539861a Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:09:03 -0700 Subject: [PATCH 092/135] A store that could not be journalled is never answered as a duplicate on retry or written by a later save store(), link() and the other writes change the rows and the index in RAM before they journal. When the append did not land - the write lock still held past its timeout by another process's long import or compaction, a disk that refused the first byte, a failed sync - the change stayed in RAM. A retry of the same store then matched it as a near-duplicate, returned its id as stored and journalled nothing, so the memory the caller believed stored was gone at the next reopen; and if no retry came, the next save wrote the record the caller had been told was not stored. A failed append now marks the handle as diverged from the file: is_stale() is true from then on, so the MCP server reopens from disk before its next call, and every write from the handle (checked at the door of store() and link(), which can answer from RAM, and under the write lock) refuses with a VaultStaleError that says to reopen. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 35 +++++++++++++++++++- tests/test_storage_durability.py | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 2284f16..1fd6a42 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -354,6 +354,11 @@ class VaultStaleError(CryptoError): """Another process wrote the vault; the caller should reopen and retry.""" +#: The disk state of a handle whose RAM holds a change its journal append +#: failed to write. Matches no file, so is_stale() is True from then on. +_DIVERGED = (-1, -1, -1, "") + + def _synchronized(fn): """Serialize vault operations: the in-RAM SQLite connection and index are shared with background threads (Hermes provider writer, auto-lock).""" @@ -423,11 +428,23 @@ def is_stale(self) -> bool: return True return self._stat_disk() != self._disk_state + def _refuse_if_diverged(self) -> None: + """Refuse to write from a handle whose journal append failed (see + _journal). Checked at the door of the writes that can answer from RAM + without journalling - a store matched as a duplicate, a link that + already exists - as well as under the write lock.""" + if self._disk_state == _DIVERGED: + raise VaultStaleError( + "An earlier write from this handle did not reach the vault " + "file, so what it holds in memory no longer matches it. " + "Reopen the vault and retry.") + def _with_file_lock(self, fn, timeout: float = 10.0): """Advisory single-writer lock: serializes journal appends and saves across processes sharing one vault (Hermes + Claude + CLI). Cross-platform via platforms.FileLock (POSIX flock / Windows msvcrt).""" with FileLock(self.path + ".flock", timeout=timeout): + self._refuse_if_diverged() if self.is_stale(): if not os.path.exists(self.path): raise VaultStaleError( @@ -947,7 +964,21 @@ def _append(): vaultfile.append_journal_entries(self.path, self.header, self._journal_seq, list(entries), self._master) - self._with_file_lock(_append) + try: + self._with_file_lock(_append) + except BaseException: + # Every caller changes the rows and the index in RAM before it + # journals, so a journal that did not land (the write lock busy + # past its timeout, a full disk, a failed sync) leaves this + # handle holding a change the file does not have. Kept, it went + # to disk with the next save, a memory the caller had been told + # was not stored; and a retry of the same store found it in RAM, + # returned it as a duplicate, and journalled nothing, so it was + # gone at the next reopen. The handle is marked as no longer + # matching the file, so every later write refuses and the server + # reopens from disk, which is the only state worth trusting. + self._disk_state = _DIVERGED + raise self._journal_seq += len(entries) def _replay(self, e: dict) -> None: @@ -1045,6 +1076,7 @@ def store(self, text: str, caller: str, namespace: str | None = None, _expiry_strict: bool = True, _gate: bool = True, _split_hint: str | None = None) -> dict: self._require_open() + self._refuse_if_diverged() text, source, discovered, namespace = ( clean_text(text), clean_text(source), clean_text(discovered), clean_text(namespace)) @@ -1875,6 +1907,7 @@ def link(self, subject: str, predicate: str, obj: str, caller: str, host model (or the user), exactly like memory curation. Idempotent: re-linking the same triple returns the existing relation.""" self._require_open() + self._refuse_if_diverged() subject, predicate, obj, namespace = ( clean_text(subject), clean_text(predicate), clean_text(obj), clean_text(namespace)) diff --git a/tests/test_storage_durability.py b/tests/test_storage_durability.py index 064eb81..8fc7df4 100644 --- a/tests/test_storage_durability.py +++ b/tests/test_storage_durability.py @@ -297,3 +297,60 @@ def replace(src, dst): v.save() assert Vault.unlock(vault_path, passphrase=PASS).status()[ "organic_records"] == 1 + + +def test_a_store_whose_journal_did_not_land_cannot_be_saved_later( + vault_path, monkeypatch): + """The store changes RAM before it journals. When the journal never + landed - here the write lock stayed busy past its timeout, as behind a + long import in another process - the record stayed in RAM: a retry of + the same store was answered "duplicate" from it and journalled nothing, + and the memory the caller believed stored was gone at the next reopen, + or a later save wrote the one the caller was told had failed.""" + from compartment import vault as vault_mod + from compartment.crypto import CryptoError + from compartment.vault import VaultStaleError + + v = Vault.create(vault_path, PASS, creator="t") + real_lock = vault_mod.FileLock + calls = [] + + def busy_once(path, timeout=10.0): + calls.append(path) + if len(calls) == 1: + raise CryptoError("Vault is busy: another process holds the " + "write lock (waited 10s)") + return real_lock(path, timeout=timeout) + + monkeypatch.setattr(vault_mod, "FileLock", busy_once) + text = "The parking spot is level 3, bay 41." + with pytest.raises(CryptoError, match="busy"): + v.store(text, caller="t") + assert v.is_stale(), "the handle no longer matches the file" + with pytest.raises(VaultStaleError, match="did not reach"): + v.store(text, caller="t") + with pytest.raises(VaultStaleError): + v.save() + + again = Vault.unlock(vault_path, passphrase=PASS) # what the server does + assert again.db.count() == 0 + out = again.store(text, caller="t") + assert not out["duplicate"] + assert Vault.unlock(vault_path, passphrase=PASS).db.count() == 1 + + +def test_a_refused_append_that_wrote_nothing_is_not_saved_later( + vault_path, monkeypatch): + """A disk already full refuses the first byte: the file is untouched, + so nothing about it says this handle is behind - but its RAM holds the + record, and the next save wrote it.""" + from compartment.vault import VaultStaleError + + v = Vault.create(vault_path, PASS, creator="t") + with monkeypatch.context() as m: + _disk_fills_during_appends(m, _Budget(room=0)) + with pytest.raises(OSError, match="No space"): + v.store("A memory refused at the first byte.", caller="t") + with pytest.raises(VaultStaleError): + v.save() + assert Vault.unlock(vault_path, passphrase=PASS).db.count() == 0 From 801945bb86da391ecd9be59fe21c6539f73796b4 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:09:36 -0700 Subject: [PATCH 093/135] On Linux, init registers start at login only for a Python that can draw the panel, and names the Tk package for this machine `_start_status_bar_app` registered start at login (an autostart entry or a systemd user unit, plus the applications menu entry) before it checked that tkinter imports. A Linux Python without Tk therefore got entries that failed at every login from then on, with nothing on screen to say why, and a menu entry that opened nothing. tkinter is now checked first: with none, nothing is registered, and the message names the install command for this machine from systray.tk_install_hint() (each Linux family, the BSDs, Termux, conda, Homebrew, uv) instead of a fixed Debian/Fedora line, and says to turn start at login on afterwards with `compartment panel --login on`. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 27 ++++++++++++++++++--------- tests/test_cli_status_bar.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 tests/test_cli_status_bar.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 83adc75..9c8abb6 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -332,6 +332,24 @@ def _start_status_bar_app(vault: str) -> None: print(" The CLI and the MCP server are ready, and `compartment " "dash` opens the vault in a browser.") return + if linux and not _linux_gui_available(): + # Before start at login is registered, not after: an autostart + # entry or a systemd user unit for a panel that cannot import + # tkinter fails at every login from then on, with nothing on screen + # to say why. + hint = getattr(app, "tk_install_hint", None) + print(" the panel needs tkinter, which this Python does not have.") + print(" Quickest fix, a Python that includes it:") + print(" uv tool install compartment") + print(" or give this Python tkinter:") + print(f" {hint()}" if hint else + " install your distribution's Tk package for Python " + "(often python3-tk or python3-tkinter)") + print(" then turn start at login on with `compartment panel " + "--login on`.") + print(" Everything else works now: the CLI, the MCP server, and " + "`compartment dash`.") + return state = "" try: state = app.set_login(True, vault) @@ -346,15 +364,6 @@ def _start_status_bar_app(vault: str) -> None: print(" applications menu entry: on") except Exception as exc: # noqa: BLE001 print(f" could not register the app ({exc})") - if linux and not _linux_gui_available(): - print(" the panel needs tkinter, which this Python does not have.") - print(" Quickest fix, a Python that includes it:") - print(" uv tool install compartment") - print(" or install your distribution's package (Debian/Ubuntu: " - "python3-tk, Fedora: python3-tkinter).") - print(" Everything else works now: the CLI, the MCP server, and " - "`compartment dash`.") - return # Let the supervisor start it where there is one, so the copy that comes # up is the copy that gets restarted if it dies. On macOS bootstrapping # the agent a moment ago already did it (RunAtLoad); on Linux and Windows diff --git a/tests/test_cli_status_bar.py b/tests/test_cli_status_bar.py new file mode 100644 index 0000000..3710d5d --- /dev/null +++ b/tests/test_cli_status_bar.py @@ -0,0 +1,30 @@ +"""What `compartment init` does to put the status bar app up: nothing +registered for a panel that cannot draw.""" +import sys +import types + +from compartment import cli + + +def test_a_python_without_tkinter_registers_nothing_to_start_at_login( + monkeypatch, capsys, tmp_path): + """Start at login was registered before tkinter was checked, so a Linux + Python without it got an autostart entry or a systemd user unit that + failed at every login, with nothing on screen to say why.""" + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setenv("DISPLAY", ":0") + monkeypatch.setattr(cli, "_linux_gui_available", lambda: False) + registered = [] + app = types.SimpleNamespace( + set_login=lambda on, vault=None: registered.append(on) or "on", + tk_install_hint=lambda: "sudo zypper install python312-tk") + monkeypatch.setattr(cli, "_tray_app", lambda: app) + monkeypatch.setattr(cli.subprocess, "Popen", lambda *a, **k: + (_ for _ in ()).throw(AssertionError("launched"))) + cli._start_status_bar_app(str(tmp_path / "v.vault")) + out = capsys.readouterr().out + assert registered == [] + assert "sudo zypper install python312-tk" in out + assert "uv tool install compartment" in out + assert "compartment panel --login on" in out + assert "start at login: on" not in out From 3d80e591038012b6957c01a98146514589722675 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:09:37 -0700 Subject: [PATCH 094/135] On Windows, init starts the tray app with pythonw.exe, so no console window opens behind it `compartment init` started the tray app as the console script compartment.exe with DETACHED_PROCESS. That keeps it off the installer's console, but compartment.exe is a console program whose Python child could still open a console window of its own behind a tray app that has no window, and closing that window quit the app. It now starts the program start at sign-in runs (systray._autostart_parts): pythonw.exe beside this interpreter, `-m compartment.cli --vault V tray`, with the console script only where there is no pythonw.exe. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 24 ++++++++++++--- tests/test_cli_status_bar.py | 59 +++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 9c8abb6..bdcf9da 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -311,6 +311,25 @@ def _linux_gui_available() -> bool: } +def _status_bar_argv(app, vault: str) -> list[str]: + """The command that starts the status bar app now. + + On Windows the program start at sign-in uses (systray._autostart_parts): + pythonw.exe beside this interpreter where there is one. compartment.exe + is a console program, and DETACHED_PROCESS only keeps it off this + terminal's console: the Python it starts could still open a console + window of its own behind a tray app that has no window.""" + if sys.platform == "win32": + parts = getattr(app, "_autostart_parts", None) + exe = parts(vault)[0] if parts else None + if exe and Path(exe).name.lower() == "pythonw.exe": + return [exe, "-m", "compartment.cli", "--vault", vault, "tray"] + exe = shutil.which("compartment") + return ([exe, "--vault", vault, "menubar"] if exe else + [sys.executable, "-m", "compartment.cli", "--vault", vault, + "menubar"]) + + def _start_status_bar_app(vault: str) -> None: """Put the status bar app up now, and again at every login. @@ -383,10 +402,7 @@ def _start_status_bar_app(vault: str) -> None: print(" start at login is registered, but no icon appeared; " "starting one now") try: - exe = shutil.which("compartment") - argv = ([exe, "--vault", vault, "menubar"] if exe else - [sys.executable, "-m", "compartment.cli", "--vault", vault, - "menubar"]) + argv = _status_bar_argv(app, vault) # stdin included. Without it the app inherits this process's stdin, # sees a terminal on the other end, decides it was typed at a prompt # and relaunches itself detached - so the copy the installer started diff --git a/tests/test_cli_status_bar.py b/tests/test_cli_status_bar.py index 3710d5d..ecc617e 100644 --- a/tests/test_cli_status_bar.py +++ b/tests/test_cli_status_bar.py @@ -1,11 +1,20 @@ """What `compartment init` does to put the status bar app up: nothing -registered for a panel that cannot draw.""" +registered for a panel that cannot draw, and no console window behind the +Windows tray app.""" +import subprocess import sys import types from compartment import cli +class _Running: + """A child still running when the installer looks.""" + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired("tray", timeout) + + def test_a_python_without_tkinter_registers_nothing_to_start_at_login( monkeypatch, capsys, tmp_path): """Start at login was registered before tkinter was checked, so a Linux @@ -28,3 +37,51 @@ def test_a_python_without_tkinter_registers_nothing_to_start_at_login( assert "uv tool install compartment" in out assert "compartment panel --login on" in out assert "start at login: on" not in out + + +def _windows(monkeypatch, tmp_path, pythonw: bool): + scripts = tmp_path / "venv" / "Scripts" + scripts.mkdir(parents=True) + names = ["python.exe", "compartment.exe"] + (["pythonw.exe"] if pythonw + else []) + for name in names: + (scripts / name).write_bytes(b"MZ") + monkeypatch.setattr(sys, "platform", "win32") + + def parts(vault=None): # systray._autostart_parts, in short + pyw = scripts / "pythonw.exe" + return ((str(pyw), "-m compartment.cli tray") if pyw.is_file() + else (str(scripts / "compartment.exe"), "tray")) + app = types.SimpleNamespace(set_login=lambda on, vault=None: "on", + start_supervised=lambda: False, + _autostart_parts=parts) + monkeypatch.setattr(cli, "_tray_app", lambda: app) + monkeypatch.setattr(cli.shutil, "which", + lambda name, *a, **k: str(scripts / f"{name}.exe")) + monkeypatch.setattr(cli, "_wait_for_status_bar_app", lambda *a, **k: True) + started = [] + monkeypatch.setattr(cli.subprocess, "Popen", lambda argv, **k: + started.append((argv, k)) or _Running()) + return scripts, started + + +def test_windows_starts_the_tray_with_pythonw(monkeypatch, tmp_path, capsys): + """The console script compartment.exe was started with DETACHED_PROCESS, + and the Python it runs could still open a console window of its own + behind a tray app that has no window. pythonw.exe opens none, and it is + what start at sign-in runs.""" + scripts, started = _windows(monkeypatch, tmp_path, pythonw=True) + vault = str(tmp_path / "Zoë Ann" / "v.vault") + cli._start_status_bar_app(vault) + (argv, kwargs), = started + assert argv == [str(scripts / "pythonw.exe"), "-m", "compartment.cli", + "--vault", vault, "tray"] + assert kwargs["stdin"] is subprocess.DEVNULL + + +def test_windows_without_pythonw_still_starts_the_console_script( + monkeypatch, tmp_path, capsys): + scripts, started = _windows(monkeypatch, tmp_path, pythonw=False) + cli._start_status_bar_app(str(tmp_path / "v.vault")) + (argv, _), = started + assert argv[0] == str(scripts / "compartment.exe") From 82278a46c021104261a660f92eadc32ed8bdf7e2 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:09:56 -0700 Subject: [PATCH 095/135] The website stops rendering the shared skill and plugin manifests as pages, and the shared plugin root is guarded by tests GitHub Pages builds the repository root with Jekyll, and _config.yml excluded server.json and glama.json but not the other manifests, nor skills/, whose SKILL.md carries YAML front matter, so Jekyll published it as a page at /skills/compartmentalize/SKILL.html and copied plugin.json, mcp.json, gemini-extension.json, GEMINI.md and the Dockerfile into the site. They are excluded now. The same root is installed whole by Claude Code, Codex, Gemini CLI, OpenClaw and Hermes, and each loads some directories from it unasked: Claude Code and OpenClaw a hooks/hooks.json, commands/, agents/, .mcp.json and settings.json; Gemini a hooks/hooks.json and commands/ in its own different formats, plus policies/. Nothing of the kind exists today, which is right; a test now names each one so that adding it is a decision about all five hosts rather than one (it is also why the Claude Code capture hook is not shipped as a root hooks/hooks.json - see docs/PLUGINS.md). Further guards: every skills/*/SKILL.md meets the Agent Skills specification that Hermes and Codex enforce, the Claude marketplace serves the plugin from the root under the same name, the Gemini manifest has what the gallery crawler indexes, and GEMINI.md is the server's handshake instructions verbatim. Before the fix: test_the_website_does_not_publish_the_plugin_files_as_pages E AssertionError: skills Co-Authored-By: Claude Opus 5.5 --- _config.yml | 9 ++++ tests/test_plugin_manifests.py | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/_config.yml b/_config.yml index 916a7ea..71116cd 100644 --- a/_config.yml +++ b/_config.yml @@ -42,3 +42,12 @@ exclude: - server.json - glama.json - RELEASING.md + # The repository root is also a plugin for Claude Code, Codex, Gemini CLI, + # OpenClaw and Hermes. Their manifests are not pages, and skills/ holds a + # SKILL.md whose YAML front matter Jekyll would otherwise render as one. + - skills + - plugin.json + - mcp.json + - gemini-extension.json + - GEMINI.md + - Dockerfile diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index dd59c49..e331268 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -432,3 +432,97 @@ def test_llms_install_gets_an_agent_past_the_common_install_failures(): assert "where compartment" in text and "which compartment" in text for client in re.findall(r"compartment integrate ([a-z0-9][a-z0-9-]*)", text): assert client in clients.CLIENTS, client + + +# ---------------------------------- the repository root is a plugin root ---- +#: What each host that installs the repository root loads from it without +#: being told to. skills/ and mcp.json are shared on purpose; everything +#: else here would be read by several hosts in several incompatible ways +#: (a Claude hooks/hooks.json is not a Gemini one; Gemini's commands/ are +#: TOML, Claude's Markdown; OpenClaw imports a root settings.json into its +#: own settings). Adding one is a decision for every host at once, which is +#: why this test names them. +HOST_AUTOLOADED = ("commands", "agents", "hooks", "policies", "output-styles", + "monitors", "themes", ".mcp.json", ".lsp.json", + "settings.json", ".codex-plugin", ".cursor-plugin", + "openclaw.plugin.json") + + +def test_the_root_holds_nothing_a_host_would_load_by_surprise(): + present = [n for n in HOST_AUTOLOADED if (ROOT / n).exists()] + assert not present, ( + f"{present} at the repository root would be auto-loaded by Claude " + "Code, Gemini CLI, Codex, OpenClaw or Hermes, each in its own format; " + "see HOST_AUTOLOADED and docs/PLUGINS.md before adding it") + + +def test_every_shared_skill_meets_the_agent_skills_specification(): + """All five hosts load skills/*/SKILL.md, and Hermes and Codex skip a + skill that fails the specification instead of loading it loosely.""" + allowed = {"name", "description", "license", "compatibility", "metadata", + "allowed-tools"} + skills = sorted((ROOT / "skills").glob("*/SKILL.md")) + assert skills + for path in skills: + text = path.read_text(encoding="utf-8") + assert text.startswith("---\n") + front = text.split("---\n", 2)[1] + keys = set(re.findall(r"^([A-Za-z][\w-]*):", front, re.M)) + assert keys <= allowed, (path, keys - allowed) + name = re.search(r"^name:\s*(\S+)", front, re.M).group(1) + assert name == path.parent.name + assert re.fullmatch(r"[a-z0-9]+(-[a-z0-9]+)*", name) and len(name) <= 64 + desc = re.search(r"^description:\s*(.+)$", front, re.M).group(1) + assert 1 <= len(desc) <= 1024 + # metadata is a map of strings: quoted, never a YAML list or number + for value in re.findall(r"^ [\w-]+:\s*(.*)$", front, re.M): + assert value.startswith('"') and value.endswith('"'), (path, value) + + +# --------------------------------------------------- Claude Code, Gemini ---- +def test_the_claude_code_marketplace_serves_the_plugin_at_the_root(): + market = _json(".claude-plugin/marketplace.json") + plugin = _json(".claude-plugin/plugin.json") + assert market["owner"]["name"] + (entry,) = market["plugins"] + assert entry["name"] == plugin["name"] == "compartment" + assert entry["source"] == "./" + assert re.fullmatch(r"[a-z][a-z0-9]*(-[a-z0-9]+)*", plugin["name"]) + assert re.fullmatch(r"\d+\.\d+\.\d+", plugin["version"]) + server = plugin["mcpServers"]["compartment"] + assert set(server) <= {"type", "command", "args", "env"} + + +def test_the_gemini_extension_is_what_the_gallery_indexes(): + """The gallery crawler wants gemini-extension.json at the repository + root with a name matching the install directory, a version and a + description; extensions may not set `trust` on a server.""" + d = _json("gemini-extension.json") + assert d["name"] == "compartment" + assert re.fullmatch(r"\d+\.\d+\.\d+", d["version"]) and d["description"] + assert (ROOT / d["contextFileName"]).is_file() + for server in d["mcpServers"].values(): + assert "trust" not in server + + +def test_gemini_md_is_the_handshake_instructions_verbatim(): + """Gemini CLI loads GEMINI.md into every session as the extension's + context, on top of the instructions the server sends in its handshake; + two versions of the rules would tell the model two things.""" + from compartment import server + flat = lambda s: re.sub(r"\s+", " ", s).strip() # noqa: E731 + body = (ROOT / "GEMINI.md").read_text(encoding="utf-8").split("\n", 1)[1] + assert flat(body) == flat(server.COMPARTMENT_INSTRUCTIONS) + + +def test_the_website_does_not_publish_the_plugin_files_as_pages(): + """GitHub Pages builds the repository root with Jekyll, which renders + any file with YAML front matter (every SKILL.md) as a page and copies + the rest; the manifests are for hosts, not for the site.""" + import pytest + yaml = pytest.importorskip("yaml") + excluded = set(yaml.safe_load((ROOT / "_config.yml").read_text( + encoding="utf-8"))["exclude"]) + for name in ("skills", "plugin.json", "mcp.json", "gemini-extension.json", + "GEMINI.md", "server.json", "glama.json", "packaging", "tools"): + assert name in excluded, name From 9b80f3813e1a9be2f71ce8d985cd5d8c83f1c47f Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:10:38 -0700 Subject: [PATCH 096/135] On native Windows, `integrate hermes` puts the provider plugin where Hermes reads it, and finds Hermes's venv Hermes Agent keeps its home in HERMES_HOME or, by default, ~/.hermes, but on Windows its default is %LOCALAPPDATA%\hermes (hermes_constants._platform_default_hermes_home), and its Windows installer does not export HERMES_HOME. `compartment integrate hermes` looked only in HERMES_HOME or ~/.hermes, so on native Windows it wrote the provider plugin to ~/.hermes\plugins, which Hermes never reads, reported it installed, and looked for Hermes's interpreter at venv/bin/python, the POSIX layout, so it never put compartment into Hermes's venv either. Uninstall and the refresh after an update looked in the same wrong place. The Hermes home is now resolved as Hermes resolves it: HERMES_HOME with ~ and variables expanded, else %LOCALAPPDATA%\hermes on Windows and ~/.hermes elsewhere, and the venv interpreter is venv\Scripts\python.exe on Windows. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 30 +++++++++++++-- tests/test_cli_hermes_home.py | 69 +++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 tests/test_cli_hermes_home.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index bdcf9da..3b74d6f 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -671,7 +671,7 @@ def cmd_uninstall(args) -> None: print(f" compartment block removed from {md}") except Exception as exc: # noqa: BLE001 print(f" CLAUDE.md: {exc}") - hermes_home = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) + hermes_home = _hermes_home() plugin = hermes_home / "plugins" / "compartment" try: files = ("__init__.py", "plugin.yaml") @@ -1856,7 +1856,7 @@ def _refresh_installed() -> list[str]: line += f" (the previous copy is kept at {r['backup']})" done.append(line) - hermes_home = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) + hermes_home = _hermes_home() plug_src = _data_dir() / "hermes-plugin" plug_dst = hermes_home / "plugins" / "compartment" if (plug_dst / "plugin.yaml").is_file(): @@ -2007,6 +2007,27 @@ def _integrate_client(client, args, brief: bool = False) -> None: #: small step for that program; one busy updating itself is no reason to #: hold an install for as long as that takes. AGENT_TOOL_TIMEOUT = 120 +def _hermes_home() -> Path: + """Hermes Agent's home, found the way Hermes finds it + (hermes_constants.get_hermes_home): HERMES_HOME, with ~ and variables + expanded, else the platform default. + + On Windows that default is %LOCALAPPDATA%\\hermes, not ~/.hermes, and the + Windows installer does not export HERMES_HOME. Looking only in ~/.hermes, + `integrate hermes` on native Windows put the provider plugin where + Hermes never reads it and reported it installed, never found the + Hermes venv to put compartment into, and uninstall and the refresh + after an update looked in the same wrong place.""" + explicit = os.environ.get("HERMES_HOME", "").strip() + if explicit: + return Path(os.path.expanduser(os.path.expandvars(explicit))) + if sys.platform == "win32": + local = os.environ.get("LOCALAPPDATA", "").strip() + return (Path(local) if local + else Path.home() / "AppData" / "Local") / "hermes" + return Path.home() / ".hermes" + + #: The `import compartment` check in Hermes's own interpreter. HERMES_IMPORT_TIMEOUT = 30 #: pip installing compartment into Hermes's own environment. @@ -2020,7 +2041,7 @@ def _integrate_target(args) -> None: vault = args.vault if target == "hermes": - hermes_home = Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) + hermes_home = _hermes_home() plug_src = _data_dir() / "hermes-plugin" plug_dst = hermes_home / "plugins" / "compartment" plug_dst.mkdir(parents=True, exist_ok=True) @@ -2028,7 +2049,8 @@ def _integrate_target(args) -> None: shutil.copy2(plug_src / f, plug_dst / f) print(f"✓ provider plugin installed → {plug_dst}") # compartment must be importable from Hermes's own venv - hermes_py = hermes_home / "hermes-agent" / "venv" / "bin" / "python" + hermes_py = hermes_home / "hermes-agent" / "venv" / ( + "Scripts/python.exe" if sys.platform == "win32" else "bin/python") if hermes_py.exists(): # No stdin for any program run here: a question one of them asks # must fail, not wait on the user's keyboard, and a failure or a diff --git a/tests/test_cli_hermes_home.py b/tests/test_cli_hermes_home.py new file mode 100644 index 0000000..b0305f5 --- /dev/null +++ b/tests/test_cli_hermes_home.py @@ -0,0 +1,69 @@ +"""Where `integrate hermes`, `uninstall` and the refresh after an update look +for Hermes Agent: where Hermes itself looks, which on native Windows is +%LOCALAPPDATA%\\hermes and not ~/.hermes.""" +import subprocess +import sys +import types + +import pytest + +from compartment import cli + + +@pytest.fixture() +def windows_hermes(tmp_path, monkeypatch): + """Windows, with Hermes installed by its own installer: no HERMES_HOME + exported, its home under LOCALAPPDATA and its venv in the Windows + layout. Every program it would run is a stub that records the call.""" + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.delenv("HERMES_HOME", raising=False) + local = tmp_path / "AppData" / "Local" + monkeypatch.setenv("LOCALAPPDATA", str(local)) + monkeypatch.setattr(cli.Path, "home", + classmethod(lambda cls: tmp_path / "home")) + venv_py = local / "hermes" / "hermes-agent" / "venv" / "Scripts" / \ + "python.exe" + venv_py.parent.mkdir(parents=True) + venv_py.write_bytes(b"MZ") + ran = [] + + def run(argv, **kwargs): + ran.append([str(a) for a in argv]) + return subprocess.CompletedProcess(argv, 0, "", "") + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(cli.shutil, "which", lambda name, *a, **k: None) + vault = tmp_path / "memory.vault" + vault.write_bytes(b"") + return types.SimpleNamespace(home=local / "hermes", venv_py=venv_py, + ran=ran, vault=str(vault), + wrong=tmp_path / "home" / ".hermes") + + +def test_integrate_hermes_on_windows_writes_where_hermes_reads( + windows_hermes, capsys): + """The plugin went to ~/.hermes/plugins, which Hermes on Windows never + reads, the Hermes venv was looked for as bin/python and never found, + and the install was reported done.""" + cli._integrate_target(types.SimpleNamespace(target="hermes", + vault=windows_hermes.vault)) + plugin = windows_hermes.home / "plugins" / "compartment" + assert (plugin / "plugin.yaml").is_file() + assert (plugin / "__init__.py").is_file() + assert not windows_hermes.wrong.exists() + assert [str(windows_hermes.venv_py), "-c", "import compartment"] in \ + windows_hermes.ran + + +def test_hermes_home_follows_hermes_rules(tmp_path, monkeypatch): + """HERMES_HOME with its variables expanded, as Hermes expands it; else + ~/.hermes, and on Windows %LOCALAPPDATA%\\hermes.""" + monkeypatch.setattr(cli.Path, "home", classmethod(lambda cls: tmp_path)) + monkeypatch.setenv("HERMES_BASE", str(tmp_path / "base")) + monkeypatch.setenv("HERMES_HOME", "$HERMES_BASE/custom-hermes") + assert cli._hermes_home() == tmp_path / "base" / "custom-hermes" + monkeypatch.delenv("HERMES_HOME") + monkeypatch.setattr(sys, "platform", "linux") + assert cli._hermes_home() == tmp_path / ".hermes" + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.delenv("LOCALAPPDATA", raising=False) + assert cli._hermes_home() == tmp_path / "AppData" / "Local" / "hermes" From 358b484286c35571c6e4d13c13819abc99fe78a6 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:11:35 -0700 Subject: [PATCH 097/135] docs/PLUGINS.md says, store by store, what the listing is, how a user installs from it, and what the maintainer does to publish it The owner asked for up-to-date plugins for every store, and nothing in the repository said which stores exist, which file each one reads, or what has to happen at release time or once by hand. docs/PLUGINS.md covers the MCP Registry, Claude Code, the Claude Desktop bundle, Codex, Gemini CLI, Hermes, OpenClaw, Smithery, Glama, Docker's MCP Catalog, Cline, Continue, goose, LM Studio, Cursor, VS Code, Kiro and Oh My Pi, with the source each format was checked against, plus three things that cut across them: why most listings run a bare `compartment` and which ones avoid the PATH problem, how the caller is chosen, and what happens when one host is wired two ways (two servers, every tool twice, except in Gemini CLI). It records why the Claude Code capture hook is not shipped through the plugin and what would let it be. The page states no client count and the true tool count, which the manifest tests check. Co-Authored-By: Claude Opus 5.5 --- docs/PLUGINS.md | 323 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 docs/PLUGINS.md diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md new file mode 100644 index 0000000..ede4af8 --- /dev/null +++ b/docs/PLUGINS.md @@ -0,0 +1,323 @@ +# Plugins, extensions and store listings + +Compartment is one Python package on PyPI and one MCP server +(`compartment serve`, stdio, 14 memory tools). Everything below is a way for a +particular host or catalogue to find that server and start it. This page says, +store by store, what the listing is, how a user installs from it, and what the +maintainer has to do to publish or update it. The formats were checked against +each host's own repository in October 2026; the source is named for each. + +`tests/test_plugin_manifests.py` checks every manifest here against the parts +of its host's format that matter, and against each other (same command, same +subcommand, one caller per host, no wrong tool count). `tests/test_version_lockstep.py` +fails the build when any manifest carrying a version disagrees with the package, +or when a file anywhere in the tree carries a version it does not check. + +## At a glance + +| Store / host | Listing in this repo | User installs with | Maintainer, each release | Maintainer, once | +|---|---|---|---|---| +| PyPI | `pyproject.toml` | `pip install compartment` | automatic (`publish.yml`) | - | +| Official MCP Registry | `server.json` | their client's registry browser, `uvx compartment serve` | automatic (`publish-registry.yml`) | - | +| Claude Code plugins | `.claude-plugin/` | `/plugin marketplace add MaxFreedomPollard/Compartment` | version is gated; nothing else | submit to Anthropic's plugin directory | +| Claude Desktop extension | `packaging/mcpb/` | the `.mcpb` from the release | build and attach the `.mcpb` | submit to Anthropic's extension directory | +| Codex plugins | root `plugin.json` + `.claude-plugin/marketplace.json` | `codex plugin marketplace add MaxFreedomPollard/Compartment` | nothing | - | +| Gemini CLI extensions | `gemini-extension.json`, `GEMINI.md` | `gemini extensions install https://github.com/MaxFreedomPollard/Compartment` | nothing (crawled daily) | add the `gemini-cli-extension` topic | +| Hermes Agent | `integrations/hermes/`, root `plugin.json` + `mcp.json` | `compartment integrate hermes` or `hermes plugins install MaxFreedomPollard/Compartment` | nothing | upstream PR (see below) | +| OpenClaw | `.claude-plugin/` (as a bundle), `skills/` | `compartment integrate openclaw` | nothing | publish the skill on ClawHub | +| Smithery | the `.mcpb` | Smithery's install command | `smithery mcp publish` | `smithery auth login` | +| Glama | `glama.json`, `Dockerfile` | Glama's page | nothing | - | +| Docker MCP Catalog | `packaging/docker-mcp/server.yaml` | Docker Desktop, MCP Toolkit | nothing (Docker bumps the pin) | PR to docker/mcp-registry | +| Cline marketplace | `llms-install.md` | Cline's MCP marketplace | nothing | issue at cline/mcp-marketplace | +| Continue hub | `packaging/continue/compartment.yaml` | hub.continue.dev, or copy the file | republish the block | publish the block | +| goose | `add/goose.html` | the Install button | nothing | PR to goose's extension directory | +| LM Studio | `add/lmstudio.html` | the Add button | nothing | - | +| Cursor | deeplink in the README | the Add to Cursor button | nothing | - | +| VS Code, Copilot | deeplinks in the README, `server.json` | the Install buttons, VS Code's MCP gallery | nothing | - | +| Oh My Pi | `integrations/omp/extension.ts` | copy the file; `compartment integrate omp` | nothing | - | + +## The command a host runs, and PATH + +Most listings start the server as the bare command `compartment`, because their +formats allow nothing else portable: Agent Plugins (`mcp.json`, read by Codex, +Hermes and OpenClaw) requires a single bare executable name or a path inside +the plugin, and a Claude Code, Gemini, Continue or LM Studio entry has no way +to discover where pip put the script. A bare name works when the host inherits +the shell's PATH, which terminal programs (Claude Code, Codex, Gemini CLI, +Oh My Pi) do. GUI apps started from the Dock, the Start menu or a desktop +launcher often do not, and then the server fails with `command not found` or +`spawn compartment ENOENT`. + +Where a format allows something safer, the listing uses it: + +- The Claude Desktop bundle runs the server through `uv`, which Claude Desktop + supplies, in an environment it builds from the bundle; no PATH lookup of + `compartment` at all. +- goose runs `uvx compartment` (goose only accepts its own launchers from a + link, and ships uvx). +- The Docker catalogue entry runs the image's own entrypoint. +- `compartment integrate ` writes the absolute path of the executable + into every config it edits, which is the reliable route for GUI clients. + +For a hand-written config in a GUI client, use the absolute path: +`which compartment` (macOS, Linux) or `where compartment` (Windows). + +## The caller + +Every listing passes `--caller `, which names the client in the audit +log and is the key per-caller ACL entries are written against. Where +`compartment integrate` wires the same host, the listing uses the same name +(`claude-code`, `claude-desktop`, `gemini`, `lmstudio`, `goose`, `continue`), +so one app never shows up as two clients. The Agent Plugins package uses +`agent-plugin`, because Codex, Hermes and OpenClaw all read it; the Docker +image uses `docker`. A test enforces all of this. + +## Using two routes at once + +A host wired two ways starts two servers, and the model sees every tool twice: + +- `compartment init` connects Claude Code when it is installed (the same as + `compartment integrate claude`). Installing the Claude Code plugin as well + adds a second server. Claude Code deduplicates a plugin server only when its + command is identical to a configured one, and integrate writes an absolute + path, so it does not here. Use one: integrate is the fuller route (capture + hook, CLAUDE.md block, Claude Desktop). +- The same holds for the Claude Desktop extension next to + `compartment integrate claude`. +- Gemini CLI is the exception: a server in `settings.json` with the same name + as an extension's replaces it, so `integrate gemini` and the extension do not + double up. + +## Store by store + +### Official MCP Registry + +- **Listing:** `server.json`, schema `2025-12-11` (the newest released schema + in modelcontextprotocol/registry), name `io.github.MaxFreedomPollard/compartment`, + one PyPI package run as `uvx compartment serve`, plus `title`, `websiteUrl` + and `icons`. Validates against the schema with `mcp-publisher validate`. +- **Users:** clients with a registry browser install it from there. VS Code's + MCP gallery and GitHub's MCP registry draw on this registry. +- **Each release:** automatic. `publish-registry.yml` publishes on every GitHub + release after the version reaches PyPI. The two version fields are + deliberately different (RELEASING.md step 5 explains why). +- The registry proves PyPI ownership by finding `mcp-name: ` in the + README PyPI serves; a test keeps the two in sync. +- Possible later: an `mcpb` package entry pointing at the release's `.mcpb`. + It needs the bundle's SHA-256 in `server.json` (`fileSha256`), so the release + workflow would have to write it in before publishing. + +### Claude Code plugin marketplace + +- **Listing:** `.claude-plugin/marketplace.json` (marketplace `maxfreedompollard`, + one plugin with `source: "./"`) and `.claude-plugin/plugin.json` (name, + display name, version, the MCP server). Because the plugin root is the + repository root, Claude Code also loads `skills/compartmentalize`, which + appears as `/compartment:compartmentalize`. Format: anthropics/claude-code + and anthropics/claude-plugins-official, schemas on SchemaStore. +- **Users:** after `pip install compartment && compartment init`: + `/plugin marketplace add MaxFreedomPollard/Compartment`, then + `/plugin install compartment@maxfreedompollard`. Updates arrive with + `/plugin marketplace update maxfreedompollard`. See "Using two routes at + once" first. +- **Each release:** the `version` in `.claude-plugin/plugin.json` moves with + the package (gated); Claude Code compares it to decide what is an update. +- **Once:** to appear in `/plugin` > Discover for everyone, submit the plugin + to Anthropic's directory (anthropics/claude-plugins-official, through the + submission form its README links, `clau.de/plugin-directory-submission`). +- **Not shipped: the capture hook.** `integrate claude` installs a + `PostToolUse` hook into `~/.claude/settings.json`, and `compartment init` + runs that on any machine with Claude Code. A plugin hook would run a second + copy of `compartment hook capture` on every Write and Edit for nearly every + plugin user. It cannot live at the root as `hooks/hooks.json` either, because + Gemini CLI and OpenClaw read that path in their own formats. It can ship once + the capture command can tell it is the plugin's copy and stand down when + settings.json already has one (for example a flag that `hook capture` + accepts and checks with `claude_hooks.is_installed()`); then the hook goes + inline in `.claude-plugin/plugin.json`. + +### Claude Desktop extension (MCP Bundle) + +- **Listing:** `packaging/mcpb/` - `manifest.json` (MCPB `manifest_version` + 0.4, server type `uv`), `pyproject.toml` pinning `compartment==`, + `.python-version`, and `server/main.py`. Format: modelcontextprotocol/mcpb + (formerly anthropics/dxt). Type `uv` is the only one that can carry + compiled dependencies (onnxruntime, PyNaCl, argon2-cffi, numpy) to every + platform: Claude Desktop has uv build the environment from PyPI once, at + install. The server it starts makes no network connection. +- **Users:** download `compartment-.mcpb` from the release and open + it, or Claude Desktop > Settings > Extensions > Install Extension. The vault + still comes from `compartment init` in a terminal (the passphrase has to be + typed somewhere). +- **Each release:** after the version is on PyPI, + + ```bash + pip install uv + python tools/build_mcpb.py # dist/compartment-.mcpb, with uv.lock + ``` + + and attach the file to the GitHub release with the other artifacts. The + build refuses a manifest or pin that is not the package version, and is + reproducible byte for byte. `uv lock` needs the release on PyPI, so this + runs after `publish.yml`, not before. +- **Once:** Claude Desktop's curated extension directory is Anthropic's; + submission is through Anthropic, not a GitHub repository. +- The interpreter is pinned (3.12, and `requires-python` below 3.15) because uv + otherwise picks the newest Python, and onnxruntime has no wheels for 3.15 + yet. Raise both when it does. + +### Codex + +- **Listing:** Codex reads `.claude-plugin/marketplace.json` (it looks for + `.agents/plugins/`, then `.claude-plugin/`, then `.cursor-plugin/`). For the + plugin at the root it loads the Agent Plugins `plugin.json` first, then + `mcp.json` and `skills/`, and takes its listing (display name, category, + prompts, logo) from `extensions["com.openai"]` in that manifest. Format: + openai/codex `codex-rs/core-plugins`. +- **Users:** `codex plugin marketplace add MaxFreedomPollard/Compartment`, then + `codex plugin add compartment@maxfreedompollard`. +- **Each release:** nothing beyond the gated version. +- OpenAI's curated directory (openai/plugins) has no public submission + process as of October 2026. + +### Gemini CLI + +- **Listing:** `gemini-extension.json` (name, version, description, + `contextFileName: GEMINI.md`, the MCP server). Gemini also loads `skills/` + from the extension. `GEMINI.md` is the server's handshake instructions, + verbatim (tested). Format: google-gemini/gemini-cli `docs/extensions/`. +- **Users:** `gemini extensions install https://github.com/MaxFreedomPollard/Compartment`. + Gemini only starts stdio servers in trusted folders (`gemini trust`). +- **Each release:** nothing; the gallery crawler indexes tagged releases daily. + Installs from a release use the source archive as long as the release has + more than one asset. A release with exactly one asset would make Gemini + download that asset as the extension, so keep the full set attached. +- **Once:** add the `gemini-cli-extension` topic to the GitHub repository + (About > Topics). That is the gallery's only requirement beyond the + manifest at the root. + +### Hermes Agent + +- **Listing:** the native memory provider in `integrations/hermes/` (the + fuller route) and the portable Agent Plugins package at the root + (`plugin.json` 1.0.0, `mcp.json`, `skills/`). Agent Plugins 1.0.0 is still + the published release; 1.1.0 is a draft no host accepts. +- **Users:** `compartment integrate hermes`, or + `hermes plugins install MaxFreedomPollard/Compartment`. +- **Once:** the upstream PR described in `integrations/hermes/upstream/PR.md`. + Its `plugin.yaml` there is stale (version 1.7.0, `compartment>=2.3`); the + lockstep test pins it as a known stale copy until it is refreshed. + +### OpenClaw + +- **Listing:** OpenClaw installs Agent Plugins, Claude, Codex and Cursor + bundles. This repository is detected as a Claude bundle (a client marker + wins over the root `plugin.json`): it gets `skills/` and the MCP server from + `.claude-plugin/plugin.json`. Format: openclaw/openclaw `docs/plugins/bundles.md`. +- **Users:** `compartment integrate openclaw` (writes the server with caller + `openclaw` and installs the skill), or + `openclaw plugins install git:github.com/MaxFreedomPollard/Compartment` + (then the caller is `claude-code`, the bundle's). +- **Once:** publish `skills/compartmentalize` to ClawHub with the `clawhub` + CLI (needs a ClawHub account). A native memory-slot plugin + (`openclaw.plugin.json`) needs in-process TypeScript and is not written; + adding that manifest at the root would make OpenClaw stop treating the + repository as a bundle. + +### Smithery + +- **Listing:** Smithery no longer builds local servers from a `smithery.yaml`; + local servers are published as MCP Bundles. Use the release's `.mcpb` + (Smithery's limit is 25 MB; ours is about 40 KB). +- **Each release / once:** `smithery auth login`, then + `smithery mcp publish dist/compartment-.mcpb -n maxfreedompollard/compartment`. + +### Glama + +- **Listing:** `glama.json` (schema and maintainers); Glama builds the + `Dockerfile` to inspect the server. Already listed; nothing to do. + +### Docker MCP Catalog + +- **Listing:** `packaging/docker-mcp/server.yaml`, the entry docker/mcp-registry + wants at `servers/compartment/server.yaml`. Docker builds and signs + `mcp/compartment` from the `Dockerfile`, mounts the user's vault folder at + `/data`, takes the passphrase from Docker Desktop's secret store as + `COMPARTMENT_PASSPHRASE`, and runs the container with no network. +- **Once:** fork docker/mcp-registry, copy the file to + `servers/compartment/server.yaml` (the header comment can go), run + `task validate -- --name compartment` and `task build -- --tools compartment`, + and open the PR with their template. The server lists its tools without a + vault, so no `tools.json` is needed; add one only if their build cannot + start the image. After the merge, Docker's pin-update workflow moves + `source.commit` with each release. +- **Users:** create the vault on the host first (`compartment init`). On + Linux the image's user (uid 10001) must be able to read the mounted vault, + whose files are mode 0600; Docker Desktop on macOS and Windows maps + ownership for you. + +### Cline marketplace + +- **Listing:** `llms-install.md`, the instructions Cline's agent follows to + install (including PEP 668 systems, Windows and PATH problems). +- **Once:** open an issue at github.com/cline/mcp-marketplace with the + repository URL, `docs/images/logo-400.png` (400x400 PNG) and the reason; + test that Cline can install from `llms-install.md` first. + +### Continue hub + +- **Listing:** `packaging/continue/compartment.yaml`, a Continue block + (`schema: v1`). Users can also copy it into `.continue/mcpServers/` (one + workspace) or `~/.continue/mcpServers/` (all of them). +- **Each release / once:** publish the block on hub.continue.dev (needs an + account). Its version is gated with the package. + +### goose + +- **Listing:** `add/goose.html`, a `goose://extension` link running + `uvx compartment --caller goose serve`. +- **Once:** goose's extension directory is `documentation/static/servers.json` + in block/goose. A PR adding: + + ```json + { + "id": "compartment", + "name": "Compartment", + "description": "Encrypted, fully offline long-term memory for AI agents", + "command": "uvx compartment --caller goose serve", + "link": "https://github.com/MaxFreedomPollard/Compartment", + "installation_notes": "Create the encrypted vault once first: pip install compartment && compartment init", + "is_builtin": false, + "endorsed": false, + "environmentVariables": [] + } + ``` + +### LM Studio, Cursor, VS Code, Kiro + +- **LM Studio:** `add/lmstudio.html`, an `lmstudio://add_mcp` link with the + entry base64-encoded (format: lmstudio-ai/docs). No directory. +- **Cursor:** the README's `cursor.com/install-mcp` button, and the existing + Cursor Directory listing. Cursor's own plugin marketplace + (`.cursor-plugin/`) is curated by Cursor and submitted through the Cursor + team; a `.cursor-plugin` manifest at the root would also read the root + `mcp.json`, which is in Agent Plugins format, and would make OpenClaw treat + the repository as a Cursor bundle, so none is shipped. +- **VS Code / Copilot:** the README's `vscode.dev/redirect/mcp/install` + buttons; the gallery side comes from the MCP Registry. +- **Kiro:** the README's `kiro.dev/launch/mcp/add` button. + +### Oh My Pi + +- **Listing:** `integrations/omp/extension.ts`, an omp extension module + (`ExtensionAPI`): automatic recall and capture, and `memory_search` / + `memory_store` tools declared with omp's approval tiers (read, write) and + loaded as essential. Format: can1357/oh-my-pi `docs/extensions.md`. +- **Users:** copy it to `~/.omp/agent/extensions/compartment.ts`; + `compartment integrate omp` adds the MCP server as well. + +### Directories that scrape + +mcp.so, MCP Market, LobeHub, mcpservers.org, PulseMCP, MCP Toplist, the +toolsdk registry and the awesome lists read GitHub, PyPI or the MCP Registry. +Nothing in this repository is specific to them. From d4ef546272d6880908f95fb0e26c8a6065cdefaa Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:11:42 -0700 Subject: [PATCH 098/135] Shell scripts, the Dockerfile and the workflows keep LF endings in a Windows checkout, so they still run in WSL and in the container Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitattributes b/.gitattributes index e26e145..f6534f4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,11 @@ src/compartment/models/** -text *.onnx -text *.mpack -text *.jsonl -text + +# Scripts a POSIX shell or a container reads. A Windows checkout with +# core.autocrlf rewrites them to CRLF, and bash then fails on `$'\r'` - the +# usual way to meet it is running install-linux.sh from that checkout inside +# WSL, or `docker build` from Windows copying CRLF files into the image. +*.sh text eol=lf +Dockerfile text eol=lf +.github/workflows/*.yml text eol=lf From afae50c31c12df3c08d33049a853a2772433f0cd Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:12:03 -0700 Subject: [PATCH 099/135] RELEASING.md lists the new versioned manifests, and the Claude plugin names its skill the way Claude Code shows it The release checklist's version list now includes the Claude Desktop bundle (manifest semver, pyproject version and the compartment== pin) and the Continue block, which the lockstep test already gates. The Claude Code plugin and marketplace descriptions said they add "the /compartmentalize skill"; a plugin's skills are namespaced, so in Claude Code it is /compartment:compartmentalize, and the descriptions now say so (the plain /compartmentalize is the copy `integrate claude` installs). Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- RELEASING.md | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 8b5f101..c8ec477 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "plugins": [ { "name": "compartment", - "description": "Encrypted, fully offline memory for Claude Code. One vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client; 14 memory tools plus the /compartmentalize skill. Requires `pip install compartment && compartment init` first.", + "description": "Encrypted, fully offline memory for Claude Code. One vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client; 14 memory tools plus the compartmentalize skill. Requires `pip install compartment && compartment init` first.", "author": { "name": "Max Freedom Pollard" }, "category": "productivity", "source": "./", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 8fe41a8..eefb12c 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "compartment", "displayName": "Compartment", "version": "4.12.0", - "description": "Encrypted, fully offline memory for Claude Code: one vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client. Adds 14 memory tools and the /compartmentalize skill. Requires `pip install compartment && compartment init` first (the vault passphrase is yours to choose).", + "description": "Encrypted, fully offline memory for Claude Code: one vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client. Adds 14 memory tools and the compartmentalize skill (/compartment:compartmentalize). Requires `pip install compartment && compartment init` first (the vault passphrase is yours to choose).", "author": { "name": "Max Freedom Pollard", "url": "https://github.com/MaxFreedomPollard" diff --git a/RELEASING.md b/RELEASING.md index 3d6fea3..9507919 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -35,6 +35,10 @@ server without the app. # gemini-extension.json version = "2.1.0" Gemini CLI extension manifest # .claude-plugin/plugin.json version = "2.1.0" Claude Code plugin (Codex, # OpenClaw read it too) +# packaging/mcpb/manifest.json version = "2.1.0" Claude Desktop extension +# packaging/mcpb/pyproject.toml version = "2.1" and its pin, +# "compartment==2.1" which is what it installs +# packaging/continue/compartment.yaml version: 2.1 Continue hub block # # The two server.json fields are deliberately NOT the same value. Step 5 # explains why; setting them both to "2.1" is what published a registry From c9cb1308fe8590f2f433e1a3b840c310dfc5aae5 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:12:04 -0700 Subject: [PATCH 100/135] Under mcp 1.x the server no longer logs a line to stderr for every request mcp 1.x's FastMCP configures logging at INFO unless told otherwise, so `compartment serve` wrote "Processing request of type CallToolRequest" (or ListToolsRequest, and so on) to stderr for every request a client made. Cursor shows each stderr line of an MCP server as an [error] in its MCP log, so a working server looked like a failing one, and the root logger it set at INFO let every library's INFO lines through as well. The 1.x server is now built with log_level="WARNING"; warnings and errors still reach stderr. mcp 2.x logs nothing per request and is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/compartment/server.py | 7 ++++++- tests/test_server_tools.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/compartment/server.py b/src/compartment/server.py index 75a89c4..8108241 100644 --- a/src/compartment/server.py +++ b/src/compartment/server.py @@ -102,7 +102,12 @@ mcp = MCPServer("compartment", instructions=COMPARTMENT_INSTRUCTIONS, version=__version__) except TypeError: # mcp 1.x - mcp = MCPServer("compartment", instructions=COMPARTMENT_INSTRUCTIONS) + # WARNING, not the 1.x default of INFO, which logged "Processing request + # of type CallToolRequest" to stderr for every request: Cursor shows + # each stderr line of a server as an [error], and the root logger it + # configures at that level passed every library's INFO lines through. + mcp = MCPServer("compartment", instructions=COMPARTMENT_INSTRUCTIONS, + log_level="WARNING") try: mcp._mcp_server.version = __version__ except Exception: # noqa: BLE001 diff --git a/tests/test_server_tools.py b/tests/test_server_tools.py index 199f96b..8dd6f82 100644 --- a/tests/test_server_tools.py +++ b/tests/test_server_tools.py @@ -4,6 +4,8 @@ Listed through the SDK the server runs on (mcp 1.x or 2.x), without an event loop, as test_server.py calls tools: listing awaits nothing.""" +import pytest + from compartment import server @@ -79,3 +81,16 @@ def test_every_input_schema_is_one_strict_clients_accept(): assert key not in schema, (t["name"], key) _typed(schema, t["name"]) assert set(schema.get("required", [])) <= set(schema["properties"]) + + +def test_mcp_1x_does_not_log_every_request(capsys): + """mcp 1.x's FastMCP logs at INFO by default, "Processing request of type + CallToolRequest" on stderr for every request, which Cursor shows as an + [error] line each time. mcp 2.x logs nothing per request.""" + import logging + if not server.MCPServer.__module__.startswith("mcp.server.fastmcp"): + pytest.skip("mcp 2.x logs nothing per request") + assert server.mcp.settings.log_level == "WARNING" + logging.getLogger("mcp.server.lowlevel.server").info( + "Processing request of type ListToolsRequest") + assert "Processing request" not in capsys.readouterr().err From 964ced5f51a17f061923d8762feed9bad670d48b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:12:43 -0700 Subject: [PATCH 101/135] `compartment serve` under mcp 1.x exits cleanly when the client hangs up, instead of ending in a ValueError traceback The flush the CLI now makes after every command, so that a closed pipe is caught inside it, met a stdout that mcp 1.x's stdio transport had already closed: that transport wraps the buffer under sys.stdout and closes it when the session ends. Every `compartment serve` on mcp 1.x then ended with "ValueError: I/O operation on closed file" and exit 1 when its client closed stdin. A closed stdout is now left alone. Found by driving the server over raw stdio on mcp 1.30 and 2.3. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 5 ++++- tests/test_cli_errors.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 3b74d6f..95a9477 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -2773,7 +2773,10 @@ def main(argv: list[str] | None = None) -> None: args.fn(args) # Inside the try: buffered output reaches a pipe here, and a reader # that has gone away fails this flush, not one at interpreter exit. - sys.stdout.flush() + # A stdout already closed is left alone: mcp 1.x's stdio transport + # closes the buffer under sys.stdout when `serve` ends. + if not sys.stdout.closed: + sys.stdout.flush() except CryptoError as exc: _die(str(exc)) except BrokenPipeError: diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index ffa4c16..37dcf74 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -207,3 +207,16 @@ def test_a_new_signing_identity_is_readable_by_its_owner_alone( out=tmp_path / "notes.mpack") for a in command]]) assert os.stat(identity).st_mode & 0o777 == 0o600 assert "generated signing identity" in capsys.readouterr().out + + +def test_serve_ending_with_stdout_closed_by_the_transport_exits_cleanly( + monkeypatch, tmp_path): + """mcp 1.x's stdio transport closes the buffer under sys.stdout when the + session ends; the flush the CLI makes after every command then raised + ValueError, and `compartment serve` ended in a traceback with exit 1 + each time a client hung up.""" + import io + out = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") + monkeypatch.setattr("sys.stdout", out) + monkeypatch.setattr(cli, "cmd_serve", lambda args: out.buffer.close()) + cli.main(["--vault", str(tmp_path / "v.vault"), "serve"]) From 097a5a08725c0906f18288705977c17900029216 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:13:02 -0700 Subject: [PATCH 102/135] A damaged vault config is reported as a damaged config, not as a locked vault The earlier change made the refusal for an unusable .config.json an AclError, and so a CryptoError. Callers read a CryptoError out of an unlock as "the vault is locked": the Claude Code capture hook reported a config cut short by a full disk as "vault locked" instead of "vault unavailable", sending the user to unlock a vault that was not locked. ConfigError is now a plain ValueError, as the json error it replaced was, so every caller classifies it exactly as before and only the message changes: it names the file and what to do. Co-Authored-By: Claude Opus 5.5 --- src/compartment/acl.py | 12 ++++++++---- tests/test_storage_windows_files.py | 7 +++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/compartment/acl.py b/src/compartment/acl.py index 5f9c16b..7b7a16a 100644 --- a/src/compartment/acl.py +++ b/src/compartment/acl.py @@ -54,10 +54,14 @@ class AclError(CryptoError): pass -class ConfigError(AclError, ValueError): - """The config file exists and cannot be used. A ValueError as well, as - the json error it replaces was, so a caller that already catches a bad - config that way (the panel's settings switch) still does.""" +class ConfigError(ValueError): + """The config file exists and cannot be used. + + A ValueError, as the json error it replaces was, and deliberately not a + CryptoError: callers read a CryptoError out of an unlock as "the vault + is locked" (the capture hook does, and says so), and a damaged settings + file is not that. Every caller that coped with the old error copes with + this one; what changes is that it says which file and what to do.""" @dataclass diff --git a/tests/test_storage_windows_files.py b/tests/test_storage_windows_files.py index 63f53a3..2bfa2c9 100644 --- a/tests/test_storage_windows_files.py +++ b/tests/test_storage_windows_files.py @@ -12,7 +12,7 @@ import pytest from compartment import curate -from compartment.acl import AclError, VaultConfig +from compartment.acl import AclError, ConfigError, VaultConfig from compartment.vault import Vault from conftest import PASS @@ -51,13 +51,16 @@ def test_a_config_that_cannot_be_used_is_refused_with_what_to_do( names the file and says what moving it aside would grant.""" with open(VaultConfig.path_for(vault_path), "wb") as f: f.write(content) - with pytest.raises(AclError) as caught: + with pytest.raises(ConfigError) as caught: VaultConfig.load(vault_path) msg = str(caught.value) assert why in msg and VaultConfig.path_for(vault_path) in msg assert "move it aside" in msg assert isinstance(caught.value, ValueError), \ "still what a caller catching the old json error catches" + from compartment.crypto import CryptoError + assert not isinstance(caught.value, CryptoError), \ + "callers read a CryptoError from an unlock as 'locked'; this is not" def test_an_export_passed_through_powershell_imports(vault): From 270a77c4eee66846643162baf7bd2758724bfa5d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:13:23 -0700 Subject: [PATCH 103/135] Tests: a session over real stdio pipes checks the frames, the exit status and stderr when the client hangs up The suite drove the server only in-process, so it could not see what a client sees first: a stray line on stdout, a traceback on the way out, an exit status that reports a crash. The new test runs `compartment serve` as a client does, sends the handshake and tools/list, closes stdin, and requires exit 0, no traceback, and nothing on stdout but the two replies. On mcp 1.x it fails on the closed-stdout flush fixed just before it. Co-Authored-By: Claude Opus 5.5 --- tests/test_server_stdio.py | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/test_server_stdio.py diff --git a/tests/test_server_stdio.py b/tests/test_server_stdio.py new file mode 100644 index 0000000..1c02588 --- /dev/null +++ b/tests/test_server_stdio.py @@ -0,0 +1,47 @@ +"""`compartment serve` driven over real pipes, as a client drives it: the +handshake, a tool list, then the client hanging up. The rest of the suite +calls the server in-process and cannot see what only a client sees: a +stray line on stdout, a traceback on the way out, an exit status that +tells the client the server crashed.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +from compartment.vault import Vault + +from conftest import PASS + +SRC = str(Path(__file__).resolve().parents[1] / "src") + + +def test_a_session_over_stdio_ends_cleanly_when_the_client_hangs_up( + tmp_path): + vault = str(tmp_path / "v.vault") + Vault.create(vault, PASS, creator="test").lock() + env = dict(os.environ, COMPARTMENT_PASSPHRASE=PASS, + COMPARTMENT_EMBED_DAEMON="0", + PYTHONPATH=SRC + os.pathsep + os.environ.get("PYTHONPATH", "")) + requests = [ + {"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2025-06-18", "capabilities": {}, + "clientInfo": {"name": "test", "version": "0"}}}, + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, + ] + proc = subprocess.run( + [sys.executable, "-m", "compartment.cli", "--vault", vault, "serve"], + input="".join(json.dumps(r) + "\n" for r in requests).encode(), + capture_output=True, env=env, timeout=120) + err = proc.stderr.decode("utf-8", "replace") + assert proc.returncode == 0, err + assert "Traceback" not in err, err + lines = proc.stdout.decode("utf-8").splitlines() + replies = [json.loads(line) for line in lines] # nothing but frames + assert [r.get("id") for r in replies] == [1, 2] + init, listed = replies + assert init["result"]["serverInfo"]["name"] == "compartment" + assert "memory_search" in init["result"]["instructions"] + assert {t["name"] for t in listed["result"]["tools"]} >= { + "memory_search", "memory_store", "memory_status"} From 660c52ae0200446f59be67707316fac76b62b309 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:13:29 -0700 Subject: [PATCH 104/135] Building the Claude Desktop bundle before the release reaches PyPI stops with what to do instead of a traceback tools/build_mcpb.py locks the bundle's environment with `uv lock`, and the lock pins the release being built, so it can only resolve once publish.yml has uploaded that version. Run earlier (as it was here, where 4.12.0 is not on PyPI yet), uv printed "no version of compartment==4.12.0" and the script died with CalledProcessError and a stack trace. It now ends with a message naming the version and saying to build after the PyPI publish, or pass --no-lock; no partial bundle is left behind. Seen by running the real build with uv 0.13.2. Before the fix: test_the_bundle_lock_failure_says_what_to_do E FileNotFoundError: [Errno 2] No such file or directory: '.../uv.lock' (the test fakes a failing uv; the real run raised CalledProcessError) Co-Authored-By: Claude Opus 5.5 --- tests/test_plugin_manifests.py | 17 +++++++++++++++++ tools/build_mcpb.py | 8 +++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_plugin_manifests.py b/tests/test_plugin_manifests.py index e331268..d2b57d7 100644 --- a/tests/test_plugin_manifests.py +++ b/tests/test_plugin_manifests.py @@ -261,6 +261,23 @@ def test_the_bundle_build_is_complete_and_reproducible(tmp_path): assert {manifest["icon"], manifest["server"]["entry_point"]} <= names +def test_the_bundle_lock_failure_says_what_to_do(tmp_path, monkeypatch): + """The lock pins the release, so it only resolves once the release is on + PyPI; a build run too early must say so, not end in a traceback.""" + import subprocess + import pytest + mod = _build_mcpb() + monkeypatch.setattr(mod.shutil, "which", lambda name: None) + with pytest.raises(SystemExit, match="--no-lock"): + mod.build(tmp_path, with_lock=True) + monkeypatch.setattr(mod.shutil, "which", lambda name: "/usr/bin/uv") + monkeypatch.setattr(mod.subprocess, "run", + lambda *a, **k: subprocess.CompletedProcess(a, 1)) + with pytest.raises(SystemExit, match="after the PyPI publish"): + mod.build(tmp_path, with_lock=True) + assert not list(tmp_path.glob("*.mcpb*")) + + def test_the_bundle_build_refuses_a_bundle_for_another_release(): import pytest mod = _build_mcpb() diff --git a/tools/build_mcpb.py b/tools/build_mcpb.py index 9bd2dad..091c6dc 100644 --- a/tools/build_mcpb.py +++ b/tools/build_mcpb.py @@ -109,7 +109,13 @@ def lock(staging: pathlib.Path) -> pathlib.Path: "pass --no-lock for a development build.") for name in ("pyproject.toml", ".python-version"): shutil.copy2(SRC / name, staging / name) - subprocess.run([uv, "lock", "--directory", str(staging)], check=True) + if subprocess.run([uv, "lock", "--directory", str(staging)]).returncode: + # Almost always the release is not on PyPI yet: the lock pins the + # version this bundle installs, so it can only be resolved after + # publish.yml has uploaded it. uv has printed its own reason above. + raise SystemExit(f"uv could not lock compartment=={package_version()}. " + "If that version is not on PyPI yet, build the bundle " + "after the PyPI publish; --no-lock skips the lock.") return staging / "uv.lock" From 70e7dfbdf0aa20d9c35a6709a29b61fbd9ee2e67 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:14:24 -0700 Subject: [PATCH 105/135] COMPARTMENT_PASSPHRASE opens the vault where the boot-session store cannot work, instead of being hidden by that store's error find_credential looked for a credential in order: an explicit passphrase, the boot-session store, the macOS Keychain, then COMPARTMENT_PASSPHRASE (or ENGRAM_PASSPHRASE). A session lookup that failed - no POSIX shared memory in a container, a session directory that cannot be read, a boot that cannot be identified inside a sandbox or on Android - raised on the spot, so the Keychain and the environment were never tried. The error itself tells the user to set COMPARTMENT_PASSPHRASE, and doing so changed nothing: the MCP server, the CLI and the dashboard all still failed to open the vault. The session error is now kept while the Keychain and the environment are tried, and raised only when neither provides a credential, so a vault with no credential at all is still reported as the error it is, never as merely locked. Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 15 ++++++++- tests/test_storage_credentials.py | 53 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 tests/test_storage_credentials.py diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 1fd6a42..fb979a9 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -788,7 +788,18 @@ def find_credential(path: str, passphrase: str | None = None from . import session if passphrase: return passphrase, None - key = session.get(path) + # A session lookup that fails (no shared memory in a container, a + # session directory that cannot be read, a boot that cannot be + # identified inside a sandbox) says nothing about the credentials + # after it. Raised on the spot, it hid a COMPARTMENT_PASSPHRASE set + # precisely because the session store does not work there - which is + # what that error's own message tells the user to do. It is kept and + # raised only when nothing else opens the vault. + session_error: Exception | None = None + try: + key = session.get(path) + except Exception as exc: # noqa: BLE001 + session_error, key = exc, None if key is not None: return None, key key = keychain_get(path) @@ -797,6 +808,8 @@ def find_credential(path: str, passphrase: str | None = None from_env = env("PASSPHRASE") if from_env: return from_env, None + if session_error is not None: + raise session_error return None @staticmethod diff --git a/tests/test_storage_credentials.py b/tests/test_storage_credentials.py new file mode 100644 index 0000000..0e5d834 --- /dev/null +++ b/tests/test_storage_credentials.py @@ -0,0 +1,53 @@ +"""Which credential opens the vault when the session store itself fails. + +The boot-session store needs shared memory, a readable session directory +and an identifiable boot. Containers, sandboxes and Android lack one or +another, and the error for that tells the user to set COMPARTMENT_PASSPHRASE; +the lookup raised before it ever reached the variable. +""" +import pytest + +from compartment import session, vault as vault_mod +from compartment.crypto import CryptoError +from compartment.vault import Vault + +from conftest import PASS + +BOOT_ERROR = ("Cannot identify this boot session (no boot clock), so a " + "stored credential cannot be checked. Use COMPARTMENT_PASSPHRASE " + "or `compartment unlock --once`.") + + +@pytest.fixture() +def broken_session_store(monkeypatch): + def get(path): + raise CryptoError(BOOT_ERROR) + monkeypatch.setattr(session, "get", get) + monkeypatch.setattr(vault_mod, "keychain_get", lambda path: None) + monkeypatch.delenv("COMPARTMENT_PASSPHRASE", raising=False) + monkeypatch.delenv("ENGRAM_PASSPHRASE", raising=False) + + +@pytest.mark.parametrize("var", ["COMPARTMENT_PASSPHRASE", + "ENGRAM_PASSPHRASE"]) +def test_a_passphrase_in_the_environment_is_used_when_the_session_store_fails( + broken_session_store, vault_path, monkeypatch, var): + Vault.create(vault_path, PASS, creator="t").lock() + monkeypatch.setenv(var, PASS) + assert Vault.find_credential(vault_path) == (PASS, None) + pw, key = Vault.resolve_credential(vault_path) + assert Vault.unlock(vault_path, passphrase=pw, raw_key=key).db.count() == 0 + + +def test_with_nothing_else_the_session_error_still_surfaces( + broken_session_store, vault_path): + """Not reported as "locked": the status panel must not say a vault is + locked when the truth is that its credential store cannot be read.""" + with pytest.raises(CryptoError, match="Cannot identify this boot"): + Vault.find_credential(vault_path) + + +def test_a_keychain_credential_is_used_when_the_session_store_fails( + broken_session_store, vault_path, monkeypatch): + monkeypatch.setattr(vault_mod, "keychain_get", lambda path: b"k" * 32) + assert Vault.find_credential(vault_path) == (None, b"k" * 32) From 2c601c583db718ff356fd0f3df5387509c16e42b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:15:26 -0700 Subject: [PATCH 106/135] `compartment init` finishes on a machine that cannot keep an unlock credential, and says the vault is created and locked Termux, OpenBSD and containers without /dev/shm have no volatile holder for the per-boot secret, and session.store refuses with CryptoError there, on purpose. init let that end the command: the vault was created and seeded, but the save after seeding, the wiring of the agents already on the machine and the app were all skipped, and init exited 1 as though nothing had been made. The refusal is now reported in place: the vault is created and stays locked, agents and commands take the passphrase from COMPARTMENT_PASSPHRASE, `compartment unlock --once` checks it, and the rest of init runs. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 18 +++++++++++++++--- tests/test_cli_init.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/test_cli_init.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 95a9477..455564b 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -213,9 +213,21 @@ def cmd_init(args) -> None: keychain_store(path, v._master) print(" keychain credential stored (persists across reboots)") elif not args.no_session: - session.store(path, v._master) - print(" unlocked: stays open until the next restart/power loss or " - "`compartment lock`") + try: + session.store(path, v._master) + print(" unlocked: stays open until the next restart/power loss " + "or `compartment lock`") + except CryptoError as exc: + # No volatile holder for the per-boot secret here (no shared + # memory: Termux, OpenBSD, a container without /dev/shm). The + # vault is made and seeded; dying at this point left the rest of + # init undone (the save, the agent wiring, the app) over a + # credential the vault does not need to exist. + print(f" the vault is created and stays locked: this machine " + f"cannot hold an unlock credential ({exc}).") + print(" Give agents and commands the passphrase in " + "COMPARTMENT_PASSPHRASE, or check it with " + "`compartment unlock --once`.") st = v.status() v.save() # Before the app appears, so the first thing anyone sees is a panel whose diff --git a/tests/test_cli_init.py b/tests/test_cli_init.py new file mode 100644 index 0000000..4a7a125 --- /dev/null +++ b/tests/test_cli_init.py @@ -0,0 +1,28 @@ +"""`compartment init` finishes on machines that cannot do everything.""" +from compartment import cli, session +from compartment.crypto import CryptoError +from compartment.vault import Vault + +from conftest import PASS + + +def test_init_finishes_where_no_unlock_credential_can_be_kept( + tmp_path, monkeypatch, capsys): + """Termux, OpenBSD and containers without /dev/shm have no volatile + holder for the per-boot secret, and session.store says so with + CryptoError. init died there half-way: the vault made and seeded, but + not saved after seeding, no agent wired, no app, and exit 1.""" + def no_holder(path, key): + raise CryptoError("No per-boot secret is available on this system.") + monkeypatch.setattr(session, "store", no_holder) + monkeypatch.setattr(cli, "_seed_blobs", lambda: []) + wired = [] + monkeypatch.setattr(cli, "connect_present_agents", + lambda path: wired.append(path) or []) + path = str(tmp_path / "new.vault") + cli.main(["--vault", path, "init", "--passphrase", PASS, "--no-app"]) + out = capsys.readouterr().out + assert "stays locked" in out and "COMPARTMENT_PASSPHRASE" in out + assert wired == [path] # the rest of init ran + assert "Vault ready" in out + Vault.unlock(path, passphrase=PASS) From 740e26c9ea40b01f40131c02641a3cbc94723e6d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:16:00 -0700 Subject: [PATCH 107/135] claude_hooks, claude_memory and agent_skill import where there is no home directory, working out Claude Code's, Hermes's and OpenClaw's paths when they are used In a container run as an arbitrary uid, with HOME unset and no passwd entry, Path.home() raises. claude_hooks (SETTINGS), claude_memory (DEFAULT_ROOT) and agent_skill (the default home of each skill target) called it at import, and every `compartment` command imports them, so every command died before it ran, including the ones that never touch Claude Code or a skills directory. SETTINGS and DEFAULT_ROOT are now module attributes worked out when they are read (PEP 562), honouring CLAUDE_CONFIG_DIR at that moment, and the modules use settings_path() and default_root(), which still take a value a caller has assigned. A skill target's default home is a function called when needed; HERMES_HOME, OPENCLAW_* and CLAUDE_CONFIG_DIR still win over it. With no home to be found, DEFAULT_ROOT names a path that holds nothing, so `import-claude --help` can show it and discovery finds no memories, rather than raising. Co-Authored-By: Claude Opus 5.5 --- src/compartment/agent_skill.py | 19 +++-- src/compartment/claude_hooks.py | 30 ++++++-- src/compartment/claude_memory.py | 39 ++++++++-- tests/test_integrations_claude_config_dir.py | 18 ++--- tests/test_integrations_homeless.py | 81 ++++++++++++++++++++ 5 files changed, 158 insertions(+), 29 deletions(-) create mode 100644 tests/test_integrations_homeless.py diff --git a/src/compartment/agent_skill.py b/src/compartment/agent_skill.py index 56ffd87..edb8584 100644 --- a/src/compartment/agent_skill.py +++ b/src/compartment/agent_skill.py @@ -67,11 +67,14 @@ def _claude_dir() -> Path | None: #: Every agent known to read the Agent Skills layout, and what relocates #: its home: the environment variable naming it, or a function working it #: out. Claude Code's is CLAUDE_CONFIG_DIR, which moves its skills/ with -#: everything else. +#: everything else. The default home is a function too, called when it is +#: needed: Path.home() raises where HOME is unset and the uid has no passwd +#: entry, and evaluated here it killed every `compartment` command at +#: import in such a container. SKILL_TARGETS = { - "claude": (_claude_dir, Path.home() / ".claude"), - "hermes": ("HERMES_HOME", Path.home() / ".hermes"), - "openclaw": (_openclaw_dir, Path.home() / ".openclaw"), + "claude": (_claude_dir, lambda: Path.home() / ".claude"), + "hermes": ("HERMES_HOME", lambda: Path.home() / ".hermes"), + "openclaw": (_openclaw_dir, lambda: Path.home() / ".openclaw"), } @@ -87,10 +90,12 @@ def agent_home(target: str) -> Path | None: return None var, default = entry if callable(var): - return var() or default - if var and os.environ.get(var): + override = var() + if override: + return override + elif var and os.environ.get(var): return Path(os.environ[var]).expanduser() - return default + return default() if callable(default) else default def skill_path(target: str) -> Path | None: diff --git a/src/compartment/claude_hooks.py b/src/compartment/claude_hooks.py index 66b0515..d90e423 100644 --- a/src/compartment/claude_hooks.py +++ b/src/compartment/claude_hooks.py @@ -25,8 +25,25 @@ from . import claude_memory, clients -#: Claude Code's user settings, which move with CLAUDE_CONFIG_DIR. -SETTINGS = claude_memory.claude_config_dir() / "settings.json" +# Claude Code's user settings, which move with CLAUDE_CONFIG_DIR, are +# SETTINGS: worked out when asked for rather than at import, since +# Path.home() raises where HOME is unset and the uid has no passwd entry +# (a container run as an arbitrary uid), and an import-time lookup killed +# every `compartment` command there. Assigning SETTINGS still pins it. + + +def settings_path() -> Path: + """Claude Code's user settings.json: SETTINGS when it has been + assigned, otherwise /settings.json, now.""" + pinned = globals().get("SETTINGS") + return Path(pinned) if pinned is not None else ( + claude_memory.claude_config_dir() / "settings.json") + + +def __getattr__(name: str): + if name == "SETTINGS": + return claude_memory.claude_config_dir() / "settings.json" + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") # Identifies OUR entries. Both tokens, not one literal string: the command # carries `--vault ...` between them when a vault is pinned, so a single # substring would stop matching and re-installing would duplicate the hook. @@ -99,7 +116,7 @@ def is_ours(command: object) -> bool: def is_installed(settings: Path | None = None) -> bool: """Whether compartment's capture hook is in settings.json. Raises ValueError when the file cannot be read as settings.""" - path = Path(settings) if settings else SETTINGS + path = Path(settings) if settings else settings_path() for group in _groups(_read(path), path): for h in group.get("hooks") or []: if is_ours(h.get("command")): @@ -150,7 +167,7 @@ def _groups(data: dict, path: Path) -> list: def install(compartment_bin: str | None = None, vault: str | None = None, settings: Path | None = None) -> dict: """Add (or refresh) compartment's capture hook. Never removes anyone else's.""" - path = Path(settings) if settings else SETTINGS + path = Path(settings) if settings else settings_path() data = _read(path) # raises on malformed groups = _groups(data, path) # and on a bad shape @@ -177,7 +194,7 @@ def install(compartment_bin: str | None = None, vault: str | None = None, def uninstall(settings: Path | None = None) -> bool: - path = Path(settings) if settings else SETTINGS + path = Path(settings) if settings else settings_path() data = _read(path) groups = _groups(data, path) if not groups: @@ -331,4 +348,5 @@ def capture(stream=None, vault_path: str | None = None) -> dict: __all__ = ["install", "uninstall", "is_installed", "is_ours", "capture", "hook_command", - "is_memory_file", "SETTINGS", "MARKER", "EVENT", "MATCHER"] + "is_memory_file", "SETTINGS", "settings_path", "MARKER", "EVENT", + "MATCHER"] diff --git a/src/compartment/claude_memory.py b/src/compartment/claude_memory.py index ca3cefb..b02384c 100644 --- a/src/compartment/claude_memory.py +++ b/src/compartment/claude_memory.py @@ -51,8 +51,35 @@ def claude_config_dir() -> Path: return Path.home() / ".claude" -# Claude Code's per-project memory lives under //memory. -DEFAULT_ROOT = claude_config_dir() / "projects" +# Claude Code's per-project memory lives under //memory, +# and is DEFAULT_ROOT. It is worked out when it is asked for, not at +# import: in a container run as an arbitrary uid, with HOME unset and no +# passwd entry, Path.home() raises, and computing it here killed every +# `compartment` command at import, including the ones that never touch +# Claude Code. A caller may still pin DEFAULT_ROOT by assigning it. + + +def _projects() -> Path: + try: + return claude_config_dir() / "projects" + except RuntimeError: + # No home directory to be found, so no Claude Code memory either: + # a path that names nothing, for discovery to find nothing in and + # `import-claude --help` to show, rather than an error. + return Path("~") / ".claude" / "projects" + + +def default_root() -> Path: + """Where Claude Code keeps per-project memory: DEFAULT_ROOT when it has + been assigned, otherwise /projects, now.""" + pinned = globals().get("DEFAULT_ROOT") + return Path(pinned) if pinned is not None else _projects() + + +def __getattr__(name: str): + if name == "DEFAULT_ROOT": + return _projects() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") INDEX_NAME = "MEMORY.md" # a table of contents, not a fact # Frontmatter `type` -> Compartment importance. Mirrors the vault's own tiers: @@ -82,7 +109,7 @@ def auto_memory_dir() -> Path | None: resolve it against.""" try: from .clients import _read_json - value = _read_json(DEFAULT_ROOT.parent / "settings.json").get( + value = _read_json(default_root().parent / "settings.json").get( "autoMemoryDirectory") except (OSError, ValueError): return None @@ -97,7 +124,7 @@ def discover(root: Path | str | None = None) -> list[Path]: `root`, or by default under DEFAULT_ROOT and in the directory autoMemoryDirectory names.""" if root is None: - out = discover(DEFAULT_ROOT) + out = discover(default_root()) extra = auto_memory_dir() if extra is not None and extra.is_dir(): out += [p for p in sorted(extra.glob("*.md")) @@ -255,9 +282,9 @@ def pending(vault, root: Path | str | None = None) -> int: def default_root_exists() -> bool: - return DEFAULT_ROOT.exists() and bool(discover()) + return default_root().exists() and bool(discover()) -__all__ = ["DEFAULT_ROOT", "discover", "parse", "import_files", "pending", +__all__ = ["DEFAULT_ROOT", "default_root", "discover", "parse", "import_files", "pending", "default_root_exists", "IMPORTANCE", "file_tag", "claude_config_dir", "auto_memory_dir"] diff --git a/tests/test_integrations_claude_config_dir.py b/tests/test_integrations_claude_config_dir.py index 8052c3a..d627a97 100644 --- a/tests/test_integrations_claude_config_dir.py +++ b/tests/test_integrations_claude_config_dir.py @@ -2,8 +2,6 @@ settings.json, the per-project memory under projects/, skills/. People set it to run a second account, or to keep the directory in a dotfiles tree under a name that is not .claude.""" -import importlib - import pytest from compartment import agent_skill, claude_hooks, claude_memory @@ -12,16 +10,16 @@ @pytest.fixture() def config_dir(monkeypatch, tmp_path): """CLAUDE_CONFIG_DIR set, as it is in the environment Claude Code and - `compartment` are started with, and the modules that read it at import - imported again; put back afterwards.""" + `compartment` are started with. SETTINGS and DEFAULT_ROOT are worked out + when used; a value an earlier test's monkeypatch left assigned on its + way out would pin them, so none is left in place for this one.""" d = tmp_path / "dotfiles" / "claude-work" monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(d)) - importlib.reload(claude_memory) - importlib.reload(claude_hooks) - yield d - monkeypatch.delenv("CLAUDE_CONFIG_DIR") - importlib.reload(claude_memory) - importlib.reload(claude_hooks) + for module, name in ((claude_memory, "DEFAULT_ROOT"), + (claude_hooks, "SETTINGS")): + if name in vars(module): + monkeypatch.delattr(module, name) + return d def test_the_hook_goes_into_the_settings_claude_code_reads(config_dir): diff --git a/tests/test_integrations_homeless.py b/tests/test_integrations_homeless.py new file mode 100644 index 0000000..0736b2a --- /dev/null +++ b/tests/test_integrations_homeless.py @@ -0,0 +1,81 @@ +"""A container run as an arbitrary uid has no HOME and no passwd entry, so +Path.home() raises there. Importing a module must not ask for it: every +`compartment` command imports these, whether or not it touches Claude Code +or a skills directory.""" +import os +import subprocess +import sys + +import pytest + +MODULES = ["clients", "claude_desktop", "claude_hooks", "claude_memory", + "agent_skill", "wire"] + +PRELUDE = """ +import pathlib, sys +try: + import pwd +except ImportError: # Windows has no passwd database + pass +else: + def _no_entry(uid): + raise KeyError(f"getpwuid(): uid not found: {uid}") + pwd.getpwuid = _no_entry +try: + pathlib.Path.home() +except RuntimeError: + pass +else: + sys.exit("the home directory could still be found") +""" + + +def _homeless_env(): + env = {k: v for k, v in os.environ.items() + if k not in ("HOME", "USERPROFILE", "HOMEDRIVE", "HOMEPATH", + "CLAUDE_CONFIG_DIR", "HERMES_HOME", "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH")} + src = os.path.join(os.path.dirname(__file__), os.pardir, "src") + env["PYTHONPATH"] = os.pathsep.join( + [os.path.abspath(src), env.get("PYTHONPATH", "")]) + return env + + +@pytest.mark.parametrize("module", MODULES) +def test_a_module_imports_where_there_is_no_home_directory(module): + """claude_hooks, claude_memory and agent_skill each called Path.home() + at import, so in such a container every command died before it ran.""" + code = PRELUDE + f"import compartment.{module}\n" + ran = subprocess.run([sys.executable, "-c", code], env=_homeless_env(), + capture_output=True, text=True, timeout=60) + assert ran.returncode == 0, ran.stderr[-2000:] + + +def test_the_import_root_has_a_value_to_show_with_no_home(): + """`import-claude --help` shows DEFAULT_ROOT; with no home it names a + path that holds nothing, and discovery finds nothing in it.""" + code = PRELUDE + ( + "from compartment import claude_memory\n" + "print(claude_memory.DEFAULT_ROOT)\n" + "assert claude_memory.discover() == []\n") + ran = subprocess.run([sys.executable, "-c", code], env=_homeless_env(), + capture_output=True, text=True, timeout=60) + assert ran.returncode == 0, ran.stderr[-2000:] + assert ".claude" in ran.stdout + + +def test_the_env_overrides_still_win_when_asked_for(monkeypatch, tmp_path): + from compartment import agent_skill, claude_hooks, claude_memory + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "c")) + monkeypatch.setenv("HERMES_HOME", str(tmp_path / "h")) + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(tmp_path / "o")) + for module, name in ((claude_memory, "DEFAULT_ROOT"), + (claude_hooks, "SETTINGS")): + if name in vars(module): + monkeypatch.delattr(module, name) + assert claude_hooks.SETTINGS == tmp_path / "c" / "settings.json" + assert claude_hooks.settings_path() == tmp_path / "c" / "settings.json" + assert claude_memory.DEFAULT_ROOT == tmp_path / "c" / "projects" + assert agent_skill.agent_home("claude") == tmp_path / "c" + assert agent_skill.agent_home("hermes") == tmp_path / "h" + assert agent_skill.agent_home("openclaw") == tmp_path / "o" From 932c8df3c3913ed9bec19e1428718c036bfe0487 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:17:27 -0700 Subject: [PATCH 108/135] A vault path starting with ~ is the home one wherever it comes from, and a named vault needs no home directory MCP client configs pass "~/.compartment/memory.vault" in args and env literally: no shell runs in between to expand it. `--vault`, the COMPARTMENT_VAULT and ENGRAM_VAULT variables, and `python -m compartment.server --vault` took it as a relative path, so the server looked for a folder named "~" in whatever directory the client started it in and found no vault (or init created one there). ~ is now expanded in all of them, before the subcommand or after it. The default vault was a module constant computed at import, so with HOME unset and no password-database entry for the uid (a container started with an arbitrary --user) every command died at import with "Could not determine home directory", even one that named its vault. It is now computed when a command runs; with no home and no --vault a command says to pass --vault PATH or set COMPARTMENT_VAULT, and the signing identity's default is found the same lazy way. agent_skill, claude_hooks and claude_memory still look up the home at import (reported separately). Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 61 +++++++++++++++++++++++++---- src/compartment/server.py | 3 ++ tests/test_cli_home.py | 81 +++++++++++++++++++++++++++++++++++++++ tests/test_server.py | 27 +++++++++++++ 4 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 tests/test_cli_home.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 455564b..608ae29 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -29,7 +29,42 @@ from .vault import Vault, keychain_clear, keychain_get, keychain_store from .vaultfile import read_vault_file, verify_manifest -DEFAULT_VAULT = env("VAULT", str(home() / "memory.vault")) +def _default_vault() -> str | None: + """COMPARTMENT_VAULT (or ENGRAM_VAULT) with ~ expanded, else + memory.vault in Compartment's home; None where there is no home to look + in. + + Asked when a command runs, not when this module is imported. It was a + module constant, so with HOME unset and no password-database entry for + the uid, as for a container started with an arbitrary --user, every + command died at import with "Could not determine home directory", even + `compartment --vault /data/memory.vault serve`, which needs no home. + And the variable was taken literally, so a "~/..." an MCP client config + passes (no shell expands it there) named a folder called "~" in + whatever directory the client started the server in.""" + explicit = env("VAULT") + if explicit: + return os.path.expanduser(explicit) + try: + return str(home() / "memory.vault") + except RuntimeError: + return None + + +def __getattr__(name: str): + # DEFAULT_VAULT stays importable for anything that read it, computed + # when it is read. + if name == "DEFAULT_VAULT": + return _default_vault() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +#: What a command says when there is neither --vault nor a home to find the +#: default vault in. +NO_HOME = ("no home directory to find the vault in (HOME is not set and " + "this user has no entry in the password database, as for a " + "container started with an arbitrary --user): pass --vault PATH, " + "or set COMPARTMENT_VAULT") def _utf8_console() -> None: @@ -791,7 +826,7 @@ def cmd_unlock(args) -> None: v.save() -def _signing_identity(path: str, creator: str) -> dict: +def _signing_identity(path: str | None, creator: str) -> dict: """The Ed25519 identity that signs a sealed vault or a pack, made on first use. @@ -799,7 +834,7 @@ def _signing_identity(path: str, creator: str) -> dict: alone: written plainly it took the umask's 0644, and any account on the machine that could read it could sign a pack this user's keys trust, or seal a vault `compartment verify` reports as signed by them.""" - ident_path = Path(path) + ident_path = Path(path) if path else home() / "identity.json" if ident_path.exists(): return json.loads(ident_path.read_text(encoding="utf-8")) identity = packs.new_identity(creator) @@ -2349,8 +2384,10 @@ def main(argv: list[str] | None = None) -> None: ap = argparse.ArgumentParser( prog="compartment", description="Compartment - high-security offline vector memory for AI agents") - ap.add_argument("--vault", default=DEFAULT_VAULT, - help=f"vault path (default {DEFAULT_VAULT})") + default_vault = _default_vault() + ap.add_argument("--vault", default=default_vault, + help="vault path (default " + f"{default_vault or 'none: no home directory'})") ap.add_argument("--caller", default="user") ap.add_argument("--keyfile", help="second-factor keyfile (only needed if `compartment 2fa " @@ -2394,7 +2431,9 @@ def main(argv: list[str] | None = None) -> None: p = sub.add_parser("lock", help="clear stored credential (vault stays sealed)") p.add_argument("--sign", action="store_true", help="seal with an Ed25519 signed manifest before locking") - p.add_argument("--identity", default=str(home() / "identity.json")) + p.add_argument("--identity", default=None, + help="signing identity (default identity.json in " + "Compartment's home)") p.add_argument("--creator", default="vault-owner") p.set_defaults(fn=cmd_lock) @@ -2697,7 +2736,9 @@ def main(argv: list[str] | None = None) -> None: p.add_argument("--version", default="1.0.0") p.add_argument("--description", default="") p.add_argument("--creator", default="pack-author") - p.add_argument("--identity", default=str(home() / "identity.json")) + p.add_argument("--identity", default=None, + help="signing identity (default identity.json in " + "Compartment's home)") p.add_argument("--encrypt", action="store_true") p.add_argument("--out") p.set_defaults(fn=cmd_pack_build) @@ -2779,6 +2820,12 @@ def main(argv: list[str] | None = None) -> None: _accept_global_options_anywhere(ap) args = ap.parse_args(argv) + if args.vault is not None: + # "~/..." from an MCP client config or a quoted shell argument: + # nothing between there and here expands it. + args.vault = os.path.expanduser(args.vault) + elif args.fn not in (cmd_embed_daemon, cmd_setup): + _die(NO_HOME) if args.assert_offline: offline_guard.activate() try: diff --git a/src/compartment/server.py b/src/compartment/server.py index 8108241..fd10128 100644 --- a/src/compartment/server.py +++ b/src/compartment/server.py @@ -986,6 +986,9 @@ def main(argv: list[str] | None = None) -> None: args = ap.parse_args(argv) if args.assert_offline: offline_guard.activate() + # `python -m compartment.server --vault ~/...` from a client config: no + # shell expands the ~ there. + args.vault = os.path.expanduser(args.vault) _state["path"] = args.vault _state["caller"] = args.caller sig = _credential_signature() diff --git a/tests/test_cli_home.py b/tests/test_cli_home.py new file mode 100644 index 0000000..e4e714a --- /dev/null +++ b/tests/test_cli_home.py @@ -0,0 +1,81 @@ +"""Where the CLI finds the vault: `~` expanded wherever a path comes from, +and no home directory needed when the vault is named.""" +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from compartment import cli +from compartment.vault import Vault + +from conftest import PASS + +SRC = str(Path(__file__).resolve().parents[1] / "src") + + +@pytest.fixture() +def home_dir(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) # expanduser, POSIX + monkeypatch.setenv("USERPROFILE", str(home)) # expanduser, Windows + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", PASS) + Vault.create(str(home / "v.vault"), PASS, creator="test").lock() + return home + + +@pytest.mark.parametrize("how", ["flag", "flag after", "COMPARTMENT_VAULT", + "ENGRAM_VAULT"]) +def test_a_vault_path_starting_with_a_tilde_is_the_home_one( + home_dir, monkeypatch, capsys, how): + """MCP client configs pass "~/..." in args and env literally, since no + shell runs in between: the CLI looked for a folder named "~" in the + client's working directory and reported no vault.""" + for name in ("COMPARTMENT_VAULT", "ENGRAM_VAULT"): + monkeypatch.delenv(name, raising=False) + argv = ["status"] + if how == "flag": + argv = ["--vault", "~/v.vault", "status"] + elif how == "flag after": + argv = ["status", "--vault", "~/v.vault"] + else: + monkeypatch.setenv(how, "~/v.vault") + cli.main(argv) + out = capsys.readouterr().out + assert str(home_dir / "v.vault") in out and '"locked": false' in out + + +def _cli_with_no_home(*argv): + """The CLI in a process where Compartment's home cannot be found, as + with HOME unset and no password-database entry for the uid.""" + code = ("import sys, compartment.home as h\n" + "def no_home():\n" + " raise RuntimeError('Could not determine home directory.')\n" + "h.home = no_home\n" + "from compartment import cli\n" + "cli.main(sys.argv[1:])\n") + env = dict(os.environ, COMPARTMENT_EMBED_DAEMON="0", + PYTHONPATH=SRC + os.pathsep + os.environ.get("PYTHONPATH", "")) + for name in ("COMPARTMENT_VAULT", "ENGRAM_VAULT"): + env.pop(name, None) + return subprocess.run([sys.executable, "-c", code, *argv], + capture_output=True, text=True, env=env, timeout=120) + + +def test_a_named_vault_needs_no_home_directory(tmp_path): + """The default vault was a module constant: with no home every command + died at import, even one that named its vault.""" + missing = str(tmp_path / "nowhere.vault") + r = _cli_with_no_home("--vault", missing, "status") + assert "Traceback" not in r.stderr, r.stderr + assert f"no vault at {missing}" in r.stderr + r = _cli_with_no_home("--version") + assert r.returncode == 0 and r.stdout.strip() + + +def test_no_vault_and_no_home_says_what_to_pass(): + r = _cli_with_no_home("status") + assert r.returncode == 1 and "Traceback" not in r.stderr + assert "--vault PATH" in r.stderr and "COMPARTMENT_VAULT" in r.stderr diff --git a/tests/test_server.py b/tests/test_server.py index eb26cd2..51efbe9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -555,6 +555,33 @@ def _serve(monkeypatch, vault_path, session, passphrase="CorrectHorse"): signal.signal(sig, handler) +def test_a_vault_path_with_a_tilde_is_the_home_one(vault, vault_path, + own_state, monkeypatch, + tmp_path): + """Run as `python -m compartment.server --vault ~/...` from a client's + config, where no shell expands the ~.""" + vault.lock() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + rel = os.path.relpath(vault_path, tmp_path) + seen = [] + monkeypatch.setattr(server, "_autolock_loop", lambda: None) + monkeypatch.setattr(server, "_retag_loop", lambda: None) + monkeypatch.setattr(server.mcp, "run", lambda *a, **k: seen.append( + (server._state["path"], server._state["vault"]))) + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", "CorrectHorse") + handled = [getattr(signal, n) for n in ("SIGTERM", "SIGHUP") + if hasattr(signal, n)] + before = {sig: signal.getsignal(sig) for sig in handled} + try: + server.main(["--vault", os.path.join("~", rel)]) + finally: + for sig, handler in before.items(): + signal.signal(sig, handler) + (path, opened), = seen + assert path == vault_path and opened is not None + + def _audited_on_disk(vault_path): v = Vault.unlock(vault_path, passphrase="CorrectHorse") return [r["op"] for r in v.db.conn.execute( From fd130cccbf963f17c50dcbd1a18351061c34bb65 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:21:42 -0700 Subject: [PATCH 109/135] Tests: the stdio session test reads its replies before it hangs up, as a client does Writing the requests and closing stdin at once raced the replies: the server read EOF and ended the session before answering tools/list, about half the time. The test now hangs up once it has both answers, and lets communicate() close stdin. Co-Authored-By: Claude Opus 5.5 --- tests/test_server_stdio.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/test_server_stdio.py b/tests/test_server_stdio.py index 1c02588..226101e 100644 --- a/tests/test_server_stdio.py +++ b/tests/test_server_stdio.py @@ -30,15 +30,26 @@ def test_a_session_over_stdio_ends_cleanly_when_the_client_hangs_up( {"jsonrpc": "2.0", "method": "notifications/initialized"}, {"jsonrpc": "2.0", "id": 2, "method": "tools/list"}, ] - proc = subprocess.run( + proc = subprocess.Popen( [sys.executable, "-m", "compartment.cli", "--vault", vault, "serve"], - input="".join(json.dumps(r) + "\n" for r in requests).encode(), - capture_output=True, env=env, timeout=120) - err = proc.stderr.decode("utf-8", "replace") + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + try: + for r in requests: + proc.stdin.write((json.dumps(r) + "\n").encode()) + proc.stdin.flush() + # The client hangs up once it has its answers, as a client does: + # closing stdin at once would race the replies. + replies = [json.loads(proc.stdout.readline()) for _ in range(2)] + rest, err = proc.communicate(timeout=60) # closes stdin first + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + err = err.decode("utf-8", "replace") assert proc.returncode == 0, err assert "Traceback" not in err, err - lines = proc.stdout.decode("utf-8").splitlines() - replies = [json.loads(line) for line in lines] # nothing but frames + assert rest == b"" # nothing but frames assert [r.get("id") for r in replies] == [1, 2] init, listed = replies assert init["result"]["serverInfo"]["name"] == "compartment" From 6e8a6f3af4d921e0165a06f8dac1041429830cd8 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:21:59 -0700 Subject: [PATCH 110/135] Windows always locks byte zero of the lock file, a save outlasts OneDrive and antivirus holding the vault, and a NaN similarity is never a duplicate msvcrt.locking locks from the handle's current position, and the lock file's "a+" handle starts at end of file, so two processes agreed on the byte only while the file stayed empty; anything written to it let both hold the "exclusive" lock. The Windows replace retry gave up after about a second, shorter than OneDrive's sync engine and some scanners hold a file; it now backs off up to five. The duplicate and opinion windows compared `score < threshold`, which a NaN from a broken model run fails, so every store came back as a duplicate of its nearest neighbour. Co-Authored-By: Claude Opus 5.5 --- src/compartment/platforms.py | 7 +++++++ src/compartment/vault.py | 8 ++++++-- src/compartment/vaultfile.py | 20 ++++++++++++++++++-- tests/test_platforms.py | 25 +++++++++++++++++++++++++ tests/test_vault_ops.py | 11 +++++++++++ 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index 0b18c4c..1b069b4 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -40,8 +40,14 @@ _BUSY = frozenset({errno.EACCES, getattr(errno, "EDEADLOCK", 36)}) + # msvcrt.locking locks bytes from the handle's CURRENT position, and the + # lock file is opened "a+", whose position is end of file. Two processes + # agree on which byte that is only while the file stays empty; anything + # that ever wrote to it (a tool, an older version, a user) would put + # them on different bytes and both would "hold" the lock. Byte 0 always. def _lock_nb(fh) -> bool: try: + fh.seek(0) msvcrt.locking(fh.fileno(), msvcrt.LK_NBLCK, 1) return True except OSError as exc: @@ -51,6 +57,7 @@ def _lock_nb(fh) -> bool: def _unlock(fh) -> None: try: + fh.seek(0) msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1) except OSError: pass diff --git a/src/compartment/vault.py b/src/compartment/vault.py index fb979a9..7721d20 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -1203,7 +1203,11 @@ def store(self, text: str, caller: str, namespace: str | None = None, # being missed. Walk a small window instead, stopping as soon as # the scores drop below the threshold (results are descending). for ikey, score in self.index.search(vec[0], DEDUP_CANDIDATES): - if score < thr: + # `not >=` rather than `<`: a NaN score (a vector from a + # broken model run) compares false both ways, and `< thr` + # read it as a match, so every store came back as a + # "duplicate" of whatever happened to be nearest. + if not score >= thr: break rid = self._id_by_ikey.get(ikey) if not rid: @@ -1277,7 +1281,7 @@ def _similar_opinions(self, qvec, ns: str, exclude: set[str], best-first. One entry per record (best window), organic only.""" best: dict[str, float] = {} for ikey, score in self.index.search(qvec, OPINION_CANDIDATES): - if score < floor: + if not score >= floor: # NaN: never a match break # results are descending rid = self._id_by_ikey.get(ikey) if rid is None or rid in exclude: diff --git a/src/compartment/vaultfile.py b/src/compartment/vaultfile.py index b0a9489..ff24383 100644 --- a/src/compartment/vaultfile.py +++ b/src/compartment/vaultfile.py @@ -126,6 +126,10 @@ def is_int(x): ) +#: How long _atomic_replace keeps retrying a Windows sharing violation. +_REPLACE_PATIENCE_S = 5.0 + + def _atomic_replace(tmp: str, path: str) -> None: """os.replace, tolerant of the transient sharing violations Windows raises when another process momentarily holds a handle to `path`. @@ -140,14 +144,26 @@ def _atomic_replace(tmp: str, path: str) -> None: if os.name != "nt": os.replace(tmp, path) return + # Backing off to ~5 s in all: a reader's handle goes in milliseconds, but + # OneDrive's sync engine and some antivirus scanners hold a file they + # are looking at for a second or more, and a save that gives up then + # fails a store the user was told succeeded once it is retried. last: OSError | None = None - for _ in range(20): # ~1s worst case + delay, waited = 0.02, 0.0 + while waited < _REPLACE_PATIENCE_S: try: os.replace(tmp, path) return except PermissionError as exc: last = exc - time.sleep(0.05) + time.sleep(delay) + waited += delay + delay = min(delay * 2, 0.5) + try: + os.replace(tmp, path) + return + except PermissionError as exc: + last = exc raise last if last is not None else OSError(f"could not replace {path}") diff --git a/tests/test_platforms.py b/tests/test_platforms.py index b179234..63361b4 100644 --- a/tests/test_platforms.py +++ b/tests/test_platforms.py @@ -420,3 +420,28 @@ def _linux(monkeypatch, *paths): monkeypatch.setattr(platforms, "_LINUX_MACHINE_ID_PATHS", tuple(str(p) for p in paths)) yield + + +def test_the_windows_lock_always_takes_byte_zero(monkeypatch, tmp_path): + """msvcrt.locking locks from the handle's position, and an "a+" handle + starts at end of file: with anything in the lock file, two processes + locked different bytes and both held the "exclusive" lock.""" + import importlib + import types + calls = [] + fake = types.SimpleNamespace( + LK_NBLCK=2, LK_UNLCK=0, + locking=lambda fd, mode, n: calls.append((mode, os.lseek(fd, 0, os.SEEK_CUR)))) + monkeypatch.setitem(sys.modules, "msvcrt", fake) + monkeypatch.setattr(os, "name", "nt") + try: + win = importlib.reload(platforms) + lockfile = tmp_path / "v.flock" + lockfile.write_text("left behind by something else") + with win.FileLock(str(lockfile), timeout=0.2): + pass + finally: + monkeypatch.setattr(os, "name", "posix" if sys.platform != "win32" else "nt") + monkeypatch.delitem(sys.modules, "msvcrt", raising=False) + importlib.reload(platforms) + assert calls and all(pos == 0 for _, pos in calls) diff --git a/tests/test_vault_ops.py b/tests/test_vault_ops.py index 6af739b..b004f29 100644 --- a/tests/test_vault_ops.py +++ b/tests/test_vault_ops.py @@ -713,3 +713,14 @@ def stuck(argv, **kwargs): assert vault_module.keychain_clear("/x/a.vault") is False with pytest.raises(vault_module.CryptoError, match="did not answer"): vault_module.keychain_store("/x/a.vault", b"\x00" * 32) + + +def test_a_nan_similarity_is_never_a_duplicate(vault, monkeypatch): + """A vector from a broken model run scores NaN against everything, and + NaN compares false both ways: `score < thr` read it as a match, so every + later store came back as a "duplicate" of whatever was nearest.""" + first = vault.store("The staging database is Postgres 16", caller="test") + monkeypatch.setattr(vault.index, "search", + lambda vec, k: [(next(iter(vault._id_by_ikey)), float("nan"))]) + second = vault.store("Deploys go out on Tuesdays", caller="test") + assert not second["duplicate"] and second["id"] != first["id"] From 8a93a76622fd01f5cc13c653fd1114164f129b72 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:22:02 -0700 Subject: [PATCH 111/135] A server pointed at a vault that does not exist starts and says so, and every tool failure reaches the model with its reason `compartment serve` with a --vault that points nowhere (a client set up before `compartment init`, or a path mistyped in its config) died in main() with FileNotFoundError whenever a passphrase was in the environment, so the client showed a server that failed to start, the reason in a log file. Without one it started, and every tool said the vault was locked and to run `compartment unlock`, which cannot make a vault that is not there. The server now starts either way; each tool answers "No vault at PATH. Create one with `compartment init` on this machine, or point --vault in this client's MCP config at the vault that exists", and memory_status reports {"exists": false} with that message. mcp 2.x passes the model the text of a ToolError only. Any other failure in a tool, a full disk, a vault file this user cannot read, the shared embedding process refusing a text, arrived as "Error executing tool memory_store" and nothing more. Every exception a tool raises is now turned into the same ToolError the vault's own errors already were, with its type and message, as mcp 1.x always reported it. Co-Authored-By: Claude Opus 5.5 --- src/compartment/server.py | 34 ++++++++++++++++++++++++++++- tests/test_server.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/compartment/server.py b/src/compartment/server.py index fd10128..c3f15ad 100644 --- a/src/compartment/server.py +++ b/src/compartment/server.py @@ -198,6 +198,10 @@ def _credential_gone(v: Vault) -> bool: return False +class VaultMissingError(CryptoError): + """The vault this server was pointed at does not exist.""" + + def _vault() -> Vault: v = _state["vault"] if v is not None and not v._locked and _credential_gone(v): @@ -216,6 +220,13 @@ def _vault() -> Vault: _state["vault"] = None stale, v = v, None if v is None or v._locked: + if not os.path.exists(_state["path"]): + # Not "locked": `compartment unlock` cannot make a vault that is + # not there, and that is what the locked message sent people to. + raise VaultMissingError( + f"No vault at {_state['path']}. Create one with `compartment " + "init` on this machine, or point --vault in this client's " + "MCP config at the vault that exists.") # try silent re-unlock via keychain/env (user intent persists until # `compartment lock` clears the credential) sig = _credential_signature() @@ -409,7 +420,18 @@ def _offload(fn): async def wrapper(*args, **kwargs): def _call(): with _toolgate: - return fn(*args, **kwargs) + try: + return fn(*args, **kwargs) + except ToolError: + raise + except Exception as exc: # noqa: BLE001 + # mcp 2.x passes the model the text of a ToolError + # only; anything else arrives as "Error executing tool + # " and nothing more. A full disk, a vault file + # this user cannot read, the shared embedding process + # refusing a text: each says what it is, so the model + # can tell the user, as mcp 1.x always let it. + raise _fail(exc) from exc return await anyio.to_thread.run_sync(_call) return wrapper @@ -822,6 +844,9 @@ def memory_status() -> str: # genuinely locked: say so, do not fail the call return json.dumps({"vault": _state["path"], "locked": True, "message": str(exc)}) + except VaultMissingError as exc: + return json.dumps({"vault": _state["path"], "exists": False, + "message": str(exc)}) except CryptoError as exc: raise _fail(exc) from exc @@ -1001,6 +1026,13 @@ def main(argv: list[str] | None = None) -> None: _read_settings() except CryptoError: _state["vault"] = None # start locked; tools will say so + except OSError as exc: + # The file is missing or unreadable. Starting anyway gives the + # client a server whose tools say what is wrong; dying here gave it + # a server that "failed to start", with the reason in a log file. + print(f"compartment: cannot open the vault at {args.vault}: {exc}", + file=sys.stderr, flush=True) + _state["vault"] = None threading.Thread(target=_autolock_loop, daemon=True).start() threading.Thread(target=_retag_loop, daemon=True).start() # SIGHUP too, where there is one: a client running in a terminal, as diff --git a/tests/test_server.py b/tests/test_server.py index 51efbe9..9fc569a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -522,6 +522,40 @@ def test_a_locked_vault_tells_the_model_what_to_run(vault_path, monkeypatch): assert "compartment unlock" in str(e.value), str(e.value) +def test_a_vault_that_is_not_there_is_not_called_locked(tmp_path, + monkeypatch): + """A client configured before `compartment init` ran, or with a --vault + that points nowhere: every tool said the vault was locked and to run + `compartment unlock`, which cannot make a vault that is not there.""" + missing = str(tmp_path / "nowhere.vault") + monkeypatch.setenv("COMPARTMENT_PASSPHRASE", "CorrectHorse") + for key, value in (("vault", None), ("path", missing), + ("caller", "agent")): + monkeypatch.setitem(server._state, key, value) + with pytest.raises(Exception) as e: + _through_the_sdk(monkeypatch, "memory_search", {"query": "anything"}) + assert f"No vault at {missing}" in str(e.value), str(e.value) + assert "compartment init" in str(e.value) + status = json.loads(server.memory_status.__wrapped__()) + assert status["exists"] is False and "compartment init" in \ + status["message"] + + +def test_a_failure_that_is_not_the_vaults_own_still_reaches_the_model( + serving, monkeypatch): + """mcp 2.x passes the model the text of a ToolError only. A full disk, a + file this user cannot read, the embedding process refusing a text, all + arrived as "Error executing tool memory_store" and nothing else.""" + def full(*a, **k): + raise OSError(28, "No space left on device") + monkeypatch.setattr(serving, "store", full) + with pytest.raises(Exception) as e: + _through_the_sdk(monkeypatch, "memory_store", + {"text": "The staging database runs Postgres 16", + "source": "from chat"}) + assert "No space left on device" in str(e.value), str(e.value) + + @pytest.fixture() def own_state(monkeypatch): """A copy of server._state for this test, so that main() leaves nothing @@ -627,6 +661,17 @@ def session(): assert ended == [(os.getpid(), signal.SIGTERM, signal.SIG_DFL, "search")] +def test_a_server_whose_vault_is_missing_still_starts(tmp_path, own_state, + monkeypatch): + """With the passphrase in the environment, a vault path that pointed + nowhere raised FileNotFoundError out of main(): the client showed a + server that failed to start, the reason in a log nobody opens.""" + ran = [] + _serve(monkeypatch, str(tmp_path / "nowhere.vault"), + lambda: ran.append(True)) + assert ran == [True] and server._state["vault"] is None + + @pytest.mark.skipif(not hasattr(signal, "SIGHUP"), reason="Windows has no SIGHUP") def test_a_server_whose_terminal_closes_saves_them_too( From 06ac458555b5e7ddcae7c740d707b79104ff32d0 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:22:51 -0700 Subject: [PATCH 112/135] Publishing a release also builds the Claude Desktop extension against the PyPI release and attaches it Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish.yml | 43 +++++++++++++++++++++++++++++++++++ RELEASING.md | 4 +++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 71f72a2..6fb5a30 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -113,3 +113,46 @@ jobs: with: packages-dir: packaging/engram-memory-vault/dist skip-existing: true # only the first release of a given version uploads + + # The Claude Desktop extension. Its uv.lock pins this release from PyPI + # with hashes, so it can only be resolved once PyPI serves the version: + # this waits for it, builds dist/compartment-.mcpb and attaches it to + # the release the rest of the downloads are already on. + claude-desktop-bundle: + runs-on: ubuntu-latest + needs: build-and-publish + if: github.event_name == 'release' + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.release.tag_name }} + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Wait for the version to reach PyPI + shell: bash + env: + TAG: ${{ github.event.release.tag_name }} + run: | + v="${TAG#v}" + for attempt in $(seq 1 60); do + code=$(curl -s -o /dev/null -w '%{http_code}' \ + "https://pypi.org/pypi/compartment/$v/json" || echo 000) + [ "$code" = "200" ] && { echo "on PyPI after $attempt attempt(s)"; exit 0; } + sleep 10 + done + echo "::error::compartment $v never appeared on PyPI"; exit 1 + + - name: Build the bundle and attach it to the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.release.tag_name }} + shell: bash + run: | + python -m pip install uv + python tools/build_mcpb.py + gh release upload "$TAG" dist/compartment-*.mcpb --clobber diff --git a/RELEASING.md b/RELEASING.md index 9344899..ca7aac9 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -20,6 +20,7 @@ GitHub's machines and attaches them to a draft, so nobody has to remember: | `compartment--py3-none-any.whl` | pip / offline installs, any OS | | `compartment-.tar.gz` | source distribution, packagers, air-gapped builds | | `SHA256SUMS.txt` | the checksums of everything above | +| `compartment-.mcpb` | Claude Desktop extension (attached by publish.yml once PyPI serves the version, because its lock file pins that release) | GitHub adds "Source code (zip/tar.gz)" on its own. The notes should also carry the `pip install compartment` line for people who want the CLI and MCP @@ -77,7 +78,8 @@ git tag vX.Y.Z && git push origin vX.Y.Z # 3. review the draft on the Releases page, then press Publish. Publishing # triggers publish.yml, which uploads the release's own wheel and sdist to -# PyPI (the same bytes as the downloads), and publish-registry.yml. +# PyPI (the same bytes as the downloads), then builds the Claude Desktop +# bundle against that PyPI release and attaches it; and publish-registry.yml. # A rebuild of an existing tag: Actions -> Release -> Run workflow -> tag. # # Building by hand still works when it has to, from a Mac: From 6a980a565017b0f47251a575e34e247b8aac8aba Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:46 -0700 Subject: [PATCH 113/135] Tests: the dashboard-reuse test waits for the browser thread and no longer depends on the runner having a display Co-Authored-By: Claude Opus 5.5 --- tests/test_menubar.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_menubar.py b/tests/test_menubar.py index bd52143..798f701 100644 --- a/tests/test_menubar.py +++ b/tests/test_menubar.py @@ -580,9 +580,13 @@ def test_the_dashboard_button_starts_dash_once_and_reuses_it(monkeypatch, "time.sleep(30)\n") monkeypatch.setattr(menubar, "_cli_argv", lambda: [sys.executable, "-c", script]) - opened = [] - monkeypatch.setattr(menubar.webbrowser, "open", - lambda u: opened.append(u) or True) + # Recorded at open_in_browser, not webbrowser.open: whether a browser + # exists is that function's business (none on a headless Linux runner), + # and the reuse path calls it on a thread, so wait for the call. + import threading + opened, called = [], threading.Event() + monkeypatch.setattr(menubar, "open_in_browser", + lambda u: (opened.append(u), called.set(), True)[-1]) menubar._DASH.update(proc=None, url=None) vault = str(tmp_path / "x.vault") url = menubar.open_dashboard(vault) @@ -591,6 +595,7 @@ def test_the_dashboard_button_starts_dash_once_and_reuses_it(monkeypatch, assert proc is not None and proc.poll() is None assert opened == [] # dash opened the browser itself assert menubar.open_dashboard(vault) == url + assert called.wait(5) assert menubar._DASH["proc"] is proc and opened == [url] menubar.stop_dashboard() assert proc.poll() is not None and menubar._DASH["proc"] is None From f090ca54cd2e06ce75ae84f47d6e78f6707ae8b2 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:25:58 -0700 Subject: [PATCH 114/135] README: where Compartment runs, an install that works on externally managed Pythons, the stores, and the full path GUI clients need The install table leads with pipx and uv, says why plain pip fails on Debian 12+, Ubuntu 23.04+, Fedora, Arch and Homebrew, and lists the install scripts. A "Where it runs" table gives each system, CPU and Python range as CI tests them, why onnxruntime sets the limits, and how Alpine, FreeBSD and Termux run it on their own onnxruntime. The Mac downloads name the Intel installer; the stores point at docs/PLUGINS.md; the client block explains the full path a Dock or Start-menu app needs; Docker gets its run line; the client count is 33; and the retrieval notes say when the exact index stays above 20,000 records. Co-Authored-By: Claude Opus 5.5 --- README.md | 98 +++++++++++++++++++++++++++++++++++++++++-------- docs/PLUGINS.md | 23 +++++------- 2 files changed, 93 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 4a5eb76..4da0c26 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ network, no telemetry. [![mcpservers.org](https://img.shields.io/badge/mcpservers.org-listed-8b5cf6)](https://mcpservers.org/servers/maxfreedompollard/compartment) [![LobeHub](https://lobehub.com/badge/mcp/maxfreedompollard-compartment)](https://lobehub.com/mcp/maxfreedompollard-compartment) -**One-click install** (after `pip install compartment && compartment init`): +**One-click install** (after [installing](#install) and running `compartment init`):

Add to Cursor @@ -381,7 +381,10 @@ sized to the requested number of results could be emptied by them while matching memories sit just past the cut. The pool starts at 200 per channel and widens up to three times when filtering leaves too few. Below 20,000 records the vector search is exact (SIMD matrix math, recall 1.0); above -that, HNSW at about 99% recall. +that, HNSW at about 99% recall, unless usearch can use no SIMD on the +machine - an agent's sandbox can hide the CPU features it probes - in which +case the exact index stays, because rebuilding HNSW without SIMD on every +open cost from 50 seconds at 25,000 records. ## Security and the lock model @@ -504,7 +507,8 @@ own size. ## Install -**No command line needed.** On a Mac, download **Compartment.pkg** from the +**No command line needed.** On a Mac, download **Compartment-<version>.pkg** +(Apple silicon) or **Compartment-<version>-intel.pkg** (Intel) from the [latest release](https://github.com/MaxFreedomPollard/Compartment/releases/latest) and open it. Python, the embedding model and every dependency are inside it. It asks you to choose a passphrase, creates the vault, and puts Compartment @@ -514,14 +518,63 @@ From the command line, on any system: | | | |---|---| -| **pip** (macOS, Linux, Windows) | `pip install compartment && compartment init` | -| **pipx / uv** | `pipx install compartment` or `uv tool install compartment`, then `compartment init` | -| **Claude Code plugin** | after `pip install compartment && compartment init`: `/plugin marketplace add MaxFreedomPollard/Compartment`, then `/plugin install compartment@maxfreedompollard`. Codex reads the same marketplace file | +| **pipx** (recommended) | `pipx install compartment && compartment init` | +| **uv** | `uv tool install compartment && compartment init` | +| **pip** | `pip install compartment && compartment init` inside a virtual environment. Debian 12+, Ubuntu 23.04+, Fedora, Arch and Homebrew refuse a system-wide `pip install` with "externally-managed-environment"; pipx and uv avoid that. On Windows, `py -m pip install compartment` | +| **Install script** | from a checkout: `bash install/install-linux.sh` (Linux, WSL, ChromeOS), `bash install/install-macos.sh`, or `powershell -ExecutionPolicy Bypass -File install\install-windows.ps1`. Each picks a Python that works on the machine, handles the externally-managed case, creates the vault and connects the agents it finds | | **Docker** | `docker build -t compartment .` from a checkout; see [Wiring each agent](#wiring-each-agent) | -The pip route needs Python 3.11 or newer. The app runs on macOS 13 or -newer, on Windows with the Microsoft Visual C++ runtime installed, and on -any Linux desktop. +Agents can also install it from their own stores - the Claude Desktop +extension (`compartment-.mcpb` on each release), the Claude Code and +Codex plugin marketplaces, Gemini CLI extensions, Hermes Agent plugins and +more. Each still needs the vault from `compartment init`, because the +passphrase is typed by you. [docs/PLUGINS.md](docs/PLUGINS.md) lists every +store and its install command. + +### Where it runs + +| System | CPU | Python for pip / pipx / uv | The app | +|---|---|---|---| +| macOS 14 or newer | Apple silicon | 3.11 - 3.14 | menu bar; `.pkg` / `.dmg` | +| macOS 13.4 or newer, and every Intel Mac | Apple silicon, Intel | 3.11 - 3.13 | menu bar; `-intel.pkg` / `-intel.dmg` on Intel | +| Windows 10 / 11 | x64 | 3.11 - 3.14 | tray icon | +| Windows 11 on ARM | ARM64 | 3.12 - 3.14 | tray icon | +| Linux with glibc 2.28+ (Ubuntu 20.04+, Debian 11+, Fedora, RHEL 8+, Arch, openSUSE), WSL 2, ChromeOS Linux | x86_64, aarch64 (incl. Raspberry Pi 5) | 3.11 - 3.14 | a window (needs Tk: `python3-tk` or your distro's equivalent) | +| Docker | amd64, arm64 | in the image | headless | + +The limits come from onnxruntime, the one dependency without wheels +everywhere: its macOS builds need macOS 14 from 1.24 on, and its last Intel +Mac build (1.23.2) stops at Python 3.13. CI runs the suite on every row and +checks the whole matrix for a dependency that would have to be compiled +(`tools/wheel_coverage.py`). On Windows, onnxruntime also needs the +[Microsoft Visual C++ runtime](https://aka.ms/vs/17/release/vc_redist.x64.exe) +([ARM64](https://aka.ms/vs/17/release/vc_redist.arm64.exe)); Compartment says +so if it is missing. + +**Elsewhere, with the system's onnxruntime.** Alpine (musl), FreeBSD and +Android (Termux) package onnxruntime themselves; install it from the system +and give Compartment a virtual environment that can see it. The install +script does this for you. + +```bash +# Alpine +apk add python3 py3-onnxruntime +# FreeBSD (default Python 3.12; the compiled dependencies come from packages) +pkg install py312-onnxruntime py312-tokenizers py312-pydantic-core py312-pynacl py312-argon2-cffi py312-numpy py312-cryptography +# Termux +pkg install python onnxruntime python-numpy rust + +# then, on any of them +python3 -m venv --system-site-packages ~/.compartment-venv +~/.compartment-venv/bin/pip install compartment && ~/.compartment-venv/bin/compartment init +``` + +On Termux there is no shared memory for the +unlock-until-restart credential, so set `COMPARTMENT_PASSPHRASE` or use +`compartment unlock --once`. A glibc older than 2.28, 32-bit systems, armv7 +and riscv64 are not supported. On WSL 2, the VM shutting down after it has +been idle clears the unlock credential, so the vault relocks between +sessions. `init` asks you to choose a passphrase, creates the vault, loads the reference facts, connects Claude Code, Hermes Agent or OpenClaw if they are @@ -534,7 +587,7 @@ To connect an agent later, or any other client: compartment integrate claude # Claude Code + Claude Desktop compartment integrate hermes # Hermes Agent compartment integrate openclaw # OpenClaw -compartment integrate --list # the 28 MCP clients it can wire: Cursor, VS Code, Cline, Roo Code, Zed, OpenCode, Codex CLI, Gemini CLI, Oh My Pi, LM Studio, AnythingLLM, BoltAI ... +compartment integrate --list # the 33 MCP clients it can wire: Cursor, VS Code, Windsurf, Cline, Roo Code, Kilo Code, Zed, OpenCode, Codex CLI, Gemini CLI, Kiro, Amazon Q, Junie, Goose, LM Studio, Continue ... compartment integrate --all # every one of them that is installed here ``` @@ -546,6 +599,11 @@ installed in their skills directories. Any other MCP client uses this block { "mcpServers": { "compartment": { "command": "compartment", "args": ["serve"] } } } ``` +An app started from the Dock, Finder or the Start menu does not see your +shell's PATH, so a bare `compartment` can fail there with "spawn compartment +ENOENT". Use the full path that `which compartment` (`where compartment` on +Windows) prints; `integrate` always writes that full path. + ## Wiring each agent None of this needs a terminal: the **Connect an agent** buttons in the app @@ -607,7 +665,7 @@ backup. Then run `openclaw gateway restart` and check with **Any MCP client** -`compartment integrate ` wires any of the 28 clients in `--list`. +`compartment integrate ` wires any of the 33 clients in `--list`. Each config write takes a byte-exact backup first, merges rather than replaces, writes atomically, and refuses to touch a file it cannot parse (it prints the block to paste instead). To do it by hand, use the block in @@ -620,10 +678,19 @@ Client-by-client walkthroughs are in **Docker** -`docker build -t compartment .` from a checkout builds a headless image: -stdio only, no port, unprivileged user, vault on a bind mount at `/data`. -Create the vault on the host first with `compartment init`, because that -step prompts for the passphrase. +`docker build -t compartment .` from a checkout builds a headless image +(amd64 and arm64): stdio only, no port, unprivileged user, the offline guard +on, vault on a bind mount at `/data`. Create the vault on the host first +with `compartment init`, because that step prompts for the passphrase, then +give the client this command: + +```bash +docker run -i --rm --network none --user "$(id -u):$(id -g)" \ + -v "$HOME/.compartment:/data" -e COMPARTMENT_PASSPHRASE compartment +``` + +`--user` lets the container read your 0600 vault on a Linux host; Docker +Desktop on macOS and Windows does not need it. ## Configuration @@ -748,6 +815,7 @@ is off unless you turn it on above. | | | |---|---| | [docs/MEMORY.md](docs/MEMORY.md) | how memory is stored, what gets remembered, and the ranking design | +| [docs/PLUGINS.md](docs/PLUGINS.md) | every plugin store and listing: how users install from it, what the maintainer does | | [docs/INTEGRATIONS.md](docs/INTEGRATIONS.md) | selecting Compartment in Hermes Agent, OpenClaw, Claude, everything else | | [docs/COMPARISON.md](docs/COMPARISON.md) | other memory servers, with sources | | [SECURITY.md](SECURITY.md) | the full threat model and its limits | diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index ede4af8..d01b2d7 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -20,9 +20,9 @@ or when a file anywhere in the tree carries a version it does not check. | PyPI | `pyproject.toml` | `pip install compartment` | automatic (`publish.yml`) | - | | Official MCP Registry | `server.json` | their client's registry browser, `uvx compartment serve` | automatic (`publish-registry.yml`) | - | | Claude Code plugins | `.claude-plugin/` | `/plugin marketplace add MaxFreedomPollard/Compartment` | version is gated; nothing else | submit to Anthropic's plugin directory | -| Claude Desktop extension | `packaging/mcpb/` | the `.mcpb` from the release | build and attach the `.mcpb` | submit to Anthropic's extension directory | +| Claude Desktop extension | `packaging/mcpb/` | the `.mcpb` from the release | automatic (`publish.yml` builds and attaches it once PyPI has the version) | submit to Anthropic's extension directory | | Codex plugins | root `plugin.json` + `.claude-plugin/marketplace.json` | `codex plugin marketplace add MaxFreedomPollard/Compartment` | nothing | - | -| Gemini CLI extensions | `gemini-extension.json`, `GEMINI.md` | `gemini extensions install https://github.com/MaxFreedomPollard/Compartment` | nothing (crawled daily) | add the `gemini-cli-extension` topic | +| Gemini CLI extensions | `gemini-extension.json`, `GEMINI.md` | `gemini extensions install https://github.com/MaxFreedomPollard/Compartment` | nothing (crawled daily) | - (the `gemini-cli-extension` topic is set) | | Hermes Agent | `integrations/hermes/`, root `plugin.json` + `mcp.json` | `compartment integrate hermes` or `hermes plugins install MaxFreedomPollard/Compartment` | nothing | upstream PR (see below) | | OpenClaw | `.claude-plugin/` (as a bundle), `skills/` | `compartment integrate openclaw` | nothing | publish the skill on ClawHub | | Smithery | the `.mcpb` | Smithery's install command | `smithery mcp publish` | `smithery auth login` | @@ -149,14 +149,11 @@ A host wired two ways starts two servers, and the model sees every tool twice: it, or Claude Desktop > Settings > Extensions > Install Extension. The vault still comes from `compartment init` in a terminal (the passphrase has to be typed somewhere). -- **Each release:** after the version is on PyPI, - - ```bash - pip install uv - python tools/build_mcpb.py # dist/compartment-.mcpb, with uv.lock - ``` - - and attach the file to the GitHub release with the other artifacts. The +- **Each release:** automatic. Publishing the release runs `publish.yml`, + whose `claude-desktop-bundle` job waits for the version on PyPI, then + runs `python tools/build_mcpb.py` (dist/compartment-.mcpb, with + uv.lock) and attaches it to the release. By hand, the same two commands + after the version is on PyPI: `pip install uv`, then the build. The build refuses a manifest or pin that is not the package version, and is reproducible byte for byte. `uv lock` needs the release on PyPI, so this runs after `publish.yml`, not before. @@ -192,9 +189,9 @@ A host wired two ways starts two servers, and the model sees every tool twice: Installs from a release use the source archive as long as the release has more than one asset. A release with exactly one asset would make Gemini download that asset as the extension, so keep the full set attached. -- **Once:** add the `gemini-cli-extension` topic to the GitHub repository - (About > Topics). That is the gallery's only requirement beyond the - manifest at the root. +- **Once:** the `gemini-cli-extension` topic on the GitHub repository + (About > Topics) is the gallery's only requirement beyond the manifest at + the root; it is set. ### Hermes Agent From 644a351743b7c2bdc394963bdb4f187d620a4ac0 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:27:01 -0700 Subject: [PATCH 115/135] `integrate openclaw` registers the server under mcp.servers, where current OpenClaw reads it, in the config OpenClaw's own variables name Current OpenClaw reads its MCP servers from `mcp.servers`, and its root schema rejects keys it does not know (src/config/zod-schema.ts and zod-schema.root-support.ts). `integrate openclaw`, the panel's Connect button (which runs it) and its "already connected" check used a top-level `mcpServers`, so the server was never loaded, the config could fail to load as a whole, and the panel reported OpenClaw connected all the same. The entry now goes under mcp.servers; one an earlier release left at the top level is moved there (the key removed once empty); uninstall takes it out of both places; only mcp.servers counts as connected; and the block printed for pasting has the current shape. The config was looked for in OPENCLAW_HOME as though that were the .openclaw folder itself. OpenClaw (src/infra/config-dir.ts) reads OPENCLAW_STATE_DIR as the folder, then the folder of OPENCLAW_CONFIG_PATH, then .openclaw under OPENCLAW_HOME or the home directory; the CLI and the panel now share that rule (agent_skill._openclaw_dir) and honour OPENCLAW_CONFIG_PATH as the file. The panel also finds Hermes by the rule the CLI now uses, so on Windows it looks in %LOCALAPPDATA%\hermes. Tests that set OPENCLAW_HOME to the folder itself now set OPENCLAW_STATE_DIR. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 73 ++++++++++++++++++------- src/compartment/menubar.py | 60 +++++++++++++++++---- tests/test_cli_openclaw.py | 90 +++++++++++++++++++++++++++++++ tests/test_clients.py | 18 +++---- tests/test_connect_environment.py | 6 ++- tests/test_uninstall.py | 6 ++- 6 files changed, 213 insertions(+), 40 deletions(-) create mode 100644 tests/test_cli_openclaw.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 608ae29..2b15c5d 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -699,19 +699,18 @@ def cmd_uninstall(args) -> None: f"compartment from {client.config_path()} by hand") except Exception as exc: # noqa: BLE001 print(f" {client.display}: {exc}") - openclaw = Path(os.environ.get("OPENCLAW_HOME", - Path.home() / ".openclaw")) / "openclaw.json" + openclaw = None try: + from . import menubar + openclaw = menubar._openclaw_config() cfg = (json.loads(openclaw.read_text(encoding="utf-8")) if openclaw.is_file() else {}) - servers = cfg.get("mcpServers") if isinstance(cfg, dict) else None - if isinstance(servers, dict) and "compartment" in servers: - del servers["compartment"] + if isinstance(cfg, dict) and _openclaw_unregister(cfg): # As integrate wrote it: backed up, then replaced whole. clients.write_config(openclaw, json.dumps(cfg, indent=2)) print(f" OpenClaw registration removed from {openclaw}") except Exception as exc: # noqa: BLE001 - print(f" OpenClaw: {openclaw} left as it is ({exc})") + print(f" OpenClaw: {openclaw or 'its config'} left as it is ({exc})") try: md = _remove_managed_claude_md() if md: @@ -2065,14 +2064,49 @@ def _hermes_home() -> Path: Hermes never reads it and reported it installed, never found the Hermes venv to put compartment into, and uninstall and the refresh after an update looked in the same wrong place.""" - explicit = os.environ.get("HERMES_HOME", "").strip() - if explicit: - return Path(os.path.expanduser(os.path.expandvars(explicit))) - if sys.platform == "win32": - local = os.environ.get("LOCALAPPDATA", "").strip() - return (Path(local) if local - else Path.home() / "AppData" / "Local") / "hermes" - return Path.home() / ".hermes" + from . import menubar # one rule for the CLI and the panel + return menubar.hermes_home() + + +def _openclaw_register(cfg: dict, entry: dict) -> None: + """Put compartment's entry where current OpenClaw reads MCP servers, + `mcp.servers`, and take back one an earlier release wrote at the top + level as `mcpServers`. + + OpenClaw's root schema rejects keys it does not know + (src/config/zod-schema.ts), so that top-level entry was never loaded as + a server, and could keep the whole config from loading.""" + mcp = cfg.setdefault("mcp", {}) + if not isinstance(mcp, dict): + raise ValueError('its "mcp" is not a JSON object') + servers = mcp.setdefault("servers", {}) + if not isinstance(servers, dict): + raise ValueError('its "mcp.servers" is not a JSON object') + servers["compartment"] = entry + _drop_legacy_openclaw_entry(cfg) + + +def _drop_legacy_openclaw_entry(cfg: dict) -> bool: + legacy = cfg.get("mcpServers") + if not isinstance(legacy, dict) or "compartment" not in legacy: + return False + del legacy["compartment"] + if not legacy: + del cfg["mcpServers"] # a key OpenClaw would refuse + return True + + +def _openclaw_unregister(cfg: dict) -> bool: + """Take compartment out of an OpenClaw config, from `mcp.servers` and + from the top-level `mcpServers` earlier releases wrote. True if + anything was removed.""" + removed = _drop_legacy_openclaw_entry(cfg) + mcp = cfg.get("mcp") + servers = mcp.get("servers") if isinstance(mcp, dict) else None + if isinstance(servers, dict) and "compartment" in servers: + del servers["compartment"] + removed = True + return removed #: The `import compartment` check in Hermes's own interpreter. @@ -2151,8 +2185,8 @@ def _integrate_target(args) -> None: compartment_bin = clients.executable() entry = {"command": compartment_bin, "args": ["--vault", vault, "--caller", "openclaw", "serve"]} - cfg_path = Path(os.environ.get("OPENCLAW_HOME", - Path.home() / ".openclaw")) / "openclaw.json" + from . import menubar + cfg_path = menubar._openclaw_config() wrote = False # An OpenClaw that has not written its config yet is a normal first # install, not a reason to hand the JSON back for pasting: it reads @@ -2164,7 +2198,7 @@ def _integrate_target(args) -> None: if existed else {}) if not isinstance(cfg, dict): # null, a list: not a config raise ValueError("not a JSON object") - cfg.setdefault("mcpServers", {})["compartment"] = entry + _openclaw_register(cfg, entry) # A byte-exact backup first, then the file replaced whole. backup = clients.write_config(cfg_path, json.dumps(cfg, indent=2)) @@ -2176,9 +2210,10 @@ def _integrate_target(args) -> None: except (ValueError, OSError) as exc: print(f" could not edit {cfg_path} automatically ({exc});") if not wrote: - print(" add this under \"mcpServers\" in ~/.openclaw/openclaw.json, " + print(f" add this to {cfg_path}, merged into what is there, " "then run `openclaw gateway restart`:") - print(json.dumps({"compartment": entry}, indent=2)) + print(json.dumps({"mcp": {"servers": {"compartment": entry}}}, + indent=2)) if not os.path.exists(vault): print(f"\n! no vault at {vault} - run `compartment init` to create one") diff --git a/src/compartment/menubar.py b/src/compartment/menubar.py index 66c470c..6d80dd2 100644 --- a/src/compartment/menubar.py +++ b/src/compartment/menubar.py @@ -1235,7 +1235,7 @@ def integration_status(vault: str) -> dict: p = _openclaw_config() if p.is_file(): cfg = json.loads(p.read_text(encoding="utf-8")) - out["openclaw"] = "compartment" in (cfg.get("mcpServers") or {}) + out["openclaw"] = "compartment" in openclaw_servers(cfg) except (OSError, ValueError): pass return out @@ -1245,14 +1245,58 @@ def _claude_code_config() -> Path: return Path.home() / ".claude.json" +def hermes_home() -> Path: + """Hermes Agent's home, found the way Hermes finds it + (hermes_constants.get_hermes_home): HERMES_HOME, with ~ and variables + expanded, else the platform default. + + On Windows that default is %LOCALAPPDATA%\\hermes, not ~/.hermes, and + Hermes's Windows installer does not export HERMES_HOME. Looking only in + ~/.hermes, `integrate hermes` put the provider plugin where Hermes on + Windows never reads it, and the panel could neither see Hermes nor tell + that it was connected.""" + explicit = os.environ.get("HERMES_HOME", "").strip() + if explicit: + return Path(os.path.expanduser(os.path.expandvars(explicit))) + if sys.platform == "win32": + local = os.environ.get("LOCALAPPDATA", "").strip() + return (Path(local) if local + else Path.home() / "AppData" / "Local") / "hermes" + return Path.home() / ".hermes" + + def _hermes_plugin_manifest() -> Path: - return (Path(os.environ.get("HERMES_HOME", Path.home() / ".hermes")) - / "plugins" / "compartment" / "plugin.yaml") + return hermes_home() / "plugins" / "compartment" / "plugin.yaml" + + +def openclaw_dir() -> Path: + """OpenClaw's config directory, by OpenClaw's own rule + (src/infra/config-dir.ts, as agent_skill._openclaw_dir reads it): + OPENCLAW_STATE_DIR, else the folder of OPENCLAW_CONFIG_PATH, else + .openclaw under OPENCLAW_HOME or the home directory. OPENCLAW_HOME was + taken for the .openclaw folder itself, one level too high.""" + from .agent_skill import _openclaw_dir + return _openclaw_dir() or Path.home() / ".openclaw" def _openclaw_config() -> Path: - return (Path(os.environ.get("OPENCLAW_HOME", Path.home() / ".openclaw")) - / "openclaw.json") + explicit = os.environ.get("OPENCLAW_CONFIG_PATH", "").strip() + if explicit: + return Path(explicit).expanduser() + return openclaw_dir() / "openclaw.json" + + +def openclaw_servers(cfg) -> dict: + """The MCP servers an OpenClaw config declares: `mcp.servers`. + + Current OpenClaw reads its servers there, and its root schema rejects + keys it does not know (src/config/zod-schema.ts), so the top-level + `mcpServers` that `integrate openclaw` used to write was never loaded + and could stop the whole config from loading. An entry left there is + therefore not a connection, and is not counted as one.""" + mcp = cfg.get("mcp") if isinstance(cfg, dict) else None + servers = mcp.get("servers") if isinstance(mcp, dict) else None + return servers if isinstance(servers, dict) else {} def integration_paths() -> list[Path]: @@ -1302,12 +1346,10 @@ def agent_present(target: str, path: str | None = None) -> bool: or claude_desktop.present()) if target == "hermes": return (bool(shutil.which("hermes", path=path)) - or Path(os.environ.get("HERMES_HOME", - Path.home() / ".hermes")).is_dir()) + or hermes_home().is_dir()) if target == "openclaw": return (bool(shutil.which("openclaw", path=path)) - or Path(os.environ.get("OPENCLAW_HOME", - Path.home() / ".openclaw")).is_dir()) + or openclaw_dir().is_dir()) except Exception: # noqa: BLE001 return False return False diff --git a/tests/test_cli_openclaw.py b/tests/test_cli_openclaw.py new file mode 100644 index 0000000..9afe6af --- /dev/null +++ b/tests/test_cli_openclaw.py @@ -0,0 +1,90 @@ +"""`integrate openclaw` and `uninstall` against the config current OpenClaw +reads: servers under `mcp.servers`, the file found by OpenClaw's own rule +for OPENCLAW_STATE_DIR, OPENCLAW_CONFIG_PATH and OPENCLAW_HOME.""" +import json +import subprocess +import types + +import pytest + +from compartment import cli, menubar + +VAULT = "/v/memory.vault" + + +@pytest.fixture() +def openclaw_home(tmp_path, monkeypatch): + for name in ("OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENCLAW_HOME", str(tmp_path)) + d = tmp_path / ".openclaw" + d.mkdir() + return d / "openclaw.json" + + +def _integrate(): + cli._integrate_target(types.SimpleNamespace(target="openclaw", + vault=VAULT)) + + +def test_the_server_goes_under_mcp_servers_and_an_old_entry_moves_there( + openclaw_home, capsys): + """OpenClaw reads `mcp.servers`, and its root schema rejects keys it does + not know: the top-level `mcpServers` integrate wrote was never loaded, + and could keep the whole config from loading.""" + openclaw_home.write_text(json.dumps({ + "gateway": {"port": 18789}, + "mcp": {"servers": {"theirs": {"command": "x"}}}, + "mcpServers": {"compartment": {"command": "old"}}}), + encoding="utf-8") + _integrate() + cfg = json.loads(openclaw_home.read_text(encoding="utf-8")) + assert "mcpServers" not in cfg + servers = cfg["mcp"]["servers"] + assert set(servers) == {"theirs", "compartment"} + assert servers["compartment"]["args"] == ["--vault", VAULT, "--caller", + "openclaw", "serve"] + assert cfg["gateway"] == {"port": 18789} + assert menubar.integration_status(VAULT)["openclaw"] is True + + +def test_the_block_to_paste_has_the_current_shape(openclaw_home, capsys): + openclaw_home.write_text("[]\n", encoding="utf-8") + _integrate() + out = capsys.readouterr().out + block = json.loads(out[out.index("{"):out.rindex("}") + 1]) + assert "compartment" in block["mcp"]["servers"] + + +@pytest.mark.parametrize("var", ["OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"]) +def test_openclaws_own_variables_name_the_config(tmp_path, monkeypatch, var): + """OPENCLAW_HOME was taken for the .openclaw folder itself, one level too + high, and the two variables that move the folder were not read.""" + d = tmp_path / "state" + monkeypatch.setenv(var, str(d if var == "OPENCLAW_STATE_DIR" + else d / "openclaw.json")) + assert menubar._openclaw_config() == d / "openclaw.json" + + +def test_uninstall_takes_it_out_of_both_places(openclaw_home, monkeypatch, + capsys): + openclaw_home.write_text(json.dumps({ + "mcp": {"servers": {"compartment": {}, "theirs": {}}}, + "mcpServers": {"compartment": {}, "also-theirs": {}}}), + encoding="utf-8") + monkeypatch.setattr(cli, "_tray_app", + lambda: (_ for _ in ()).throw(RuntimeError("no app"))) + monkeypatch.setattr(menubar, "_is_generated", lambda path: False) + monkeypatch.setattr(subprocess, "run", lambda argv, **k: + subprocess.CompletedProcess(argv, 1, "", "")) + for name in ("unregister", "is_registered"): + monkeypatch.setattr(cli.clients, name, lambda *a, **k: False) + monkeypatch.setattr(cli.claude_desktop, "unregister", lambda: False) + monkeypatch.setattr(cli.claude_hooks, "is_installed", lambda: False) + monkeypatch.setattr(cli.agent_skill, "SKILL_TARGETS", {}) + monkeypatch.setattr(cli, "_remove_managed_claude_md", lambda: None) + cli.cmd_uninstall(types.SimpleNamespace(purge=False, vault=VAULT)) + cfg = json.loads(openclaw_home.read_text(encoding="utf-8")) + assert cfg == {"mcp": {"servers": {"theirs": {}}}, + "mcpServers": {"also-theirs": {}}} + assert "OpenClaw registration removed" in capsys.readouterr().out diff --git a/tests/test_clients.py b/tests/test_clients.py index 07f6a1c..79813e9 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -613,10 +613,10 @@ def test_an_openclaw_write_cut_short_leaves_its_config_whole_with_a_backup( with every other server in it, cut short as well.""" home = tmp_path / "openclaw-home" home.mkdir() - monkeypatch.setenv("OPENCLAW_HOME", str(home)) + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(home)) cfg = home / "openclaw.json" - cfg.write_text(json.dumps({"mcpServers": {f"server-{i}": OTHER - for i in range(100)}}, + cfg.write_text(json.dumps({"mcp": {"servers": {f"server-{i}": OTHER + for i in range(100)}}}, indent=2), encoding="utf-8") before = cfg.read_bytes() args = types.SimpleNamespace(target="openclaw", vault=VAULT) @@ -629,7 +629,7 @@ def test_an_openclaw_write_cut_short_leaves_its_config_whole_with_a_backup( assert backup.read_bytes() == before cli._integrate_target(args) - servers = json.loads(cfg.read_text(encoding="utf-8"))["mcpServers"] + servers = json.loads(cfg.read_text(encoding="utf-8"))["mcp"]["servers"] assert "compartment" in servers and servers["server-99"] == OTHER assert backup.read_bytes() == before @@ -642,7 +642,7 @@ def test_an_openclaw_config_that_is_json_but_not_an_object_is_refused( readers leave such a file alone and the user gets the block to paste.""" home = tmp_path / "openclaw-home" home.mkdir() - monkeypatch.setenv("OPENCLAW_HOME", str(home)) + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(home)) cfg = home / "openclaw.json" cfg.write_text(raw, encoding="utf-8") cli._integrate_target(types.SimpleNamespace(target="openclaw", @@ -708,8 +708,8 @@ def test_a_claude_md_or_openclaw_config_kept_in_dotfiles_stays_a_link( write = cli._write_managed_claude_md else: link = tmp_path / "openclaw-home" / name - monkeypatch.setenv("OPENCLAW_HOME", str(link.parent)) - real.write_text(json.dumps({"mcpServers": {"theirs": OTHER}}), + monkeypatch.setenv("OPENCLAW_STATE_DIR", str(link.parent)) + real.write_text(json.dumps({"mcp": {"servers": {"theirs": OTHER}}}), encoding="utf-8") write = lambda: cli._integrate_target( # noqa: E731 types.SimpleNamespace(target="openclaw", vault=VAULT)) @@ -722,8 +722,8 @@ def test_a_claude_md_or_openclaw_config_kept_in_dotfiles_stays_a_link( text = real.read_text(encoding="utf-8") assert ("kept in my dotfiles" in text and cli._CLAUDE_MD_BEGIN in text if name == "CLAUDE.md" - else set(json.loads(text)["mcpServers"]) == {"theirs", - "compartment"}) + else set(json.loads(text)["mcp"]["servers"]) == { + "theirs", "compartment"}) backup = link.with_name(link.name + clients.BACKUP_SUFFIX) assert backup.read_bytes() == before assert os.listdir(dot) == [name] # nothing left beside it diff --git a/tests/test_connect_environment.py b/tests/test_connect_environment.py index 10c0e5f..3a66d22 100644 --- a/tests/test_connect_environment.py +++ b/tests/test_connect_environment.py @@ -240,8 +240,12 @@ def test_openclaw_counts_as_connected_once_it_is_in_the_config(fake_home, d = fake_home / ".openclaw" d.mkdir() (d / "openclaw.json").write_text( - json.dumps({"mcpServers": {"compartment": {}}})) + json.dumps({"mcp": {"servers": {"compartment": {}}}})) assert menubar.integration_status("/v")["openclaw"] is True + # Where earlier releases wrote it: a key current OpenClaw never loads. + (d / "openclaw.json").write_text( + json.dumps({"mcpServers": {"compartment": {}}})) + assert menubar.integration_status("/v")["openclaw"] is False def test_a_broken_config_file_is_not_a_crash(fake_home, monkeypatch): diff --git a/tests/test_uninstall.py b/tests/test_uninstall.py index be27180..ae2c9f0 100644 --- a/tests/test_uninstall.py +++ b/tests/test_uninstall.py @@ -33,7 +33,9 @@ def machine(tmp_path, monkeypatch): "openclaw": ("OPENCLAW_HOME", home / ".openclaw"), }) monkeypatch.setenv("HERMES_HOME", str(home / ".hermes")) - monkeypatch.setenv("OPENCLAW_HOME", str(home / ".openclaw")) + # OPENCLAW_HOME stands in for the home directory, as OpenClaw reads it: + # its config is .openclaw/openclaw.json under it. + monkeypatch.setenv("OPENCLAW_HOME", str(home)) monkeypatch.setattr(cli, "_tray_app", lambda: (_ for _ in ()).throw(RuntimeError("no app"))) monkeypatch.setattr(menubar, "_is_generated", lambda path: False) @@ -118,7 +120,7 @@ def test_uninstall_takes_back_every_registration_integrate_wrote(machine, assert json.loads(w.cursor.config_path().read_text(encoding="utf-8")) == { "mcpServers": {"theirs": {"command": "x"}}} assert json.loads(w.openclaw.read_text(encoding="utf-8")) == { - "gateway": {"port": 18789}, "mcpServers": {}} + "gateway": {"port": 18789}, "mcp": {"servers": {}}} assert w.md.read_text(encoding="utf-8") == "# My notes\nkeep me\n" assert not w.plugin.exists() for said in ("Claude Desktop registration removed", From aa0b99b89babe282004607afbcd9b9f77a15c30b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:27:40 -0700 Subject: [PATCH 116/135] The CLAUDE.md block is written where Claude Code reads it when CLAUDE_CONFIG_DIR moves Claude Code's directory Claude Code keeps its user CLAUDE.md in its own directory, which CLAUDE_CONFIG_DIR moves (a second account, a dotfiles tree). `integrate claude`, the refresh after an update and uninstall always used ~/.claude/CLAUDE.md, which such a Claude Code never loads, so the memory instructions never reached it and uninstall looked in the wrong file. The path now comes from claude_memory.claude_config_dir(), the rule the hook and the importer already follow; CLAUDE_MD still overrides it. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 18 +++++++++++++++--- tests/test_cli_claude_md.py | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 tests/test_cli_claude_md.py diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 2b15c5d..59b4458 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -1722,13 +1722,25 @@ def cmd_pack_export(args) -> None: "`memory_search` does not return, store it into compartment.") +def _claude_md_path() -> Path: + """The user CLAUDE.md Claude Code loads: CLAUDE_MD when set (an + override for tests and unusual setups), else CLAUDE.md in Claude + Code's own directory, which CLAUDE_CONFIG_DIR moves. The block was + always written to ~/.claude/CLAUDE.md, which a Claude Code run with + CLAUDE_CONFIG_DIR (a second account, a dotfiles tree) never reads.""" + explicit = os.environ.get("CLAUDE_MD") + if explicit: + return Path(explicit) + return claude_memory.claude_config_dir() / "CLAUDE.md" + + def _write_managed_claude_md() -> Path: """Write an idempotent, sentinel-fenced compartment block into the user's CLAUDE.md. Only ever rewrites BETWEEN the markers - the user's own text above and below is never touched; re-running updates the block in place. (Belt-and-suspenders: the authoritative behavioral instruction rides the MCP `initialize` handshake; this covers hosts that under-weight it.)""" - md = Path(os.environ.get("CLAUDE_MD", Path.home() / ".claude" / "CLAUDE.md")) + md = _claude_md_path() md.parent.mkdir(parents=True, exist_ok=True) block = f"{_CLAUDE_MD_BEGIN}\n{_CLAUDE_MD_BODY}\n{_CLAUDE_MD_END}" text = md.read_text(encoding="utf-8") if md.exists() else "" @@ -1748,7 +1760,7 @@ def _remove_managed_claude_md() -> Path | None: """Take the fenced block back out of the user's CLAUDE.md: the inverse of _write_managed_claude_md, leaving everything outside the markers as it is. Returns the file, or None when it holds no block.""" - md = Path(os.environ.get("CLAUDE_MD", Path.home() / ".claude" / "CLAUDE.md")) + md = _claude_md_path() if not md.is_file(): return None text = md.read_text(encoding="utf-8") @@ -1877,7 +1889,7 @@ def _refresh_installed() -> list[str]: the user had turned off), no registration, no import. An edited skill is backed up before it is replaced, as `integrate` does.""" done: list[str] = [] - md = Path(os.environ.get("CLAUDE_MD", Path.home() / ".claude" / "CLAUDE.md")) + md = _claude_md_path() try: before = md.read_text(encoding="utf-8") if md.is_file() else "" if _CLAUDE_MD_BEGIN in before and _CLAUDE_MD_END in before: diff --git a/tests/test_cli_claude_md.py b/tests/test_cli_claude_md.py new file mode 100644 index 0000000..136afb8 --- /dev/null +++ b/tests/test_cli_claude_md.py @@ -0,0 +1,23 @@ +"""The CLAUDE.md block goes where Claude Code reads its user CLAUDE.md.""" +from compartment import cli + + +def test_the_block_follows_claude_config_dir(tmp_path, monkeypatch): + """With CLAUDE_CONFIG_DIR set, Claude Code reads CLAUDE.md from there; + the block went to ~/.claude/CLAUDE.md, which it never loads.""" + monkeypatch.delenv("CLAUDE_MD", raising=False) + monkeypatch.setattr(cli.Path, "home", + classmethod(lambda cls: tmp_path / "home")) + moved = tmp_path / "dotfiles" / "claude" + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(moved)) + md = cli._write_managed_claude_md() + assert md == moved / "CLAUDE.md" + assert cli._CLAUDE_MD_BEGIN in md.read_text(encoding="utf-8") + assert not (tmp_path / "home" / ".claude" / "CLAUDE.md").exists() + assert cli._remove_managed_claude_md() == md + + +def test_claude_md_still_overrides(tmp_path, monkeypatch): + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "moved")) + monkeypatch.setenv("CLAUDE_MD", str(tmp_path / "explicit.md")) + assert cli._claude_md_path() == tmp_path / "explicit.md" From f5e14951047dd197d545fd5c1f67f6fe19b9a131 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:28:22 -0700 Subject: [PATCH 117/135] After connecting Claude Desktop, integrate says to quit it completely and reopen it, and how, instead of "Restart Claude Desktop" "Restart Claude Desktop" was read as closing its window, which leaves it running (in the notification area on Windows, in the Dock on macOS): it never rereads its config, and a copy still running can write the config it holds back over the one just written. The line now says to quit it completely, how on this platform (the notification-area icon's Quit on Windows, Cmd-Q on macOS), and that closing the window leaves it running. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 18 +++++++++++++++++- tests/test_cli_claude_md.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 59b4458..fa83580 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -2065,6 +2065,22 @@ def _integrate_client(client, args, brief: bool = False) -> None: #: small step for that program; one busy updating itself is no reason to #: hold an install for as long as that takes. AGENT_TOOL_TIMEOUT = 120 +def _quit_claude_desktop(platform: str | None = None) -> str: + """How to make Claude Desktop load a config just written. + + Quit, not close: closing its window leaves Claude Desktop running (in + the notification area on Windows), so it never rereads the file, and a + copy still running can write the config it holds back over this one. + "Restart Claude Desktop" read as closing the window.""" + platform = platform or sys.platform + where = ("right-click its icon in the notification area and choose " + "Quit" if platform == "win32" else + "Claude > Quit Claude, or Cmd-Q" if platform == "darwin" + else "quit it") + return (f"Quit Claude Desktop completely ({where}; closing its window " + "leaves it running), then open it again to load it.") + + def _hermes_home() -> Path: """Hermes Agent's home, found the way Hermes finds it (hermes_constants.get_hermes_home): HERMES_HOME, with ~ and variables @@ -2271,7 +2287,7 @@ def _integrate_target(args) -> None: print(f" ✓ registered in {reg['config']}") if reg["backup"]: print(f" (previous config backed up to {reg['backup']})") - print(" Restart Claude Desktop to load it.") + print(f" {_quit_claude_desktop()}") except ValueError as exc: # malformed config - never guess print(f" ! {exc}") print(" fix the file, then run `compartment integrate " diff --git a/tests/test_cli_claude_md.py b/tests/test_cli_claude_md.py index 136afb8..009560e 100644 --- a/tests/test_cli_claude_md.py +++ b/tests/test_cli_claude_md.py @@ -21,3 +21,32 @@ def test_claude_md_still_overrides(tmp_path, monkeypatch): monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "moved")) monkeypatch.setenv("CLAUDE_MD", str(tmp_path / "explicit.md")) assert cli._claude_md_path() == tmp_path / "explicit.md" + + + +import pytest # noqa: E402 + + +@pytest.mark.parametrize("platform, words", [ + ("win32", "notification area"), ("darwin", "Cmd-Q"), ("linux", "quit")]) +def test_the_desktop_hint_says_to_quit_it_not_close_it(platform, words): + """"Restart Claude Desktop" read as closing its window, which leaves it + running (in the notification area on Windows): it never rereads its + config, and a copy still running can write the old one back.""" + hint = cli._quit_claude_desktop(platform) + assert hint.startswith("Quit Claude Desktop completely") and words in hint + + +def test_connecting_claude_desktop_prints_that_hint(tmp_path, monkeypatch, + capsys): + from compartment import claude_desktop + monkeypatch.setattr(cli.shutil, "which", lambda name, *a, **k: None) + monkeypatch.setattr(claude_desktop, "present", lambda: True) + monkeypatch.setattr(claude_desktop, "register", lambda exe, args: { + "config": str(tmp_path / "claude_desktop_config.json"), + "backup": None}) + monkeypatch.setenv("CLAUDE_MD", str(tmp_path / "CLAUDE.md")) + cli._integrate_target(cli.argparse.Namespace( + target="claude", vault=str(tmp_path / "v.vault"), no_import=True, + no_hooks=True)) + assert cli._quit_claude_desktop() in capsys.readouterr().out From b55265eac32232f1cf36581607e2b6d26dd562a6 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:28:44 -0700 Subject: [PATCH 118/135] Tests: the failed-replace test checks the retry budget rather than a fixed count of tries Co-Authored-By: Claude Opus 5.5 --- tests/test_vault_ops.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_vault_ops.py b/tests/test_vault_ops.py index b004f29..8cb43e6 100644 --- a/tests/test_vault_ops.py +++ b/tests/test_vault_ops.py @@ -520,7 +520,10 @@ def nap(seconds): m.setattr(vaultfile, "time", _With(time, sleep=nap)) with pytest.raises(PermissionError): vault.save() - assert len(tries) == 20, "the Windows retries all ran" + # The retries ran for the whole patience budget, one try per nap plus + # the last, before giving up. + assert sum(naps) >= vaultfile._REPLACE_PATIENCE_S, "the Windows retries all ran" + assert len(tries) == len(naps) + 1 assert not os.path.exists(vault_path + ".tmp") assert Vault.unlock(vault_path, passphrase=PASS).db.count() == 1 From b6fe852a1f77fc27b28f1a243b37a97538d5a6d8 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:29:06 -0700 Subject: [PATCH 119/135] `compartment hook status` and `hook uninstall` name a Claude Code settings.json that is not JSON instead of ending in a traceback claude_hooks refuses a settings.json it cannot read as settings (hand edited, cut short) with a ValueError that names the file. `hook install` turned that into one line; `hook status`, bare `hook` and `hook uninstall` let it out as a traceback. All of them now print it and exit 1. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 14 ++++++++++++-- tests/test_cli_errors.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index fa83580..aa57601 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -1168,12 +1168,22 @@ def cmd_hook(args) -> None: print(" Claude Code memory writes now land in the vault " "automatically (restart Claude Code to load it)") elif args.hook_cmd == "uninstall": - print("capture hook removed" if claude_hooks.uninstall() + # A settings.json that is not JSON (hand-edited, cut short) is + # refused with ValueError naming the file, as install already said. + try: + removed = claude_hooks.uninstall() + except ValueError as exc: + _die(str(exc)) + print("capture hook removed" if removed else "no compartment hook was installed") else: # No subcommand given (bare `compartment hook`) behaves like `status`, # then points at the rest of the group instead of failing. - print("installed" if claude_hooks.is_installed() else "not installed") + try: + installed = claude_hooks.is_installed() + except ValueError as exc: + _die(str(exc)) + print("installed" if installed else "not installed") if getattr(args, "hook_cmd", None) is None: print(" subcommands: status | install | uninstall | capture") diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 37dcf74..3888dfe 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -220,3 +220,20 @@ def test_serve_ending_with_stdout_closed_by_the_transport_exits_cleanly( monkeypatch.setattr("sys.stdout", out) monkeypatch.setattr(cli, "cmd_serve", lambda args: out.buffer.close()) cli.main(["--vault", str(tmp_path / "v.vault"), "serve"]) + + +@pytest.mark.parametrize("command", [["hook", "status"], ["hook", "uninstall"], + ["hook"]]) +def test_a_claude_settings_file_that_is_not_json_is_named(tmp_path, + monkeypatch, capsys, + command): + """`hook status` and `hook uninstall` let claude_hooks' ValueError for a + malformed settings.json out as a traceback; `hook install` already + said it in one line.""" + from compartment import claude_hooks + settings = tmp_path / "settings.json" + settings.write_text('{"hooks": {', encoding="utf-8") + monkeypatch.setattr(claude_hooks, "SETTINGS", settings, raising=False) + err = _ends(command, capsys) + assert str(settings) in err and "Traceback" not in err + From de50a6d893cfd4987421dbad0a771c8a7dc45d5d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:29:10 -0700 Subject: [PATCH 120/135] A vault config file that cannot be used ends a CLI command with its own message instead of a traceback acl.ConfigError, raised for a .config.json that is not JSON or not a settings object, names the file and what to do. It is a ValueError by design (an unlock caller must not read it as "locked"), and the CLI caught CryptoError alone, so every command that read such a config ended in a traceback. main() now prints it as "error: ..." with exit 1. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 7 +++++-- tests/test_cli_errors.py | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index aa57601..8ff07f5 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -23,7 +23,7 @@ from . import (__version__, agent_skill, audit, claude_desktop, claude_hooks, claude_memory, clients, gate, offline_guard, packs, selftest, session) -from .acl import VaultConfig +from .acl import ConfigError, VaultConfig from .crypto import CryptoError from .embed import DEFAULT_MODEL, OPTIONAL_MODELS, Embedder, get_embedder, user_model_dir from .vault import Vault, keychain_clear, keychain_get, keychain_store @@ -2909,7 +2909,10 @@ def main(argv: list[str] | None = None) -> None: # closes the buffer under sys.stdout when `serve` ends. if not sys.stdout.closed: sys.stdout.flush() - except CryptoError as exc: + except (CryptoError, ConfigError) as exc: + # ConfigError: a .config.json that cannot be used, which + # names the file and what to do; a ValueError, so it was a + # traceback. _die(str(exc)) except BrokenPipeError: # `compartment recent --all | head`: the reader took what it wanted diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 3888dfe..49b3216 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -237,3 +237,12 @@ def test_a_claude_settings_file_that_is_not_json_is_named(tmp_path, err = _ends(command, capsys) assert str(settings) in err and "Traceback" not in err + +def test_a_vault_config_that_cannot_be_used_is_named(open_vault, capsys): + """A .config.json that is not JSON raises acl.ConfigError, which + names the file and what to do, and is a ValueError: the CLI, which + caught CryptoError alone, printed it as a traceback.""" + with open(open_vault + ".config.json", "w", encoding="utf-8") as f: + f.write('{"settings": {') + err = _ends(["--vault", open_vault, "search", "anything"], capsys) + assert ".config.json" in err and "Traceback" not in err From fa7ece6954657583afa68894c52c21539ca30b7c Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:29:35 -0700 Subject: [PATCH 121/135] `compartment import` also reads UTF-32 and UTF-16 written without a byte-order mark The import now finds a file's encoding the way json does (json.detect_encoding): the BOM where there is one, otherwise the zero bytes the opening "{" leaves in UTF-16 or UTF-32, otherwise UTF-8. A UTF-16 export with no BOM, which some editors and converters write, and any UTF-32 one were still refused as not UTF-8. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 12 +++++++----- tests/test_cli_errors.py | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 8ff07f5..6517300 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -1316,13 +1316,15 @@ def _read_text_file(path: str) -> str: utf8`) and UTF-16 for a plain `>` redirect; Notepad wrote a BOM for years. Read as bare UTF-8, the first gave JSONDecodeError "Unexpected UTF-8 BOM" and the second UnicodeDecodeError, both as tracebacks. The - BOM says which it is; with none, the file is UTF-8, as `export` writes - it, and anything else is refused by name.""" + encoding is found as json finds it (UTF-8 with or without a BOM, + UTF-16, UTF-32), and anything that does not decode is refused by + name.""" raw = Path(path).read_bytes() try: - if raw.startswith((b"\xff\xfe", b"\xfe\xff")): - return raw.decode("utf-16") - return raw.decode("utf-8-sig") + # json's own detection: the BOM where there is one, else the zero + # bytes an ASCII "{" leaves in UTF-16 or UTF-32, else UTF-8. + enc = json.detect_encoding(raw) + return raw.decode("utf-8-sig" if enc == "utf-8" else enc) except UnicodeDecodeError as exc: _die(f"{path} is not UTF-8 text (byte {exc.start}): save it as " "UTF-8 and import it again") diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 49b3216..0108f26 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -136,7 +136,8 @@ def open_vault(vault, vault_path, monkeypatch): return vault_path -@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16"]) +@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16", "utf-16-le", + "utf-32", "utf-32-be"]) def test_an_import_saved_by_a_windows_tool_is_read(open_vault, tmp_path, capsys, encoding): """PowerShell 5.1 writes UTF-8 with a BOM, or UTF-16 for `>`: the first From 2198849d0780c3b31b9866b4e7d23e78183f3879 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:30:17 -0700 Subject: [PATCH 122/135] Where there is no session directory for the embedding daemon's socket, or no home directory at all, every path embeds in-process or says why instead of raising The daemon's socket lives in the session directory. Where that cannot be made (a read-only home) a client already fell back to its own model, but with no home directory at all (HOME unset and no password-database entry, a container's arbitrary uid) finding it raised RuntimeError, which escaped get_embedder and failed the vault's first search or store. And `embed-daemon run` raised a traceback in either case. Both errors now mean "no daemon" for a client, status and stop report it, and `embed-daemon run` says there is no session directory and to pass --socket. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed_daemon.py | 18 ++++++++++++++---- tests/test_embed_daemon.py | 27 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/compartment/embed_daemon.py b/src/compartment/embed_daemon.py index 4903e89..8767692 100644 --- a/src/compartment/embed_daemon.py +++ b/src/compartment/embed_daemon.py @@ -589,7 +589,14 @@ def run(path: Path | None = None, idle: int | None = None) -> int: print("compartment: the shared embedding daemon needs a POSIX system " "with Unix domain sockets", file=sys.stderr) return 2 - return _Daemon(path or socket_path(), idle or idle_seconds()).run() + try: + path = path or socket_path() + except (OSError, RuntimeError) as exc: + print(f"compartment: no session directory to hold the embedding " + f"daemon's socket ({exc}); pass --socket PATH", + file=sys.stderr) + return 1 + return _Daemon(path, idle or idle_seconds()).run() # ------------------------------------------------------------ starting it up @@ -799,9 +806,12 @@ def connect(cls, model_name: str = DEFAULT_MODEL, *, path: Path | None = None, return None try: return cls(model_name, path or socket_path(), spawn=spawn, wait=wait) - except (Unavailable, OSError): + except (Unavailable, OSError, RuntimeError): # OSError: the session directory or the lock files cannot be # made here (a read-only home), which is no daemon either. + # RuntimeError: there is no home directory at all (HOME unset + # and no password-database entry, as for a container's + # arbitrary uid), so no session directory to put a socket in. return None # -- properties the vault reports -------------------------------------- @@ -981,7 +991,7 @@ def status(path: Path | None = None) -> dict: "embeds with a model of its own"} try: path = path or socket_path() - except OSError as exc: + except (OSError, RuntimeError) as exc: return {"running": False, "socket": None, "reason": f"no session directory to hold its socket ({exc})"} s = _connect(path, timeout=2.0) @@ -1013,7 +1023,7 @@ def stop(path: Path | None = None, wait: float = 5.0) -> dict: return {"stopped": False, "socket": None} try: path = path or socket_path() - except OSError: + except (OSError, RuntimeError): return {"stopped": False, "socket": None} pid = None s = _connect(path, timeout=2.0) diff --git a/tests/test_embed_daemon.py b/tests/test_embed_daemon.py index 20f8b5c..1c7bfc1 100644 --- a/tests/test_embed_daemon.py +++ b/tests/test_embed_daemon.py @@ -878,3 +878,30 @@ def popen(cmd, **kwargs): capture_output=True, text=True, timeout=120) assert "shadowed" not in out.stdout + out.stderr assert out.returncode == 0 and "--socket" in out.stdout + + +@pytest.mark.parametrize("error", [ + PermissionError(13, "Permission denied"), + RuntimeError("Could not determine home directory."), +]) +def test_no_session_directory_means_a_local_model_not_a_crash(monkeypatch, + capsys, error): + """The socket lives in the session directory. Where that cannot be made + (a read-only home) or there is no home at all (HOME unset and no + password-database entry, as for a container's arbitrary uid), the + daemon is given up and this process embeds with its own model; status, + stop and `embed-daemon run` say why instead of raising.""" + from compartment import session + + def no_dir(): + raise error + monkeypatch.setenv("COMPARTMENT_EMBED_DAEMON", "1") + monkeypatch.delenv("COMPARTMENT_EMBED_SOCKET", raising=False) + monkeypatch.setattr(session, "_session_dir", no_dir) + assert RemoteEmbedder.connect() is None + assert isinstance(get_embedder(), Embedder) + st = embed_daemon.status() + assert st["running"] is False and "session directory" in st["reason"] + assert embed_daemon.stop()["stopped"] is False + assert embed_daemon.run() == 1 + assert "--socket" in capsys.readouterr().err From eaf015a293480481a219f31203e29ad5f43352dd Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:31:42 -0700 Subject: [PATCH 123/135] A client config keeps its own line endings on every system, and the Windows-only test expectations match what Windows writes write_config wrote in text mode, so the same `integrate` produced LF on macOS and Linux but CRLF on Windows, whatever the file had. It now writes verbatim: CRLF when the file had CRLF (detected through its byte order mark for UTF-16 files), LF otherwise and for a new file. Real Windows CI also showed three tests asserting POSIX results: the capture hook's vault path is written with forward slashes there (so Git Bash does not eat the backslashes), and Goose's default config lives under AppData; the tests now expect what each system really does. Co-Authored-By: Claude Opus 5.5 --- src/compartment/clients.py | 27 ++++++++++++++++++++---- tests/test_claude_hooks_and_integrate.py | 5 ++++- tests/test_integrations_clients.py | 5 +++-- tests/test_integrations_encodings.py | 5 ++++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/compartment/clients.py b/src/compartment/clients.py index 9afef8b..c4043b6 100644 --- a/src/compartment/clients.py +++ b/src/compartment/clients.py @@ -744,9 +744,8 @@ def read_config_text(path: Path) -> str: clients use, rejects the mark and cannot read UTF-16 at all, so a marked file was already broken for most of them. - Line endings come back as \\n, as read_text gave them: write_config - writes in text mode, and a \\r\\n kept in the text would be written - back as \\r\\r\\n on Windows. + Line endings come back as \\n; write_config puts back the ones the file + had. Raises OSError when the file cannot be read and UnicodeDecodeError (a ValueError) when it is not text in any of these encodings. @@ -1128,12 +1127,32 @@ def write_config(path: Path, text: str) -> Path | None: raise PermissionError(errno.EACCES, os.strerror(errno.EACCES), str(target)) backup = None + crlf = False if existed: backup = path.with_suffix(path.suffix + BACKUP_SUFFIX) shutil.copy2(target, backup) + try: + raw = target.read_bytes() + for bom, codec in _BOMS: # CRLF inside UTF-16 too + if raw.startswith(bom): + raw = raw.decode(codec, "replace").encode("utf-8") + break + crlf = b"\r\n" in raw + except OSError: + pass + # The file keeps its own line endings, on every system. Text comes in + # with "\n" (the reader normalises), and a text-mode write used to turn + # every "\n" into "\r\n" on Windows only, so the same `integrate` gave + # a different file depending on where it ran, and a CRLF file that kept + # a "\r\n" through some other path came out with "\r\r\n", which is + # not valid TOML. Written verbatim: CRLF if the file had CRLF, LF + # otherwise and for a new file. + text = text.replace("\r\n", "\n") + if crlf: + text = text.replace("\n", "\r\n") tmp = target.with_name(f"{target.name}.{secrets.token_hex(8)}.tmp") try: - with open(tmp, "w", encoding="utf-8") as f: + with open(tmp, "w", encoding="utf-8", newline="") as f: if existed and os.name != "nt": shutil.copymode(target, tmp) # before any text is in it f.write(text) diff --git a/tests/test_claude_hooks_and_integrate.py b/tests/test_claude_hooks_and_integrate.py index 18080e4..36177d7 100644 --- a/tests/test_claude_hooks_and_integrate.py +++ b/tests/test_claude_hooks_and_integrate.py @@ -177,7 +177,10 @@ def test_the_capture_hook_names_a_compartment_that_exists( cmd = _installed_command(home.settings) assert cmd == claude_hooks.hook_command(exe, vault_path) assert claude_hooks.hook_command(vault=vault_path) == cmd # hook install - assert f'--vault "{vault_path}"' in cmd + # Windows: written with forward slashes, which Git Bash, cmd and + # PowerShell all read as the same path (see claude_hooks._shell_word). + shown = vault_path.replace("\\", "/") if os.name == "nt" else vault_path + assert f'--vault "{shown}"' in cmd if os.name != "nt": # run it the way Claude Code does edit = json.dumps({"tool_name": "Edit", diff --git a/tests/test_integrations_clients.py b/tests/test_integrations_clients.py index 498e31f..a866c2e 100644 --- a/tests/test_integrations_clients.py +++ b/tests/test_integrations_clients.py @@ -279,12 +279,13 @@ def test_goose_on_windows_is_under_appdata_block(home, monkeypatch): def test_goose_path_root_moves_its_config(home, monkeypatch, tmp_path): + monkeypatch.delenv("GOOSE_PATH_ROOT", raising=False) + default = clients.CLIENTS["goose"].config_path() # per-OS default monkeypatch.setenv("GOOSE_PATH_ROOT", str(tmp_path / "g")) assert clients.CLIENTS["goose"].config_path() == ( tmp_path / "g" / "config" / "config.yaml") monkeypatch.setenv("GOOSE_PATH_ROOT", "relative") # ignored by Goose - assert clients.CLIENTS["goose"].config_path() == ( - home / ".config" / "goose" / "config.yaml") + assert clients.CLIENTS["goose"].config_path() == default # ---------------------------------------------------------- newer clients diff --git a/tests/test_integrations_encodings.py b/tests/test_integrations_encodings.py index 1cc8e95..ea9929c 100644 --- a/tests/test_integrations_encodings.py +++ b/tests/test_integrations_encodings.py @@ -122,7 +122,10 @@ def test_a_codex_config_saved_by_a_windows_editor_is_registered(how, assert doc["model"] == "o3" and doc["mcp_servers"]["other"] == { "command": "x"} assert doc["mcp_servers"]["compartment"]["args"][1] == VAULT - assert raw.decode("utf-8").startswith("# mine\n") + text = raw.decode("utf-8") + assert text.replace("\r\n", "\n").startswith("# mine\n") + # The file keeps the line endings it had, on every system. + assert ("\r\n" in text) == ("CRLF" in how) assert clients.is_registered(codex, path=p) is True assert clients.unregister(codex, path=p) is True assert "compartment" not in tomllib.loads( From 8f8ceb67ab834ec56139f4aeb4a6ceecbafa8605 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:35:14 -0700 Subject: [PATCH 124/135] COMPARTMENT_MODEL_DIR finds downloaded models where there is no home directory user_model_dir() computed its ~/.compartment/models default even when COMPARTMENT_MODEL_DIR was set, so with HOME unset and no password-database entry (a container's arbitrary uid) naming the folder did not help: the lookup raised "Could not determine home directory" before the variable was read. The home is now looked up only without it. Co-Authored-By: Claude Opus 5.5 --- src/compartment/embed.py | 7 ++++++- tests/test_server_embed_runtime.py | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/compartment/embed.py b/src/compartment/embed.py index dd3b51f..6b59bcc 100644 --- a/src/compartment/embed.py +++ b/src/compartment/embed.py @@ -114,7 +114,12 @@ def bundled_model_dir() -> Path: def user_model_dir() -> Path: - return Path(env("MODEL_DIR", home() / "models")) + # COMPARTMENT_MODEL_DIR first, and the home only without it: the default + # was computed even when the variable was set, so with no home directory + # (HOME unset and no password-database entry) naming the folder did not + # help, and a downloaded model could not be found. + explicit = env("MODEL_DIR") + return Path(explicit) if explicit else home() / "models" def resolve_model_dir(name: str) -> Path: diff --git a/tests/test_server_embed_runtime.py b/tests/test_server_embed_runtime.py index 84c4b7f..c75d5eb 100644 --- a/tests/test_server_embed_runtime.py +++ b/tests/test_server_embed_runtime.py @@ -107,3 +107,12 @@ def test_the_embedder_runs_with_that_many_threads(monkeypatch): e = embed.Embedder() assert e.sess.get_session_options().intra_op_num_threads == 2 assert e.embed_query("router passphrase").shape == (e.dim,) + + +def test_a_model_folder_named_in_the_environment_needs_no_home(tmp_path, + monkeypatch): + def no_home(): + raise RuntimeError("Could not determine home directory.") + monkeypatch.setattr(embed, "home", no_home) + monkeypatch.setenv("COMPARTMENT_MODEL_DIR", str(tmp_path / "models")) + assert embed.user_model_dir() == tmp_path / "models" From 7cc2f66a927a9b788dc991eb91493e7b038a1a75 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:37:09 -0700 Subject: [PATCH 125/135] COMPARTMENT_SESSION_DIR works without a home folder: the session directory is read from the environment before home() is asked Co-Authored-By: Claude Opus 5.5 --- src/compartment/home.py | 15 ++++++++++++++- src/compartment/platforms.py | 8 ++++---- src/compartment/session.py | 6 +++--- tests/test_platforms.py | 16 ++++++++++++++++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/compartment/home.py b/src/compartment/home.py index 1ac95ca..b435fc2 100644 --- a/src/compartment/home.py +++ b/src/compartment/home.py @@ -50,4 +50,17 @@ def env(suffix: str, default: str | None = None) -> str | None: or default) -__all__ = ["home", "file", "env", "NAME", "LEGACY_NAME"] +def session_dir() -> Path: + """Where the unlock credential and the per-boot markers live: + COMPARTMENT_SESSION_DIR, else home()/session. + + The variable is read first and home() only when it is unset. Writing + env("SESSION_DIR", home() / "session") evaluated home() every time, and + in a container run as a uid with no passwd entry and no HOME, + Path.home() raises: setting COMPARTMENT_SESSION_DIR, the documented way + to put the session somewhere writable, could not get past it.""" + given = env("SESSION_DIR") + return Path(given) if given else home() / "session" + + +__all__ = ["home", "file", "env", "session_dir", "NAME", "LEGACY_NAME"] diff --git a/src/compartment/platforms.py b/src/compartment/platforms.py index 1b069b4..b7194c0 100644 --- a/src/compartment/platforms.py +++ b/src/compartment/platforms.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from .home import env, home +from .home import env, home, session_dir import errno import json import os @@ -278,7 +278,7 @@ def _clock_boot_estimate() -> tuple[float, float] | None: def _posix_marker() -> Path: """The POSIX pin. Windows keeps its own `.winboot` - renaming a marker that live installs already rely on would re-key their credentials.""" - base = Path(env("SESSION_DIR", home() / "session")) + base = session_dir() return base / ".boottime" @@ -316,7 +316,7 @@ def _pinned_boot_token(computed: float, uptime_s: float, marker: Path) -> str: def _win_marker() -> Path: - base = Path(env("SESSION_DIR", home() / "session")) + base = session_dir() return base / ".winboot" @@ -389,7 +389,7 @@ def _windows_boot_token() -> str: def _machine_marker() -> Path: - base = Path(env("SESSION_DIR", home() / "session")) + base = session_dir() return base / ".machineid" diff --git a/src/compartment/session.py b/src/compartment/session.py index 3267ce1..28919ed 100644 --- a/src/compartment/session.py +++ b/src/compartment/session.py @@ -71,7 +71,7 @@ """ from __future__ import annotations -from .home import env, home +from .home import env, home, session_dir import ctypes import ctypes.util import errno @@ -118,7 +118,7 @@ def _session_dir() -> Path: session directory fails with EPERM. Calling it unconditionally made every `compartment search` the agent ran die with PermissionError before it could look at the credential the user had unlocked.""" - d = Path(env("SESSION_DIR", home() / "session")) + d = session_dir() d.mkdir(parents=True, exist_ok=True) if stat.S_IMODE(d.stat().st_mode) != 0o700: os.chmod(d, 0o700) @@ -519,7 +519,7 @@ def credential_path(vault_path: str) -> Path: unlocked in a terminal by looking at the file's modification time - the one check cheap enough to make on every click. Unlike `_file_for`, it never creates the session directory or changes its mode.""" - return Path(env("SESSION_DIR", home() / "session")) / _name_for(vault_path) + return session_dir() / _name_for(vault_path) def watch_signature(vault_path: str) -> tuple: diff --git a/tests/test_platforms.py b/tests/test_platforms.py index 63361b4..cb960e1 100644 --- a/tests/test_platforms.py +++ b/tests/test_platforms.py @@ -445,3 +445,19 @@ def test_the_windows_lock_always_takes_byte_zero(monkeypatch, tmp_path): monkeypatch.delitem(sys.modules, "msvcrt", raising=False) importlib.reload(platforms) assert calls and all(pos == 0 for _, pos in calls) + + +def test_a_session_dir_from_the_environment_needs_no_home_folder( + monkeypatch, tmp_path): + """A container run as a uid with no passwd entry and no HOME: Path.home() + raises, and setting COMPARTMENT_SESSION_DIR - the documented way to put + the session somewhere writable - still went through it.""" + from compartment import home as home_mod, session + + def no_home(cls=None): + raise RuntimeError("Could not determine home directory.") + monkeypatch.setattr(home_mod.Path, "home", classmethod(no_home)) + monkeypatch.setenv("COMPARTMENT_SESSION_DIR", str(tmp_path / "s")) + assert home_mod.session_dir() == tmp_path / "s" + assert platforms._posix_marker().parent == tmp_path / "s" + assert session.credential_path(str(tmp_path / "v.vault")).parent == tmp_path / "s" From 0654db9249949677b5eeaf4562aafc5ec9934d51 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:38:13 -0700 Subject: [PATCH 126/135] Creating a vault records the model from its pinned description instead of loading it or starting the shared embedding process Co-Authored-By: Claude Opus 5.5 --- src/compartment/vault.py | 15 +++++++++------ tests/test_vault_ops.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/compartment/vault.py b/src/compartment/vault.py index 7721d20..b571dc7 100644 --- a/src/compartment/vault.py +++ b/src/compartment/vault.py @@ -474,18 +474,23 @@ def create(cls, path: str, passphrase: str, creator: str = "user", raise CryptoError("Empty passphrase refused - the user sets it") master = crypto.new_key() slot_pw = crypto.make_passphrase_slot(master, passphrase) - emb = get_embedder(model_name) + # The model's pinned description is all a new vault records. Building + # an embedder here loaded the model (or started the machine's shared + # embedding process) before anything needed a vector, and ignored the + # embed_daemon setting the vault's own embedder property honours; it + # is built on first use instead, like an opened vault's. + emb = model_info(model_name) header = vaultfile.VaultHeader( vault_id=uuid.uuid4().hex, created=datetime.datetime.now(datetime.UTC).isoformat(), keyslots=[slot_pw], payload_len=0, - model={"name": model_name, "sha256": emb.model_sha256, "dim": emb.dim}, + model={"name": model_name, "sha256": emb.sha256, "dim": emb.dim}, extra={"creator": creator, "wire": wire.WIRE_FORMAT}, ) db = Store() db.set_meta("model_name", model_name) - db.set_meta("model_sha256", emb.model_sha256) + db.set_meta("model_sha256", emb.sha256) audit.append(db.conn, creator, "init", f"vault created (model {model_name})") # Anchor at creation, not just on save: the anchor is mandatory, so a # vault has to carry one from its very first entry onwards or its own @@ -495,9 +500,7 @@ def create(cls, path: str, passphrase: str, creator: str = "user", vaultfile.write_vault_file(path, header, {"sqlite": db.serialize()}, master) config.save(path) - v = cls(path, header, db, master, config) - v._embedder = emb - return v + return cls(path, header, db, master, config) # ---------------------------------------------------------------- unlock diff --git a/tests/test_vault_ops.py b/tests/test_vault_ops.py index 8cb43e6..97acc6a 100644 --- a/tests/test_vault_ops.py +++ b/tests/test_vault_ops.py @@ -727,3 +727,17 @@ def test_a_nan_similarity_is_never_a_duplicate(vault, monkeypatch): lambda vec, k: [(next(iter(vault._id_by_ikey)), float("nan"))]) second = vault.store("Deploys go out on Tuesdays", caller="test") assert not second["duplicate"] and second["id"] != first["id"] + + +def test_creating_a_vault_does_not_load_the_model(tmp_path, monkeypatch): + """Vault.create needs only the model's pinned hash and dimension; it used + to build a whole embedder for them, loading the model or starting the + shared embedding process before anything needed a vector.""" + from compartment import vault as vault_mod + + def boom(*a, **k): + raise AssertionError("create built an embedder") + monkeypatch.setattr(vault_mod, "get_embedder", boom) + v = Vault.create(str(tmp_path / "new.vault"), "pw-not-a-secret") + assert v.header.model["dim"] == 384 and len(v.header.model["sha256"]) == 64 + assert v._embedder is None From 183dae88e4fbd13a4e7a8e283e4063756642ff6c Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:39:46 -0700 Subject: [PATCH 127/135] A new 2FA keyfile is created private and exclusive, a model download cannot hang, and the MCP smoke test fails on an unclean exit `2fa enable` wrote the keyfile and chmod'ed it after, leaving the second factor at the umask's mode for that moment and writing through anything planted at the path; it is now created 0600 with O_EXCL. `setup download-model` bounds its download at 120 s instead of waiting for ever. tools/mcp_smoke.py now also requires exit 0 and no traceback once the client hangs up, which a shutdown regression would otherwise pass. Co-Authored-By: Claude Opus 5.5 --- src/compartment/cli.py | 15 ++++++++++++--- tests/test_twofa.py | 29 +++++++++++++++++++++++++++++ tools/mcp_smoke.py | 9 +++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/compartment/cli.py b/src/compartment/cli.py index 6517300..615bcbd 100644 --- a/src/compartment/cli.py +++ b/src/compartment/cli.py @@ -1581,8 +1581,15 @@ def cmd_2fa(args) -> None: from . import crypto as _crypto kf = _secrets.token_bytes(_crypto.KEYFILE_LEN) p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(kf) - os.chmod(p, 0o600) + # Created 0600 and exclusively: written first and chmod'ed after, + # the second factor sat at the umask's mode (0644 on most systems) + # for that moment, readable by every local account, and a file + # that appeared at the path in between would have been written + # through. O_BINARY keeps Windows from translating its bytes. + fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL + | getattr(os, "O_BINARY", 0), 0o600) + with os.fdopen(fd, "wb") as f: + f.write(kf) print(f"keyfile written → {p}") print("BACK THIS FILE UP (it is random bytes, not derived from " "anything).\nLose it and the passphrase alone will NOT " @@ -2402,7 +2409,9 @@ def cmd_setup(args) -> None: hashes = {} for fname, url in spec["files"].items(): print(f" downloading {fname} …") - with urllib.request.urlopen(url) as r: + # Bounded: a stalled connection otherwise holds the command for + # ever, with nothing on screen after "downloading". + with urllib.request.urlopen(url, timeout=120) as r: data = r.read() (d / fname).write_bytes(data) hashes[fname] = hashlib.sha256(data).hexdigest() diff --git a/tests/test_twofa.py b/tests/test_twofa.py index d785af5..7ffa1d9 100644 --- a/tests/test_twofa.py +++ b/tests/test_twofa.py @@ -102,3 +102,32 @@ def test_legacy_recovery_slot_still_opens(vault_path): v.lock() v2 = Vault.unlock(vault_path, passphrase=" ".join(words)) assert v2.status()["records"] == 0 + + +def test_a_new_keyfile_is_created_private_and_exclusive(vault, vault_path, + tmp_path, monkeypatch): + """`2fa enable` wrote the keyfile and chmod'ed it afterwards, so the + second factor sat at the umask's mode (0644 on most systems) for that + moment, and a file planted at the path in between was written through. + It is now created 0600 with O_EXCL in one call.""" + import os + from compartment import cli + vault.lock() + keyfile = tmp_path / "factor.key" + seen = [] + real_open = os.open + + def spy(path, flags, mode=0o777, *a, **k): + if str(path) == str(keyfile): + seen.append((flags, mode)) + return real_open(path, flags, mode, *a, **k) + monkeypatch.setattr(os, "open", spy) + monkeypatch.delenv("COMPARTMENT_PASSPHRASE", raising=False) + cli.main(["--vault", vault_path, "2fa", "enable", "--passphrase", PASS, + "--keyfile", str(keyfile)]) + assert seen, "the keyfile was not created through os.open" + flags, mode = seen[0] + assert flags & os.O_EXCL and flags & os.O_CREAT and mode == 0o600 + assert len(keyfile.read_bytes()) == crypto.KEYFILE_LEN + if os.name != "nt": + assert (keyfile.stat().st_mode & 0o777) == 0o600 diff --git a/tools/mcp_smoke.py b/tools/mcp_smoke.py index 2e6f96e..2a3ac7f 100644 --- a/tools/mcp_smoke.py +++ b/tools/mcp_smoke.py @@ -135,6 +135,15 @@ def send(obj: dict) -> None: except subprocess.TimeoutExpired: proc.kill() raise SystemExit("the server did not exit after its stdin closed") + # A client that hangs up is the normal end of a session, so it has to be + # a clean one: exit 0 and no traceback on stderr. A regression that + # crashed on shutdown would otherwise pass every request above. + stderr = "".join(err_tail) + if proc.returncode != 0: + raise SystemExit(f"the server exited {proc.returncode} after its stdin " + f"closed\nstderr:\n{stderr}") + if "Traceback" in stderr: + raise SystemExit(f"the server printed a traceback\nstderr:\n{stderr}") print("MCP smoke test passed") return 0 From bdb46c325927e5dc7a772bf75b936a68494b74b1 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:41:27 -0700 Subject: [PATCH 128/135] Release 4.13.0 Every version field moves to 4.13.0, including the Hermes plugin.yaml prepared for the upstream PR, which sat at 1.7.0 and is now gated like the rest. The release notes the Release workflow attaches are in .github/release-notes/v4.13.0.md. 4.12.0 was merged but never published, so its fixes ship in this release. Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/plugin.json | 2 +- .github/release-notes/v4.13.0.md | 111 ++++++++++++++++++ gemini-extension.json | 2 +- integrations/hermes/compartment/plugin.yaml | 2 +- integrations/hermes/upstream/plugin.yaml | 4 +- packaging/continue/compartment.yaml | 2 +- packaging/mcpb/manifest.json | 2 +- packaging/mcpb/pyproject.toml | 4 +- plugin.json | 2 +- pyproject.toml | 2 +- server.json | 4 +- src/compartment/__init__.py | 2 +- .../data/hermes-plugin/plugin.yaml | 2 +- tests/test_version_lockstep.py | 19 ++- 14 files changed, 139 insertions(+), 21 deletions(-) create mode 100644 .github/release-notes/v4.13.0.md diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index eefb12c..e64c93b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "compartment", "displayName": "Compartment", - "version": "4.12.0", + "version": "4.13.0", "description": "Encrypted, fully offline memory for Claude Code: one vault on your machine, shared with Claude Desktop, Cursor, Hermes and any other MCP client. Adds 14 memory tools and the compartmentalize skill (/compartment:compartmentalize). Requires `pip install compartment && compartment init` first (the vault passphrase is yours to choose).", "author": { "name": "Max Freedom Pollard", diff --git a/.github/release-notes/v4.13.0.md b/.github/release-notes/v4.13.0.md new file mode 100644 index 0000000..f57b4b0 --- /dev/null +++ b/.github/release-notes/v4.13.0.md @@ -0,0 +1,111 @@ +Compartment 4.13.0 is a compatibility release: it is about Compartment installing, starting and staying connected on every system and in every agent, including the places it used to fail quietly - agents' sandboxed shells, Windows on ARM, Intel Macs, containers, and GUI apps that cannot see your PATH. 4.12.0 was merged but never published, so its 79 stability fixes ship here too. + +## Downloads + +| File | For | +|---|---| +| `Compartment-4.13.0.pkg` | macOS on Apple silicon, one click, with the menu bar app | +| `Compartment-4.13.0.dmg` | macOS on Apple silicon, drag to Applications | +| `Compartment-4.13.0-intel.pkg` | **new:** macOS on Intel, one click | +| `Compartment-4.13.0-intel.dmg` | **new:** macOS on Intel, drag to Applications | +| `compartment-4.13.0.mcpb` | **new:** Claude Desktop extension (attached a few minutes after publishing) | +| `compartment-4.13.0-py3-none-any.whl` | pip and offline installs, any OS | +| `compartment-4.13.0.tar.gz` | source | +| `SHA256SUMS.txt` | checksums of all of the above | + +Everything here was built from this tag on GitHub's machines, and PyPI carries the very same wheel and sdist. + +```bash +pipx install compartment && compartment init # or: uv tool install compartment +``` + +The Mac app needs macOS 13.4 or newer. The README's new **Where it runs** table gives every system, CPU and Python version CI tests. + +## Highlights + +- **Works from an agent's own shell.** Claude Code and Codex run commands in a sandbox that blocks the macOS boot clock, Unix sockets and some file modes. Every `compartment` command an agent ran there used to fail with "Cannot identify this boot session". It now uses the vault you unlocked, embeds in-process, and never hangs. +- **Clients can always start the server.** `compartment integrate` writes the full path of the installed command. GUI apps started from the Dock or Start menu do not inherit your PATH, and a bare `compartment` failed there with "spawn compartment ENOENT". +- **`pip install compartment` works on Windows on ARM and Intel Macs again.** Two indirect dependencies (cryptography, argon2-cffi-bindings) stopped shipping wheels there, and pip tried to compile them. A new CI check fails if any supported platform would need a compiler again. +- **Intel Mac installer.** The Mac app is built for Apple silicon and Intel, and no longer bundles libraries that need a newer macOS than it says it supports. +- **A plugin in every store.** + - New: a Claude Desktop extension (`.mcpb`). + - Updated: the Claude Code plugin now carries a version, plus better Codex and MCP Registry listings, and corrected Gemini CLI and LM Studio listings. + - Ready to submit: Docker MCP Catalog and Continue entries. + - The full list is in the new `docs/PLUGINS.md`. + +## Agents and clients + +- 33 clients, adding Kiro, Amazon Q Developer CLI, Junie, Factory and Crush. Continue gets its own block file, and Goose gets a real YAML extension entry. +- **New config locations:** Claude Desktop's Microsoft Store build, Cline 4, Kilo Code 7, Cline/Roo/Kilo inside Cursor, Windsurf, VSCodium and Flathub VS Code, and Zed on macOS. +- **Moved config homes followed:** `CODEX_HOME`, `GEMINI_CLI_HOME`, `QWEN_HOME`, `COPILOT_HOME`, `OPENCODE_CONFIG_DIR`, `GOOSE_PATH_ROOT`, `CLAUDE_CONFIG_DIR`, and OpenClaw's state and config variables. +- **OpenClaw:** the server is written under `mcp.servers`, where current OpenClaw reads it. An old entry is moved there. +- **Config edits keep what is already there:** + - Comments in Zed and VS Code files, and the user's own keys in a Codex entry. + - Each file's own line endings. + - Files saved by Notepad or PowerShell (byte order mark, UTF-16) are read correctly. + - `integrate codex` can no longer corrupt config.toml. +- **Claude Code capture hook:** quoted correctly for Git Bash on Windows and non-ASCII user names. It also follows `CLAUDE_CONFIG_DIR` and `autoMemoryDirectory`. +- **MCP:** + - Every tool says it never reaches the internet. + - Tool errors reach the model with their reason. + - A missing vault says "run `compartment init`" instead of "locked". + - `compartment serve --vault PATH` works. + - Closing a client's terminal keeps the session's audit rows. + - mcp 1.x no longer logs a stderr line per request. + - The real floor is mcp 1.21.2: nothing older runs beside today's pydantic. + +## Operating systems + +- **Windows:** + - A machine up for more than 24.9 days no longer relocks the vault and deletes the unlock credential. + - A process started without USERNAME no longer deletes everyone's credential. + - No console windows behind the tray or the panel's buttons, and start at sign-in uses pythonw. + - With fast user switching, another user's tray is not taken for yours. + - Saves outlast OneDrive and antivirus holding the vault. + - The lock always takes byte zero. + - The Visual C++ message names the ARM64 installer on ARM. +- **macOS:** + - An acknowledged store survives a power cut (F_FULLFSYNC). + - The boot time is read in-process instead of forking `sysctl` on every vault open. +- **Linux:** + - Start at login keeps working on Xfce, MATE, Cinnamon and LXQt. + - It is refused, with a reason, inside Flatpak or a strict Snap. + - A missing Tk names the exact package for your distribution. + - `compartment dash` works on WSL (it opens the Windows browser) and on headless machines (it prints how to reach it). +- **Containers:** + - A container run as an arbitrary user with no home folder works, with `COMPARTMENT_PASSPHRASE` or `COMPARTMENT_SESSION_DIR`. + - Inference threads respect CPU quotas. + - The Docker image reads a host user's vault, runs with `--network none` and keeps the offline guard on. +- **BSD, Alpine, Termux:** + - FreeBSD, NetBSD, DragonFly and OpenBSD get the boot clock. + - OpenBSD's on-disk shared memory is refused for the unlock secret. + - The installers point Alpine, FreeBSD and Termux at their own onnxruntime package. +- **Installers:** + - They handle "externally-managed-environment" Pythons (Debian 12+, Ubuntu 23.04+, Fedora, Arch, Homebrew). + - They pick a Python that works on the machine. + - They survive being re-run or piped into a shell. +- **Offline guard:** it now refuses every socket family that can reach another machine, not just IP: Bluetooth, VSOCK/Hyper-V (WSL 2's host) and packet sockets. + +## Storage + +- A disk that fills up or a sync that fails part way through a store no longer leaves a vault that will not open. +- Text containing half an emoji (a lone surrogate) is stored and searched instead of failing. +- Imports with odd dates, NaN, or Unicode line separators no longer break search. +- A save by another process is noticed even on filesystems with coarse timestamps. +- A store whose journal write failed is never treated as written. +- A vault on a filesystem that cannot lock says so at once. +- Above 20,000 records the exact index stays when usearch has no SIMD (as inside a sandbox). Rebuilding HNSW there took 50 s at 25,000 records on every open. +- A Python whose SQLite lacks FTS5 gets a clear message. +- `COMPARTMENT_PASSPHRASE` is still tried when the boot-session lookup fails. + +## For maintainers + +- **Release workflow:** a tag now builds every download, Intel included, and drafts the release. Publishing it uploads those same files to PyPI, then builds and attaches the Claude Desktop extension. +- **CI coverage:** + - Every declared Python (3.11-3.14) on Windows, macOS and Linux, plus ARM64 Linux, Windows on ARM and an Intel Mac. + - The macOS app on both architectures. + - The built wheel installed with pipx and driven over MCP on each system. + - The suite at the lowest dependency versions the bounds allow. + - A weekly wheel-coverage check. +- **Failure reports:** a red job states its failing tests on the run page. +- **One-time store submissions** that need an account: listed in `docs/PLUGINS.md`. diff --git a/gemini-extension.json b/gemini-extension.json index c8d88c4..e15ca56 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "compartment", - "version": "4.12.0", + "version": "4.13.0", "description": "Encrypted, fully offline long-term memory for Gemini CLI. One vault on your machine, shared by every MCP client. Requires `pip install compartment && compartment init` first.", "contextFileName": "GEMINI.md", "mcpServers": { diff --git a/integrations/hermes/compartment/plugin.yaml b/integrations/hermes/compartment/plugin.yaml index 8e338f1..3874ec7 100644 --- a/integrations/hermes/compartment/plugin.yaml +++ b/integrations/hermes/compartment/plugin.yaml @@ -1,5 +1,5 @@ name: compartment -version: 4.12.0 +version: 4.13.0 description: "Compartment - high-security, fully offline, encrypted vector memory. AEAD-encrypted vault, locked-by-default after restart, native Hermes provider, no API key." # The engine. `hermes plugins install compartment` (the plugin catalog) and # `hermes memory setup` install it into the Hermes environment from this diff --git a/integrations/hermes/upstream/plugin.yaml b/integrations/hermes/upstream/plugin.yaml index 3633e8e..54e1e44 100644 --- a/integrations/hermes/upstream/plugin.yaml +++ b/integrations/hermes/upstream/plugin.yaml @@ -3,10 +3,10 @@ # `hermes memory setup` auto-installs compartment when the user selects # Compartment in the picker. Use THIS file in the upstream PR. name: compartment -version: 1.7.0 +version: 4.13.0 description: "Compartment - high-security, fully offline, encrypted vector memory. AEAD-encrypted vault, locked-by-default after restart, native Hermes provider, no API key." pip_dependencies: - - "compartment>=2.3" + - "compartment>=4.9.3" requires_env: [] hooks: - on_session_switch diff --git a/packaging/continue/compartment.yaml b/packaging/continue/compartment.yaml index 5d62fd9..ccc39de 100644 --- a/packaging/continue/compartment.yaml +++ b/packaging/continue/compartment.yaml @@ -11,7 +11,7 @@ # version moves with the package (tests/test_version_lockstep.py), so the # hub shows the release the block was published from. name: Compartment -version: 4.12.0 +version: 4.13.0 schema: v1 mcpServers: - name: Compartment diff --git a/packaging/mcpb/manifest.json b/packaging/mcpb/manifest.json index 33d3364..9c8e177 100644 --- a/packaging/mcpb/manifest.json +++ b/packaging/mcpb/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "0.4", "name": "compartment", "display_name": "Compartment", - "version": "4.12.0", + "version": "4.13.0", "description": "Encrypted, fully offline memory for Claude: one vault on your machine, shared with every MCP client. No network, no API key.", "long_description": "Compartment is persistent memory for AI agents, stored on your own computer and encrypted at rest. What Claude learns in one conversation is available in every later one, and to Claude Code, Cursor, Codex, Gemini CLI, Hermes Agent, OpenClaw and any other MCP client on the same machine. Nothing leaves it: the embedding model runs locally, the server speaks stdio only and opens no ports, and there is no account.\n\n**Before the first run**, create the vault once in a terminal: `pip install compartment && compartment init` (or `pipx install compartment`, or `uv tool install compartment`). It asks for a passphrase you choose; that passphrase is the only key and Compartment cannot recover it. If the vault is locked after a restart, run `compartment unlock`.\n\nIf you already connected Claude Desktop with `compartment integrate claude` or the app's Connect button, you do not need this extension as well: both would start the same server and Claude would see every tool twice.", "author": { diff --git a/packaging/mcpb/pyproject.toml b/packaging/mcpb/pyproject.toml index 82c4f94..4e241e8 100644 --- a/packaging/mcpb/pyproject.toml +++ b/packaging/mcpb/pyproject.toml @@ -15,7 +15,7 @@ [project] name = "compartment-claude-desktop" -version = "4.12.0" +version = "4.13.0" description = "Compartment as a Claude Desktop extension (MCP Bundle)" # uv runs the newest interpreter it can find or download unless told # otherwise, and the compiled dependencies trail new Python releases: @@ -25,7 +25,7 @@ description = "Compartment as a Claude Desktop extension (MCP Bundle)" # here makes a host that overrides it fail with a clear message instead of # a missing-wheel error. Raise both when onnxruntime ships wheels for more. requires-python = ">=3.11,<3.15" -dependencies = ["compartment==4.12.0"] +dependencies = ["compartment==4.13.0"] [tool.uv] package = false diff --git a/plugin.json b/plugin.json index e877fd5..183385d 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "compartment", - "version": "4.12.0", + "version": "4.13.0", "description": "High-security, fully offline, encrypted vector memory for AI agents. AEAD-encrypted vault, locked by default after restart, no API key and no network at runtime.", "author": { "name": "Max Freedom Pollard", diff --git a/pyproject.toml b/pyproject.toml index 690d8ac..9efb9de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "compartment" -version = "4.12.0" +version = "4.13.0" description = "Compartment - high-security, fully offline, encrypted vector memory for AI agents (MCP)" readme = "README.md" license = { text = "Apache-2.0" } diff --git a/server.json b/server.json index aaa6143..95a0ad8 100644 --- a/server.json +++ b/server.json @@ -18,13 +18,13 @@ "source": "github", "id": "1306651494" }, - "version": "4.12.0", + "version": "4.13.0", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "compartment", - "version": "4.12.0", + "version": "4.13.0", "runtimeHint": "uvx", "transport": { "type": "stdio" diff --git a/src/compartment/__init__.py b/src/compartment/__init__.py index 6b639b0..ef89784 100644 --- a/src/compartment/__init__.py +++ b/src/compartment/__init__.py @@ -1,6 +1,6 @@ """Compartment - high-security, fully offline, encrypted vector memory for AI agents.""" -__version__ = "4.12.0" +__version__ = "4.13.0" from . import offline_guard as _og diff --git a/src/compartment/data/hermes-plugin/plugin.yaml b/src/compartment/data/hermes-plugin/plugin.yaml index 8e338f1..3874ec7 100644 --- a/src/compartment/data/hermes-plugin/plugin.yaml +++ b/src/compartment/data/hermes-plugin/plugin.yaml @@ -1,5 +1,5 @@ name: compartment -version: 4.12.0 +version: 4.13.0 description: "Compartment - high-security, fully offline, encrypted vector memory. AEAD-encrypted vault, locked-by-default after restart, native Hermes provider, no API key." # The engine. `hermes plugins install compartment` (the plugin catalog) and # `hermes memory setup` install it into the Hermes environment from this diff --git a/tests/test_version_lockstep.py b/tests/test_version_lockstep.py index c727e29..f3a71e9 100644 --- a/tests/test_version_lockstep.py +++ b/tests/test_version_lockstep.py @@ -130,6 +130,7 @@ def test_the_continue_block_carries_this_release(): "packaging/mcpb/manifest.json", "packaging/mcpb/pyproject.toml", "packaging/continue/compartment.yaml", + "integrations/hermes/upstream/plugin.yaml", } #: Files whose version field is their own, not the package's. @@ -146,12 +147,7 @@ def test_the_continue_block_carries_this_release(): #: Known drift in a file this test does not own the fix for. Strict, so the #: day it is fixed the marker has to go and the file joins GATED. -KNOWN_STALE = { - # The plugin.yaml prepared for a PR to hermes-agent still says 1.7.0 and - # compartment>=2.3. It is not shipped by anything in this repository, - # which is why nothing caught it. - "integrations/hermes/upstream/plugin.yaml", -} +KNOWN_STALE: set[str] = set() _SKIP_DIRS = {".git", "tests", "models", "build", "dist", ".venv", "venv", "node_modules", "__pycache__"} @@ -225,3 +221,14 @@ def test_the_shipped_plugin_is_the_one_in_integrations(): something no one has read.""" a, b = (p.read_text(encoding="utf-8") for p in PLUGIN_YAMLS) assert a == b + + +def test_the_upstream_hermes_plugin_yaml_matches_the_package(): + """The copy prepared for a PR to hermes-agent sat at 1.7.0 and + compartment>=2.3 while the package shipped 4.12.0, because nothing read + it. It names this release and the same engine floor as the shipped one.""" + up = ROOT / "integrations" / "hermes" / "upstream" / "plugin.yaml" + assert _yaml_version(up) == _version() + shipped = PLUGIN_YAMLS[0].read_text(encoding="utf-8") + floor = re.search(r'"compartment>=([0-9.]+)"', shipped).group(1) + assert f'"compartment>={floor}"' in up.read_text(encoding="utf-8") From 7314e3df7aa1697c21843e920a01cdaa831568e4 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:48:37 -0700 Subject: [PATCH 129/135] Tests: the mistyped-precision index test accepts HNSW's approximate top-1, which on some CI CPUs is the runner-up Co-Authored-By: Claude Opus 5.5 --- tests/test_vindex.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_vindex.py b/tests/test_vindex.py index 6e3630f..202913d 100644 --- a/tests/test_vindex.py +++ b/tests/test_vindex.py @@ -181,8 +181,12 @@ def test_a_mistyped_precision_setting_still_builds_an_index(capsys): for setting in ("int-8", 8, None, "b1", ["f32"]): idx = build_index(16, list(range(1, 51)), mat, precision=setting, force="hnsw") - assert idx.kind == "hnsw/usearch (f32)" - assert idx.search(mat[7], 1)[0][0] == 8 + assert idx.kind == "hnsw/usearch (f32)" and len(idx) == 50 + # HNSW is approximate: on some CI CPUs a 50-node graph answered + # vector 8 with its runner-up, 20 (cosine 0.58). The point here is + # that the index was built and searches, so the query has to be + # among the closest few, not necessarily first. + assert 8 in [k for k, _ in idx.search(mat[7], 3)] assert "index_precision" in capsys.readouterr().err for spelled, kind in (("INT8", "i8"), (" i8", "i8"), ("float16", "f16")): assert build_index(16, list(range(1, 51)), mat, precision=spelled, From 356c3987ac7b0edb6c32f9180cc7ee123e4d5403 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:49:12 -0700 Subject: [PATCH 130/135] CI keeps one run per branch: the newest push or pull-request run cancels the ones it supersedes Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bc8aff..35faa74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,15 @@ on: - cron: "17 6 * * 1" workflow_dispatch: +# One run per branch at a time. A push to a pull request's branch fires both +# `push` and `pull_request`, and every new commit used to leave the previous +# runs going too, so the macOS jobs - the slowest runners to come free - +# queued behind runs nobody would read. Keyed on the branch for both events, +# the newest run cancels the rest; the weekly run has its own group. +concurrency: + group: ci-${{ github.event_name == 'schedule' && 'weekly' || (github.head_ref || github.ref_name) }} + cancel-in-progress: true + jobs: test-offline: # The full suite - including init, seed-pack install, selftest, store, From 4555dd3f487e7dde64703f5656e70c086485e658 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:01:28 -0700 Subject: [PATCH 131/135] Tests and the Windows tray job match what Windows really does The tray job counted processes named "compartment", but init now starts the tray with pythonw.exe so no console window opens; it counts by command line instead. The tilde test searched JSON for a raw Windows path whose backslashes JSON escapes, and now compares parsed paths; the Hermes test built ~/.hermes where native Windows Hermes lives under %LOCALAPPDATA%, and now asks hermes_home() under a test LOCALAPPDATA; the unreadable-vault test skips on Windows, where chmod cannot take read access away. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 17 +++++++++++++---- tests/test_cli_errors.py | 2 ++ tests/test_cli_home.py | 9 +++++++-- tests/test_connect_environment.py | 5 ++++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35faa74..8a2fd77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -338,16 +338,25 @@ jobs: # `compartment init` above already started the tray app, which is the # whole point of it. So a launch now must hand off and exit 0, and # the copy that was already there must still be running afterwards. + # Counted by command line, not by process name: init starts the tray + # with pythonw.exe (no console window), so there is no process called + # "compartment" to count. shell: pwsh run: | - $before = @(Get-Process compartment -ErrorAction SilentlyContinue).Count + function Get-Tray { + @(Get-CimInstance Win32_Process | Where-Object { + $_.CommandLine -and $_.CommandLine -match 'compartment' -and + $_.CommandLine -match '\btray\b' }) + } + $before = (Get-Tray).Count + Get-Tray | ForEach-Object { Write-Host " running: $($_.Name): $($_.CommandLine)" } if ($before -lt 1) { Write-Error "init did not start the tray app"; exit 1 } compartment tray --show if ($LASTEXITCODE -ne 0) { Write-Error "the second copy failed instead of standing down"; exit 1 } - $after = @(Get-Process compartment -ErrorAction SilentlyContinue).Count + $after = (Get-Tray).Count if ($after -ne $before) { Write-Error "second copy changed the count: $before -> $after"; exit 1 } - Write-Host "one copy before, one copy after" - Get-Process compartment -ErrorAction SilentlyContinue | Stop-Process -Force + Write-Host "the same $before process(es) before and after" + Get-Tray | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } Start-Sleep -Seconds 2 - name: Start the tray app and prove it is still alive diff --git a/tests/test_cli_errors.py b/tests/test_cli_errors.py index 0108f26..e510772 100644 --- a/tests/test_cli_errors.py +++ b/tests/test_cli_errors.py @@ -89,6 +89,8 @@ def test_a_vault_found_but_unreadable_names_the_file(vault, vault_path, capsys, monkeypatch): if getattr(os, "geteuid", lambda: 1)() == 0: pytest.skip("root reads any file") + if os.name == "nt": + pytest.skip("chmod cannot take read access away on Windows") vault.lock() os.chmod(vault_path, 0) try: diff --git a/tests/test_cli_home.py b/tests/test_cli_home.py index e4e714a..c24042a 100644 --- a/tests/test_cli_home.py +++ b/tests/test_cli_home.py @@ -1,5 +1,6 @@ """Where the CLI finds the vault: `~` expanded wherever a path comes from, and no home directory needed when the vault is named.""" +import json import os import subprocess import sys @@ -43,8 +44,12 @@ def test_a_vault_path_starting_with_a_tilde_is_the_home_one( else: monkeypatch.setenv(how, "~/v.vault") cli.main(argv) - out = capsys.readouterr().out - assert str(home_dir / "v.vault") in out and '"locked": false' in out + status = json.loads(capsys.readouterr().out) + # Compared as paths: JSON escapes a Windows path's backslashes, and + # expanduser there joins with "/" after the profile folder. + assert os.path.normcase(os.path.normpath(status["vault"])) == \ + os.path.normcase(os.path.normpath(str(home_dir / "v.vault"))) + assert status["locked"] is False def _cli_with_no_home(*argv): diff --git a/tests/test_connect_environment.py b/tests/test_connect_environment.py index 3a66d22..6abbb72 100644 --- a/tests/test_connect_environment.py +++ b/tests/test_connect_environment.py @@ -202,6 +202,9 @@ def test_without_the_path_fix_it_would_have_written_a_bare_name(monkeypatch, @pytest.fixture() def fake_home(tmp_path, monkeypatch): monkeypatch.setattr(menubar.Path, "home", classmethod(lambda cls: tmp_path)) + # Hermes on native Windows lives under %LOCALAPPDATA%, which must not be + # the runner's real one. + monkeypatch.setenv("LOCALAPPDATA", str(tmp_path / "AppData" / "Local")) monkeypatch.delenv("HERMES_HOME", raising=False) monkeypatch.delenv("OPENCLAW_HOME", raising=False) monkeypatch.setattr(menubar, "integration_status", @@ -226,7 +229,7 @@ def test_hermes_counts_as_connected_once_the_plugin_is_there(fake_home, monkeypatch): from compartment import claude_hooks monkeypatch.setattr(claude_hooks, "is_installed", lambda *a, **k: False) - plug = fake_home / ".hermes" / "plugins" / "compartment" + plug = menubar.hermes_home() / "plugins" / "compartment" # per platform plug.mkdir(parents=True) (plug / "plugin.yaml").write_text("name: compartment\n") st = menubar.integration_status("/v") From 502a1893019780ad6f7231a9fbbf5ad8c4e7d79d Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:36:09 -0700 Subject: [PATCH 132/135] CI pins setup-uv to an exact release, and the mistyped-precision test no longer asks an approximate index for an exact answer setup-uv publishes no floating major tags, so `@v10` failed both macOS app builds at "Set up job"; it is pinned to v10.3.0. The index test asked a 50-vector HNSW graph built in parallel for each row's own vector; such a graph leaves a few rows unreachable (measured: 1-4 of 50, and 0.3% of self-lookups at 25,000, in line with HNSW's documented ~99% recall), so it now checks what it is about: the index is built and searches. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- tests/test_vindex.py | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a2fd77..381c389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v10 + - uses: astral-sh/setup-uv@v10.3.0 # exact: no floating major tags are published - name: A standalone, non-framework Python to embed shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e96804..32a614b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,7 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ needs.check.outputs.tag }} - - uses: astral-sh/setup-uv@v10 + - uses: astral-sh/setup-uv@v10.3.0 # exact: no floating major tags are published - name: A standalone, non-framework Python to embed # tools/build_macos_app.py explains why it must not be a framework # build: the app would then report itself as org.python.python. diff --git a/tests/test_vindex.py b/tests/test_vindex.py index 202913d..32bd3df 100644 --- a/tests/test_vindex.py +++ b/tests/test_vindex.py @@ -182,11 +182,7 @@ def test_a_mistyped_precision_setting_still_builds_an_index(capsys): idx = build_index(16, list(range(1, 51)), mat, precision=setting, force="hnsw") assert idx.kind == "hnsw/usearch (f32)" and len(idx) == 50 - # HNSW is approximate: on some CI CPUs a 50-node graph answered - # vector 8 with its runner-up, 20 (cosine 0.58). The point here is - # that the index was built and searches, so the query has to be - # among the closest few, not necessarily first. - assert 8 in [k for k, _ in idx.search(mat[7], 3)] + assert len(idx.search(mat[7], 3)) == 3 # built, and it searches assert "index_precision" in capsys.readouterr().err for spelled, kind in (("INT8", "i8"), (" i8", "i8"), ("float16", "f16")): assert build_index(16, list(range(1, 51)), mat, precision=spelled, @@ -243,3 +239,4 @@ class _Simd(usearch_index.Index): idx = vindex.build_index(16, list(range(1, 41)), mat) assert idx.kind.startswith("hnsw") assert idx.search(mat[3], 1)[0][0] == 4 + From f37718dcf37a5cfadac08b04a4a90c27a51a51b9 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:11:05 -0700 Subject: [PATCH 133/135] CI fails a hung test with its stack after five minutes, and no test job can hold a runner past 45 Every macOS job sat in the test step for hours: one test hangs on a real Mac (not in the Linux or Windows jobs), and each stuck job also held one of the few macOS runners, so the Intel app build behind them never started. pytest-timeout (now in the dev extras) fails the test that hangs, with where it hung, and the job timeout caps whatever is left. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 11 ++++++++--- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 381c389..f661202 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,11 @@ jobs: - os: macos-15-intel python: "3.13" runs-on: ${{ matrix.os }} + # The suite takes minutes. A test that hangs (a pipe never closed, a + # child never reaped) used to hold a runner for six hours - and on macOS, + # where runners are scarce, every other macOS job behind it. pytest's + # --timeout below fails the one test with its stack; this caps the rest. + timeout-minutes: 45 steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 @@ -58,12 +63,12 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: python tools/ci_report.py tests/ -q --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/ -q --timeout=300 --cov=compartment --cov-report=xml --cov-append - name: Dashboard tests (loopback socket; offline guard intentionally off) # `compartment dash` binds 127.0.0.1 by design and is never run under the # offline guard, so its tests run here with the guard off. - run: python tools/ci_report.py tests/test_dash.py -q --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/test_dash.py -q --timeout=300 --cov=compartment --cov-report=xml --cov-append - name: Coverage to Codecov # Never fail the build on a reporting service: the offline guard and the @@ -202,7 +207,7 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: python tools/ci_report.py tests/ -q --ignore=tests/test_dash.py + run: python tools/ci_report.py tests/ -q --timeout=300 --ignore=tests/test_dash.py linux-panel: # Linux has no notification area worth relying on, so it draws the same diff --git a/pyproject.toml b/pyproject.toml index 9efb9de..0fb848a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,7 +111,7 @@ hnsw = ["usearch>=2.9"] # failing on an unknown extra. They add nothing. menubar = [] tray = [] -dev = ["pytest>=8", "pytest-cov>=5", "usearch>=2.9"] +dev = ["pytest>=8", "pytest-cov>=5", "pytest-timeout>=2.3", "usearch>=2.9"] [tool.setuptools.packages.find] where = ["src"] From a06ee3a8e16630a0c51a08c723389b73119251f7 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:18:35 -0700 Subject: [PATCH 134/135] A stalled test reports itself while the job is still alive, and the daemon and stdio tests run on their own on every system The macOS suite ran into its job timeout with no verdict: a test hangs on a real Mac, and the report only came at exit. tools/ci_report.py now watches for silence and annotates the running test and the last lines (with every thread's stack, via faulthandler_timeout) while the job is alive. A new real-sockets job runs the embedding-daemon and stdio-server tests - the ones agent sandboxes cannot run - verbose, on macOS, Linux and Windows. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 29 ++++++++++++-- tools/ci_report.py | 82 +++++++++++++++++++++++++++++++--------- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f661202..0024e0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,12 +63,12 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: python tools/ci_report.py tests/ -q --timeout=300 --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/ -q --timeout=300 -o faulthandler_timeout=120 --cov=compartment --cov-report=xml --cov-append - name: Dashboard tests (loopback socket; offline guard intentionally off) # `compartment dash` binds 127.0.0.1 by design and is never run under the # offline guard, so its tests run here with the guard off. - run: python tools/ci_report.py tests/test_dash.py -q --timeout=300 --cov=compartment --cov-report=xml --cov-append + run: python tools/ci_report.py tests/test_dash.py -q --timeout=300 -o faulthandler_timeout=120 --cov=compartment --cov-report=xml --cov-append - name: Coverage to Codecov # Never fail the build on a reporting service: the offline guard and the @@ -81,6 +81,29 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false + real-sockets: + # The shared embedding daemon and the stdio server need real Unix sockets + # and pipes, which agent sandboxes refuse, so these are the tests that + # only ever run on a CI runner. They run here on their own, verbose, with + # every thread's stack dumped for a test that stalls, so a hang names + # itself within minutes instead of holding a runner until its timeout. + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: pip install -e .[dev] + - name: Daemon and stdio tests, verbose + env: + CI_REPORT_STALL_S: "75" + run: python tools/ci_report.py tests/test_embed_daemon.py tests/test_server_stdio.py -v --timeout=120 -o faulthandler_timeout=60 + wheel-smoke: # The suite runs against an editable checkout, which hides packaging # mistakes: a model file or the starter pack missing from the wheel, an @@ -207,7 +230,7 @@ jobs: - name: Test suite with offline guard active env: COMPARTMENT_ASSERT_OFFLINE: "1" - run: python tools/ci_report.py tests/ -q --timeout=300 --ignore=tests/test_dash.py + run: python tools/ci_report.py tests/ -q --timeout=300 -o faulthandler_timeout=120 --ignore=tests/test_dash.py linux-panel: # Linux has no notification area worth relying on, so it draws the same diff --git a/tools/ci_report.py b/tools/ci_report.py index 369d82e..4464a86 100644 --- a/tools/ci_report.py +++ b/tools/ci_report.py @@ -1,22 +1,36 @@ -"""Run pytest and, when it fails, surface the reason where it can be read. +"""Run pytest and, when it fails or stalls, surface why where it can be read. python tools/ci_report.py [pytest args...] A red CI job's log is only readable by someone signed in to GitHub, and pytest's verdict (which test, which assertion, which import failed during collection) sits at the end of thousands of lines. This runs pytest with the -given arguments, streams its output unchanged, and on failure also emits the -short test summary and the last lines as GitHub Actions error annotations: -they appear on the run's summary page, inline on a pull request, and in the -check-run annotations API. Outside GitHub Actions it is plain pytest. +given arguments, streams its output unchanged, and under GitHub Actions also +emits error annotations - they appear on the run's summary page, inline on a +pull request, and in the check-run annotations API: + +* when pytest fails: the short test summary and the last lines; +* when pytest falls silent for STALL_S seconds (a hung test): the last lines + as they stand, the unfinished line included - with `-v` that line names + the test that is running, and with `-o faulthandler_timeout=N` the stacks + of every thread are already in it. + +A job that a hung test makes run into its timeout used to end with no +verdict at all; the stall annotation is written while the job is alive. +Outside GitHub Actions it is plain pytest. """ from __future__ import annotations import os import subprocess import sys +import threading +import time TAIL_LINES = 60 +STALL_S = int(os.environ.get("CI_REPORT_STALL_S", "150")) +MAX_STALL_REPORTS = 4 +ON_ACTIONS = os.environ.get("GITHUB_ACTIONS") == "true" def _escape(text: str) -> str: @@ -25,24 +39,58 @@ def _escape(text: str) -> str: return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") +def _annotate(title: str, text: str) -> None: + sys.stdout.write(f"::error title={title}::{_escape(text)}\n") + sys.stdout.flush() + + def main(argv: list[str]) -> int: proc = subprocess.Popen([sys.executable, "-m", "pytest", "-rfE", *argv], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - lines: list[str] = [] assert proc.stdout is not None - for raw in proc.stdout: - line = raw.decode("utf-8", "replace") - sys.stdout.write(line) - lines.append(line.rstrip("\n")) - rc = proc.wait() - if rc != 0 and os.environ.get("GITHUB_ACTIONS") == "true": + buf: list[str] = [] # complete lines so far + partial = [""] # the line being written now + last = [time.monotonic()] + lock = threading.Lock() + + def reader() -> None: + fd = proc.stdout.fileno() + while True: + chunk = os.read(fd, 4096) + if not chunk: + return + text = chunk.decode("utf-8", "replace") + sys.stdout.write(text) + sys.stdout.flush() + with lock: + last[0] = time.monotonic() + pieces = (partial[0] + text).split("\n") + partial[0] = pieces.pop() + buf.extend(pieces) + del buf[:-2000] + + t = threading.Thread(target=reader, daemon=True) + t.start() + reports = 0 + while proc.poll() is None: + time.sleep(5) + with lock: + idle = time.monotonic() - last[0] + tail = buf[-TAIL_LINES:] + ([partial[0]] if partial[0] else []) + if ON_ACTIONS and idle >= STALL_S and reports < MAX_STALL_REPORTS: + reports += 1 + _annotate(f"pytest silent for {int(idle)}s (still running)", + "\n".join(tail)) + with lock: + last[0] = time.monotonic() # next report after another stall + t.join(timeout=10) + rc = proc.returncode + lines = buf + ([partial[0]] if partial[0] else []) + if rc != 0 and ON_ACTIONS: summary = [ln for ln in lines if ln.startswith(("FAILED ", "ERROR "))] if summary: - print("::error title=pytest failures::" - + _escape("\n".join(summary[:50]))) - print(f"::error title=pytest exited {rc} (last lines)::" - + _escape("\n".join(lines[-TAIL_LINES:]))) - sys.stdout.flush() + _annotate("pytest failures", "\n".join(summary[:50])) + _annotate(f"pytest exited {rc} (last lines)", "\n".join(lines[-TAIL_LINES:])) return rc From 9815d5557f321d20cd2f6f2c99a2d10bf09a5d56 Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:08:33 -0700 Subject: [PATCH 135/135] The installer tests can no longer reach the real Mac's Python: the macOS script's well-known interpreter paths are replaceable, and the suite empties them install-macos.sh tries Homebrew's and python.org's absolute paths after PATH. The tests build a stand-in machine through PATH alone, so on a real Mac (a GitHub macOS runner) the "only Python 3.14 on an Intel Mac" case skipped the stand-in, found the runner's real Python 3.13, ran a real install and then waited for a passphrase until the job timed out - and on a developer's Mac it would have installed into their Homebrew Python. CMPT_EXTRA_PYTHONS now replaces the list when set (users get the same default), and the suite sets it empty. Co-Authored-By: Claude Opus 5.5 --- install/install-macos.sh | 14 ++++++++------ tests/test_platform_installers.py | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/install/install-macos.sh b/install/install-macos.sh index 9c0fa63..5d37c5a 100755 --- a/install/install-macos.sh +++ b/install/install-macos.sh @@ -72,15 +72,17 @@ PY="" NEWEST="" PYARCH="" TOO_OLD="" +# Where Homebrew and the python.org installer put interpreters that are not +# on PATH. CMPT_EXTRA_PYTHONS replaces the list when it is set, even to +# nothing: the test suite sets it empty, so the stand-in machine it builds +# can never reach the real interpreters of the Mac running the tests (which +# then installed into one of them and sat waiting for a passphrase). +EXTRA_PYTHONS="${CMPT_EXTRA_PYTHONS-/opt/homebrew/bin/python3.13 /opt/homebrew/bin/python3.12 /opt/homebrew/bin/python3.11 /usr/local/bin/python3.13 /usr/local/bin/python3.12 /usr/local/bin/python3.11 /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 /Library/Frameworks/Python.framework/Versions/3.12/bin/python3 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3}" if [ -n "${PYTHON:-}" ]; then set -- "$PYTHON" else - set -- python3 python3.13 python3.12 python3.11 python3.14 \ - /opt/homebrew/bin/python3.13 /opt/homebrew/bin/python3.12 /opt/homebrew/bin/python3.11 \ - /usr/local/bin/python3.13 /usr/local/bin/python3.12 /usr/local/bin/python3.11 \ - /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 \ - /Library/Frameworks/Python.framework/Versions/3.12/bin/python3 \ - /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 + # shellcheck disable=SC2086 # one word per path, none has a space + set -- python3 python3.13 python3.12 python3.11 python3.14 $EXTRA_PYTHONS fi for cand in "$@"; do usable_name "$cand" || continue diff --git a/tests/test_platform_installers.py b/tests/test_platform_installers.py index 3209114..57b9b56 100644 --- a/tests/test_platform_installers.py +++ b/tests/test_platform_installers.py @@ -203,7 +203,11 @@ def run(script, pythons=("python3",), facts=None, tools=(), env=None, "STUB_COMPARTMENT": str(comp), "STUB_USER_BIN": str(home / ".local" / "bin"), "STUB_VENV_BIN": str(root / "active-venv" / "bin"), - "STUB_TOOL_BIN": str(home / ".local" / "bin")} + "STUB_TOOL_BIN": str(home / ".local" / "bin"), + # The macOS script also tries Homebrew's and python.org's + # absolute paths; on a real Mac those are this machine's own + # interpreters, which a test must never install into. + "CMPT_EXTRA_PYTHONS": ""} e.update(env or {}) master = slave = None if stdin_script: @@ -506,3 +510,14 @@ def test_the_windows_installer_picks_a_python_inside_the_wheel_matrix( assert (code == 0) is usable, out if not usable: assert out + + +@needs_bash +def test_the_macos_script_tries_the_well_known_paths_unless_told_otherwise(): + """The absolute interpreter paths are on by default for users and + replaceable, so the suite's stand-in Mac never reaches a real one: on a + GitHub macOS runner it found the real Python 3.13, ran a real install + and then waited for a passphrase until the job timed out.""" + script = MACOS.read_text(encoding="utf-8") + assert "${CMPT_EXTRA_PYTHONS-/opt/homebrew/bin/python3.13" in script + assert "$EXTRA_PYTHONS" in script