refactor(pruner): table-driven LRU engine with per-slot budget (13 stores)

tests/lib.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Shared helpers for tests/pruner-*.sh. Sourced, not run. Needs GNU coreutils,
+# find, flock (Linux, or `docker run --rm -v "$PWD:/w" -w /w ubuntu:24.04 bash tests/<t>.sh`).
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+
+fails=0
+fail() { echo "FAIL: $*"; fails=$((fails + 1)); }
+
+# shellcheck source=pruner.sh
+source "$here/../pruner.sh"
+RUNNER_DATA_DIR="$work/data"
+ACTION_CACHE_DIR="$work/no-action-cache"
+CACHE_BUDGET_PCT=100 # the byte budget decides, whatever the test fs is
+
+# mkslot <id>: create slot <id> and print its cache dir.
+mkslot() {
+  mkdir -p "$RUNNER_DATA_DIR/slots/$1/cache" "$RUNNER_DATA_DIR/slots/$1/work"
+  touch "$RUNNER_DATA_DIR/slots/$1/.lock"
+  echo "$RUNNER_DATA_DIR/slots/$1/cache"
+}
+
+sz() { du -sb "$1" | awk '{printf "%.0f\n", $1}'; }
+
+# stores_bytes <cache dir>: what the pruner counts for a slot.
+stores_bytes() {
+  local d total=0
+  for d in "$1"/*/; do total=$((total + $(sz "$d"))); done
+  echo "$total"
+}
+
+# mkfile <path> <bytes> <atime age in days>
+mkfile() {
+  mkdir -p "$(dirname "$1")"
+  head -c "$2" /dev/urandom >"$1"
+  touch -a -d "$3 days ago" "$1"
+}
+
+# mkdirfile <dir> <bytes> <age in days>: a unit with one file of that age.
+mkunit() {
+  mkfile "$1/data.bin" "$2" "$3"
+}
+
+# budget_without <cache dir> <unit>...: a budget that makes exactly <unit>s
+# (the oldest ones) go and nothing more: now minus them, plus slack that is far
+# below any unit's size.
+budget_without() {
+  local cache="$1" total u
+  shift
+  total=$(stores_bytes "$cache")
+  for u in "$@"; do total=$((total - $(sz "$u"))); done
+  echo $((total + 20000))
+}
+
+gone() { [ ! -e "$1" ] && [ ! -L "$1" ] || fail "$1 should be evicted"; }
+kept() { [ -e "$1" ] || [ -L "$1" ] || fail "$1 should survive"; }
+
+finish() {
+  if [ "$fails" -gt 0 ]; then
+    echo "$fails check(s) failed"
+    exit 1
+  fi
+  echo "PASS: $1"
+}

tests/pruner-budget.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Budget: min(CACHE_BUDGET_BYTES, CACHE_BUDGET_PCT% of the filesystem) split
+# evenly over the slots, each slot ranked on its own (not one LRU across slots).
+# Needs GNU tools:
+#   bash tests/pruner-budget.sh
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
+
+# slot_budget arithmetic.
+mkdir -p "$RUNNER_DATA_DIR"
+CACHE_BUDGET_BYTES=1000 CACHE_BUDGET_PCT=100
+[ "$(slot_budget 2)" = 500 ] || fail "1000B over 2 slots: $(slot_budget 2)"
+fs=$(df -B1 --output=size "$RUNNER_DATA_DIR" | awk 'NR == 2 {print $1}')
+CACHE_BUDGET_BYTES=$((fs * 2)) CACHE_BUDGET_PCT=10
+[ "$(slot_budget 4)" = $((fs * 10 / 100 / 4)) ] || fail "10% of the fs over 4 slots: $(slot_budget 4)"
+
+# Two slots, three 100000 B units each. Slot 2's units are all newer than
+# slot 1's: a global LRU would empty slot 1 first; per slot, each loses just
+# its own oldest unit.
+for id in 1 2; do
+  c="$(mkslot "$id")"
+  if [ "$id" = 1 ]; then ages=(90 80 70); else ages=(30 20 10); fi
+  for i in 0 1 2; do mkunit "$c/mise-data/installs/tool/v$i" 100000 "${ages[$i]}"; done
+done
+per_slot=$(budget_without "$RUNNER_DATA_DIR/slots/1/cache" "$RUNNER_DATA_DIR/slots/1/cache/mise-data/installs/tool/v0")
+CACHE_BUDGET_BYTES=$((per_slot * 2)) CACHE_BUDGET_PCT=100
+out=$(prune_caches)
+for id in 1 2; do
+  u="$RUNNER_DATA_DIR/slots/$id/cache/mise-data/installs/tool"
+  gone "$u/v0"
+  kept "$u/v1/data.bin"
+  kept "$u/v2/data.bin"
+  grep -q "slot $id: budget ${per_slot}B" <<<"$out" || fail "slot $id budget not ${per_slot}B"
+done
+
+finish "budget split evenly across slots"

tests/pruner-bun.sh

@@ -1,21 +1,17 @@
 #!/bin/bash
 # pruner.sh must evict whole bun packages, never single files: bun reuses a
-# cache folder as complete whenever it exists. Builds a fake /root/.bun volume
-# (packages holding 0-byte files with old atimes, a scoped package, the
-# <pkg>/<ver> symlinks, @t@ piles and bin/bun), runs prune_caches under a small
-# cap, then checks every surviving package is complete and every evicted one
-# is fully gone. Needs GNU find/stat/du (Linux, or `docker run ubuntu`):
+# cache folder as complete whenever it exists. Builds a fake /root/.bun store
+# in a slot (packages holding 0-byte files with old atimes, a scoped package,
+# the <pkg>/<ver> symlinks, @t@ piles and bin/bun), runs prune_caches under a
+# small budget, then checks every surviving package is complete and every
+# evicted one is fully gone. Needs GNU tools (Linux, or `docker run ubuntu`):
 #   bash tests/pruner-bun.sh
-set -euo pipefail
-
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-work="$(mktemp -d)"
-trap 'rm -rf "$work"' EXIT
-
-fails=0
-fail() { echo "FAIL: $*"; fails=$((fails + 1)); }
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
 
-bun="$work/caches/1/bun"
+cache0="$(mkslot 1)"
+bun="$cache0/bun"
 cache="$bun/install/cache"
 mkdir -p "$bun/bin" "$cache/@t@"
 head -c 300000 /dev/zero >"$bun/bin/bun"
@@ -56,14 +52,11 @@ touch -d '2 days ago' "$cache/.tmp/stale"
 head -c 10 /dev/zero >"$cache/half@1.0.0@@@1.tgz.part"
 touch -d '2 days ago' "$cache/half@1.0.0@@@1.tgz.part"
 
-# Cap = volume minus the three oldest packages (old-a, old-b, @scope/old-c),
-# plus half a package of slack: exactly those three must go, nothing else.
-sz() { du -sb "$1" | awk '{print $1}'; }
-cap=$(($(sz "$bun") - $(sz "$cache/old-a@1.0.0@@@1") - $(sz "$cache/old-b@2.0.0@@@1") \
-  - $(sz "$cache/@scope/old-c@3.0.0@@@1") - 2 * 4096 + 50000))
-# shellcheck source=pruner.sh
-source "$here/../pruner.sh"
-CACHES_ROOT="$work/caches" CACHE_CAP_BYTES="$cap" prune_caches
+# Budget = volume minus the three oldest packages (old-a, old-b, @scope/old-c):
+# exactly those three must go, nothing else.
+CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$cache/old-a@1.0.0@@@1" "$cache/old-b@2.0.0@@@1" \
+  "$cache/@scope/old-c@3.0.0@@@1")
+prune_caches >/dev/null
 
 for entry in 'old-a@1.0.0@@@1' 'old-b@2.0.0@@@1' '@scope/old-c@3.0.0@@@1'; do
   base="${entry##*/}"
@@ -93,11 +86,7 @@ done
 [ -e "$cache/.tmp/live" ] || fail "live .tmp entry removed"
 [ ! -e "$cache/half@1.0.0@@@1.tgz.part" ] || fail "stale .part kept"
 
-size=$(sz "$bun")
-[ "$size" -le "$cap" ] || fail "volume ${size}B still over the ${cap}B cap"
+size=$(stores_bytes "$cache0")
+[ "$size" -le "$CACHE_BUDGET_BYTES" ] || fail "slot ${size}B still over the ${CACHE_BUDGET_BYTES}B budget"
 
-if [ "$fails" -gt 0 ]; then
-  echo "$fails check(s) failed"
-  exit 1
-fi
-echo "PASS: bun cache evicted by whole package (volume ${size}B)"
+finish "bun cache evicted by whole package (slot ${size}B)"

tests/pruner-busy.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Job safety: a slot with a fresh `busy` marker is skipped, one with a stale
+# marker (> 6 h) is pruned, and a slot whose .prune.lock is held is skipped.
+# Needs GNU tools and flock:
+#   bash tests/pruner-busy.sh
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
+
+unit() { echo "$RUNNER_DATA_DIR/slots/$1/cache/mise-data/installs/node/20.0.0"; }
+for id in 1 2 3 4; do
+  mkslot "$id" >/dev/null
+  mkunit "$(unit "$id")" 100000 30
+done
+touch "$RUNNER_DATA_DIR/slots/1/busy"                  # job running
+touch -d '7 hours ago' "$RUNNER_DATA_DIR/slots/2/busy" # crashed job's leftover
+# slot 3: idle
+mkdir -p "$RUNNER_DATA_DIR/slots/c1" && touch "$RUNNER_DATA_DIR/slots/c1/.lock" # control slot, no cache/
+( flock 8; sleep 60 ) 8>"$RUNNER_DATA_DIR/slots/4/.prune.lock" & # another pruner
+holder=$!
+sleep 1
+
+CACHE_BUDGET_BYTES=1
+out=$(prune_caches)
+kill "$holder" 2>/dev/null || true
+wait "$holder" 2>/dev/null || true
+
+kept "$(unit 1)/data.bin"
+gone "$(unit 2)"
+gone "$(unit 3)"
+kept "$(unit 4)/data.bin"
+[ ! -e "$RUNNER_DATA_DIR/slots/c1/cache" ] || fail "control slot got a cache dir"
+grep -q 'slot 1: job running' <<<"$out" || fail "no log line for the busy slot"
+grep -q 'slot 2: stale busy' <<<"$out" || fail "no log line for the stale busy marker"
+grep -q 'slot 4: another prune' <<<"$out" || fail "no log line for the locked slot"
+grep -q 'slot 3: took [0-9]*s' <<<"$out" || fail "no elapsed log line"
+[ -e "$RUNNER_DATA_DIR/slots/1/busy" ] || fail "busy marker removed by the pruner"
+
+finish "busy slot skipped, stale busy pruned, locked slot skipped"

tests/pruner-cargo.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Cargo: .crate files, extracted crates and git checkouts/dbs are evicted;
+# bin/ (cargo install'ed tools) and the registry index never are. Needs GNU tools:
+#   bash tests/pruner-cargo.sh
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
+
+cache0="$(mkslot 1)"
+c="$cache0/cargo"
+
+mkfile "$c/bin/cargo-nextest" 200000 900 # oldest thing in the store
+mkfile "$c/registry/index/idx/.cache/ab/serde" 50000 900
+mkfile "$c/registry/cache/idx/old-1.0.0.crate" 100000 90
+mkfile "$c/registry/cache/idx/fresh-1.0.0.crate" 100000 1
+mkunit "$c/registry/src/idx/old-1.0.0" 100000 80
+mkunit "$c/registry/src/idx/fresh-1.0.0" 100000 1
+mkunit "$c/git/checkouts/oldrepo-aaa" 100000 70
+mkunit "$c/git/checkouts/freshrepo-bbb" 100000 1
+mkunit "$c/git/db/oldrepo-aaa" 100000 60
+mkunit "$c/git/db/freshrepo-bbb" 100000 1
+
+CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$c/registry/cache/idx/old-1.0.0.crate" "$c/registry/src/idx/old-1.0.0" \
+  "$c/git/checkouts/oldrepo-aaa" "$c/git/db/oldrepo-aaa")
+prune_caches >/dev/null
+
+gone "$c/registry/cache/idx/old-1.0.0.crate"
+gone "$c/registry/src/idx/old-1.0.0"
+gone "$c/git/checkouts/oldrepo-aaa"
+gone "$c/git/db/oldrepo-aaa"
+kept "$c/registry/cache/idx/fresh-1.0.0.crate"
+kept "$c/registry/src/idx/fresh-1.0.0/data.bin"
+kept "$c/git/checkouts/freshrepo-bbb/data.bin"
+kept "$c/git/db/freshrepo-bbb/data.bin"
+[ -s "$c/bin/cargo-nextest" ] || fail "bin/cargo-nextest removed"
+[ -s "$c/registry/index/idx/.cache/ab/serde" ] || fail "registry index removed"
+
+# Even with no budget at all, bin/ and the index stay.
+CACHE_BUDGET_BYTES=1
+prune_caches >/dev/null
+[ -s "$c/bin/cargo-nextest" ] || fail "bin/cargo-nextest removed under a tiny budget"
+[ -s "$c/registry/index/idx/.cache/ab/serde" ] || fail "registry index removed under a tiny budget"
+[ ! -e "$c/registry/cache/idx/fresh-1.0.0.crate" ] || fail "fresh crate should go under a tiny budget"
+
+finish "cargo evicted caches, bin/ protected"

tests/pruner-gradle.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Gradle: whole module versions / transform dirs / wrapper dists / jars dirs
+# are evicted, never single files inside them, and the metadata Gradle keeps
+# beside them (modules-2/metadata-*) is left alone. Needs GNU tools:
+#   bash tests/pruner-gradle.sh
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
+
+cache0="$(mkslot 1)"
+g="$cache0/gradle"
+fs="$g/caches/modules-2/files-2.1"
+
+mkunit "$fs/org.old/lib/1.0.0/h1" 100000 90      # oldest version of a module
+mkfile "$fs/org.old/lib/1.0.0/h1/lib-1.0.0.pom" 50000 400
+mkunit "$fs/org.old/lib/1.1.0/h2" 100000 2       # newer version of the same module
+mkunit "$fs/org.fresh/app/2.0.0/h3" 100000 1
+mkunit "$g/caches/transforms-4/oldhash" 100000 80
+mkunit "$g/caches/transforms-4/freshhash" 100000 1
+mkunit "$g/wrapper/dists/gradle-7.0-bin/abc" 100000 70
+mkunit "$g/wrapper/dists/gradle-8.5-bin/def" 100000 1
+mkunit "$g/caches/jars-9/oldjars" 100000 60
+mkfile "$g/caches/build-cache-1/oldentry" 100000 50
+mkfile "$g/caches/build-cache-1/freshentry" 100000 1
+mkdir -p "$g/caches/modules-2/metadata-2.107"
+echo meta >"$g/caches/modules-2/metadata-2.107/descriptors.bin"
+touch -a -d '500 days ago' "$g/caches/modules-2/metadata-2.107/descriptors.bin"
+
+CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$fs/org.old/lib/1.0.0" "$g/caches/transforms-4/oldhash" \
+  "$g/wrapper/dists/gradle-7.0-bin" "$g/caches/jars-9/oldjars" "$g/caches/build-cache-1/oldentry")
+prune_caches >/dev/null
+
+gone "$fs/org.old/lib/1.0.0"
+gone "$g/caches/transforms-4/oldhash"
+gone "$g/wrapper/dists/gradle-7.0-bin"
+gone "$g/caches/jars-9/oldjars"
+gone "$g/caches/build-cache-1/oldentry"
+kept "$fs/org.old/lib/1.1.0/h2/data.bin"
+kept "$fs/org.fresh/app/2.0.0/h3/data.bin"
+kept "$g/caches/transforms-4/freshhash/data.bin"
+kept "$g/wrapper/dists/gradle-8.5-bin/def/data.bin"
+kept "$g/caches/build-cache-1/freshentry"
+kept "$g/caches/modules-2/metadata-2.107/descriptors.bin"
+[ -d "$fs/org.old/lib" ] || fail "module dir removed while a version is left"
+[ -d "$fs" ] || fail "files-2.1 removed"
+[ -d "$g/caches/build-cache-1" ] || fail "build-cache-1 removed"
+
+size=$(stores_bytes "$cache0")
+[ "$size" -le "$CACHE_BUDGET_BYTES" ] || fail "slot ${size}B still over the ${CACHE_BUDGET_BYTES}B budget"
+
+finish "gradle evicted whole versions, metadata untouched"

tests/pruner-pnpm.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# pnpm: the store is all-or-nothing, ranked by its newest atime. Never a file
+# or two out of a store (pnpm trusts the files its index lists). Needs GNU tools:
+#   bash tests/pruner-pnpm.sh
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
+
+# mkstore <cache dir> <newest atime age in days>: a store with one very old
+# file and one file of the given age.
+mkstore() {
+  mkfile "$1/pnpm/store/v10/files/aa/old-file" 100000 300
+  mkfile "$1/pnpm/cache/metadata/newest-file" 100000 "$2"
+}
+
+# Slot 1: the pnpm dir (newest atime 10 d) is older than a gradle unit (1 d):
+# all of it (store/ and cache/) must go, even though dropping its 300 d file would do.
+c1="$(mkslot 1)"
+mkstore "$c1" 10
+mkunit "$c1/gradle/caches/transforms-4/h" 100000 1
+CACHE_BUDGET_BYTES=$(($(stores_bytes "$c1") - 150000))
+prune_caches >/dev/null
+gone "$c1/pnpm/store/v10"
+gone "$c1/pnpm/cache/metadata"
+if [ ! -d "$c1/pnpm/store" ] || [ ! -d "$c1/pnpm/cache" ]; then fail "pnpm store/ and cache/ dirs must stay (symlink targets)"; fi
+kept "$c1/gradle/caches/transforms-4/h/data.bin"
+
+# Slot 2: the store is the most recently used, so the gradle unit goes and the
+# store stays untouched, including its 300 d file.
+rm -rf "$RUNNER_DATA_DIR/slots/1"
+c2="$(mkslot 2)"
+mkstore "$c2" 1
+mkunit "$c2/gradle/caches/transforms-4/h" 100000 10
+CACHE_BUDGET_BYTES=$(($(stores_bytes "$c2") - 50000))
+prune_caches >/dev/null
+gone "$c2/gradle/caches/transforms-4/h"
+kept "$c2/pnpm/store/v10/files/aa/old-file"
+kept "$c2/pnpm/cache/metadata/newest-file"
+
+finish "pnpm store evicted whole or not at all"

tests/pruner-rustup.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# rustup: whole toolchains are evicted, except the default one named in
+# settings.toml (which may be given by its short name). Needs GNU tools:
+#   bash tests/pruner-rustup.sh
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tests/lib.sh
+source "$here/lib.sh"
+
+# mkrustup <cache dir> <default_toolchain>
+mkrustup() {
+  local r="$1/rustup"
+  mkdir -p "$r"
+  printf 'version = "12"\ndefault_toolchain = "%s"\nprofile = "minimal"\n' "$2" >"$r/settings.toml"
+  mkunit "$r/toolchains/stable-x86_64-unknown-linux-gnu" 200000 400 # oldest, but the default
+  mkunit "$r/toolchains/1.70.0-x86_64-unknown-linux-gnu" 100000 90
+  mkunit "$r/toolchains/nightly-2024-01-01-x86_64-unknown-linux-gnu" 100000 80
+  mkunit "$r/toolchains/1.80.0-x86_64-unknown-linux-gnu" 100000 1
+}
+
+for default in stable-x86_64-unknown-linux-gnu stable; do
+  rm -rf "$RUNNER_DATA_DIR/slots"
+  c="$(mkslot 1)"
+  mkrustup "$c" "$default"
+  t="$c/rustup/toolchains"
+  CACHE_BUDGET_BYTES=$(budget_without "$c" "$t/1.70.0-x86_64-unknown-linux-gnu" \
+    "$t/nightly-2024-01-01-x86_64-unknown-linux-gnu")
+  prune_caches >/dev/null
+  gone "$t/1.70.0-x86_64-unknown-linux-gnu"
+  gone "$t/nightly-2024-01-01-x86_64-unknown-linux-gnu"
+  kept "$t/1.80.0-x86_64-unknown-linux-gnu/data.bin"
+  kept "$t/stable-x86_64-unknown-linux-gnu/data.bin"
+
+  # No budget at all: everything but the default goes.
+  CACHE_BUDGET_BYTES=1
+  prune_caches >/dev/null
+  gone "$t/1.80.0-x86_64-unknown-linux-gnu"
+  kept "$t/stable-x86_64-unknown-linux-gnu/data.bin"
+  kept "$c/rustup/settings.toml"
+done
+
+finish "rustup default toolchain protected"