diff --git a/pruner.sh b/pruner.sh index 9dedb1b..7ef627e 100755 --- a/pruner.sh +++ b/pruner.sh @@ -1,17 +1,26 @@ #!/bin/bash -# Nightly pruner. Run by the `pruner` compose service (same image, this script -# as the entrypoint), NOT by a runner job hook. +# Pruner. Run by the `pruner` compose service (same image, this script as the +# entrypoint), NOT by a runner job hook. # # These scans used to sit in cleanup.sh and ran after every job, costing # 21-39 s of runner time each time and freeing nothing (issue #16). Here they -# run once a day, off the job path: -# 1. LRU size caps on the per-runner cache volumes (mounted under -# $CACHES_ROOT//, see docker-compose.yml). Caches whose -# entries are only valid whole (bun packages, mise tool versions) evict -# whole entries; the rest evict single files. +# run off the job path: +# 1. A size budget per runner slot ($RUNNER_DATA_DIR/slots//cache/), +# enforced with one global LRU across all the stores of the slot. What is +# evictable is described by STORE_TABLE below: caches whose entries are only +# valid whole (bun packages, gradle versions, mise tool versions, ...) evict +# whole directories, the rest evict single files. # 2. docker system prune / builder prune on the host daemon (docker.sock). # 3. Hourly, in the background: fill the shared action archive cache. # +# Job safety: a slot is pruned only while holding its `.prune.lock` (pre-job.sh +# waits on it) and only when it is not mid-job: a `busy` marker younger than 6 h +# means a job is running there, so the slot is skipped; an older one is stale. +# +# Adaptive schedule: every PRUNE_CHECK_SECONDS a cheap df runs; a full pass +# happens when free space is below PRUNE_FREE_PCT, or when PRUNE_INTERVAL_SECONDS +# have passed since the last pass. Under disk pressure docker prune is tighter. +# # Action archive cache: the runner copies # $ACTION_CACHE_DIR/_/.tar.gz instead of downloading when it # exists, but never writes it. prefetch_actions() fills it as the GitHub App, so @@ -20,152 +29,324 @@ # `pruner.sh prefetch` runs it once (rollout warm-up). # # Env: -# PRUNE_INTERVAL_SECONDS sleep between runs (default 86400) -# PRUNE_INITIAL_DELAY sleep before the first run (default 300; keeps a +# RUNNER_DATA_DIR data root holding slots// (/var/lib/github-runner) +# CACHE_BUDGET_BYTES total cache budget over all slots (60 GiB) +# CACHE_BUDGET_PCT ... capped to this % of the filesystem holding +# RUNNER_DATA_DIR (40); the smaller wins and is split +# evenly between the slots +# CACHE_CAP_BYTES DEPRECATED alias of CACHE_BUDGET_BYTES +# PRUNE_CHECK_SECONDS df check period (3600) +# PRUNE_FREE_PCT full pass when free space is below (15) +# PRUNE_INTERVAL_SECONDS full pass at least this often (86400) +# PRUNE_INITIAL_DELAY sleep before the first check (300; keeps a # `docker compose up -d` from running a scan at once) -# PRUNE_RUN_ONCE=true run once and exit (manual / testing) -# CACHES_ROOT where the cache volumes are mounted (/caches) -# CACHE_CAP_BYTES cap for each file-level cache dir (20 GiB) -# MISE_DATA_CAP_BYTES cap for each mise installs tree (20 GiB) +# PRUNE_RUN_ONCE=true run one pass and exit (manual / testing) # ACTION_CACHE_CAP_BYTES cap for the action archive cache (2 GiB) # PREFETCH_SOURCES / PREFETCH_EXTRA "owner/repo@ref ..." to scan / fetch as-is # PREFETCH_INTERVAL_SECONDS prefetch period (default 3600; 0 = off) -# DOCKER_PRUNE_UNTIL / DOCKER_BUILDER_PRUNE_UNTIL (24h / 72h) +# DOCKER_PRUNE_UNTIL / DOCKER_PRUNE_UNTIL_LOW / DOCKER_BUILDER_PRUNE_UNTIL +# (24h / 2h when below PRUNE_FREE_PCT / 72h) # # Intentionally not using set -e: one failing step must not stop the others. -CACHES_ROOT="${CACHES_ROOT:-/caches}" -CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB -MISE_DATA_CAP_BYTES="${MISE_DATA_CAP_BYTES:-21474836480}" # 20 GiB +RUNNER_DATA_DIR="${RUNNER_DATA_DIR:-/var/lib/github-runner}" +CACHE_BUDGET_BYTES="${CACHE_BUDGET_BYTES:-}" +CACHE_BUDGET_PCT="${CACHE_BUDGET_PCT:-40}" +PRUNE_CHECK_SECONDS="${PRUNE_CHECK_SECONDS:-3600}" +PRUNE_FREE_PCT="${PRUNE_FREE_PCT:-15}" +PRUNE_INTERVAL_SECONDS="${PRUNE_INTERVAL_SECONDS:-86400}" +BUSY_STALE_MINUTES=360 # a `busy` marker older than 6 h is a crashed job's leftover ACTION_CACHE_DIR=/opt/actionarchivecache # mount point in docker-compose.yml ACTION_CACHE_CAP_BYTES="${ACTION_CACHE_CAP_BYTES:-2147483648}" # 2 GiB PREFETCH_SOURCES="${PREFETCH_SOURCES:-}" PREFETCH_EXTRA="${PREFETCH_EXTRA:-}" GITHUB_API="https://api.github.com" DOCKER_PRUNE_UNTIL="${DOCKER_PRUNE_UNTIL:-24h}" +DOCKER_PRUNE_UNTIL_LOW="${DOCKER_PRUNE_UNTIL_LOW:-2h}" DOCKER_BUILDER_PRUNE_UNTIL="${DOCKER_BUILDER_PRUNE_UNTIL:-72h}" log() { echo "[pruner] $(date -u +%FT%TZ) $*"; } -dir_bytes() { - du -sb "$1" 2>/dev/null | awk '{print $1}' -} - -# Print how many bytes dir $1 is over cap $2. Returns 1 (printing nothing) when -# the dir is missing, unreadable, or already within the cap. -excess_bytes() { - local dir="$1" cap="$2" size - [ -d "$dir" ] || return 1 - size=$(dir_bytes "$dir") - [ -n "$size" ] || return 1 - if [ "$size" -le "$cap" ]; then - log "$dir ${size}B <= cap ${cap}B" >&2 - return 1 +if [ -z "$CACHE_BUDGET_BYTES" ]; then + if [ -n "${CACHE_CAP_BYTES:-}" ]; then + CACHE_BUDGET_BYTES="$CACHE_CAP_BYTES" + log "CACHE_CAP_BYTES is deprecated; using it as CACHE_BUDGET_BYTES (now the total over all slots)" + else + CACHE_BUDGET_BYTES=64424509440 # 60 GiB fi - log "$dir is ${size}B (cap ${cap}B)" >&2 - echo $((size - cap)) +fi + +# Evictable units per store, one rule per line: store|kind|glob|skip +# kind d: every directory matching glob (relative to the store dir, never a +# symlink) is one unit, removed whole; skip is a case pattern for the +# relative path of units to leave alone. Glob "." makes the whole +# store dir one unit (pnpm: store/ + cache/ are all-or-nothing); it is +# emptied, but its subdirs are kept, as container symlinks point there. +# kind f: every regular file named skip (a find -name pattern) under the dir +# glob is one unit. +# Anything no rule matches (bun's bin/ and @t@ piles, cargo's bin/ and registry +# index, rustup's settings.toml, ...) is never touched. Hooks for the few stores +# needing more than a rule are in unit_key / unit_protected / unit_pre_evict / +# unit_post_evict / store_sweep. +STORE_TABLE=' +bun|d|install/cache/*@*|install/cache/@* +bun|d|install/cache/@*/*@*|install/cache/@t@/* +npm|f|_cacache|* +pnpm|d|.| +gradle|d|caches/modules-2/files-2.1/*/*/*| +gradle|d|caches/transforms-*/*| +gradle|d|wrapper/dists/*| +gradle|d|caches/jars-*/*| +gradle|f|caches/build-cache-1|* +cargo|f|registry/cache|*.crate +cargo|d|registry/src/*/*| +cargo|d|git/checkouts/*| +cargo|d|git/db/*| +rustup|d|toolchains/*| +konan|d|dependencies/*| +konan|d|kotlin-native-prebuilt-*| +mise-data|d|installs/*/*| +pub|d|hosted/*/*| +pub|d|git/*| +toolcache|d|*/*| +mise-cache|f|.|* +hk|f|.|* +actions|f|.|* +' + +# Stores of a runner slot (everything but the shared action archive cache). +slot_stores() { + awk -F'|' 'NF > 3 && $1 != "actions" && !seen[$1]++ {print $1}' <<<"$STORE_TABLE" } -# LRU-evict regular files, least recently accessed first, until $1 is at or -# under $2 bytes. Ordered by atime (a read counts as use; Gradle never touches -# mtime on read, so mtime order evicted the most-used dependencies first). -# Volumes mounted relatime still refresh atime on the first read after a -# change. Does not wholesale wipe the tree. -lru_cap() { - local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" - local excess freed=0 rec fsize path - excess=$(excess_bytes "$dir" "$cap") || return 0 - log "evicting least recently used files from $dir" - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - rec="${rec#* }" # drop atime - fsize="${rec%% *}" - path="${rec#* }" - if [ -z "$path" ] || [ -z "$fsize" ]; then continue; fi - rm -f "$path" 2>/dev/null || true - freed=$((freed + fsize)) - done < <(find "$dir" -type f -printf '%A@\t%s\t%p\0' 2>/dev/null | sort -z -n) - find "$dir" -type d -empty -delete 2>/dev/null || true - log "$dir freed ~${freed}B" +dir_bytes() { + du -sb "$1" 2>/dev/null | awk '{printf "%.0f\n", $1}' } -# mise installs (installs//) are only useful whole: deleting -# single files would leave a tool that exists but is broken. Evict whole -# / directories instead, oldest install first, until the -# installs tree fits under the cap. -lru_cap_mise_installs() { - local data="$1" cap="${2:-$MISE_DATA_CAP_BYTES}" - local installs="$data/installs" - local excess freed=0 rec path dsize - excess=$(excess_bytes "$installs" "$cap") || return 0 - log "evicting oldest tool versions from $installs" - while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - path="${rec#* }" - [ -d "$path" ] || continue - dsize=$(dir_bytes "$path") - rm -rf "$path" 2>/dev/null || true - freed=$((freed + ${dsize:-0})) - done < <(find "$installs" -mindepth 2 -maxdepth 2 -type d -printf '%T@\t%p\0' 2>/dev/null | sort -z -n) - log "$installs freed ~${freed}B" +# default_toolchain from rustup's settings.toml ($1 = RUSTUP_HOME). +rustup_default() { + sed -n 's/^default_toolchain[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1/settings.toml" 2>/dev/null | head -n 1 } # bun treats install/cache/@@@@/ as complete whenever the folder # exists, so a file evicted from it is silently missing from every later # node_modules (next@16.3.8 lost dist/compiled/server-only/empty.js, breaking # every Next build). 0-byte files went first: copying them never reads, so -# their atime never moved. Evict whole package folders instead, least recently -# used first by the atime of their package.json (read on every install and -# import), until the whole volume ($1, mounted from /root/.bun) fits under the -# cap. Never touches $1/bin (the bun binary itself) or the @t@ manifest piles. -lru_cap_bun() { - local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" - local cache="$dir/install/cache" - local excess freed=0 rec path dsize base t +# their atime never moved. Hence whole packages, least recently used first by +# the atime of their package.json (read on every install and import). + +# Before measuring a store ($1 = name, $2 = dir). +store_sweep() { + [ "$1" = bun ] || return 0 + local cache="$2/install/cache" [ -d "$cache" ] || return 0 # Leftovers of interrupted extractions; the age guard spares live installs. find "$cache" -mindepth 1 -maxdepth 2 ! -name .tmp \( -name '*.tmp' -o -name '*.part' \) \ -mmin +60 -exec rm -rf {} + 2>/dev/null find "$cache/.tmp" -mindepth 1 -maxdepth 1 -mmin +60 -exec rm -rf {} + 2>/dev/null - excess=$(excess_bytes "$dir" "$cap") || return 0 - log "evicting least recently used packages from $cache" +} + +# LRU key override for dir unit $2 of store $1; prints nothing to keep the +# newest-atime-of-any-file key. +unit_key() { + [ "$1" = bun ] || return 0 + stat -c %X "$2/package.json" 2>/dev/null || stat -c %Y "$2" 2>/dev/null +} + +# Succeeds when unit $3 (relative to dir $2 of store $1) must not be evicted. +unit_protected() { + local def + [ "$1" = rustup ] || return 1 + def=$(rustup_default "$2") + [ -n "$def" ] || return 1 + case "${3##*/}" in + "$def" | "$def"-*) return 0 ;; # `stable` is the dir stable- + esac + return 1 +} + +# toolcache: /.complete says / is whole. Drop the markers +# first so an interrupted removal never leaves a "complete" half tree. +unit_pre_evict() { + [ "$1" = toolcache ] || return 0 + find "$2" -name '*.complete' -delete 2>/dev/null +} + +# After unit $2 of store $1 is gone. +unit_post_evict() { + local base + [ "$1" = bun ] || return 0 + # Drop the /@@@ symlink pointing at it, and the dir if empty. + base="${2##*/}" + rm -f "${2%/*}/${base%%@*}/${base#*@}" 2>/dev/null + rmdir "${2%/*}/${base%%@*}" 2>/dev/null + return 0 +} + +# Print NUL-terminated "keysizestorestoppath" records, one +# per evictable unit of store $1 living in dir $2. key is the LRU order (atime, +# seconds); stop is the dir above which emptied parents are not removed. +collect_units() { + local store="$1" sd="$2" s kind glob skip pre stop + [ -d "$sd" ] || return 0 + while IFS='|' read -r s kind glob skip; do + [ "$s" = "$store" ] || continue + if [ "$kind" = f ]; then + [ -d "$sd/$glob" ] || continue + find "$sd/$glob" -type f -name "$skip" -printf "%A@\t%s\t$store\t-\t%p\0" 2>/dev/null + continue + fi + pre="${glob%%[*?[]*}" + case "$pre" in + */*) pre="${pre%/*}" ;; + *) pre="" ;; + esac + stop="$sd${pre:+/$pre}" + ( + cd "$sd" 2>/dev/null || exit 0 + shopt -s nullglob + # shellcheck disable=SC2086 # glob expansion is the point + for rel in $glob; do + if [ ! -d "$rel" ] || [ -L "$rel" ]; then continue; fi + # shellcheck disable=SC2254 # skip is a pattern + case "$rel" in $skip) continue ;; esac + if unit_protected "$store" "$sd" "$rel"; then continue; fi + stats=$(find "$rel" -printf '%y\t%A@\t%s\n' 2>/dev/null | + awk -F'\t' '{ s += $3; if ($1 == "f" && $2 + 0 > m) m = $2 + 0 } END { printf "%.6f\t%.0f\n", m, s }') + key="${stats%%$'\t'*}" + t=$(unit_key "$store" "$rel") + [ -z "$t" ] || key="$t" + printf '%s\t%s\t%s\t%s\t%s\0' "$key" "${stats#*$'\t'}" "$store" "$stop" "$sd/$rel" + done + ) + done <<<"$STORE_TABLE" +} + +# Remove unit $3 of store $1 (stop dir $2): a whole directory (then its emptied +# parents up to $2), or a single file. +evict_unit() { + local store="$1" stop="$2" path="$3" p + if [ "${path##*/}" = . ]; then + find "${path%/.}" -mindepth 2 -delete 2>/dev/null + find "${path%/.}" -mindepth 1 -maxdepth 1 ! -type d -delete 2>/dev/null + elif [ -d "$path" ] && [ ! -L "$path" ]; then + unit_pre_evict "$store" "$path" + rm -rf "$path" 2>/dev/null + unit_post_evict "$store" "$path" + p="${path%/*}" + while [ "$p" != "$stop" ] && [ "${p#"$stop"/}" != "$p" ]; do + rmdir "$p" 2>/dev/null || break + p="${p%/*}" + done + else + rm -f "$path" 2>/dev/null + fi +} + +# Keep the stores given as = pairs ($2...) within $1 bytes in total, +# evicting units across all of them in global LRU order, least recently used +# first (a read counts as use; Gradle never touches mtime on read, so mtime +# order evicted the most-used dependencies first). Volumes mounted relatime +# still refresh atime on the first read after a change. +lru_entries() { + local budget="$1" spec store sd total=0 sz excess freed=0 n=0 + local rec key size stop path + local -A dirs=() touched=() + shift + for spec in "$@"; do + store="${spec%%=*}" + sd="${spec#*=}" + [ -d "$sd" ] || continue + store_sweep "$store" "$sd" + dirs[$store]="$sd" + sz=$(dir_bytes "$sd") + total=$((total + ${sz:-0})) + done + if [ "$total" -le "$budget" ]; then + log "${total}B <= budget ${budget}B" + return 0 + fi + excess=$((total - budget)) + log "${total}B (budget ${budget}B), evicting least recently used entries" while IFS= read -r -d '' rec; do - [ "$freed" -ge "$excess" ] && break - path="${rec#* }" - [ -d "$path" ] || continue - dsize=$(dir_bytes "$path") - rm -rf "$path" 2>/dev/null || true - freed=$((freed + ${dsize:-0})) - # Drop the /@@@ symlink pointing at it, and the dir if empty. - base="${path##*/}" - rm -f "${path%/*}/${base%%@*}/${base#*@}" 2>/dev/null - rmdir "${path%/*}/${base%%@*}" 2>/dev/null + if [ "$freed" -ge "$excess" ]; then break; fi + IFS=$'\t' read -r key size store stop path <<<"$rec" + if [ -z "$path" ] || [ -z "$size" ]; then continue; fi + evict_unit "$store" "$stop" "$path" + freed=$((freed + size)) + n=$((n + 1)) + touched[$store]=1 done < <( - # @... at the top level, @scope/@... one level down. - find "$cache" -mindepth 1 -maxdepth 2 -type d -name '*@*' ! -name '@*' \ - \( -path "$cache/@*/*" -o ! -path "$cache/*/*" \) -print0 2>/dev/null | - while IFS= read -r -d '' path; do - t=$(stat -c %X "$path/package.json" 2>/dev/null || stat -c %Y "$path" 2>/dev/null) || continue - printf '%s\t%s\0' "$t" "$path" - done | sort -z -n + for store in "${!dirs[@]}"; do collect_units "$store" "${dirs[$store]}"; done | + LC_ALL=C sort -z -n ) - find "$cache" -mindepth 1 -maxdepth 1 -name '@*' ! -name '@t@' -type d -empty -delete 2>/dev/null || true - log "$cache freed ~${freed}B" + # Emptied directories under file-level stores. + for store in "${!touched[@]}"; do + while IFS='|' read -r s kind glob _; do + if [ "$s" = "$store" ] && [ "$kind" = f ] && [ -d "${dirs[$store]}/$glob" ]; then + find "${dirs[$store]}/$glob" -mindepth 1 -type d -empty -delete 2>/dev/null + fi + done <<<"$STORE_TABLE" + done + if [ "$freed" -lt "$excess" ]; then + log "evicted $n entries, ~${freed}B freed; still ~$((excess - freed))B over budget (protected or untracked data)" + else + log "evicted $n entries, ~${freed}B freed" + fi +} + +# Total cache budget in bytes of one slot, with $1 slots sharing it. +slot_budget() { + local slots="$1" fs total + total="$CACHE_BUDGET_BYTES" + fs=$(df -B1 --output=size "$RUNNER_DATA_DIR" 2>/dev/null | awk 'NR == 2 {print $1}') + if [ -n "$fs" ] && [ $((fs * CACHE_BUDGET_PCT / 100)) -lt "$total" ]; then + total=$((fs * CACHE_BUDGET_PCT / 100)) + fi + echo $((total / slots)) +} + +# Free space of the filesystem holding RUNNER_DATA_DIR, in whole percent. +free_pct() { + local dir="$RUNNER_DATA_DIR" + [ -d "$dir" ] || dir=/ + df -B1 --output=avail,size "$dir" 2>/dev/null | awk 'NR == 2 && $2 > 0 {print int($1 * 100 / $2)}' +} + +# Prune one slot ($1 = slot dir) down to $2 bytes, unless a job is running in +# it. The flock keeps jobs from starting meanwhile: pre-job.sh waits on it. +prune_slot() { + local slot="$1" budget="$2" id start="$SECONDS" + id="${slot##*/}" + ( + flock -n 9 || { log "slot $id: another prune holds the lock; skipping"; exit 0; } + if [ -e "$slot/busy" ]; then + if [ -n "$(find "$slot/busy" -maxdepth 0 -mmin "-$BUSY_STALE_MINUTES" 2>/dev/null)" ]; then + log "slot $id: job running (busy marker); skipping" + exit 0 + fi + log "slot $id: stale busy marker (> ${BUSY_STALE_MINUTES} min); pruning anyway" + fi + args=() + for store in $(slot_stores); do args+=("$store=$slot/cache/$store"); done + log "slot $id: budget ${budget}B" + lru_entries "$budget" "${args[@]}" + ) 9>"$slot/.prune.lock" + log "slot $id: took $((SECONDS - start))s" } prune_caches() { - local runner - for runner in "$CACHES_ROOT"/*/; do - [ -d "$runner" ] || continue - runner="${runner%/}" - log "capping caches under $runner" - lru_cap "$runner/gradle" - lru_cap_bun "$runner/bun" - lru_cap "$runner/mise-cache" - lru_cap "$runner/hk" # configs/.json + pklr/packages/.zip: whole files - lru_cap_mise_installs "$runner/mise-data" + local caches=() cache budget + for cache in "$RUNNER_DATA_DIR"/slots/*/cache; do + [ -d "$cache" ] && caches+=("$cache") done - lru_cap "$ACTION_CACHE_DIR" "$ACTION_CACHE_CAP_BYTES" + if [ "${#caches[@]}" -gt 0 ]; then + budget=$(slot_budget "${#caches[@]}") + for cache in "${caches[@]}"; do prune_slot "${cache%/cache}" "$budget"; done + else + log "no slots under $RUNNER_DATA_DIR/slots" + fi + lru_entries "$ACTION_CACHE_CAP_BYTES" "actions=$ACTION_CACHE_DIR" } gh_curl() { @@ -253,13 +434,16 @@ prefetch_actions() { GH_TOKEN="" } +# $1 = low: free space is below PRUNE_FREE_PCT, so prune docker harder. prune_docker() { + local until="$DOCKER_PRUNE_UNTIL" + [ "${1:-}" = low ] && until="$DOCKER_PRUNE_UNTIL_LOW" if ! docker info >/dev/null 2>&1; then log "docker daemon unreachable; skipping docker prune" return 0 fi - log "docker system prune -af --filter until=${DOCKER_PRUNE_UNTIL}" - docker system prune -af --filter "until=${DOCKER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' + log "docker system prune -af --filter until=${until}" + docker system prune -af --filter "until=${until}" 2>&1 | sed 's/^/[pruner] /' log "docker builder prune -af --filter until=${DOCKER_BUILDER_PRUNE_UNTIL}" docker builder prune -af --filter "until=${DOCKER_BUILDER_PRUNE_UNTIL}" 2>&1 | sed 's/^/[pruner] /' } @@ -268,7 +452,7 @@ run_once() { local before after before=$(df -h / | awk 'NR==2 {print $4}') prune_caches - prune_docker + prune_docker "${1:-}" after=$(df -h / | awk 'NR==2 {print $4}') log "done. Disk free: ${before} -> ${after}" } @@ -297,7 +481,20 @@ if [ "${PREFETCH_INTERVAL_SECONDS:-3600}" -gt 0 ]; then fi sleep "${PRUNE_INITIAL_DELAY:-300}" +last=0 while true; do - run_once - sleep "${PRUNE_INTERVAL_SECONDS:-86400}" + free=$(free_pct) + low="" + now=$(date +%s) + if [ -n "$free" ] && [ "$free" -lt "$PRUNE_FREE_PCT" ]; then + low=low + log "free space ${free}% < ${PRUNE_FREE_PCT}%: full pass" + elif [ $((now - last)) -ge "$PRUNE_INTERVAL_SECONDS" ]; then + log "free space ${free:-?}%, ${PRUNE_INTERVAL_SECONDS}s since the last pass: full pass" + fi + if [ -n "$low" ] || [ $((now - last)) -ge "$PRUNE_INTERVAL_SECONDS" ]; then + run_once "$low" + last=$(date +%s) + fi + sleep "$PRUNE_CHECK_SECONDS" done diff --git a/tests/lib.sh b/tests/lib.sh new file mode 100755 index 0000000..8e12b82 --- /dev/null +++ b/tests/lib.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Shared helpers for tests/pruner-*.sh. Sourced, not run. Needs GNU coreutils, +# find, flock (Linux, or `docker run --rm -v "$PWD:/w" -w /w ubuntu:24.04 bash tests/.sh`). +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } + +# shellcheck source=pruner.sh +source "$here/../pruner.sh" +RUNNER_DATA_DIR="$work/data" +ACTION_CACHE_DIR="$work/no-action-cache" +CACHE_BUDGET_PCT=100 # the byte budget decides, whatever the test fs is + +# mkslot : create slot and print its cache dir. +mkslot() { + mkdir -p "$RUNNER_DATA_DIR/slots/$1/cache" "$RUNNER_DATA_DIR/slots/$1/work" + touch "$RUNNER_DATA_DIR/slots/$1/.lock" + echo "$RUNNER_DATA_DIR/slots/$1/cache" +} + +sz() { du -sb "$1" | awk '{printf "%.0f\n", $1}'; } + +# stores_bytes : what the pruner counts for a slot. +stores_bytes() { + local d total=0 + for d in "$1"/*/; do total=$((total + $(sz "$d"))); done + echo "$total" +} + +# mkfile +mkfile() { + mkdir -p "$(dirname "$1")" + head -c "$2" /dev/urandom >"$1" + touch -a -d "$3 days ago" "$1" +} + +# mkdirfile : a unit with one file of that age. +mkunit() { + mkfile "$1/data.bin" "$2" "$3" +} + +# budget_without ...: a budget that makes exactly s +# (the oldest ones) go and nothing more: now minus them, plus slack that is far +# below any unit's size. +budget_without() { + local cache="$1" total u + shift + total=$(stores_bytes "$cache") + for u in "$@"; do total=$((total - $(sz "$u"))); done + echo $((total + 20000)) +} + +gone() { [ ! -e "$1" ] && [ ! -L "$1" ] || fail "$1 should be evicted"; } +kept() { [ -e "$1" ] || [ -L "$1" ] || fail "$1 should survive"; } + +finish() { + if [ "$fails" -gt 0 ]; then + echo "$fails check(s) failed" + exit 1 + fi + echo "PASS: $1" +} diff --git a/tests/pruner-budget.sh b/tests/pruner-budget.sh new file mode 100755 index 0000000..e4aaa1f --- /dev/null +++ b/tests/pruner-budget.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Budget: min(CACHE_BUDGET_BYTES, CACHE_BUDGET_PCT% of the filesystem) split +# evenly over the slots, each slot ranked on its own (not one LRU across slots). +# Needs GNU tools: +# bash tests/pruner-budget.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +# slot_budget arithmetic. +mkdir -p "$RUNNER_DATA_DIR" +CACHE_BUDGET_BYTES=1000 CACHE_BUDGET_PCT=100 +[ "$(slot_budget 2)" = 500 ] || fail "1000B over 2 slots: $(slot_budget 2)" +fs=$(df -B1 --output=size "$RUNNER_DATA_DIR" | awk 'NR == 2 {print $1}') +CACHE_BUDGET_BYTES=$((fs * 2)) CACHE_BUDGET_PCT=10 +[ "$(slot_budget 4)" = $((fs * 10 / 100 / 4)) ] || fail "10% of the fs over 4 slots: $(slot_budget 4)" + +# Two slots, three 100000 B units each. Slot 2's units are all newer than +# slot 1's: a global LRU would empty slot 1 first; per slot, each loses just +# its own oldest unit. +for id in 1 2; do + c="$(mkslot "$id")" + if [ "$id" = 1 ]; then ages=(90 80 70); else ages=(30 20 10); fi + for i in 0 1 2; do mkunit "$c/mise-data/installs/tool/v$i" 100000 "${ages[$i]}"; done +done +per_slot=$(budget_without "$RUNNER_DATA_DIR/slots/1/cache" "$RUNNER_DATA_DIR/slots/1/cache/mise-data/installs/tool/v0") +CACHE_BUDGET_BYTES=$((per_slot * 2)) CACHE_BUDGET_PCT=100 +out=$(prune_caches) +for id in 1 2; do + u="$RUNNER_DATA_DIR/slots/$id/cache/mise-data/installs/tool" + gone "$u/v0" + kept "$u/v1/data.bin" + kept "$u/v2/data.bin" + grep -q "slot $id: budget ${per_slot}B" <<<"$out" || fail "slot $id budget not ${per_slot}B" +done + +finish "budget split evenly across slots" diff --git a/tests/pruner-bun.sh b/tests/pruner-bun.sh index aca92af..675394a 100755 --- a/tests/pruner-bun.sh +++ b/tests/pruner-bun.sh @@ -1,21 +1,17 @@ #!/bin/bash # pruner.sh must evict whole bun packages, never single files: bun reuses a -# cache folder as complete whenever it exists. Builds a fake /root/.bun volume -# (packages holding 0-byte files with old atimes, a scoped package, the -# / symlinks, @t@ piles and bin/bun), runs prune_caches under a small -# cap, then checks every surviving package is complete and every evicted one -# is fully gone. Needs GNU find/stat/du (Linux, or `docker run ubuntu`): +# cache folder as complete whenever it exists. Builds a fake /root/.bun store +# in a slot (packages holding 0-byte files with old atimes, a scoped package, +# the / symlinks, @t@ piles and bin/bun), runs prune_caches under a +# small budget, then checks every surviving package is complete and every +# evicted one is fully gone. Needs GNU tools (Linux, or `docker run ubuntu`): # bash tests/pruner-bun.sh -set -euo pipefail - here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -work="$(mktemp -d)" -trap 'rm -rf "$work"' EXIT - -fails=0 -fail() { echo "FAIL: $*"; fails=$((fails + 1)); } +# shellcheck source=tests/lib.sh +source "$here/lib.sh" -bun="$work/caches/1/bun" +cache0="$(mkslot 1)" +bun="$cache0/bun" cache="$bun/install/cache" mkdir -p "$bun/bin" "$cache/@t@" head -c 300000 /dev/zero >"$bun/bin/bun" @@ -56,14 +52,11 @@ touch -d '2 days ago' "$cache/.tmp/stale" head -c 10 /dev/zero >"$cache/half@1.0.0@@@1.tgz.part" touch -d '2 days ago' "$cache/half@1.0.0@@@1.tgz.part" -# Cap = volume minus the three oldest packages (old-a, old-b, @scope/old-c), -# plus half a package of slack: exactly those three must go, nothing else. -sz() { du -sb "$1" | awk '{print $1}'; } -cap=$(($(sz "$bun") - $(sz "$cache/old-a@1.0.0@@@1") - $(sz "$cache/old-b@2.0.0@@@1") \ - - $(sz "$cache/@scope/old-c@3.0.0@@@1") - 2 * 4096 + 50000)) -# shellcheck source=pruner.sh -source "$here/../pruner.sh" -CACHES_ROOT="$work/caches" CACHE_CAP_BYTES="$cap" prune_caches +# Budget = volume minus the three oldest packages (old-a, old-b, @scope/old-c): +# exactly those three must go, nothing else. +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$cache/old-a@1.0.0@@@1" "$cache/old-b@2.0.0@@@1" \ + "$cache/@scope/old-c@3.0.0@@@1") +prune_caches >/dev/null for entry in 'old-a@1.0.0@@@1' 'old-b@2.0.0@@@1' '@scope/old-c@3.0.0@@@1'; do base="${entry##*/}" @@ -93,11 +86,7 @@ done [ -e "$cache/.tmp/live" ] || fail "live .tmp entry removed" [ ! -e "$cache/half@1.0.0@@@1.tgz.part" ] || fail "stale .part kept" -size=$(sz "$bun") -[ "$size" -le "$cap" ] || fail "volume ${size}B still over the ${cap}B cap" +size=$(stores_bytes "$cache0") +[ "$size" -le "$CACHE_BUDGET_BYTES" ] || fail "slot ${size}B still over the ${CACHE_BUDGET_BYTES}B budget" -if [ "$fails" -gt 0 ]; then - echo "$fails check(s) failed" - exit 1 -fi -echo "PASS: bun cache evicted by whole package (volume ${size}B)" +finish "bun cache evicted by whole package (slot ${size}B)" diff --git a/tests/pruner-busy.sh b/tests/pruner-busy.sh new file mode 100755 index 0000000..d5951f6 --- /dev/null +++ b/tests/pruner-busy.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Job safety: a slot with a fresh `busy` marker is skipped, one with a stale +# marker (> 6 h) is pruned, and a slot whose .prune.lock is held is skipped. +# Needs GNU tools and flock: +# bash tests/pruner-busy.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +unit() { echo "$RUNNER_DATA_DIR/slots/$1/cache/mise-data/installs/node/20.0.0"; } +for id in 1 2 3 4; do + mkslot "$id" >/dev/null + mkunit "$(unit "$id")" 100000 30 +done +touch "$RUNNER_DATA_DIR/slots/1/busy" # job running +touch -d '7 hours ago' "$RUNNER_DATA_DIR/slots/2/busy" # crashed job's leftover +# slot 3: idle +mkdir -p "$RUNNER_DATA_DIR/slots/c1" && touch "$RUNNER_DATA_DIR/slots/c1/.lock" # control slot, no cache/ +( flock 8; sleep 60 ) 8>"$RUNNER_DATA_DIR/slots/4/.prune.lock" & # another pruner +holder=$! +sleep 1 + +CACHE_BUDGET_BYTES=1 +out=$(prune_caches) +kill "$holder" 2>/dev/null || true +wait "$holder" 2>/dev/null || true + +kept "$(unit 1)/data.bin" +gone "$(unit 2)" +gone "$(unit 3)" +kept "$(unit 4)/data.bin" +[ ! -e "$RUNNER_DATA_DIR/slots/c1/cache" ] || fail "control slot got a cache dir" +grep -q 'slot 1: job running' <<<"$out" || fail "no log line for the busy slot" +grep -q 'slot 2: stale busy' <<<"$out" || fail "no log line for the stale busy marker" +grep -q 'slot 4: another prune' <<<"$out" || fail "no log line for the locked slot" +grep -q 'slot 3: took [0-9]*s' <<<"$out" || fail "no elapsed log line" +[ -e "$RUNNER_DATA_DIR/slots/1/busy" ] || fail "busy marker removed by the pruner" + +finish "busy slot skipped, stale busy pruned, locked slot skipped" diff --git a/tests/pruner-cargo.sh b/tests/pruner-cargo.sh new file mode 100755 index 0000000..c0ff9ef --- /dev/null +++ b/tests/pruner-cargo.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Cargo: .crate files, extracted crates and git checkouts/dbs are evicted; +# bin/ (cargo install'ed tools) and the registry index never are. Needs GNU tools: +# bash tests/pruner-cargo.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +cache0="$(mkslot 1)" +c="$cache0/cargo" + +mkfile "$c/bin/cargo-nextest" 200000 900 # oldest thing in the store +mkfile "$c/registry/index/idx/.cache/ab/serde" 50000 900 +mkfile "$c/registry/cache/idx/old-1.0.0.crate" 100000 90 +mkfile "$c/registry/cache/idx/fresh-1.0.0.crate" 100000 1 +mkunit "$c/registry/src/idx/old-1.0.0" 100000 80 +mkunit "$c/registry/src/idx/fresh-1.0.0" 100000 1 +mkunit "$c/git/checkouts/oldrepo-aaa" 100000 70 +mkunit "$c/git/checkouts/freshrepo-bbb" 100000 1 +mkunit "$c/git/db/oldrepo-aaa" 100000 60 +mkunit "$c/git/db/freshrepo-bbb" 100000 1 + +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$c/registry/cache/idx/old-1.0.0.crate" "$c/registry/src/idx/old-1.0.0" \ + "$c/git/checkouts/oldrepo-aaa" "$c/git/db/oldrepo-aaa") +prune_caches >/dev/null + +gone "$c/registry/cache/idx/old-1.0.0.crate" +gone "$c/registry/src/idx/old-1.0.0" +gone "$c/git/checkouts/oldrepo-aaa" +gone "$c/git/db/oldrepo-aaa" +kept "$c/registry/cache/idx/fresh-1.0.0.crate" +kept "$c/registry/src/idx/fresh-1.0.0/data.bin" +kept "$c/git/checkouts/freshrepo-bbb/data.bin" +kept "$c/git/db/freshrepo-bbb/data.bin" +[ -s "$c/bin/cargo-nextest" ] || fail "bin/cargo-nextest removed" +[ -s "$c/registry/index/idx/.cache/ab/serde" ] || fail "registry index removed" + +# Even with no budget at all, bin/ and the index stay. +CACHE_BUDGET_BYTES=1 +prune_caches >/dev/null +[ -s "$c/bin/cargo-nextest" ] || fail "bin/cargo-nextest removed under a tiny budget" +[ -s "$c/registry/index/idx/.cache/ab/serde" ] || fail "registry index removed under a tiny budget" +[ ! -e "$c/registry/cache/idx/fresh-1.0.0.crate" ] || fail "fresh crate should go under a tiny budget" + +finish "cargo evicted caches, bin/ protected" diff --git a/tests/pruner-gradle.sh b/tests/pruner-gradle.sh new file mode 100755 index 0000000..e92f8ae --- /dev/null +++ b/tests/pruner-gradle.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Gradle: whole module versions / transform dirs / wrapper dists / jars dirs +# are evicted, never single files inside them, and the metadata Gradle keeps +# beside them (modules-2/metadata-*) is left alone. Needs GNU tools: +# bash tests/pruner-gradle.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +cache0="$(mkslot 1)" +g="$cache0/gradle" +fs="$g/caches/modules-2/files-2.1" + +mkunit "$fs/org.old/lib/1.0.0/h1" 100000 90 # oldest version of a module +mkfile "$fs/org.old/lib/1.0.0/h1/lib-1.0.0.pom" 50000 400 +mkunit "$fs/org.old/lib/1.1.0/h2" 100000 2 # newer version of the same module +mkunit "$fs/org.fresh/app/2.0.0/h3" 100000 1 +mkunit "$g/caches/transforms-4/oldhash" 100000 80 +mkunit "$g/caches/transforms-4/freshhash" 100000 1 +mkunit "$g/wrapper/dists/gradle-7.0-bin/abc" 100000 70 +mkunit "$g/wrapper/dists/gradle-8.5-bin/def" 100000 1 +mkunit "$g/caches/jars-9/oldjars" 100000 60 +mkfile "$g/caches/build-cache-1/oldentry" 100000 50 +mkfile "$g/caches/build-cache-1/freshentry" 100000 1 +mkdir -p "$g/caches/modules-2/metadata-2.107" +echo meta >"$g/caches/modules-2/metadata-2.107/descriptors.bin" +touch -a -d '500 days ago' "$g/caches/modules-2/metadata-2.107/descriptors.bin" + +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$fs/org.old/lib/1.0.0" "$g/caches/transforms-4/oldhash" \ + "$g/wrapper/dists/gradle-7.0-bin" "$g/caches/jars-9/oldjars" "$g/caches/build-cache-1/oldentry") +prune_caches >/dev/null + +gone "$fs/org.old/lib/1.0.0" +gone "$g/caches/transforms-4/oldhash" +gone "$g/wrapper/dists/gradle-7.0-bin" +gone "$g/caches/jars-9/oldjars" +gone "$g/caches/build-cache-1/oldentry" +kept "$fs/org.old/lib/1.1.0/h2/data.bin" +kept "$fs/org.fresh/app/2.0.0/h3/data.bin" +kept "$g/caches/transforms-4/freshhash/data.bin" +kept "$g/wrapper/dists/gradle-8.5-bin/def/data.bin" +kept "$g/caches/build-cache-1/freshentry" +kept "$g/caches/modules-2/metadata-2.107/descriptors.bin" +[ -d "$fs/org.old/lib" ] || fail "module dir removed while a version is left" +[ -d "$fs" ] || fail "files-2.1 removed" +[ -d "$g/caches/build-cache-1" ] || fail "build-cache-1 removed" + +size=$(stores_bytes "$cache0") +[ "$size" -le "$CACHE_BUDGET_BYTES" ] || fail "slot ${size}B still over the ${CACHE_BUDGET_BYTES}B budget" + +finish "gradle evicted whole versions, metadata untouched" diff --git a/tests/pruner-pnpm.sh b/tests/pruner-pnpm.sh new file mode 100755 index 0000000..c42ddc1 --- /dev/null +++ b/tests/pruner-pnpm.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# pnpm: the store is all-or-nothing, ranked by its newest atime. Never a file +# or two out of a store (pnpm trusts the files its index lists). Needs GNU tools: +# bash tests/pruner-pnpm.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +# mkstore : a store with one very old +# file and one file of the given age. +mkstore() { + mkfile "$1/pnpm/store/v10/files/aa/old-file" 100000 300 + mkfile "$1/pnpm/cache/metadata/newest-file" 100000 "$2" +} + +# Slot 1: the pnpm dir (newest atime 10 d) is older than a gradle unit (1 d): +# all of it (store/ and cache/) must go, even though dropping its 300 d file would do. +c1="$(mkslot 1)" +mkstore "$c1" 10 +mkunit "$c1/gradle/caches/transforms-4/h" 100000 1 +CACHE_BUDGET_BYTES=$(($(stores_bytes "$c1") - 150000)) +prune_caches >/dev/null +gone "$c1/pnpm/store/v10" +gone "$c1/pnpm/cache/metadata" +if [ ! -d "$c1/pnpm/store" ] || [ ! -d "$c1/pnpm/cache" ]; then fail "pnpm store/ and cache/ dirs must stay (symlink targets)"; fi +kept "$c1/gradle/caches/transforms-4/h/data.bin" + +# Slot 2: the store is the most recently used, so the gradle unit goes and the +# store stays untouched, including its 300 d file. +rm -rf "$RUNNER_DATA_DIR/slots/1" +c2="$(mkslot 2)" +mkstore "$c2" 1 +mkunit "$c2/gradle/caches/transforms-4/h" 100000 10 +CACHE_BUDGET_BYTES=$(($(stores_bytes "$c2") - 50000)) +prune_caches >/dev/null +gone "$c2/gradle/caches/transforms-4/h" +kept "$c2/pnpm/store/v10/files/aa/old-file" +kept "$c2/pnpm/cache/metadata/newest-file" + +finish "pnpm store evicted whole or not at all" diff --git a/tests/pruner-rustup.sh b/tests/pruner-rustup.sh new file mode 100755 index 0000000..d51569d --- /dev/null +++ b/tests/pruner-rustup.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# rustup: whole toolchains are evicted, except the default one named in +# settings.toml (which may be given by its short name). Needs GNU tools: +# bash tests/pruner-rustup.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +# mkrustup +mkrustup() { + local r="$1/rustup" + mkdir -p "$r" + printf 'version = "12"\ndefault_toolchain = "%s"\nprofile = "minimal"\n' "$2" >"$r/settings.toml" + mkunit "$r/toolchains/stable-x86_64-unknown-linux-gnu" 200000 400 # oldest, but the default + mkunit "$r/toolchains/1.70.0-x86_64-unknown-linux-gnu" 100000 90 + mkunit "$r/toolchains/nightly-2024-01-01-x86_64-unknown-linux-gnu" 100000 80 + mkunit "$r/toolchains/1.80.0-x86_64-unknown-linux-gnu" 100000 1 +} + +for default in stable-x86_64-unknown-linux-gnu stable; do + rm -rf "$RUNNER_DATA_DIR/slots" + c="$(mkslot 1)" + mkrustup "$c" "$default" + t="$c/rustup/toolchains" + CACHE_BUDGET_BYTES=$(budget_without "$c" "$t/1.70.0-x86_64-unknown-linux-gnu" \ + "$t/nightly-2024-01-01-x86_64-unknown-linux-gnu") + prune_caches >/dev/null + gone "$t/1.70.0-x86_64-unknown-linux-gnu" + gone "$t/nightly-2024-01-01-x86_64-unknown-linux-gnu" + kept "$t/1.80.0-x86_64-unknown-linux-gnu/data.bin" + kept "$t/stable-x86_64-unknown-linux-gnu/data.bin" + + # No budget at all: everything but the default goes. + CACHE_BUDGET_BYTES=1 + prune_caches >/dev/null + gone "$t/1.80.0-x86_64-unknown-linux-gnu" + kept "$t/stable-x86_64-unknown-linux-gnu/data.bin" + kept "$c/rustup/settings.toml" +done + +finish "rustup default toolchain protected" diff --git a/tests/pruner-toolcache.sh b/tests/pruner-toolcache.sh new file mode 100755 index 0000000..39aa561 --- /dev/null +++ b/tests/pruner-toolcache.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# toolcache: / is evicted whole, taking its .complete marker +# with it so setup-* actions never see a "complete" tool that is gone. Needs GNU tools: +# bash tests/pruner-toolcache.sh +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/lib.sh +source "$here/lib.sh" + +cache0="$(mkslot 1)" +t="$cache0/toolcache" + +# mktool +mktool() { + mkfile "$t/$1/$2/x64/bin/$1" 100000 "$3" + touch -a -d "$3 days ago" "$t/$1/$2/x64.complete" +} +mktool node 18.0.0 90 +mktool node 20.0.0 1 +mktool Python 3.9.0 80 # the only Python version + +CACHE_BUDGET_BYTES=$(budget_without "$cache0" "$t/node/18.0.0" "$t/Python/3.9.0") +prune_caches >/dev/null + +gone "$t/node/18.0.0" +gone "$t/node/18.0.0/x64.complete" +gone "$t/Python" +kept "$t/node/20.0.0/x64/bin/node" +kept "$t/node/20.0.0/x64.complete" + +finish "toolcache version evicted with its .complete marker"