diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8d47f2..d7f0f64 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,20 @@ concurrency: cancel-in-progress: true jobs: + scripts: + name: Shell scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: shellcheck + run: shellcheck -x pruner.sh tests/*.sh + + - name: tests + run: | + set -euo pipefail + for t in tests/*.sh; do bash "$t"; done + matrix: name: Generate build matrix runs-on: ubuntu-latest @@ -83,14 +97,14 @@ jobs: gate: name: All builds passed - needs: [matrix, build] + needs: [scripts, matrix, build] if: always() runs-on: ubuntu-latest steps: - name: Check that all required jobs succeeded run: | - if [ "${{ needs.matrix.result }}" != "success" ] || [ "${{ needs.build.result }}" != "success" ]; then - echo "::error::A required job failed or was cancelled (matrix: ${{ needs.matrix.result }}, build: ${{ needs.build.result }})" + if [ "${{ needs.scripts.result }}" != "success" ] || [ "${{ needs.matrix.result }}" != "success" ] || [ "${{ needs.build.result }}" != "success" ]; then + echo "::error::A required job failed or was cancelled (scripts: ${{ needs.scripts.result }}, matrix: ${{ needs.matrix.result }}, build: ${{ needs.build.result }})" exit 1 fi echo "All builds succeeded" diff --git a/README.md b/README.md index 562c42d..24a7093 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,12 @@ The `pruner` service ([`pruner.sh`](pruner.sh)) keeps slow work off the job path: - **Nightly:** trims the Gradle, bun, mise and hk caches to `CACHE_CAP_BYTES` - each (default 20 GiB), dropping the least recently used files first. It then + each (default 20 GiB), least recently used first. The bun cache drops whole + package folders (`install/cache/@@@@1`, by the atime of their + `package.json`) and never `bin/`: bun reuses a package folder as complete + whenever it exists, so a single evicted file would go missing from every + later install. mise tool versions are dropped whole too; the Gradle, mise + download and hk caches drop single files. It then runs `docker system prune --filter until=24h` and `docker builder prune --filter until=72h`. Named volumes are never pruned. - **Hourly:** fills the action archive cache, capped at 2 GiB. diff --git a/pruner.sh b/pruner.sh index 8b701ae..9dedb1b 100755 --- a/pruner.sh +++ b/pruner.sh @@ -6,7 +6,9 @@ # 21-39 s of runner time each time and freeing nothing (issue #16). Here they # run once a day, off the job path: # 1. LRU size caps on the per-runner cache volumes (mounted under -# $CACHES_ROOT//, see docker-compose.yml). +# $CACHES_ROOT//, see docker-compose.yml). Caches whose +# entries are only valid whole (bun packages, mise tool versions) evict +# whole entries; the rest evict single files. # 2. docker system prune / builder prune on the host daemon (docker.sock). # 3. Hourly, in the background: fill the shared action archive cache. # @@ -79,7 +81,7 @@ lru_cap() { rec="${rec#* }" # drop atime fsize="${rec%% *}" path="${rec#* }" - [ -n "$path" ] && [ -n "$fsize" ] || continue + if [ -z "$path" ] || [ -z "$fsize" ]; then continue; fi rm -f "$path" 2>/dev/null || true freed=$((freed + fsize)) done < <(find "$dir" -type f -printf '%A@\t%s\t%p\0' 2>/dev/null | sort -z -n) @@ -108,6 +110,49 @@ lru_cap_mise_installs() { log "$installs freed ~${freed}B" } +# bun treats install/cache/@@@@/ as complete whenever the folder +# exists, so a file evicted from it is silently missing from every later +# node_modules (next@16.3.8 lost dist/compiled/server-only/empty.js, breaking +# every Next build). 0-byte files went first: copying them never reads, so +# their atime never moved. Evict whole package folders instead, least recently +# used first by the atime of their package.json (read on every install and +# import), until the whole volume ($1, mounted from /root/.bun) fits under the +# cap. Never touches $1/bin (the bun binary itself) or the @t@ manifest piles. +lru_cap_bun() { + local dir="$1" cap="${2:-$CACHE_CAP_BYTES}" + local cache="$dir/install/cache" + local excess freed=0 rec path dsize base t + [ -d "$cache" ] || return 0 + # Leftovers of interrupted extractions; the age guard spares live installs. + find "$cache" -mindepth 1 -maxdepth 2 ! -name .tmp \( -name '*.tmp' -o -name '*.part' \) \ + -mmin +60 -exec rm -rf {} + 2>/dev/null + find "$cache/.tmp" -mindepth 1 -maxdepth 1 -mmin +60 -exec rm -rf {} + 2>/dev/null + excess=$(excess_bytes "$dir" "$cap") || return 0 + log "evicting least recently used packages from $cache" + while IFS= read -r -d '' rec; do + [ "$freed" -ge "$excess" ] && break + path="${rec#* }" + [ -d "$path" ] || continue + dsize=$(dir_bytes "$path") + rm -rf "$path" 2>/dev/null || true + freed=$((freed + ${dsize:-0})) + # Drop the /@@@ symlink pointing at it, and the dir if empty. + base="${path##*/}" + rm -f "${path%/*}/${base%%@*}/${base#*@}" 2>/dev/null + rmdir "${path%/*}/${base%%@*}" 2>/dev/null + done < <( + # @... at the top level, @scope/@... one level down. + find "$cache" -mindepth 1 -maxdepth 2 -type d -name '*@*' ! -name '@*' \ + \( -path "$cache/@*/*" -o ! -path "$cache/*/*" \) -print0 2>/dev/null | + while IFS= read -r -d '' path; do + t=$(stat -c %X "$path/package.json" 2>/dev/null || stat -c %Y "$path" 2>/dev/null) || continue + printf '%s\t%s\0' "$t" "$path" + done | sort -z -n + ) + find "$cache" -mindepth 1 -maxdepth 1 -name '@*' ! -name '@t@' -type d -empty -delete 2>/dev/null || true + log "$cache freed ~${freed}B" +} + prune_caches() { local runner for runner in "$CACHES_ROOT"/*/; do @@ -115,9 +160,9 @@ prune_caches() { runner="${runner%/}" log "capping caches under $runner" lru_cap "$runner/gradle" - lru_cap "$runner/bun" + lru_cap_bun "$runner/bun" lru_cap "$runner/mise-cache" - lru_cap "$runner/hk" + lru_cap "$runner/hk" # configs/.json + pklr/packages/.zip: whole files lru_cap_mise_installs "$runner/mise-data" done lru_cap "$ACTION_CACHE_DIR" "$ACTION_CACHE_CAP_BYTES" @@ -146,8 +191,11 @@ prefetch_one() { local name="$1" ref="$2" sha="$2" dir file tmp dir="$ACTION_CACHE_DIR/${name//\//_}" if ! [[ "$sha" =~ ^[0-9a-f]{40}$ ]]; then - sha=$(gh_curl -H 'Accept: application/vnd.github.sha' "$GITHUB_API/repos/$name/commits/$ref") && - [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { log "prefetch: cannot resolve $name@$ref" >&2; return 1; } + sha=$(gh_curl -H 'Accept: application/vnd.github.sha' "$GITHUB_API/repos/$name/commits/$ref") + if ! [[ "$sha" =~ ^[0-9a-f]{40}$ ]]; then + log "prefetch: cannot resolve $name@$ref" >&2 + return 1 + fi fi file="$dir/$sha.tar.gz" [ -s "$file" ] && return 0 @@ -191,7 +239,7 @@ prefetch_actions() { refs=$(sort -u "$work/refs") rm -rf "$work" while read -r name ref; do - [ -n "$name" ] && [ -n "$ref" ] || continue + if [ -z "$name" ] || [ -z "$ref" ]; then continue; fi n=$((n + 1)) if out=$(prefetch_one "$name" "$ref"); then [ "$out" = 1 ] && fetched=$((fetched + 1)) @@ -225,6 +273,9 @@ run_once() { log "done. Disk free: ${before} -> ${after}" } +# Sourced (tests/): define the functions only. +[ "${BASH_SOURCE[0]}" = "$0" ] || return 0 + if [ "${1:-}" = prefetch ]; then prefetch_actions exit 0 diff --git a/tests/pruner-bun.sh b/tests/pruner-bun.sh new file mode 100755 index 0000000..aca92af --- /dev/null +++ b/tests/pruner-bun.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# pruner.sh must evict whole bun packages, never single files: bun reuses a +# cache folder as complete whenever it exists. Builds a fake /root/.bun volume +# (packages holding 0-byte files with old atimes, a scoped package, the +# / symlinks, @t@ piles and bin/bun), runs prune_caches under a small +# cap, then checks every surviving package is complete and every evicted one +# is fully gone. Needs GNU find/stat/du (Linux, or `docker run ubuntu`): +# bash tests/pruner-bun.sh +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +fails=0 +fail() { echo "FAIL: $*"; fails=$((fails + 1)); } + +bun="$work/caches/1/bun" +cache="$bun/install/cache" +mkdir -p "$bun/bin" "$cache/@t@" +head -c 300000 /dev/zero >"$bun/bin/bun" +ln -s /root/.bun/bin/bun "$bun/bin/bunx" +head -c 2000 /dev/zero >"$cache/@t@/0061322e0ad01aa4.pile" + +# make_pkg +make_pkg() { + local entry="$1" days="$2" d="$cache/$1" base + mkdir -p "$d/dist/compiled/server-only" + head -c 100000 /dev/urandom >"$d/index.js" + echo '{"name":"x"}' >"$d/package.json" + : >"$d/dist/compiled/server-only/empty.js" # 0-byte, atime never refreshes + : >"$d/LICENSE.empty" + # 0-byte files are the oldest thing in the volume, as in production. + find "$d" -type f -empty -exec touch -a -d '400 days ago' {} + + touch -a -d "$days days ago" "$d/package.json" + base="${entry##*/}" + mkdir -p "$(dirname "$d")/${base%%@*}" + ln -s "/root/.bun/install/cache/$entry" "$(dirname "$d")/${base%%@*}/${base#*@}" + find "$d" -type f | sed "s|^$d/||" | sort >"$work/${entry//\//_}.files" +} + +make_pkg 'old-a@1.0.0@@@1' 90 +make_pkg 'old-b@2.0.0@@@1' 80 +make_pkg '@scope/old-c@3.0.0@@@1' 70 +make_pkg 'next@16.3.8@@@1' 1 +make_pkg 'next@16.4.2@@@1' 2 +make_pkg '@scope/fresh@1.0.0@@@1' 3 +make_pkg 'nopkgjson@1.0.0@@@1' 0 +rm "$cache/nopkgjson@1.0.0@@@1/package.json" # falls back to dir mtime +sed -i '/^package.json$/d' "$work/nopkgjson@1.0.0@@@1.files" +touch -d '1 hour ago' "$cache/nopkgjson@1.0.0@@@1" + +# Stale and live partial downloads. +mkdir -p "$cache/.tmp/stale" "$cache/.tmp/live" +touch -d '2 days ago' "$cache/.tmp/stale" +head -c 10 /dev/zero >"$cache/half@1.0.0@@@1.tgz.part" +touch -d '2 days ago' "$cache/half@1.0.0@@@1.tgz.part" + +# Cap = volume minus the three oldest packages (old-a, old-b, @scope/old-c), +# plus half a package of slack: exactly those three must go, nothing else. +sz() { du -sb "$1" | awk '{print $1}'; } +cap=$(($(sz "$bun") - $(sz "$cache/old-a@1.0.0@@@1") - $(sz "$cache/old-b@2.0.0@@@1") \ + - $(sz "$cache/@scope/old-c@3.0.0@@@1") - 2 * 4096 + 50000)) +# shellcheck source=pruner.sh +source "$here/../pruner.sh" +CACHES_ROOT="$work/caches" CACHE_CAP_BYTES="$cap" prune_caches + +for entry in 'old-a@1.0.0@@@1' 'old-b@2.0.0@@@1' '@scope/old-c@3.0.0@@@1'; do + base="${entry##*/}" + [ ! -e "$cache/$entry" ] || fail "$entry should be evicted" + [ ! -L "$(dirname "$cache/$entry")/${base%%@*}/${base#*@}" ] || fail "$entry symlink left behind" + [ ! -e "$(dirname "$cache/$entry")/${base%%@*}" ] || fail "$entry name dir left behind" +done + +for entry in 'next@16.3.8@@@1' 'next@16.4.2@@@1' '@scope/fresh@1.0.0@@@1' 'nopkgjson@1.0.0@@@1'; do + d="$cache/$entry" + if [ ! -d "$d" ]; then + fail "$entry should survive" + continue + fi + diff <(find "$d" -type f | sed "s|^$d/||" | sort) "$work/${entry//\//_}.files" >/dev/null || + fail "$entry is incomplete" + base="${entry##*/}" + [ -L "$(dirname "$d")/${base%%@*}/${base#*@}" ] || fail "$entry symlink removed" +done + +[ -f "$cache/next@16.3.8@@@1/dist/compiled/server-only/empty.js" ] || fail "next lost server-only/empty.js" +[ -s "$bun/bin/bun" ] || fail "bin/bun removed" +[ -L "$bun/bin/bunx" ] || fail "bin/bunx removed" +[ -f "$cache/@t@/0061322e0ad01aa4.pile" ] || fail "@t@ pile removed" +[ -d "$cache/@scope" ] || fail "@scope dir removed while it still holds a package" +[ ! -e "$cache/.tmp/stale" ] || fail "stale .tmp entry kept" +[ -e "$cache/.tmp/live" ] || fail "live .tmp entry removed" +[ ! -e "$cache/half@1.0.0@@@1.tgz.part" ] || fail "stale .part kept" + +size=$(sz "$bun") +[ "$size" -le "$cap" ] || fail "volume ${size}B still over the ${cap}B cap" + +if [ "$fails" -gt 0 ]; then + echo "$fails check(s) failed" + exit 1 +fi +echo "PASS: bun cache evicted by whole package (volume ${size}B)"