From 103c6228558c86662aefb0383dd91bf9023b4a75 Mon Sep 17 00:00:00 2001 From: lkdvos Date: Fri, 2 Oct 2026 15:29:54 -0400 Subject: [PATCH] Make ClockCache hits lock-free ClockCache is now an open-addressing table of immutable entries, published atomically and read with acquire loads, so hits take no lock and write no shared cache line (hits and misses are counted per thread). Misses and all other mutations still take the lock. Shared hits stay at ~25 ns at 8 threads (was ~500 ns) and are 2x faster single-threaded. Task-local caches keep the previous locked implementation, now the internal LocalClockCache, to avoid the per-thread counters in every task's cache. Also: iteration has unknown length (collect could throw under threads), and get! forwards to _get! with a specialized default (no allocation on 1.10). Adds concurrency stress tests, benchmark/contention.jl and research/lockfree-clock.md. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 +- benchmark/Project.toml | 1 + benchmark/contention.jl | 134 +++++++++++++++++ research/design.md | 18 ++- research/lockfree-clock.md | 239 ++++++++++++++++++++++++++++++ src/Cached.jl | 1 + src/cachestyle.jl | 4 +- src/call.jl | 5 +- src/containers/clock.jl | 278 ++++++++++++++++++++++++++++++----- src/containers/interface.jl | 31 ++-- src/containers/localclock.jl | 79 ++++++++++ test/Project.toml | 2 + test/test_concurrency.jl | 117 +++++++++++++++ test/test_containers.jl | 8 +- 14 files changed, 860 insertions(+), 63 deletions(-) create mode 100644 benchmark/contention.jl create mode 100644 research/lockfree-clock.md create mode 100644 src/containers/localclock.jl create mode 100644 test/test_concurrency.jl diff --git a/CLAUDE.md b/CLAUDE.md index 5866018..99e16a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,7 +44,11 @@ julia --project=/tmp/runic -e 'using Pkg; Pkg.add("Runic"); using Runic; Runic.m - `src/registry.jl`: global caches, one untyped `C{Any,Any}` per function (its budget), looked up lock-free in an atomically published `IdDict` snapshot. - `src/containers/`: `LRU` and `ClockCache`, sharing the `AbstractCache` interface. Keys are - stored as `Key{Any}` and probed with a concretely typed `Key{K}`, so hits never box. + stored as `Key{Any}` (or `Any` plus hash) and probed with a concretely typed `Key{K}`, so hits + never box. `LRU` takes its lock for every operation; `ClockCache` (`clock.jl`) has lock-free + hits, through an atomically published open-addressing table of immutable entries, and keeps + the three invariants listed in its source (`research/lockfree-clock.md`). Task-local caches + use `LocalClockCache` (`localclock.jl`), the locked `ClockCache`, via `_localtype` in `call.jl`. - `src/preferences.jl`: default settings from Preferences.jl, resolved once per function (runtime > function > package > `[Cached]` > built-in), and `set_cache_preferences!`. - `src/api.jl`: `cache_info`, `empty_caches!`, `set_cache_size!`, and the `cache_dashboard` and diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 05191f6..1ac0d4a 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -4,6 +4,7 @@ Cached = "1b238080-9255-4fe9-b224-89eb24efe93b" LRUCache = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [sources] Cached = {path = ".."} diff --git a/benchmark/contention.jl b/benchmark/contention.jl new file mode 100644 index 0000000..5192dc1 --- /dev/null +++ b/benchmark/contention.jl @@ -0,0 +1,134 @@ +# Cost of a lookup when all threads share one cache, against task-local caches. +# "locked Clock" is the `ClockCache` of task-local caches (`Cached.LocalClockCache`), whose hits +# take the lock, shared here for comparison. +# +# for t in 1 2 4 8; do julia --heap-size-hint=4G --project=benchmark -t $t benchmark/contention.jl; done +# +# One task per thread, each doing `M` lookups from its own key sequence. Reports the median over +# `REPS` runs of the time per lookup within a task (ns/lookup, lower is better; perfect scaling +# keeps it constant as threads are added), and the bytes allocated per lookup. + +using Random, Printf, Statistics +using Cached: Cached, ClockCache, LocalClockCache, LRU, @cached, GlobalCache, TaskLocalCache + +const M = 200_000 +const REPS = 7 + +const CONTAINERS = [ + "ClockCache" => n -> ClockCache{Any, Any}(; maxsize = n), + "locked Clock" => n -> LocalClockCache{Any, Any}(; maxsize = n), + "LRU" => n -> LRU{Any, Any}(; maxsize = n), + "task-local Clock" => nothing, # one locked Clock per task, uncontended +] + +function zipf(rng, n, s, len) + cdf = cumsum(1 ./ (1:n) .^ s) + cdf ./= cdf[end] + return [searchsortedfirst(cdf, rand(rng)) for _ in 1:len] +end + +tuplekey(i) = (i, i % 7, :leg, Float64(i)) +compute(k::Int) = 3k +compute(k::Tuple) = k[1] + +# name => (key sequence per task, cache capacity, prefill keys) +function workloads(ntasks) + rngs = [Xoshiro(t) for t in 1:ntasks] + return [ + "hits, Int keys (1k)" => ([rand(r, 1:1_000, M) for r in rngs], 10_000, 1:1_000), + "hits, 4-tuple keys (1k)" => ([tuplekey.(rand(r, 1:1_000, M)) for r in rngs], 10_000, tuplekey.(1:1_000)), + "hits, one hot Int key" => ([fill(7, M) for r in rngs], 10_000, 7:7), + "zipf 1.0 (100k keys, cap 10k)" => ([zipf(r, 100_000, 1.0, M) for r in rngs], 10_000, 1:0), + "uniform (20k keys, cap 10k)" => ([rand(r, 1:20_000, M) for r in rngs], 10_000, 1:0), + ] +end + +function run!(c, seq) + acc = 0 + for k in seq + acc += get!(() -> compute(k), c, k)::Int + end + return acc +end + +# ns per lookup in each task, and total bytes allocated +function threaded(make, seqs, cap, prefill) + shared = make === nothing ? nothing : make(cap) + caches = [shared === nothing ? LocalClockCache{Any, Any}(; maxsize = cap) : shared for _ in seqs] + for c in unique(caches), k in prefill + get!(() -> compute(k), c, k) + end + times = zeros(length(seqs)) + start = Threads.Atomic{Int}(0) + bytes = @allocated @sync for (t, seq) in enumerate(seqs) + Threads.@spawn begin + Threads.atomic_add!(start, 1) + while start[] < length(seqs) # start together + ccall(:jl_cpu_pause, Cvoid, ()) + end + t0 = time_ns() + run!(caches[t], seq) + times[t] = (time_ns() - t0) / length(seq) + end + end + return mean(times), bytes / sum(length, seqs) +end + +# ns per hit through `@cached` functions using each cache style, Int and 4-tuple keys +@cached f_clock(x) = compute(x) +@cached f_locked(x) = compute(x) +@cached f_lru(x) = compute(x) +@cached f_local(x) = compute(x) +Cached.CacheStyle(::typeof(f_clock), x) = GlobalCache{ClockCache}() +Cached.CacheStyle(::typeof(f_locked), x) = GlobalCache{LocalClockCache}() +Cached.CacheStyle(::typeof(f_lru), x) = GlobalCache{LRU}() +Cached.CacheStyle(::typeof(f_local), x) = TaskLocalCache{ClockCache}() +const CACHED = ["ClockCache" => f_clock, "locked Clock" => f_locked, "LRU" => f_lru, "task-local Clock" => f_local] + +function callall(f::F, seq) where {F} + acc = 0 + for k in seq + acc += f(k)::Int + end + return acc +end + +function threaded_cached(f, seqs, keys) + times = zeros(length(seqs)) + bytes = @allocated @sync for (t, seq) in enumerate(seqs) + Threads.@spawn begin + callall(f, keys) # fill this task's cache (task-local) + t0 = time_ns() + callall(f, seq) + times[t] = (time_ns() - t0) / length(seq) + end + end + return mean(times), bytes / sum(length, seqs) +end + +nt = Threads.nthreads() +println("Julia ", VERSION, ", ", nt, " threads (", nt, " tasks), ", M, " lookups per task, median of ", REPS, " runs") +println("load average: ", read(`cat /proc/loadavg`, String)) +@printf("%-30s %-17s %10s %10s %8s\n", "workload", "container", "ns/lookup", "(min)", "B/lookup") +for (wname, (seqs, cap, prefill)) in workloads(nt) + for (cname, make) in CONTAINERS + threaded(make, seqs, cap, prefill) # compile + rs = [threaded(make, seqs, cap, prefill) for _ in 1:REPS] + @printf( + "%-30s %-17s %10.1f %10.1f %8.2f\n", wname, cname, + median(first.(rs)), minimum(first.(rs)), median(last.(rs)) + ) + end +end +for (kname, keys) in ("@cached hits, Int" => 1:1_000, "@cached hits, 4-tuple" => tuplekey.(1:1_000)) + seqs = [rand(Xoshiro(t), keys, M) for t in 1:nt] + for (cname, f) in CACHED + foreach(f, keys) + threaded_cached(f, seqs, keys) + rs = [threaded_cached(f, seqs, keys) for _ in 1:REPS] + @printf( + "%-30s %-17s %10.1f %10.1f %8.2f\n", kname, cname, + median(first.(rs)), minimum(first.(rs)), median(last.(rs)) + ) + end +end diff --git a/research/design.md b/research/design.md index 2f2e81f..2979a56 100644 --- a/research/design.md +++ b/research/design.md @@ -77,18 +77,20 @@ Cached provides its own containers, so it no longer depends on LRUCache.jl. Both implement the same small interface: `get!`, `get`, `haskey`, `empty!`, `resize!`, `length`, and hit/miss statistics. - **`LRU{K,V}`** is array-backed. It uses a `Dict` index, slots stored in vectors, and `prev`/`next` stored as integer vectors. Nodes are never allocated, and eviction is exact LRU. -- **The index is a `Dict{Key{Any},Int}`.** Each stored `Key{Any}` holds the key and its hash. Lookups probe it with a concretely typed `Key{K}`, whose `isequal` checks `s.key isa K` before comparing, so the comparison is static and the key is never boxed. A `Key{Any}` cannot serve as the probe: storing the key in an `Any` field boxes it (96 B per lookup on TensorKit-like keys). +- **The index of `LRU` is a `Dict{Key{Any},Int}`.** Each stored `Key{Any}` holds the key and its hash. Lookups probe it with a concretely typed `Key{K}`, whose `isequal` checks `s.key isa K` before comparing, so the comparison is static and the key is never boxed. A `Key{Any}` cannot serve as the probe: storing the key in an `Any` field boxes it (96 B per lookup on TensorKit-like keys). - This holds for `C{Any,Any}` as well, where a plain `Dict{Any,…}` would allocate on every hit. - Eviction never re-hashes, because the hash is stored. - Keys of different types are different entries, even when `isequal` (`f(3)` and `f(3.0)`). Lookups on a typed cache convert the key to `K` first. -- **`ClockCache{K,V}`** uses second-chance eviction: a ring of slots with a reference bit. A hit only sets the bit and never reorders the ring, which makes it cheap for read-heavy shared caches. -- Each container takes a size limit, either a **count** or **bytes** measured by a `by` function. Each one holds its own lock. Task-local containers skip the lock. +- **`ClockCache{K,V}`** uses second-chance eviction: a ring of entries with a reference bit. A hit only sets the bit (if it is clear) and never reorders the ring, so **hits take no lock** (see `lockfree-clock.md`). + - Its index is a fixed-size open-addressing table of immutable entries (each holding the key as `Any` with its hash, probed with a `Key{K}` as above), replaced as a whole when it fills up and published atomically. Readers load it and its slots with acquire ordering; every mutation (miss, insertion, eviction, `delete!`, `resize!`, `empty!`) still takes the lock. + - Hits and misses are counted per thread, in separate cache lines, so hits write no shared memory. + - A lookup racing with a removal may return the removed entry; the value always belongs to its key. +- Each container takes a size limit, either a **count** or **bytes** measured by a `by` function, and holds its own lock. +- **Task-local caches** of the default container use `LocalClockCache`, an internal `ClockCache` with the `Dict` index above and locked hits (uncontended, so cheap), which saves the per-thread counters (64 B per thread) in every task's cache. It evicts in the same order. - **Default for `GlobalCache`: `ClockCache`.** - - On synthetic workloads (`benchmark/containers.jl`), it beat `LRU` everywhere single-threaded (28 vs 32 ns for an all-hit lookup) and had a slightly better hit rate under Zipf access. Both were 1.5–3× faster than LRUCache.jl. - - On the TensorKit and SUNRepresentations integration, the two were indistinguishable. - - It stays the default because a hit only sets a bit, which makes lock-free hits possible. -- **open**: contention. With 8 threads, every container's per-lookup cost *rises* (all hits: about 50–75 ns, against 28–32 ns single-threaded), because every lookup takes the cache's single lock. - Sharding by key hash, or lock-free reads for `ClockCache`, are the candidate fixes. `TaskLocalCache` is the workaround meanwhile. + - On synthetic workloads (`benchmark/containers.jl`), the locked `ClockCache` beat `LRU` everywhere single-threaded (28 vs 32 ns for an all-hit lookup) and had a slightly better hit rate under Zipf access. Both were 1.5–3× faster than LRUCache.jl. On the TensorKit and SUNRepresentations integration, the two were indistinguishable. + - With lock-free hits it is about 2× faster again single-threaded (14 vs 31 ns, `Int` keys), and shared hits scale: about 25 ns at 8 threads, against about 500 ns with a lock (`benchmark/contention.jl`). That is faster than task-local caches. +- **open**: contended misses. Misses still serialize on the cache's lock, so miss-heavy shared workloads (all threads computing new keys) scale poorly; `TaskLocalCache` or a sharded cache would help there (see `lockfree-clock.md`). `LRU` hits still take the lock. ## Limits diff --git a/research/lockfree-clock.md b/research/lockfree-clock.md new file mode 100644 index 0000000..b34f69e --- /dev/null +++ b/research/lockfree-clock.md @@ -0,0 +1,239 @@ +# Lock-free hits for ClockCache + +Investigation of 2026-10-02 (branch `lockfree-clock`), adopted: `ClockCache` is now the lock-free design described here, and the locked design lives on as the internal `LocalClockCache` for task-local caches. + +## Summary + +- **`ClockCache`** (`src/containers/clock.jl`) is an open-addressing hash table of immutable entries with a CLOCK ring next to it. Readers take no lock and write no shared cache line. Every mutation still runs under the one `ReentrantLock`. +- **Hits**: + - They scale: a shared-cache hit stays at 24–45 ns from 1 to 16 threads. The locked design went from 31 to about 1000 ns. + - They are faster than task-local caches. + - They are also **2× faster single-threaded** (14 vs 31 ns for `Int` keys, 24 vs 39 ns for 4-tuples), because a hit no longer takes and releases a lock. + - Hits stay inferred and allocate nothing, on 1.10 through 1.13. +- **Misses** still serialize on the lock. With trivial computations they are only 1.3–1.6× faster than with the locked design (for example 3.9 vs 6.4 µs per lookup at 16 threads on Zipf). A miss allocates about 45 B more (one `Entry` object), and a stored entry uses about 25 B more. +- **Sharding the lock (16 or 64 locked caches) does not fix hits.** Every thread touches every shard's lock, so the cache lines still bounce. At 8+ threads, 16 shards were slower than one lock, and 64 shards about equal. Sharding does help miss-heavy workloads (2–5×). The `ShardedCache` prototype was dropped; its numbers are kept below. +- Sharding *lock-free* caches gave the best mixed-workload numbers in a quick experiment (below). It costs a per-shard budget, so it is a follow-up, not adopted. +- **Task-local caches** keep the locked design (`LocalClockCache`): a single task gains nothing from lock-free hits, and the per-thread counters would cost 64 B per thread in every task's cache. +- **Complexity**: about 200 lines of code plus about 60 lines of comments in `clock.jl`, against 86 lines for the locked design (`localclock.jl`), which also uses most of `interface.jl`. There are 3 core invariants and a few local rules (listed below). It needs no new dependencies. + +## Q1: why the `Dict` index cannot be read without the lock + +A lock-free read of the `Dict` is **memory-unsafe**, not just stale. The Julia manual says so in general: "If data-races are introduced, Julia is not memory safe". The `Dict` code makes it concrete. + +- **`ht_keyindex` on 1.13** (`base/dict.jl:238`) reads `sz = length(h.keys)` and `keys = h.keys` once. Inside an `@inbounds` loop annotated `:noub`, it reads `h.slots` on every probe step (`isslotempty(h, index)`). + - `rehash!` replaces `h.slots`, `h.keys` and `h.vals` with new `Memory`s. + - It is called from `_setindex!` when `(count + ndel)*3 > 2sz`, with the new size `max(count*4, 4)`. After many deletions this is *smaller* than the old size, which is exactly the steady churn of a full cache. + - A reader then indexes the new, smaller `slots` with an index computed for the old size. That is an out-of-bounds read. +- **`empty!` on 1.10** does `empty!(h.keys); resize!(h.keys, sz)`. On 1.11+ it `_unsetindex!`es every slot. +- **The stored keys are inline `Key{Any}`s** (a pointer and a hash, two words). A concurrent write is not atomic, so a reader can see a torn key or a null pointer. + +**A seqlock (optimistic read, then validate a version counter) does not help.** Validation happens after the read, but the read itself can already crash. A seqlock needs a structure whose reads are memory-safe under concurrent mutation, and once a table has that property (as below), immutable entries make the version counter unnecessary. + +**No Base structure gives safe concurrent reads on 1.10.** + +- `Dict`, `IdDict` and `WeakKeyDict` all lock or are unsafe in the same way. +- `Base.Lockable` is just a lock. +- `Base.PersistentDict` (a HAMT, 1.11+, internal, used by ScopedValues) is immutable, so a published snapshot would be safe to read. It is not on 1.10, every insert copies a path and allocates, and the CLOCK ring would still be separate. +- ConcurrentCollections.jl would add a dependency. + +## Q2: candidate designs + +| design | hits lock-free | memory-safe on 1.10 | miss cost | verdict | +|:-|:-|:-|:-|:-| +| (a) seqlock over the `Dict` | yes | **no** (see Q1) | unchanged | rejected | +| (b) fixed-size open-addressing table of immutable entries, replaced whole when full; acquire loads | yes | yes | +1 allocation (48 B), rare O(n) rebuild | **adopted** | +| (c) copy-on-write snapshot of the index on every write | yes | yes | O(capacity) per miss: a 10k-entry rebuild takes 0.2–0.35 ms (measured with `_rebuild!`), against about 0.1 µs now | rejected for caches with frequent misses | +| (d) sharded locks (16 or 64 locked caches) | no | yes | lower contention | prototyped (`ShardedCache`); does not fix hits | +| (e) reader–writer lock | no | yes | — | not built: every reader still does an atomic RMW on the shared reader count, which bounces the same cache line as the lock does now | +| (b) + (d): shards of lock-free caches | yes | yes | best under contention | measured as an experiment; per-shard budgets | + +**Counters.** A shared `hits += 1`, atomic or not, bounces a cache line on every hit. In my numbers that alone keeps a "lock-free" design from scaling. + +- `ClockCache` counts hits and misses **per thread**, in separate 64-byte lines (`counts[8*(tid-1) + which]`), using a relaxed load and a relaxed store. +- The counts are exact: two tasks on one thread cannot interleave between the load and the store, since neither yields. +- Threads adopted after the cache was created (foreign threads) wrap around and may lose a few counts. +- The cost is `64 × maxthreadid()` bytes per cache, about 1 KB at 16 threads. That matters only if there are very many caches, so task-local caches use the locked `LocalClockCache`. + +## Q3: the reference bit + +The bit moved into the entry, as an `@atomic ref::Bool` field of the mutable `Entry`. This sidesteps the "no atomic vector elements on 1.10" problem entirely. + +- A hit does `(@atomic :monotonic e.ref) || (@atomic :monotonic e.ref = true)`. It only writes when the bit is clear, so hot entries cause no writes at all. +- The CLOCK hand clears bits under the lock. Readers may set a bit while the hand clears it, which is benign. +- Because readers can keep re-setting bits, the sweep is bounded: after two sweeps without finding a clear bit, the hand evicts the entry under it. + +The alternatives, if the bits stay in a vector: + +- `Vector{UInt8}` with `unsafe_load`/`unsafe_store!(p, x, :monotonic)` works on 1.10. Orderings on these functions are supported since 1.10. It is safe because the elements are bits types, so no GC barrier is involved. +- `AtomicMemory` with `@atomic m[i]` needs 1.12. The type exists on 1.11, but the indexing syntax does not. +- `Threads.Atomic` per slot costs an allocation and an indirection per slot. +- UnsafeAtomics.jl is not needed, since Base's orderings cover this. + +## Q4: consistent key/value pairs, reallocation, GC safety + +In the current layout (parallel `keys`/`vals` vectors, `push!` growth, `_reuseslot!`, `_unsetindex!`), a lock-free reader could: + +- read a reallocated (freed) buffer after a `push!`; +- see a null `Key{Any}` after `_unsetindex!`; +- pair the key of one entry with the value of the next entry in the reused slot. + +Validating after the read catches the last case only, and only after the unsafe read has already happened. `ClockCache` avoids all three by construction: + +- **One immutable `Entry` per key/value** (`key::Any`, `hash`, `val::V`, `size`, `pos` are all `const`; only `ref` is `@atomic`). A reader that loads an entry sees that entry's own key and value. +- **Tables never change length.** A table that would exceed half load (live entries plus tombstones) is replaced by a new one, published with `@atomic :release c.table = t`. Readers load the table with `:acquire` and keep using that one, so every index is in bounds. +- **Slot loads use `unsafe_load(pointer(t, i), :acquire)`** on `t::Vector{Any}`, under `GC.@preserve t`. The LLVM IR shows `load atomic … acquire` on 1.10, 1.11, 1.12 and 1.13, with 0 allocations. +- **Slot stores use ordinary `t[i] = x`.** It emits `store atomic … release` *and* the GC write barrier on all four versions, also verified in the IR. + - Writing references through `unsafe_store!` or `atomic_pointerset` would skip the write barrier. The GC could then free a young entry stored into an old table, so that is never done. +- **Nothing is unset.** Removal stores `TOMBSTONE`, or `nothing` (see invariant 3), so readers never see an undefined slot. +- **Removed entries stay alive** while a reader holds them (a rooted local), and are collected afterwards. The "freed slots release their values" test passes for the new type. + +## The final design + +**Files:** + +- `src/containers/clock.jl`: `ClockCache{K,V}`, the lock-free design. Its constructor, `show`, `cache_stats`, `resize!`, `delete!`, `empty!`, iteration and second-chance order are those of the locked design. +- `src/containers/localclock.jl`: `LocalClockCache{K,V}`, the previous locked `ClockCache`, internal. `_localtype` (`src/call.jl`) maps `TaskLocalCache{ClockCache}` (and `TaskLocalCache()` while `ClockCache` is the default container) to it. `TaskLocalCache{ClockCache{Any,Any}}` asks for the concrete type and gets the lock-free one. +- `src/containers/interface.jl`: + - `get!` forwards to `_get!(default, c, p::Key)`, which containers implement, with the key already converted and hashed. + - `default::F` forces specialization: without it, 1.10 boxed the closure on hits. + - `IteratorSize = SizeUnknown()` fixes a pre-existing race. `collect(c)` sized its result with `length(c)` and then iterated a different snapshot, which threw `ArgumentError` under `-t 8`, for every container. + - `_probe` (the key converted to `K`, or `nothing`) is shared by the `Dict`-indexed containers and `ClockCache`. +- **Tests:** + - `test/test_containers.jl` runs the whole container suite on `LRU`, `ClockCache` and `LocalClockCache`. + - `test/test_concurrency.jl`: 4×nthreads tasks mix `get!`/`get`/`setindex!`/`delete!`/`resize!`/`empty!`/iteration, on mixed key types (`Int`, 4-tuples, strings, vector-holding tuples), for count and size limits, on all three containers. Every returned value is checked against its key, and the stats are checked to be exact. + - The same file has a hot-key reader test under constant table rebuilds, `empty!` and `resize!`, and checks that `@cached` hits through global and task-local `ClockCache`s are inferred and allocate 0 bytes, and that the task-local one is a `LocalClockCache`. +- **Benchmark:** `benchmark/contention.jl` compares `ClockCache`, the locked design shared ("locked Clock"), `LRU`, and task-local caches. + +**Invariants a maintainer must keep** (also written in the source): + +1. Entries are immutable apart from `ref`. +2. A published table is never resized or `push!`ed. Growth means building a new table and doing a release store into `c.table`. +3. In a published table a slot changes only, under the lock: + - from `nothing` to an entry; + - from an entry to `TOMBSTONE`; + - from `TOMBSTONE` to an entry; + - from either to `nothing` when the *next* slot is `nothing`, in which case no probe continues past it anyway. + + Entries never move, so a present key is always found. Without the trailing cleanup, steady churn rebuilt the table every 10k–20k misses; with it, `used` stays at about 1.1× the entry count. + +**Local rules:** + +- Writers store slots only with `setindex!`, never through pointers (GC barrier). +- Readers load only with `_loadslot` (acquire). +- Every ring entry is in the current table, since `_remove!` searches for it by identity. +- The table stays at most half full, so probes end. + +**Semantic change.** A lookup that races with a mutation may return an entry that is being evicted, deleted or emptied at the same moment. A lookup during a rebuild may also see the previous table, and so miss a just-inserted key; that goes to the locked miss path, which checks again. The returned value always belongs to its key, and a task always sees its own writes. For memoization this is harmless. + +**Risks:** + +- **Memory model.** + - The acquire load is a documented API: `unsafe_load` with an ordering, since 1.10. + - The *release* half relies on `setindex!` of a reference into an `Array`/`Memory` compiling to `store atomic release`. Julia does this for memory safety, and I verified it on 1.10–1.13, but it is codegen behaviour, not documented API. A `Threads.atomic_fence()` before each store would make it independent of that at a small cost on misses. +- **x86 only measured.** On ARM the acquire/release pair is what makes it correct, and nothing relies on TSO. It is untested there, though. +- **Memory.** About 127 B per entry, against 103 B for the locked design (`summarysize`, 10k `Int=>Int` entries in `{Any,Any}`). Each miss allocates about 120 B, against about 75 B. Each cache has `64 × maxthreadid()` bytes of counters. +- **Misses are still serialized.** Each insertion takes the global lock and touches shared lines. With realistic (µs+) computations this matters much less than in the benchmark below, where computing costs nothing. + +## Benchmarks + +Setup: + +- Machine: `ccqlin038`, 32 cores, shared. The load average was 5–17 during the runs, so the numbers are noisy. +- `benchmark/contention.jl`: one task per thread, 200k lookups per task, median of 7 runs, as ns per lookup within a task. Constant across thread counts means perfect scaling. +- All caches are `{Any,Any}`, as in `@cached`. +- "task-local" means one locked cache per task: uncontended, but every task fills its own. +- The 8- and 16-thread sweeps were run twice (values `a/b`). Lock-free hits agreed within about 15%, contended paths within about 20%. +- Julia 1.13.1 unless noted. +- The 16-thread rows were measured before the machine's users asked for runs of at most 8 threads, so they were not repeated after the final code changes. The last change touched only `const pos` and the tests. + +In the tables below, `ClockCache` is the locked design (now `LocalClockCache`) and `LockFree` the current `ClockCache`; they were measured on the prototype, before the rename. + +**Shared hits, `Int` keys (1k keys), ns/lookup:** + +| threads | ClockCache | LockFree | Sharded(16) | Sharded(64) | task-local | +|-:|-:|-:|-:|-:|-:| +| 1 | 30.8 | **13.8** | 32.4 | 34.7 | 30.8 | +| 2 | 309 | **29** | 263 | 213 | 34 | +| 4 | 367 | **28** | 483 | 323 | 35 | +| 8 | 504/493 | **26/24** | 992 | 536/529 | 44/38 | +| 16 | 1002/1000 | **45/38** | 2130 | 1197/1270 | 64/46 | + +**Shared hits, 4-tuple keys `(i, i%7, :leg, Float64(i))`, ns/lookup:** + +| threads | ClockCache | LockFree | Sharded(16) | Sharded(64) | task-local | +|-:|-:|-:|-:|-:|-:| +| 1 | 39.2 | **24.0** | 41.3 | 43.9 | 39.6 | +| 2 | 397 | **39** | 262 | 232 | 43 | +| 4 | 439 | **38** | 478 | 351 | 42 | +| 8 | 556/639 | **31/29** | 934 | 553/556 | 66/48 | +| 16 | 1115/1239 | **45/36** | 1815 | 1168/1303 | 67/72 | + +**One hot key shared by all threads**, where sharding cannot help: + +| threads | ClockCache | LockFree | Sharded(64) | task-local | +|-:|-:|-:|-:|-:| +| 1 | 29.9 | **11.1** | 30.0 | 29.5 | +| 8 | 378/385 | **10.9/11.9** | 367/404 | 30.0 | +| 16 | 760/817 | **15.4/16.0** | 864/855 | 46.9/40.2 | + +**Through `@cached`** (registry lookup, `CacheStyle`, `::V` assert; task-local includes the `task_local_storage` lookup): + +| threads | Int: Clock | Int: LockFree | Int: task-local | 4-tuple: Clock | 4-tuple: LockFree | 4-tuple: task-local | +|-:|-:|-:|-:|-:|-:|-:| +| 1 | 36.3 | **19.7** | 63.3 | 54.0 | **39.1** | 74.3 | +| 2 | 453 | **28.6** | 68.5 | 606 | **40.0** | 74.4 | +| 4 | 354 | **29.1** | 70.8 | 380 | **42.4** | 76.7 | +| 8 | 538/550 | **28/32** | 87/72 | 793/763 | **43/41** | 114/99 | +| 16 | 1273/1081 | **46/32** | 111/103 | 1753/1467 | **76/51** | 148/120 | + +All hit rows allocate 0 B per lookup for shared caches. Task-local rows show about 1.3 B per lookup from filling each task's cache. + +**Mixed hit/miss workloads**, where the computation is trivial, so misses are pure insertion and eviction cost. ns/lookup, and B/lookup at 1 thread: + +| workload | threads | ClockCache | LockFree | Sharded(16) | Sharded(64) | task-local | +|:-|-:|-:|-:|-:|-:|-:| +| Zipf 1.0, 100k keys, cap 10k | 1 | 92 (28 B) | **66** (35 B) | 82 | 94 | 76 | +| | 2 | 1030 | 620 | 244 | **211** | 90 | +| | 4 | 1591 | 1016 | 890 | **615** | 182 | +| | 8 | 2845/2928 | 2036/2050 | 1731 | **922/990** | 252/241 | +| | 16 | 6529/6335 | 3912/3980 | 4116 | **2410/2441** | 340/390 | +| uniform, 20k keys, cap 10k (~50% misses) | 1 | 145 (40 B) | **113** (62 B) | 139 | 151 | 137 | +| | 8 | 4027/4792 | 3777/3232 | 2025 | **1035/1048** | 414/391 | +| | 16 | 11786/11671 | 7649/7513 | 4930 | **2186/2193** | 476/475 | + +**Experiment, not kept:** `ShardedCache` over lock-free shards (one-line change): + +| workload | threads | 16 shards | 64 shards | +|:-|-:|-:|-:| +| hits, Int keys | 8 | 57 | 42 | +| hits, Int keys | 16 | 43 | 52 | +| Zipf | 8 | 738 | 452 | +| Zipf | 16 | 1412 | 624 | +| uniform | 8 | 1307 | 788 | +| uniform | 16 | 2874 | 1239 | + +These are the best contended-miss numbers measured. The cost is per-shard budgets and per-shard counter arrays. + +**Other runs:** + +- **Julia 1.10.12**: hits at 1 thread are 43.8 ns for `ClockCache` and 14.9 ns for LockFree; `@cached` `Int` hits 52.0 and 22.3 ns. At 8 threads, `Int` hits are 2181 vs 26 ns, and `@cached` 2873 vs 31 ns. `ClockCache` contention is about 4× worse on 1.10. +- **A `Threads.SpinLock` in place of `ReentrantLock`** for the writer lock made contended misses *worse*: Zipf 2548 ns at 8 threads, 5140 ns at 16. It was reverted. +- **Single-thread BenchmarkTools** (`@belapsed`, 1.13): `Int` hits 30.1 → 13.5 ns, 4-tuple 37.8 → 23.2 ns. `LRU` is 33.2 and 41.7 ns. + +**After the refactor** (final code, 1.13.1, load average about 5, ns/lookup): + +| workload | threads | ClockCache | locked Clock | LRU | task-local | +|:-|-:|-:|-:|-:|-:| +| hits, `Int` keys | 1 | 13.5 | 30.9 | 34.6 | 30.9 | +| | 8 | 23.3 | 484 | 1084 | 34.9 | +| `@cached` hits, `Int` | 1 | 20.0 | 37.0 | 40.6 | 66.0 | +| | 8 | 28.1 | 529 | 1022 | 80.1 | +| `@cached` hits, 4-tuple | 1 | 40.4 | 53.5 | 58.2 | 74.9 | +| | 8 | 41.0 | 853 | 1844 | 104 | +| Zipf | 8 | 2086 | 2874 | 3955 | 223 | + +## Follow-ups + +1. **`AtomicMemory`, once 1.10 support is dropped** (or behind `@static VERSION >= v"1.12"`). The table can become `AtomicMemory{Any}` with `@atomic :acquire t[i]` / `@atomic :release t[i] = x`. That is the fully official API, removes the reliance on `setindex!` being a release store, and the pointer code (`_loadslot`, and the counters' `unsafe_load`/`unsafe_store!`) would go away. +2. **Shards of lock-free caches for miss-heavy shared workloads.** The experiment above gave 2–4× better contended misses, at the cost of per-shard budgets (one shard can fill while others are empty, and small limits cannot be honoured exactly) and per-shard counter arrays. Worth it only if such workloads matter in practice. +3. **One implementation for both.** The per-thread counter array could be sized by the caller (one 64-byte line for a task-local cache, which a single task updates exactly), which would let task-local caches use the lock-free `ClockCache` too and drop `LocalClockCache`; the price is about 25 B more per entry and 45 B more per miss. diff --git a/src/Cached.jl b/src/Cached.jl index a20a320..97365eb 100644 --- a/src/Cached.jl +++ b/src/Cached.jl @@ -18,6 +18,7 @@ using Preferences: @load_preference, load_preference, has_preference, set_prefer include("containers/interface.jl") include("containers/lru.jl") include("containers/clock.jl") +include("containers/localclock.jl") include("registry.jl") include("preferences.jl") diff --git a/src/cachestyle.jl b/src/cachestyle.jl index 37566c2..958e5a7 100644 --- a/src/cachestyle.jl +++ b/src/cachestyle.jl @@ -54,7 +54,9 @@ Strategy that stores results in caches local to the current task, of container t `C` is either an `AbstractCache` or an `AbstractDict` type such as `Dict`; if it is not already concrete, it is completed to `C{K, V}`. Without `C`, the container is the one set by the `container` preference. Task-local caches need no locking but are not shared, not bounded -unless `C` is, and are not visible to [`cache_info`](@ref). +unless `C` is, and are not visible to [`cache_info`](@ref). `TaskLocalCache{ClockCache}` uses +an internal variant of [`ClockCache`](@ref) whose hits take the lock, which costs nothing +uncontended and saves the memory of the per-thread counters. """ struct TaskLocalCache{C} <: CacheStyle end diff --git a/src/call.jl b/src/call.jl index c564a4f..9402d87 100644 --- a/src/call.jl +++ b/src/call.jl @@ -69,7 +69,10 @@ end # One untyped cache per function for the containers of Cached; other dictionaries (e.g. `Dict`) # would box untyped keys, so they get one typed dictionary per key and value type. -_localtype(C, K, V) = C isa UnionAll ? (C <: AbstractCache ? C{Any, Any} : C{K, V}) : C +_localtype(C, K, V) = C isa UnionAll ? (C <: AbstractCache ? _localcache(C){Any, Any} : C{K, V}) : C +# A task-local cache is used by one task only, so lock-free hits gain nothing there, while the +# per-thread counters of `ClockCache` would cost memory in every task. +_localcache(C) = C === ClockCache ? LocalClockCache : C # Task-local caches follow the size settings of `f`, but are not registered. function _newlocal(f, ::Type{T}) where {T <: AbstractCache} fc = @lock REGISTRY.lock _functioncaches!(f) diff --git a/src/containers/clock.jl b/src/containers/clock.jl index 627e493..92e7239 100644 --- a/src/containers/clock.jl +++ b/src/containers/clock.jl @@ -1,3 +1,16 @@ +# An entry of a `ClockCache`; see the rules below it. +mutable struct Entry{V} + const key::Any + const hash::UInt + const val::V + const size::Int + @atomic ref::Bool + const pos::Int # slot in `ring` +end + +struct Tombstone end +const TOMBSTONE = Tombstone() + """ ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) @@ -8,79 +21,268 @@ never hit again are evicted first. `maxsize` limits the number of entries, or, when `by` is given, the sum of `by(value)` over all entries. + +Hits take no lock, and write to the entry only to set a clear reference bit, so a cache +shared between threads scales with the number of threads on hits. Misses, insertions, +evictions and every other mutation take the lock. A lookup that races with a removal +(eviction, `delete!` or `empty!`) may still return the removed entry, and one that races with +an insertion may miss it; the returned value always belongs to its key. """ mutable struct ClockCache{K, V} <: AbstractCache{K, V} - const index::Dict{Key{Any}, Int} - const keys::Vector{Key{Any}} - const vals::Vector{V} - const sizes::Vector{Int} - const ref::Vector{Bool} - const live::Vector{Bool} - const free::Vector{Int} - hand::Int # last inspected slot + @atomic table::Vector{Any} # open-addressing hash table of `Entry{V}`s, see `_find` + used::Int # entries and tombstones in `table` + const ring::Vector{Union{Nothing, Entry{V}}} # the clock, with `nothing` for a free slot + const free::Vector{Int} # free slots of `ring` + hand::Int # last inspected slot of `ring` + count::Int currentsize::Int maxsize::Int const by::Any - hits::Int - misses::Int + const counts::Vector{Int} # hits and misses per thread, see `_count!` const lock::ReentrantLock end +# Lock-free reads rest on three rules, which every method below keeps: +# +# 1. An `Entry` is immutable apart from its reference bit, so a reader that loads one sees a key +# with its own value. +# 2. A table is never resized: it is replaced as a whole by a new one, published with a release +# store to `c.table`. Readers load the table and then each slot with acquire ordering, so +# they see fully constructed entries, and their indices are always in bounds. +# 3. In a published table, a slot only changes from `nothing` to an entry, from an entry to +# `TOMBSTONE`, or from `TOMBSTONE` to an entry, all under the lock; or to `nothing` when the +# next slot is `nothing`, so that no probe passes it anyway. Entries never move, so a probe +# for a key that is present and not concurrently removed always finds it. +# +# Writers store slots with `setindex!`, which is a release store with a GC write barrier on every +# supported Julia version. (Storing through a pointer would skip the barrier, and the GC could +# free the entry.) Removed entries are dropped from the table rather than unset, so readers +# never see an undefined slot, and the GC collects them once no reader holds them. + +const MINTABLE = 16 +_emptytable(n::Int = MINTABLE) = fill!(Vector{Any}(undef, n), nothing) + function ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) return ClockCache{K, V}( - Dict{Key{Any}, Int}(), Key{Any}[], V[], Int[], Bool[], Bool[], Int[], - 0, 0, maxsize, by, 0, 0, ReentrantLock() + _emptytable(), 0, Union{Nothing, Entry{V}}[], Int[], 0, 0, 0, maxsize, by, + zeros(Int, COUNTSTRIDE * Threads.maxthreadid()), ReentrantLock() ) end -_touch!(c::ClockCache, i::Int) = (c.ref[i] = true; c) +@inline function _loadslot(t::Vector{Any}, i::Int) + return GC.@preserve t unsafe_load(pointer(t, i), :acquire) +end + +# The entry for the probe `p` in table `t`, or `nothing`. Linear probing from the hash; a +# table is at most half full, so the probe ends at an empty slot. +@inline function _find(t::Vector{Any}, p::Key{K}, ::Type{V}) where {K, V} + mask = length(t) - 1 + i = p.hash % Int + for _ in 0:mask # bounds the loop also if the table is changed while probing + e = _loadslot(t, (i & mask) + 1) + e === nothing && return nothing + if e isa Entry{V} && e.hash == p.hash + k = e.key + k isa K && isequal(p.key, k::K) && return e + end + i += 1 + end + return nothing +end +_find(c::ClockCache{K, V}, p::Key) where {K, V} = _find((@atomic :acquire c.table), p, V) + +# Hits and misses are counted per thread, in separate cache lines, with relaxed loads and stores: +# a shared counter would bounce between cores on every hit. Two tasks on one thread cannot +# interleave between the load and the store, since neither yields; threads adopted after the +# cache was created share counters, which may then lose a few counts. +const COUNTSTRIDE = 8 # 64 bytes +@inline function _count!(c::ClockCache, which::Int) + v = c.counts + i = mod(Threads.threadid() - 1, length(v) ÷ COUNTSTRIDE) * COUNTSTRIDE + which + GC.@preserve v begin + p = pointer(v, i) + unsafe_store!(p, unsafe_load(p, :monotonic) + 1, :monotonic) + end + return nothing +end +function _counted(c::ClockCache, which::Int) + v = c.counts + return GC.@preserve v sum(i -> unsafe_load(pointer(v, i), :monotonic), which:COUNTSTRIDE:length(v)) +end + +# Hits only write the reference bit if it is clear, so hot entries are not written at all. +@inline _touch!(e::Entry) = ((@atomic :monotonic e.ref) || (@atomic :monotonic e.ref = true); e) -function _place!(c::ClockCache, i::Int) - c.ref[i] = false - c.live[i] = true +function _get!(default::F, c::ClockCache{K, V}, p::Key) where {F, K, V} + e = _find(c, p) + if e !== nothing + _count!(c, 1) + return _touch!(e).val + end + _count!(c, 2) + v = convert(V, default())::V + sz = _entrysize(c, v) + @lock c.lock begin + # another task may have filled the key while we were computing + e = _find(c, p) + e === nothing || return e.val + _insert!(c, p, v, sz) + end + return v +end + +_lookup(c::ClockCache, key) = (p = _probe(c, key); p === nothing ? nothing : _find(c, p)) + +function Base.get(c::ClockCache, key, default) + e = _lookup(c, key) + return e === nothing ? default : _touch!(e).val +end +function Base.getindex(c::ClockCache, key) + e = _lookup(c, key) + e === nothing && throw(KeyError(key)) + return _touch!(e).val +end +Base.haskey(c::ClockCache, key) = _lookup(c, key) !== nothing +Base.length(c::ClockCache) = @lock c.lock c.count +Base.isempty(c::ClockCache) = length(c) == 0 + +# Mutations, all with the lock held. + +function _insert!(c::ClockCache{K, V}, p::Key, v, sz::Int) where {K, V} + sz > c.maxsize && return c + while c.currentsize + sz > c.maxsize && c.count > 0 + _evict_one!(c) + end + 2 * (c.used + 1) > length(c.table) && _rebuild!(c, c.count + 1) + isempty(c.free) && push!(c.ring, nothing) + pos = isempty(c.free) ? length(c.ring) : pop!(c.free) + e = Entry{V}(p.key, p.hash, v, sz, false, pos) + c.ring[pos] = e + _place!(c.table, e) === nothing && (c.used += 1) + c.count += 1 + c.currentsize += sz return c end -function _newslot!(c::ClockCache, k, v, sz::Int) - push!(c.keys, k) - push!(c.vals, v) - push!(c.sizes, sz) - push!(c.ref, false) - push!(c.live, false) - return length(c.keys) +# Store `e` in the first free slot of its probe sequence; returns what was there. +function _place!(t::Vector{Any}, e::Entry) + mask = length(t) - 1 + i = e.hash % Int + while true + j = (i & mask) + 1 + old = t[j] + if !(old isa Entry) + t[j] = e + return old + end + i += 1 + end + return end -function _remove!(c::ClockCache, i::Int) - c.live[i] = false - return _freeslot!(c, i) +# Replace the table by one without tombstones, with room for `n` entries at half load or less. +function _rebuild!(c::ClockCache, n::Int) + t = _emptytable(max(MINTABLE, nextpow(2, 4n))) + for e in c.ring + e === nothing || _place!(t, e) + end + c.used = c.count + @atomic :release c.table = t + return c +end + +function _remove!(c::ClockCache, e::Entry) + t = c.table + mask = length(t) - 1 + i = e.hash % Int + while t[(i & mask) + 1] !== e + i += 1 + end + if t[((i + 1) & mask) + 1] === nothing + # No probe continues past the next slot, so this slot, and the tombstones before it, + # can end probes as well: they become empty, which saves rebuilds. + while true + t[(i & mask) + 1] = nothing + c.used -= 1 + i -= 1 + t[(i & mask) + 1] === TOMBSTONE || break + end + else + t[(i & mask) + 1] = TOMBSTONE + end + c.ring[e.pos] = nothing + push!(c.free, e.pos) + c.count -= 1 + c.currentsize -= e.size + return c end -# Terminates within two sweeps, since the first sweep clears every reference bit. +# Readers may set reference bits while the hand clears them, so after two sweeps without +# finding a clear bit the entry under the hand is evicted anyway. function _evict_one!(c::ClockCache) - n = length(c.keys) - while true + n = length(c.ring) + for sweep in 1:(2n + 1) c.hand = c.hand >= n ? 1 : c.hand + 1 - i = c.hand - c.live[i] || continue - if c.ref[i] - c.ref[i] = false + e = c.ring[c.hand] + e === nothing && continue + if sweep <= 2n && (@atomic :monotonic e.ref) + @atomic :monotonic e.ref = false else - return _remove!(c, i) + return _remove!(c, e) end end - return + error("unreachable: no entry to evict") +end + +function Base.setindex!(c::ClockCache{K, V}, v, key) where {K, V} + p = _newkey(c, key) + val = convert(V, v)::V + sz = _entrysize(c, val) + @lock c.lock begin + e = _find(c, p) + e === nothing || _remove!(c, e) + _insert!(c, p, val, sz) + end + return c +end + +function Base.delete!(c::ClockCache, key) + @lock c.lock begin + e = _lookup(c, key) + e === nothing || _remove!(c, e) + end + return c +end + +function Base.resize!(c::ClockCache; maxsize::Integer) + maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) + @lock c.lock begin + c.maxsize = maxsize + while c.currentsize > c.maxsize && c.count > 0 + _evict_one!(c) + end + end + return c end function Base.empty!(c::ClockCache) @lock c.lock begin - foreach(empty!, (c.index, c.keys, c.vals, c.sizes, c.ref, c.live, c.free)) - c.hand = c.currentsize = 0 + @atomic :release c.table = _emptytable() + empty!(c.ring) + empty!(c.free) + c.used = c.hand = c.count = c.currentsize = 0 end return c end +function cache_stats(c::ClockCache) + return @lock c.lock (; + hits = _counted(c, 1), misses = _counted(c, 2), length = c.count, c.currentsize, c.maxsize, + ) +end + # Entries in slot order. Call with the lock held. function _pairs(c::ClockCache{K, V}) where {K, V} - return Pair{K, V}[c.keys[i].key::K => c.vals[i] for i in eachindex(c.keys) if c.live[i]] + return Pair{K, V}[e.key::K => e.val for e in c.ring if e isa Entry{V}] end diff --git a/src/containers/interface.jl b/src/containers/interface.jl index f37f4d6..09ae9d1 100644 --- a/src/containers/interface.jl +++ b/src/containers/interface.jl @@ -12,9 +12,12 @@ Keys of different types never match, even when they are `isequal`: in a cache wi abstract key type, `1` and `1.0` are different entries. Lookups with a key that is not of type `K` convert it to `K` first. -Subtypes provide the fields `index::Dict{Key{Any},Int}`, `keys::Vector{Key{Any}}`, `vals`, -`sizes`, `currentsize`, `maxsize`, `by`, `hits`, `misses` and `lock`, and implement the eviction -policy through `_touch!`, `_evict_one!`, `_place!` and `_remove!`. +The default methods, used by `LRU` (and the task-local `ClockCache`), take the lock for every +operation. They need the fields `index::Dict{Key{Any},Int}`, `keys::Vector{Key{Any}}`, `vals`, +`sizes`, `currentsize`, `maxsize`, `by`, `hits`, `misses` and `lock`, and the eviction policy +through `_touch!`, `_evict_one!`, `_place!` and `_remove!`. `ClockCache` implements `_get!` +and the other lookups and mutations itself, for lock-free hits; it shares `get!`, `show` and +iteration, through the fields `by` and `lock`, `cache_stats` and `_pairs`. """ abstract type AbstractCache{K, V} <: AbstractDict{K, V} end @@ -35,17 +38,20 @@ Base.isequal(a::Key{Any}, b::Key{Any}) = a.hash == b.hash && typeof(a.key) === t # The probe for a key that is inserted: converted to `K`, which throws if that is not possible. _newkey(::AbstractCache{K}, key) where {K} = Key(key isa K ? key : convert(K, key)::K) -# The slot of `key` in `c`, or 0. A key that cannot be converted to `K` is not in the cache. -function _slot(c::AbstractCache{K}, key) where {K} - key isa K && return get(c.index, Key(key), 0) +# The probe for `key`, or `nothing` if it cannot be converted to `K` (and so is not in `c`). +function _probe(::AbstractCache{K}, key) where {K} + key isa K && return Key(key) k = try convert(K, key)::K catch - return 0 + return nothing end - return get(c.index, Key(k), 0) + return Key(k) end +# The slot of `key` in `c`, or 0. +_slot(c::AbstractCache, key) = (p = _probe(c, key); p === nothing ? 0 : get(c.index, p, 0)) + _entrysize(c::AbstractCache, v) = c.by === nothing ? 1 : Int(c.by(v))::Int # Insert a new key (not currently present), evicting until it fits. @@ -89,8 +95,11 @@ function _reuseslot!(c::AbstractCache, i::Int, k, v, sz::Int) return i end -function Base.get!(default::Base.Callable, c::AbstractCache{K, V}, key) where {K, V} - p = _newkey(c, key) +# `default::F` makes Julia specialize on it, which it skips for functions only passed along. +Base.get!(default::F, c::AbstractCache, key) where {F <: Base.Callable} = _get!(default, c, _newkey(c, key)) + +# `get!` with the probe `p` of the key: containers implement this rather than `get!`. +function _get!(default::F, c::AbstractCache{K, V}, p::Key) where {F, K, V} @lock c.lock begin i = get(c.index, p, 0) if i != 0 @@ -133,6 +142,8 @@ Base.isempty(c::AbstractCache) = @lock c.lock isempty(c.index) # Iteration walks a snapshot copied under the lock, so it is safe while other tasks use the # cache (holding the lock across `iterate` calls would deadlock on an early `break`). +# The length may change between `length` and `iterate`, so `collect` must not rely on it. +Base.IteratorSize(::Type{<:AbstractCache}) = Base.SizeUnknown() function Base.iterate(c::AbstractCache) snapshot = @lock c.lock _pairs(c) return iterate(c, (snapshot, 1)) diff --git a/src/containers/localclock.jl b/src/containers/localclock.jl new file mode 100644 index 0000000..ef7d6ab --- /dev/null +++ b/src/containers/localclock.jl @@ -0,0 +1,79 @@ +# `ClockCache` for a single task, where lock-free hits gain nothing: every operation takes the +# (uncontended) lock, hits and misses are plain counters, and the slots live in vectors indexed +# by a `Dict` (see `AbstractCache`). `TaskLocalCache{ClockCache}` uses it, see `_localtype`; +# it evicts in the same order as `ClockCache`. +mutable struct LocalClockCache{K, V} <: AbstractCache{K, V} + const index::Dict{Key{Any}, Int} + const keys::Vector{Key{Any}} + const vals::Vector{V} + const sizes::Vector{Int} + const ref::Vector{Bool} + const live::Vector{Bool} + const free::Vector{Int} + hand::Int # last inspected slot + currentsize::Int + maxsize::Int + const by::Any + hits::Int + misses::Int + const lock::ReentrantLock +end + +function LocalClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} + maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) + return LocalClockCache{K, V}( + Dict{Key{Any}, Int}(), Key{Any}[], V[], Int[], Bool[], Bool[], Int[], + 0, 0, maxsize, by, 0, 0, ReentrantLock() + ) +end + +_touch!(c::LocalClockCache, i::Int) = (c.ref[i] = true; c) + +function _place!(c::LocalClockCache, i::Int) + c.ref[i] = false + c.live[i] = true + return c +end + +function _newslot!(c::LocalClockCache, k, v, sz::Int) + push!(c.keys, k) + push!(c.vals, v) + push!(c.sizes, sz) + push!(c.ref, false) + push!(c.live, false) + return length(c.keys) +end + +function _remove!(c::LocalClockCache, i::Int) + c.live[i] = false + return _freeslot!(c, i) +end + +# Terminates within two sweeps, since the first sweep clears every reference bit. +function _evict_one!(c::LocalClockCache) + n = length(c.keys) + while true + c.hand = c.hand >= n ? 1 : c.hand + 1 + i = c.hand + c.live[i] || continue + if c.ref[i] + c.ref[i] = false + else + return _remove!(c, i) + end + end + return +end + +function Base.empty!(c::LocalClockCache) + @lock c.lock begin + foreach(empty!, (c.index, c.keys, c.vals, c.sizes, c.ref, c.live, c.free)) + c.hand = c.currentsize = 0 + end + return c +end + +# Entries in slot order. Call with the lock held. +function _pairs(c::LocalClockCache{K, V}) where {K, V} + return Pair{K, V}[c.keys[i].key::K => c.vals[i] for i in eachindex(c.keys) if c.live[i]] +end diff --git a/test/Project.toml b/test/Project.toml index 2d5162c..5b9d5b4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -3,6 +3,7 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Cached = "1b238080-9255-4fe9-b224-89eb24efe93b" ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Tachikoma = "468859d6-42d8-48b7-8ad9-1d312e0e3b0a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" @@ -13,6 +14,7 @@ Cached = {path = ".."} [compat] Aqua = "0.8.9" ParallelTestRunner = "2.2.0" +Random = "1.10" Tachikoma = "2.7" Test = "1.10" TimerOutputs = "0.5, 1" diff --git a/test/test_concurrency.jl b/test/test_concurrency.jl new file mode 100644 index 0000000..ccafb10 --- /dev/null +++ b/test/test_concurrency.jl @@ -0,0 +1,117 @@ +using Test +using Random +using Cached +using Cached: cache_stats, LocalClockCache + +# Every value is a function of its key, so any value returned for the wrong key is detected. +expected(k::Int) = 3k +expected(k::NTuple{4, Int}) = (sum(k), k[1]) +expected(k::String) = length(k) +expected(k::Tuple{Vector{Int}, Symbol}) = (sum(k[1]), k[2]) + +function randkey(rng, n) + i = rand(rng, 1:n) + r = rand(rng) + return r < 0.4 ? i : r < 0.7 ? (i, i + 1, -i, 2i) : r < 0.85 ? string(i) : ([i, 1], :s) +end + +# Many tasks doing every operation at once, checking each returned value; returns the number +# of `get!` calls and whether all values were correct. +function stress!(c, ntasks, niter; nkeys = 500, maxsizes = (8, 64, 256)) + gets = Threads.Atomic{Int}(0) + ok = Threads.Atomic{Bool}(true) + @sync for t in 1:ntasks + Threads.@spawn begin + rng = Xoshiro(t) + n = 0 + for _ in 1:niter + k = randkey(rng, nkeys) + r = rand(rng) + if r < 0.8 + n += 1 + isequal(get!(() -> expected(k), c, k), expected(k)) || (ok[] = false) + elseif r < 0.9 + v = get(c, k, nothing) + v === nothing || isequal(v, expected(k)) || (ok[] = false) + elseif r < 0.95 + c[k] = expected(k) + elseif r < 0.99 + delete!(c, k) + elseif r < 0.995 + resize!(c; maxsize = rand(rng, maxsizes)) + elseif r < 0.997 + empty!(c) + else + all(((k, v),) -> isequal(v, expected(k)), c) || (ok[] = false) + end + end + Threads.atomic_add!(gets, n) + end + end + return gets[], ok[] +end + +const NTASKS = 4 * Threads.nthreads() + +@testset "$C: concurrent stress" for C in (LRU, ClockCache, LocalClockCache) + for by in (nothing, x -> 1 + (x isa Tuple ? 1 : 0)) + c = C{Any, Any}(; maxsize = 64, by) + gets, ok = stress!(c, NTASKS, 20_000) + @test ok + s = cache_stats(c) + @test s.hits + s.misses == gets + @test s.currentsize <= s.maxsize + @test length(c) == length(collect(c)) == s.length + @test all(((k, v),) -> isequal(v, expected(k)), c) + @test s.currentsize == (by === nothing ? length(c) : sum(by(v) for (k, v) in c; init = 0)) + end +end + +# Many readers of a few hot keys while a writer churns the rest of the cache, so that the +# lock-free table is rebuilt and entries are evicted under the readers. +@testset "ClockCache: hits during rebuilds" begin + c = ClockCache{Any, Any}(; maxsize = 2_000) + ok = Threads.Atomic{Bool}(true) + done = Threads.Atomic{Bool}(false) + @sync begin + for t in 1:max(1, Threads.nthreads() - 1) + Threads.@spawn while !done[] + k = mod(t, 8) + get!(() -> 3k, c, k) == 3k || (ok[] = false) + v = get(c, (k, k + 1, -k, 2k), nothing) + v === nothing || v == expected((k, k + 1, -k, 2k)) || (ok[] = false) + yield() # lets the writer run when there is a single thread + end + end + Threads.@spawn begin + rng = Xoshiro(0) + for i in 1:100_000 + k = (i, i + 1, -i, 2i) + get!(() -> expected(k), c, k) + i % 25_000 == 0 && empty!(c) + i % 7_919 == 0 && resize!(c; maxsize = rand(rng, 100:3_000)) + end + done[] = true + end + end + @test ok[] + @test all(((k, v),) -> isequal(v, expected(k)), c) +end + +@cached shared(x::Int) = x^2 +Cached.CacheStyle(::typeof(shared), ::Int) = GlobalCache{ClockCache}() +@cached tasklocal(x::Int) = x^2 +Cached.CacheStyle(::typeof(tasklocal), ::Int) = TaskLocalCache{ClockCache}() + +@testset "@cached with $f" for f in (shared, tasklocal) + @test @inferred(f(3)) == 9 + f(4) + @test (@allocated f(4)) == 0 +end + +@testset "containers of @cached" begin + c = only(cache_info(shared)).second + @test c isa ClockCache{Any, Any} + @test cache_stats(c).misses == 2 && cache_stats(c).hits >= 1 + @test Cached._container(tasklocal, TaskLocalCache{ClockCache}(), (1,), Int) isa LocalClockCache{Any, Any} +end diff --git a/test/test_containers.jl b/test/test_containers.jl index e83b854..d269017 100644 --- a/test/test_containers.jl +++ b/test/test_containers.jl @@ -1,8 +1,8 @@ using Test using Cached -using Cached: AbstractCache, cache_stats +using Cached: AbstractCache, cache_stats, LocalClockCache -const CACHETYPES = (LRU, ClockCache) +const CACHETYPES = (LRU, ClockCache, LocalClockCache) @testset "$C: basic interface" for C in CACHETYPES c = C{Int, String}(; maxsize = 3) @@ -56,8 +56,8 @@ end @test first.(collect(c)) == [4, 1, 3] # iteration is most to least recent end -@testset "ClockCache: second chance" begin - c = ClockCache{Int, Int}(; maxsize = 3) +@testset "$C: second chance" for C in (ClockCache, LocalClockCache) + c = C{Int, Int}(; maxsize = 3) for i in 1:3 c[i] = i end