diff --git a/CLAUDE.md b/CLAUDE.md index 5866018..99e16a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,7 +44,11 @@ julia --project=/tmp/runic -e 'using Pkg; Pkg.add("Runic"); using Runic; Runic.m - `src/registry.jl`: global caches, one untyped `C{Any,Any}` per function (its budget), looked up lock-free in an atomically published `IdDict` snapshot. - `src/containers/`: `LRU` and `ClockCache`, sharing the `AbstractCache` interface. Keys are - stored as `Key{Any}` and probed with a concretely typed `Key{K}`, so hits never box. + stored as `Key{Any}` (or `Any` plus hash) and probed with a concretely typed `Key{K}`, so hits + never box. `LRU` takes its lock for every operation; `ClockCache` (`clock.jl`) has lock-free + hits, through an atomically published open-addressing table of immutable entries, and keeps + the three invariants listed in its source (`research/lockfree-clock.md`). Task-local caches + use `LocalClockCache` (`localclock.jl`), the locked `ClockCache`, via `_localtype` in `call.jl`. - `src/preferences.jl`: default settings from Preferences.jl, resolved once per function (runtime > function > package > `[Cached]` > built-in), and `set_cache_preferences!`. - `src/api.jl`: `cache_info`, `empty_caches!`, `set_cache_size!`, and the `cache_dashboard` and diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 05191f6..1ac0d4a 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -4,6 +4,7 @@ Cached = "1b238080-9255-4fe9-b224-89eb24efe93b" LRUCache = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [sources] Cached = {path = ".."} diff --git a/benchmark/contention.jl b/benchmark/contention.jl new file mode 100644 index 0000000..5192dc1 --- /dev/null +++ b/benchmark/contention.jl @@ -0,0 +1,134 @@ +# Cost of a lookup when all threads share one cache, against task-local caches. +# "locked Clock" is the `ClockCache` of task-local caches (`Cached.LocalClockCache`), whose hits +# take the lock, shared here for comparison. +# +# for t in 1 2 4 8; do julia --heap-size-hint=4G --project=benchmark -t $t benchmark/contention.jl; done +# +# One task per thread, each doing `M` lookups from its own key sequence. Reports the median over +# `REPS` runs of the time per lookup within a task (ns/lookup, lower is better; perfect scaling +# keeps it constant as threads are added), and the bytes allocated per lookup. + +using Random, Printf, Statistics +using Cached: Cached, ClockCache, LocalClockCache, LRU, @cached, GlobalCache, TaskLocalCache + +const M = 200_000 +const REPS = 7 + +const CONTAINERS = [ + "ClockCache" => n -> ClockCache{Any, Any}(; maxsize = n), + "locked Clock" => n -> LocalClockCache{Any, Any}(; maxsize = n), + "LRU" => n -> LRU{Any, Any}(; maxsize = n), + "task-local Clock" => nothing, # one locked Clock per task, uncontended +] + +function zipf(rng, n, s, len) + cdf = cumsum(1 ./ (1:n) .^ s) + cdf ./= cdf[end] + return [searchsortedfirst(cdf, rand(rng)) for _ in 1:len] +end + +tuplekey(i) = (i, i % 7, :leg, Float64(i)) +compute(k::Int) = 3k +compute(k::Tuple) = k[1] + +# name => (key sequence per task, cache capacity, prefill keys) +function workloads(ntasks) + rngs = [Xoshiro(t) for t in 1:ntasks] + return [ + "hits, Int keys (1k)" => ([rand(r, 1:1_000, M) for r in rngs], 10_000, 1:1_000), + "hits, 4-tuple keys (1k)" => ([tuplekey.(rand(r, 1:1_000, M)) for r in rngs], 10_000, tuplekey.(1:1_000)), + "hits, one hot Int key" => ([fill(7, M) for r in rngs], 10_000, 7:7), + "zipf 1.0 (100k keys, cap 10k)" => ([zipf(r, 100_000, 1.0, M) for r in rngs], 10_000, 1:0), + "uniform (20k keys, cap 10k)" => ([rand(r, 1:20_000, M) for r in rngs], 10_000, 1:0), + ] +end + +function run!(c, seq) + acc = 0 + for k in seq + acc += get!(() -> compute(k), c, k)::Int + end + return acc +end + +# ns per lookup in each task, and total bytes allocated +function threaded(make, seqs, cap, prefill) + shared = make === nothing ? nothing : make(cap) + caches = [shared === nothing ? LocalClockCache{Any, Any}(; maxsize = cap) : shared for _ in seqs] + for c in unique(caches), k in prefill + get!(() -> compute(k), c, k) + end + times = zeros(length(seqs)) + start = Threads.Atomic{Int}(0) + bytes = @allocated @sync for (t, seq) in enumerate(seqs) + Threads.@spawn begin + Threads.atomic_add!(start, 1) + while start[] < length(seqs) # start together + ccall(:jl_cpu_pause, Cvoid, ()) + end + t0 = time_ns() + run!(caches[t], seq) + times[t] = (time_ns() - t0) / length(seq) + end + end + return mean(times), bytes / sum(length, seqs) +end + +# ns per hit through `@cached` functions using each cache style, Int and 4-tuple keys +@cached f_clock(x) = compute(x) +@cached f_locked(x) = compute(x) +@cached f_lru(x) = compute(x) +@cached f_local(x) = compute(x) +Cached.CacheStyle(::typeof(f_clock), x) = GlobalCache{ClockCache}() +Cached.CacheStyle(::typeof(f_locked), x) = GlobalCache{LocalClockCache}() +Cached.CacheStyle(::typeof(f_lru), x) = GlobalCache{LRU}() +Cached.CacheStyle(::typeof(f_local), x) = TaskLocalCache{ClockCache}() +const CACHED = ["ClockCache" => f_clock, "locked Clock" => f_locked, "LRU" => f_lru, "task-local Clock" => f_local] + +function callall(f::F, seq) where {F} + acc = 0 + for k in seq + acc += f(k)::Int + end + return acc +end + +function threaded_cached(f, seqs, keys) + times = zeros(length(seqs)) + bytes = @allocated @sync for (t, seq) in enumerate(seqs) + Threads.@spawn begin + callall(f, keys) # fill this task's cache (task-local) + t0 = time_ns() + callall(f, seq) + times[t] = (time_ns() - t0) / length(seq) + end + end + return mean(times), bytes / sum(length, seqs) +end + +nt = Threads.nthreads() +println("Julia ", VERSION, ", ", nt, " threads (", nt, " tasks), ", M, " lookups per task, median of ", REPS, " runs") +println("load average: ", read(`cat /proc/loadavg`, String)) +@printf("%-30s %-17s %10s %10s %8s\n", "workload", "container", "ns/lookup", "(min)", "B/lookup") +for (wname, (seqs, cap, prefill)) in workloads(nt) + for (cname, make) in CONTAINERS + threaded(make, seqs, cap, prefill) # compile + rs = [threaded(make, seqs, cap, prefill) for _ in 1:REPS] + @printf( + "%-30s %-17s %10.1f %10.1f %8.2f\n", wname, cname, + median(first.(rs)), minimum(first.(rs)), median(last.(rs)) + ) + end +end +for (kname, keys) in ("@cached hits, Int" => 1:1_000, "@cached hits, 4-tuple" => tuplekey.(1:1_000)) + seqs = [rand(Xoshiro(t), keys, M) for t in 1:nt] + for (cname, f) in CACHED + foreach(f, keys) + threaded_cached(f, seqs, keys) + rs = [threaded_cached(f, seqs, keys) for _ in 1:REPS] + @printf( + "%-30s %-17s %10.1f %10.1f %8.2f\n", kname, cname, + median(first.(rs)), minimum(first.(rs)), median(last.(rs)) + ) + end +end diff --git a/research/design.md b/research/design.md index 2f2e81f..2979a56 100644 --- a/research/design.md +++ b/research/design.md @@ -77,18 +77,20 @@ Cached provides its own containers, so it no longer depends on LRUCache.jl. Both implement the same small interface: `get!`, `get`, `haskey`, `empty!`, `resize!`, `length`, and hit/miss statistics. - **`LRU{K,V}`** is array-backed. It uses a `Dict` index, slots stored in vectors, and `prev`/`next` stored as integer vectors. Nodes are never allocated, and eviction is exact LRU. -- **The index is a `Dict{Key{Any},Int}`.** Each stored `Key{Any}` holds the key and its hash. Lookups probe it with a concretely typed `Key{K}`, whose `isequal` checks `s.key isa K` before comparing, so the comparison is static and the key is never boxed. A `Key{Any}` cannot serve as the probe: storing the key in an `Any` field boxes it (96 B per lookup on TensorKit-like keys). +- **The index of `LRU` is a `Dict{Key{Any},Int}`.** Each stored `Key{Any}` holds the key and its hash. Lookups probe it with a concretely typed `Key{K}`, whose `isequal` checks `s.key isa K` before comparing, so the comparison is static and the key is never boxed. A `Key{Any}` cannot serve as the probe: storing the key in an `Any` field boxes it (96 B per lookup on TensorKit-like keys). - This holds for `C{Any,Any}` as well, where a plain `Dict{Any,…}` would allocate on every hit. - Eviction never re-hashes, because the hash is stored. - Keys of different types are different entries, even when `isequal` (`f(3)` and `f(3.0)`). Lookups on a typed cache convert the key to `K` first. -- **`ClockCache{K,V}`** uses second-chance eviction: a ring of slots with a reference bit. A hit only sets the bit and never reorders the ring, which makes it cheap for read-heavy shared caches. -- Each container takes a size limit, either a **count** or **bytes** measured by a `by` function. Each one holds its own lock. Task-local containers skip the lock. +- **`ClockCache{K,V}`** uses second-chance eviction: a ring of entries with a reference bit. A hit only sets the bit (if it is clear) and never reorders the ring, so **hits take no lock** (see `lockfree-clock.md`). + - Its index is a fixed-size open-addressing table of immutable entries (each holding the key as `Any` with its hash, probed with a `Key{K}` as above), replaced as a whole when it fills up and published atomically. Readers load it and its slots with acquire ordering; every mutation (miss, insertion, eviction, `delete!`, `resize!`, `empty!`) still takes the lock. + - Hits and misses are counted per thread, in separate cache lines, so hits write no shared memory. + - A lookup racing with a removal may return the removed entry; the value always belongs to its key. +- Each container takes a size limit, either a **count** or **bytes** measured by a `by` function, and holds its own lock. +- **Task-local caches** of the default container use `LocalClockCache`, an internal `ClockCache` with the `Dict` index above and locked hits (uncontended, so cheap), which saves the per-thread counters (64 B per thread) in every task's cache. It evicts in the same order. - **Default for `GlobalCache`: `ClockCache`.** - - On synthetic workloads (`benchmark/containers.jl`), it beat `LRU` everywhere single-threaded (28 vs 32 ns for an all-hit lookup) and had a slightly better hit rate under Zipf access. Both were 1.5–3× faster than LRUCache.jl. - - On the TensorKit and SUNRepresentations integration, the two were indistinguishable. - - It stays the default because a hit only sets a bit, which makes lock-free hits possible. -- **open**: contention. With 8 threads, every container's per-lookup cost *rises* (all hits: about 50–75 ns, against 28–32 ns single-threaded), because every lookup takes the cache's single lock. - Sharding by key hash, or lock-free reads for `ClockCache`, are the candidate fixes. `TaskLocalCache` is the workaround meanwhile. + - On synthetic workloads (`benchmark/containers.jl`), the locked `ClockCache` beat `LRU` everywhere single-threaded (28 vs 32 ns for an all-hit lookup) and had a slightly better hit rate under Zipf access. Both were 1.5–3× faster than LRUCache.jl. On the TensorKit and SUNRepresentations integration, the two were indistinguishable. + - With lock-free hits it is about 2× faster again single-threaded (14 vs 31 ns, `Int` keys), and shared hits scale: about 25 ns at 8 threads, against about 500 ns with a lock (`benchmark/contention.jl`). That is faster than task-local caches. +- **open**: contended misses. Misses still serialize on the cache's lock, so miss-heavy shared workloads (all threads computing new keys) scale poorly; `TaskLocalCache` or a sharded cache would help there (see `lockfree-clock.md`). `LRU` hits still take the lock. ## Limits diff --git a/research/lockfree-clock.md b/research/lockfree-clock.md new file mode 100644 index 0000000..b34f69e --- /dev/null +++ b/research/lockfree-clock.md @@ -0,0 +1,239 @@ +# Lock-free hits for ClockCache + +Investigation of 2026-10-02 (branch `lockfree-clock`), adopted: `ClockCache` is now the lock-free design described here, and the locked design lives on as the internal `LocalClockCache` for task-local caches. + +## Summary + +- **`ClockCache`** (`src/containers/clock.jl`) is an open-addressing hash table of immutable entries with a CLOCK ring next to it. Readers take no lock and write no shared cache line. Every mutation still runs under the one `ReentrantLock`. +- **Hits**: + - They scale: a shared-cache hit stays at 24–45 ns from 1 to 16 threads. The locked design went from 31 to about 1000 ns. + - They are faster than task-local caches. + - They are also **2× faster single-threaded** (14 vs 31 ns for `Int` keys, 24 vs 39 ns for 4-tuples), because a hit no longer takes and releases a lock. + - Hits stay inferred and allocate nothing, on 1.10 through 1.13. +- **Misses** still serialize on the lock. With trivial computations they are only 1.3–1.6× faster than with the locked design (for example 3.9 vs 6.4 µs per lookup at 16 threads on Zipf). A miss allocates about 45 B more (one `Entry` object), and a stored entry uses about 25 B more. +- **Sharding the lock (16 or 64 locked caches) does not fix hits.** Every thread touches every shard's lock, so the cache lines still bounce. At 8+ threads, 16 shards were slower than one lock, and 64 shards about equal. Sharding does help miss-heavy workloads (2–5×). The `ShardedCache` prototype was dropped; its numbers are kept below. +- Sharding *lock-free* caches gave the best mixed-workload numbers in a quick experiment (below). It costs a per-shard budget, so it is a follow-up, not adopted. +- **Task-local caches** keep the locked design (`LocalClockCache`): a single task gains nothing from lock-free hits, and the per-thread counters would cost 64 B per thread in every task's cache. +- **Complexity**: about 200 lines of code plus about 60 lines of comments in `clock.jl`, against 86 lines for the locked design (`localclock.jl`), which also uses most of `interface.jl`. There are 3 core invariants and a few local rules (listed below). It needs no new dependencies. + +## Q1: why the `Dict` index cannot be read without the lock + +A lock-free read of the `Dict` is **memory-unsafe**, not just stale. The Julia manual says so in general: "If data-races are introduced, Julia is not memory safe". The `Dict` code makes it concrete. + +- **`ht_keyindex` on 1.13** (`base/dict.jl:238`) reads `sz = length(h.keys)` and `keys = h.keys` once. Inside an `@inbounds` loop annotated `:noub`, it reads `h.slots` on every probe step (`isslotempty(h, index)`). + - `rehash!` replaces `h.slots`, `h.keys` and `h.vals` with new `Memory`s. + - It is called from `_setindex!` when `(count + ndel)*3 > 2sz`, with the new size `max(count*4, 4)`. After many deletions this is *smaller* than the old size, which is exactly the steady churn of a full cache. + - A reader then indexes the new, smaller `slots` with an index computed for the old size. That is an out-of-bounds read. +- **`empty!` on 1.10** does `empty!(h.keys); resize!(h.keys, sz)`. On 1.11+ it `_unsetindex!`es every slot. +- **The stored keys are inline `Key{Any}`s** (a pointer and a hash, two words). A concurrent write is not atomic, so a reader can see a torn key or a null pointer. + +**A seqlock (optimistic read, then validate a version counter) does not help.** Validation happens after the read, but the read itself can already crash. A seqlock needs a structure whose reads are memory-safe under concurrent mutation, and once a table has that property (as below), immutable entries make the version counter unnecessary. + +**No Base structure gives safe concurrent reads on 1.10.** + +- `Dict`, `IdDict` and `WeakKeyDict` all lock or are unsafe in the same way. +- `Base.Lockable` is just a lock. +- `Base.PersistentDict` (a HAMT, 1.11+, internal, used by ScopedValues) is immutable, so a published snapshot would be safe to read. It is not on 1.10, every insert copies a path and allocates, and the CLOCK ring would still be separate. +- ConcurrentCollections.jl would add a dependency. + +## Q2: candidate designs + +| design | hits lock-free | memory-safe on 1.10 | miss cost | verdict | +|:-|:-|:-|:-|:-| +| (a) seqlock over the `Dict` | yes | **no** (see Q1) | unchanged | rejected | +| (b) fixed-size open-addressing table of immutable entries, replaced whole when full; acquire loads | yes | yes | +1 allocation (48 B), rare O(n) rebuild | **adopted** | +| (c) copy-on-write snapshot of the index on every write | yes | yes | O(capacity) per miss: a 10k-entry rebuild takes 0.2–0.35 ms (measured with `_rebuild!`), against about 0.1 µs now | rejected for caches with frequent misses | +| (d) sharded locks (16 or 64 locked caches) | no | yes | lower contention | prototyped (`ShardedCache`); does not fix hits | +| (e) reader–writer lock | no | yes | — | not built: every reader still does an atomic RMW on the shared reader count, which bounces the same cache line as the lock does now | +| (b) + (d): shards of lock-free caches | yes | yes | best under contention | measured as an experiment; per-shard budgets | + +**Counters.** A shared `hits += 1`, atomic or not, bounces a cache line on every hit. In my numbers that alone keeps a "lock-free" design from scaling. + +- `ClockCache` counts hits and misses **per thread**, in separate 64-byte lines (`counts[8*(tid-1) + which]`), using a relaxed load and a relaxed store. +- The counts are exact: two tasks on one thread cannot interleave between the load and the store, since neither yields. +- Threads adopted after the cache was created (foreign threads) wrap around and may lose a few counts. +- The cost is `64 × maxthreadid()` bytes per cache, about 1 KB at 16 threads. That matters only if there are very many caches, so task-local caches use the locked `LocalClockCache`. + +## Q3: the reference bit + +The bit moved into the entry, as an `@atomic ref::Bool` field of the mutable `Entry`. This sidesteps the "no atomic vector elements on 1.10" problem entirely. + +- A hit does `(@atomic :monotonic e.ref) || (@atomic :monotonic e.ref = true)`. It only writes when the bit is clear, so hot entries cause no writes at all. +- The CLOCK hand clears bits under the lock. Readers may set a bit while the hand clears it, which is benign. +- Because readers can keep re-setting bits, the sweep is bounded: after two sweeps without finding a clear bit, the hand evicts the entry under it. + +The alternatives, if the bits stay in a vector: + +- `Vector{UInt8}` with `unsafe_load`/`unsafe_store!(p, x, :monotonic)` works on 1.10. Orderings on these functions are supported since 1.10. It is safe because the elements are bits types, so no GC barrier is involved. +- `AtomicMemory` with `@atomic m[i]` needs 1.12. The type exists on 1.11, but the indexing syntax does not. +- `Threads.Atomic` per slot costs an allocation and an indirection per slot. +- UnsafeAtomics.jl is not needed, since Base's orderings cover this. + +## Q4: consistent key/value pairs, reallocation, GC safety + +In the current layout (parallel `keys`/`vals` vectors, `push!` growth, `_reuseslot!`, `_unsetindex!`), a lock-free reader could: + +- read a reallocated (freed) buffer after a `push!`; +- see a null `Key{Any}` after `_unsetindex!`; +- pair the key of one entry with the value of the next entry in the reused slot. + +Validating after the read catches the last case only, and only after the unsafe read has already happened. `ClockCache` avoids all three by construction: + +- **One immutable `Entry` per key/value** (`key::Any`, `hash`, `val::V`, `size`, `pos` are all `const`; only `ref` is `@atomic`). A reader that loads an entry sees that entry's own key and value. +- **Tables never change length.** A table that would exceed half load (live entries plus tombstones) is replaced by a new one, published with `@atomic :release c.table = t`. Readers load the table with `:acquire` and keep using that one, so every index is in bounds. +- **Slot loads use `unsafe_load(pointer(t, i), :acquire)`** on `t::Vector{Any}`, under `GC.@preserve t`. The LLVM IR shows `load atomic … acquire` on 1.10, 1.11, 1.12 and 1.13, with 0 allocations. +- **Slot stores use ordinary `t[i] = x`.** It emits `store atomic … release` *and* the GC write barrier on all four versions, also verified in the IR. + - Writing references through `unsafe_store!` or `atomic_pointerset` would skip the write barrier. The GC could then free a young entry stored into an old table, so that is never done. +- **Nothing is unset.** Removal stores `TOMBSTONE`, or `nothing` (see invariant 3), so readers never see an undefined slot. +- **Removed entries stay alive** while a reader holds them (a rooted local), and are collected afterwards. The "freed slots release their values" test passes for the new type. + +## The final design + +**Files:** + +- `src/containers/clock.jl`: `ClockCache{K,V}`, the lock-free design. Its constructor, `show`, `cache_stats`, `resize!`, `delete!`, `empty!`, iteration and second-chance order are those of the locked design. +- `src/containers/localclock.jl`: `LocalClockCache{K,V}`, the previous locked `ClockCache`, internal. `_localtype` (`src/call.jl`) maps `TaskLocalCache{ClockCache}` (and `TaskLocalCache()` while `ClockCache` is the default container) to it. `TaskLocalCache{ClockCache{Any,Any}}` asks for the concrete type and gets the lock-free one. +- `src/containers/interface.jl`: + - `get!` forwards to `_get!(default, c, p::Key)`, which containers implement, with the key already converted and hashed. + - `default::F` forces specialization: without it, 1.10 boxed the closure on hits. + - `IteratorSize = SizeUnknown()` fixes a pre-existing race. `collect(c)` sized its result with `length(c)` and then iterated a different snapshot, which threw `ArgumentError` under `-t 8`, for every container. + - `_probe` (the key converted to `K`, or `nothing`) is shared by the `Dict`-indexed containers and `ClockCache`. +- **Tests:** + - `test/test_containers.jl` runs the whole container suite on `LRU`, `ClockCache` and `LocalClockCache`. + - `test/test_concurrency.jl`: 4×nthreads tasks mix `get!`/`get`/`setindex!`/`delete!`/`resize!`/`empty!`/iteration, on mixed key types (`Int`, 4-tuples, strings, vector-holding tuples), for count and size limits, on all three containers. Every returned value is checked against its key, and the stats are checked to be exact. + - The same file has a hot-key reader test under constant table rebuilds, `empty!` and `resize!`, and checks that `@cached` hits through global and task-local `ClockCache`s are inferred and allocate 0 bytes, and that the task-local one is a `LocalClockCache`. +- **Benchmark:** `benchmark/contention.jl` compares `ClockCache`, the locked design shared ("locked Clock"), `LRU`, and task-local caches. + +**Invariants a maintainer must keep** (also written in the source): + +1. Entries are immutable apart from `ref`. +2. A published table is never resized or `push!`ed. Growth means building a new table and doing a release store into `c.table`. +3. In a published table a slot changes only, under the lock: + - from `nothing` to an entry; + - from an entry to `TOMBSTONE`; + - from `TOMBSTONE` to an entry; + - from either to `nothing` when the *next* slot is `nothing`, in which case no probe continues past it anyway. + + Entries never move, so a present key is always found. Without the trailing cleanup, steady churn rebuilt the table every 10k–20k misses; with it, `used` stays at about 1.1× the entry count. + +**Local rules:** + +- Writers store slots only with `setindex!`, never through pointers (GC barrier). +- Readers load only with `_loadslot` (acquire). +- Every ring entry is in the current table, since `_remove!` searches for it by identity. +- The table stays at most half full, so probes end. + +**Semantic change.** A lookup that races with a mutation may return an entry that is being evicted, deleted or emptied at the same moment. A lookup during a rebuild may also see the previous table, and so miss a just-inserted key; that goes to the locked miss path, which checks again. The returned value always belongs to its key, and a task always sees its own writes. For memoization this is harmless. + +**Risks:** + +- **Memory model.** + - The acquire load is a documented API: `unsafe_load` with an ordering, since 1.10. + - The *release* half relies on `setindex!` of a reference into an `Array`/`Memory` compiling to `store atomic release`. Julia does this for memory safety, and I verified it on 1.10–1.13, but it is codegen behaviour, not documented API. A `Threads.atomic_fence()` before each store would make it independent of that at a small cost on misses. +- **x86 only measured.** On ARM the acquire/release pair is what makes it correct, and nothing relies on TSO. It is untested there, though. +- **Memory.** About 127 B per entry, against 103 B for the locked design (`summarysize`, 10k `Int=>Int` entries in `{Any,Any}`). Each miss allocates about 120 B, against about 75 B. Each cache has `64 × maxthreadid()` bytes of counters. +- **Misses are still serialized.** Each insertion takes the global lock and touches shared lines. With realistic (µs+) computations this matters much less than in the benchmark below, where computing costs nothing. + +## Benchmarks + +Setup: + +- Machine: `ccqlin038`, 32 cores, shared. The load average was 5–17 during the runs, so the numbers are noisy. +- `benchmark/contention.jl`: one task per thread, 200k lookups per task, median of 7 runs, as ns per lookup within a task. Constant across thread counts means perfect scaling. +- All caches are `{Any,Any}`, as in `@cached`. +- "task-local" means one locked cache per task: uncontended, but every task fills its own. +- The 8- and 16-thread sweeps were run twice (values `a/b`). Lock-free hits agreed within about 15%, contended paths within about 20%. +- Julia 1.13.1 unless noted. +- The 16-thread rows were measured before the machine's users asked for runs of at most 8 threads, so they were not repeated after the final code changes. The last change touched only `const pos` and the tests. + +In the tables below, `ClockCache` is the locked design (now `LocalClockCache`) and `LockFree` the current `ClockCache`; they were measured on the prototype, before the rename. + +**Shared hits, `Int` keys (1k keys), ns/lookup:** + +| threads | ClockCache | LockFree | Sharded(16) | Sharded(64) | task-local | +|-:|-:|-:|-:|-:|-:| +| 1 | 30.8 | **13.8** | 32.4 | 34.7 | 30.8 | +| 2 | 309 | **29** | 263 | 213 | 34 | +| 4 | 367 | **28** | 483 | 323 | 35 | +| 8 | 504/493 | **26/24** | 992 | 536/529 | 44/38 | +| 16 | 1002/1000 | **45/38** | 2130 | 1197/1270 | 64/46 | + +**Shared hits, 4-tuple keys `(i, i%7, :leg, Float64(i))`, ns/lookup:** + +| threads | ClockCache | LockFree | Sharded(16) | Sharded(64) | task-local | +|-:|-:|-:|-:|-:|-:| +| 1 | 39.2 | **24.0** | 41.3 | 43.9 | 39.6 | +| 2 | 397 | **39** | 262 | 232 | 43 | +| 4 | 439 | **38** | 478 | 351 | 42 | +| 8 | 556/639 | **31/29** | 934 | 553/556 | 66/48 | +| 16 | 1115/1239 | **45/36** | 1815 | 1168/1303 | 67/72 | + +**One hot key shared by all threads**, where sharding cannot help: + +| threads | ClockCache | LockFree | Sharded(64) | task-local | +|-:|-:|-:|-:|-:| +| 1 | 29.9 | **11.1** | 30.0 | 29.5 | +| 8 | 378/385 | **10.9/11.9** | 367/404 | 30.0 | +| 16 | 760/817 | **15.4/16.0** | 864/855 | 46.9/40.2 | + +**Through `@cached`** (registry lookup, `CacheStyle`, `::V` assert; task-local includes the `task_local_storage` lookup): + +| threads | Int: Clock | Int: LockFree | Int: task-local | 4-tuple: Clock | 4-tuple: LockFree | 4-tuple: task-local | +|-:|-:|-:|-:|-:|-:|-:| +| 1 | 36.3 | **19.7** | 63.3 | 54.0 | **39.1** | 74.3 | +| 2 | 453 | **28.6** | 68.5 | 606 | **40.0** | 74.4 | +| 4 | 354 | **29.1** | 70.8 | 380 | **42.4** | 76.7 | +| 8 | 538/550 | **28/32** | 87/72 | 793/763 | **43/41** | 114/99 | +| 16 | 1273/1081 | **46/32** | 111/103 | 1753/1467 | **76/51** | 148/120 | + +All hit rows allocate 0 B per lookup for shared caches. Task-local rows show about 1.3 B per lookup from filling each task's cache. + +**Mixed hit/miss workloads**, where the computation is trivial, so misses are pure insertion and eviction cost. ns/lookup, and B/lookup at 1 thread: + +| workload | threads | ClockCache | LockFree | Sharded(16) | Sharded(64) | task-local | +|:-|-:|-:|-:|-:|-:|-:| +| Zipf 1.0, 100k keys, cap 10k | 1 | 92 (28 B) | **66** (35 B) | 82 | 94 | 76 | +| | 2 | 1030 | 620 | 244 | **211** | 90 | +| | 4 | 1591 | 1016 | 890 | **615** | 182 | +| | 8 | 2845/2928 | 2036/2050 | 1731 | **922/990** | 252/241 | +| | 16 | 6529/6335 | 3912/3980 | 4116 | **2410/2441** | 340/390 | +| uniform, 20k keys, cap 10k (~50% misses) | 1 | 145 (40 B) | **113** (62 B) | 139 | 151 | 137 | +| | 8 | 4027/4792 | 3777/3232 | 2025 | **1035/1048** | 414/391 | +| | 16 | 11786/11671 | 7649/7513 | 4930 | **2186/2193** | 476/475 | + +**Experiment, not kept:** `ShardedCache` over lock-free shards (one-line change): + +| workload | threads | 16 shards | 64 shards | +|:-|-:|-:|-:| +| hits, Int keys | 8 | 57 | 42 | +| hits, Int keys | 16 | 43 | 52 | +| Zipf | 8 | 738 | 452 | +| Zipf | 16 | 1412 | 624 | +| uniform | 8 | 1307 | 788 | +| uniform | 16 | 2874 | 1239 | + +These are the best contended-miss numbers measured. The cost is per-shard budgets and per-shard counter arrays. + +**Other runs:** + +- **Julia 1.10.12**: hits at 1 thread are 43.8 ns for `ClockCache` and 14.9 ns for LockFree; `@cached` `Int` hits 52.0 and 22.3 ns. At 8 threads, `Int` hits are 2181 vs 26 ns, and `@cached` 2873 vs 31 ns. `ClockCache` contention is about 4× worse on 1.10. +- **A `Threads.SpinLock` in place of `ReentrantLock`** for the writer lock made contended misses *worse*: Zipf 2548 ns at 8 threads, 5140 ns at 16. It was reverted. +- **Single-thread BenchmarkTools** (`@belapsed`, 1.13): `Int` hits 30.1 → 13.5 ns, 4-tuple 37.8 → 23.2 ns. `LRU` is 33.2 and 41.7 ns. + +**After the refactor** (final code, 1.13.1, load average about 5, ns/lookup): + +| workload | threads | ClockCache | locked Clock | LRU | task-local | +|:-|-:|-:|-:|-:|-:| +| hits, `Int` keys | 1 | 13.5 | 30.9 | 34.6 | 30.9 | +| | 8 | 23.3 | 484 | 1084 | 34.9 | +| `@cached` hits, `Int` | 1 | 20.0 | 37.0 | 40.6 | 66.0 | +| | 8 | 28.1 | 529 | 1022 | 80.1 | +| `@cached` hits, 4-tuple | 1 | 40.4 | 53.5 | 58.2 | 74.9 | +| | 8 | 41.0 | 853 | 1844 | 104 | +| Zipf | 8 | 2086 | 2874 | 3955 | 223 | + +## Follow-ups + +1. **`AtomicMemory`, once 1.10 support is dropped** (or behind `@static VERSION >= v"1.12"`). The table can become `AtomicMemory{Any}` with `@atomic :acquire t[i]` / `@atomic :release t[i] = x`. That is the fully official API, removes the reliance on `setindex!` being a release store, and the pointer code (`_loadslot`, and the counters' `unsafe_load`/`unsafe_store!`) would go away. +2. **Shards of lock-free caches for miss-heavy shared workloads.** The experiment above gave 2–4× better contended misses, at the cost of per-shard budgets (one shard can fill while others are empty, and small limits cannot be honoured exactly) and per-shard counter arrays. Worth it only if such workloads matter in practice. +3. **One implementation for both.** The per-thread counter array could be sized by the caller (one 64-byte line for a task-local cache, which a single task updates exactly), which would let task-local caches use the lock-free `ClockCache` too and drop `LocalClockCache`; the price is about 25 B more per entry and 45 B more per miss. diff --git a/src/Cached.jl b/src/Cached.jl index a20a320..97365eb 100644 --- a/src/Cached.jl +++ b/src/Cached.jl @@ -18,6 +18,7 @@ using Preferences: @load_preference, load_preference, has_preference, set_prefer include("containers/interface.jl") include("containers/lru.jl") include("containers/clock.jl") +include("containers/localclock.jl") include("registry.jl") include("preferences.jl") diff --git a/src/cachestyle.jl b/src/cachestyle.jl index 37566c2..958e5a7 100644 --- a/src/cachestyle.jl +++ b/src/cachestyle.jl @@ -54,7 +54,9 @@ Strategy that stores results in caches local to the current task, of container t `C` is either an `AbstractCache` or an `AbstractDict` type such as `Dict`; if it is not already concrete, it is completed to `C{K, V}`. Without `C`, the container is the one set by the `container` preference. Task-local caches need no locking but are not shared, not bounded -unless `C` is, and are not visible to [`cache_info`](@ref). +unless `C` is, and are not visible to [`cache_info`](@ref). `TaskLocalCache{ClockCache}` uses +an internal variant of [`ClockCache`](@ref) whose hits take the lock, which costs nothing +uncontended and saves the memory of the per-thread counters. """ struct TaskLocalCache{C} <: CacheStyle end diff --git a/src/call.jl b/src/call.jl index c564a4f..9402d87 100644 --- a/src/call.jl +++ b/src/call.jl @@ -69,7 +69,10 @@ end # One untyped cache per function for the containers of Cached; other dictionaries (e.g. `Dict`) # would box untyped keys, so they get one typed dictionary per key and value type. -_localtype(C, K, V) = C isa UnionAll ? (C <: AbstractCache ? C{Any, Any} : C{K, V}) : C +_localtype(C, K, V) = C isa UnionAll ? (C <: AbstractCache ? _localcache(C){Any, Any} : C{K, V}) : C +# A task-local cache is used by one task only, so lock-free hits gain nothing there, while the +# per-thread counters of `ClockCache` would cost memory in every task. +_localcache(C) = C === ClockCache ? LocalClockCache : C # Task-local caches follow the size settings of `f`, but are not registered. function _newlocal(f, ::Type{T}) where {T <: AbstractCache} fc = @lock REGISTRY.lock _functioncaches!(f) diff --git a/src/containers/clock.jl b/src/containers/clock.jl index 627e493..92e7239 100644 --- a/src/containers/clock.jl +++ b/src/containers/clock.jl @@ -1,3 +1,16 @@ +# An entry of a `ClockCache`; see the rules below it. +mutable struct Entry{V} + const key::Any + const hash::UInt + const val::V + const size::Int + @atomic ref::Bool + const pos::Int # slot in `ring` +end + +struct Tombstone end +const TOMBSTONE = Tombstone() + """ ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) @@ -8,79 +21,268 @@ never hit again are evicted first. `maxsize` limits the number of entries, or, when `by` is given, the sum of `by(value)` over all entries. + +Hits take no lock, and write to the entry only to set a clear reference bit, so a cache +shared between threads scales with the number of threads on hits. Misses, insertions, +evictions and every other mutation take the lock. A lookup that races with a removal +(eviction, `delete!` or `empty!`) may still return the removed entry, and one that races with +an insertion may miss it; the returned value always belongs to its key. """ mutable struct ClockCache{K, V} <: AbstractCache{K, V} - const index::Dict{Key{Any}, Int} - const keys::Vector{Key{Any}} - const vals::Vector{V} - const sizes::Vector{Int} - const ref::Vector{Bool} - const live::Vector{Bool} - const free::Vector{Int} - hand::Int # last inspected slot + @atomic table::Vector{Any} # open-addressing hash table of `Entry{V}`s, see `_find` + used::Int # entries and tombstones in `table` + const ring::Vector{Union{Nothing, Entry{V}}} # the clock, with `nothing` for a free slot + const free::Vector{Int} # free slots of `ring` + hand::Int # last inspected slot of `ring` + count::Int currentsize::Int maxsize::Int const by::Any - hits::Int - misses::Int + const counts::Vector{Int} # hits and misses per thread, see `_count!` const lock::ReentrantLock end +# Lock-free reads rest on three rules, which every method below keeps: +# +# 1. An `Entry` is immutable apart from its reference bit, so a reader that loads one sees a key +# with its own value. +# 2. A table is never resized: it is replaced as a whole by a new one, published with a release +# store to `c.table`. Readers load the table and then each slot with acquire ordering, so +# they see fully constructed entries, and their indices are always in bounds. +# 3. In a published table, a slot only changes from `nothing` to an entry, from an entry to +# `TOMBSTONE`, or from `TOMBSTONE` to an entry, all under the lock; or to `nothing` when the +# next slot is `nothing`, so that no probe passes it anyway. Entries never move, so a probe +# for a key that is present and not concurrently removed always finds it. +# +# Writers store slots with `setindex!`, which is a release store with a GC write barrier on every +# supported Julia version. (Storing through a pointer would skip the barrier, and the GC could +# free the entry.) Removed entries are dropped from the table rather than unset, so readers +# never see an undefined slot, and the GC collects them once no reader holds them. + +const MINTABLE = 16 +_emptytable(n::Int = MINTABLE) = fill!(Vector{Any}(undef, n), nothing) + function ClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) return ClockCache{K, V}( - Dict{Key{Any}, Int}(), Key{Any}[], V[], Int[], Bool[], Bool[], Int[], - 0, 0, maxsize, by, 0, 0, ReentrantLock() + _emptytable(), 0, Union{Nothing, Entry{V}}[], Int[], 0, 0, 0, maxsize, by, + zeros(Int, COUNTSTRIDE * Threads.maxthreadid()), ReentrantLock() ) end -_touch!(c::ClockCache, i::Int) = (c.ref[i] = true; c) +@inline function _loadslot(t::Vector{Any}, i::Int) + return GC.@preserve t unsafe_load(pointer(t, i), :acquire) +end + +# The entry for the probe `p` in table `t`, or `nothing`. Linear probing from the hash; a +# table is at most half full, so the probe ends at an empty slot. +@inline function _find(t::Vector{Any}, p::Key{K}, ::Type{V}) where {K, V} + mask = length(t) - 1 + i = p.hash % Int + for _ in 0:mask # bounds the loop also if the table is changed while probing + e = _loadslot(t, (i & mask) + 1) + e === nothing && return nothing + if e isa Entry{V} && e.hash == p.hash + k = e.key + k isa K && isequal(p.key, k::K) && return e + end + i += 1 + end + return nothing +end +_find(c::ClockCache{K, V}, p::Key) where {K, V} = _find((@atomic :acquire c.table), p, V) + +# Hits and misses are counted per thread, in separate cache lines, with relaxed loads and stores: +# a shared counter would bounce between cores on every hit. Two tasks on one thread cannot +# interleave between the load and the store, since neither yields; threads adopted after the +# cache was created share counters, which may then lose a few counts. +const COUNTSTRIDE = 8 # 64 bytes +@inline function _count!(c::ClockCache, which::Int) + v = c.counts + i = mod(Threads.threadid() - 1, length(v) ÷ COUNTSTRIDE) * COUNTSTRIDE + which + GC.@preserve v begin + p = pointer(v, i) + unsafe_store!(p, unsafe_load(p, :monotonic) + 1, :monotonic) + end + return nothing +end +function _counted(c::ClockCache, which::Int) + v = c.counts + return GC.@preserve v sum(i -> unsafe_load(pointer(v, i), :monotonic), which:COUNTSTRIDE:length(v)) +end + +# Hits only write the reference bit if it is clear, so hot entries are not written at all. +@inline _touch!(e::Entry) = ((@atomic :monotonic e.ref) || (@atomic :monotonic e.ref = true); e) -function _place!(c::ClockCache, i::Int) - c.ref[i] = false - c.live[i] = true +function _get!(default::F, c::ClockCache{K, V}, p::Key) where {F, K, V} + e = _find(c, p) + if e !== nothing + _count!(c, 1) + return _touch!(e).val + end + _count!(c, 2) + v = convert(V, default())::V + sz = _entrysize(c, v) + @lock c.lock begin + # another task may have filled the key while we were computing + e = _find(c, p) + e === nothing || return e.val + _insert!(c, p, v, sz) + end + return v +end + +_lookup(c::ClockCache, key) = (p = _probe(c, key); p === nothing ? nothing : _find(c, p)) + +function Base.get(c::ClockCache, key, default) + e = _lookup(c, key) + return e === nothing ? default : _touch!(e).val +end +function Base.getindex(c::ClockCache, key) + e = _lookup(c, key) + e === nothing && throw(KeyError(key)) + return _touch!(e).val +end +Base.haskey(c::ClockCache, key) = _lookup(c, key) !== nothing +Base.length(c::ClockCache) = @lock c.lock c.count +Base.isempty(c::ClockCache) = length(c) == 0 + +# Mutations, all with the lock held. + +function _insert!(c::ClockCache{K, V}, p::Key, v, sz::Int) where {K, V} + sz > c.maxsize && return c + while c.currentsize + sz > c.maxsize && c.count > 0 + _evict_one!(c) + end + 2 * (c.used + 1) > length(c.table) && _rebuild!(c, c.count + 1) + isempty(c.free) && push!(c.ring, nothing) + pos = isempty(c.free) ? length(c.ring) : pop!(c.free) + e = Entry{V}(p.key, p.hash, v, sz, false, pos) + c.ring[pos] = e + _place!(c.table, e) === nothing && (c.used += 1) + c.count += 1 + c.currentsize += sz return c end -function _newslot!(c::ClockCache, k, v, sz::Int) - push!(c.keys, k) - push!(c.vals, v) - push!(c.sizes, sz) - push!(c.ref, false) - push!(c.live, false) - return length(c.keys) +# Store `e` in the first free slot of its probe sequence; returns what was there. +function _place!(t::Vector{Any}, e::Entry) + mask = length(t) - 1 + i = e.hash % Int + while true + j = (i & mask) + 1 + old = t[j] + if !(old isa Entry) + t[j] = e + return old + end + i += 1 + end + return end -function _remove!(c::ClockCache, i::Int) - c.live[i] = false - return _freeslot!(c, i) +# Replace the table by one without tombstones, with room for `n` entries at half load or less. +function _rebuild!(c::ClockCache, n::Int) + t = _emptytable(max(MINTABLE, nextpow(2, 4n))) + for e in c.ring + e === nothing || _place!(t, e) + end + c.used = c.count + @atomic :release c.table = t + return c +end + +function _remove!(c::ClockCache, e::Entry) + t = c.table + mask = length(t) - 1 + i = e.hash % Int + while t[(i & mask) + 1] !== e + i += 1 + end + if t[((i + 1) & mask) + 1] === nothing + # No probe continues past the next slot, so this slot, and the tombstones before it, + # can end probes as well: they become empty, which saves rebuilds. + while true + t[(i & mask) + 1] = nothing + c.used -= 1 + i -= 1 + t[(i & mask) + 1] === TOMBSTONE || break + end + else + t[(i & mask) + 1] = TOMBSTONE + end + c.ring[e.pos] = nothing + push!(c.free, e.pos) + c.count -= 1 + c.currentsize -= e.size + return c end -# Terminates within two sweeps, since the first sweep clears every reference bit. +# Readers may set reference bits while the hand clears them, so after two sweeps without +# finding a clear bit the entry under the hand is evicted anyway. function _evict_one!(c::ClockCache) - n = length(c.keys) - while true + n = length(c.ring) + for sweep in 1:(2n + 1) c.hand = c.hand >= n ? 1 : c.hand + 1 - i = c.hand - c.live[i] || continue - if c.ref[i] - c.ref[i] = false + e = c.ring[c.hand] + e === nothing && continue + if sweep <= 2n && (@atomic :monotonic e.ref) + @atomic :monotonic e.ref = false else - return _remove!(c, i) + return _remove!(c, e) end end - return + error("unreachable: no entry to evict") +end + +function Base.setindex!(c::ClockCache{K, V}, v, key) where {K, V} + p = _newkey(c, key) + val = convert(V, v)::V + sz = _entrysize(c, val) + @lock c.lock begin + e = _find(c, p) + e === nothing || _remove!(c, e) + _insert!(c, p, val, sz) + end + return c +end + +function Base.delete!(c::ClockCache, key) + @lock c.lock begin + e = _lookup(c, key) + e === nothing || _remove!(c, e) + end + return c +end + +function Base.resize!(c::ClockCache; maxsize::Integer) + maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) + @lock c.lock begin + c.maxsize = maxsize + while c.currentsize > c.maxsize && c.count > 0 + _evict_one!(c) + end + end + return c end function Base.empty!(c::ClockCache) @lock c.lock begin - foreach(empty!, (c.index, c.keys, c.vals, c.sizes, c.ref, c.live, c.free)) - c.hand = c.currentsize = 0 + @atomic :release c.table = _emptytable() + empty!(c.ring) + empty!(c.free) + c.used = c.hand = c.count = c.currentsize = 0 end return c end +function cache_stats(c::ClockCache) + return @lock c.lock (; + hits = _counted(c, 1), misses = _counted(c, 2), length = c.count, c.currentsize, c.maxsize, + ) +end + # Entries in slot order. Call with the lock held. function _pairs(c::ClockCache{K, V}) where {K, V} - return Pair{K, V}[c.keys[i].key::K => c.vals[i] for i in eachindex(c.keys) if c.live[i]] + return Pair{K, V}[e.key::K => e.val for e in c.ring if e isa Entry{V}] end diff --git a/src/containers/interface.jl b/src/containers/interface.jl index f37f4d6..09ae9d1 100644 --- a/src/containers/interface.jl +++ b/src/containers/interface.jl @@ -12,9 +12,12 @@ Keys of different types never match, even when they are `isequal`: in a cache wi abstract key type, `1` and `1.0` are different entries. Lookups with a key that is not of type `K` convert it to `K` first. -Subtypes provide the fields `index::Dict{Key{Any},Int}`, `keys::Vector{Key{Any}}`, `vals`, -`sizes`, `currentsize`, `maxsize`, `by`, `hits`, `misses` and `lock`, and implement the eviction -policy through `_touch!`, `_evict_one!`, `_place!` and `_remove!`. +The default methods, used by `LRU` (and the task-local `ClockCache`), take the lock for every +operation. They need the fields `index::Dict{Key{Any},Int}`, `keys::Vector{Key{Any}}`, `vals`, +`sizes`, `currentsize`, `maxsize`, `by`, `hits`, `misses` and `lock`, and the eviction policy +through `_touch!`, `_evict_one!`, `_place!` and `_remove!`. `ClockCache` implements `_get!` +and the other lookups and mutations itself, for lock-free hits; it shares `get!`, `show` and +iteration, through the fields `by` and `lock`, `cache_stats` and `_pairs`. """ abstract type AbstractCache{K, V} <: AbstractDict{K, V} end @@ -35,17 +38,20 @@ Base.isequal(a::Key{Any}, b::Key{Any}) = a.hash == b.hash && typeof(a.key) === t # The probe for a key that is inserted: converted to `K`, which throws if that is not possible. _newkey(::AbstractCache{K}, key) where {K} = Key(key isa K ? key : convert(K, key)::K) -# The slot of `key` in `c`, or 0. A key that cannot be converted to `K` is not in the cache. -function _slot(c::AbstractCache{K}, key) where {K} - key isa K && return get(c.index, Key(key), 0) +# The probe for `key`, or `nothing` if it cannot be converted to `K` (and so is not in `c`). +function _probe(::AbstractCache{K}, key) where {K} + key isa K && return Key(key) k = try convert(K, key)::K catch - return 0 + return nothing end - return get(c.index, Key(k), 0) + return Key(k) end +# The slot of `key` in `c`, or 0. +_slot(c::AbstractCache, key) = (p = _probe(c, key); p === nothing ? 0 : get(c.index, p, 0)) + _entrysize(c::AbstractCache, v) = c.by === nothing ? 1 : Int(c.by(v))::Int # Insert a new key (not currently present), evicting until it fits. @@ -89,8 +95,11 @@ function _reuseslot!(c::AbstractCache, i::Int, k, v, sz::Int) return i end -function Base.get!(default::Base.Callable, c::AbstractCache{K, V}, key) where {K, V} - p = _newkey(c, key) +# `default::F` makes Julia specialize on it, which it skips for functions only passed along. +Base.get!(default::F, c::AbstractCache, key) where {F <: Base.Callable} = _get!(default, c, _newkey(c, key)) + +# `get!` with the probe `p` of the key: containers implement this rather than `get!`. +function _get!(default::F, c::AbstractCache{K, V}, p::Key) where {F, K, V} @lock c.lock begin i = get(c.index, p, 0) if i != 0 @@ -133,6 +142,8 @@ Base.isempty(c::AbstractCache) = @lock c.lock isempty(c.index) # Iteration walks a snapshot copied under the lock, so it is safe while other tasks use the # cache (holding the lock across `iterate` calls would deadlock on an early `break`). +# The length may change between `length` and `iterate`, so `collect` must not rely on it. +Base.IteratorSize(::Type{<:AbstractCache}) = Base.SizeUnknown() function Base.iterate(c::AbstractCache) snapshot = @lock c.lock _pairs(c) return iterate(c, (snapshot, 1)) diff --git a/src/containers/localclock.jl b/src/containers/localclock.jl new file mode 100644 index 0000000..ef7d6ab --- /dev/null +++ b/src/containers/localclock.jl @@ -0,0 +1,79 @@ +# `ClockCache` for a single task, where lock-free hits gain nothing: every operation takes the +# (uncontended) lock, hits and misses are plain counters, and the slots live in vectors indexed +# by a `Dict` (see `AbstractCache`). `TaskLocalCache{ClockCache}` uses it, see `_localtype`; +# it evicts in the same order as `ClockCache`. +mutable struct LocalClockCache{K, V} <: AbstractCache{K, V} + const index::Dict{Key{Any}, Int} + const keys::Vector{Key{Any}} + const vals::Vector{V} + const sizes::Vector{Int} + const ref::Vector{Bool} + const live::Vector{Bool} + const free::Vector{Int} + hand::Int # last inspected slot + currentsize::Int + maxsize::Int + const by::Any + hits::Int + misses::Int + const lock::ReentrantLock +end + +function LocalClockCache{K, V}(; maxsize::Integer = 10_000, by = nothing) where {K, V} + maxsize >= 0 || throw(ArgumentError("maxsize must be non-negative")) + return LocalClockCache{K, V}( + Dict{Key{Any}, Int}(), Key{Any}[], V[], Int[], Bool[], Bool[], Int[], + 0, 0, maxsize, by, 0, 0, ReentrantLock() + ) +end + +_touch!(c::LocalClockCache, i::Int) = (c.ref[i] = true; c) + +function _place!(c::LocalClockCache, i::Int) + c.ref[i] = false + c.live[i] = true + return c +end + +function _newslot!(c::LocalClockCache, k, v, sz::Int) + push!(c.keys, k) + push!(c.vals, v) + push!(c.sizes, sz) + push!(c.ref, false) + push!(c.live, false) + return length(c.keys) +end + +function _remove!(c::LocalClockCache, i::Int) + c.live[i] = false + return _freeslot!(c, i) +end + +# Terminates within two sweeps, since the first sweep clears every reference bit. +function _evict_one!(c::LocalClockCache) + n = length(c.keys) + while true + c.hand = c.hand >= n ? 1 : c.hand + 1 + i = c.hand + c.live[i] || continue + if c.ref[i] + c.ref[i] = false + else + return _remove!(c, i) + end + end + return +end + +function Base.empty!(c::LocalClockCache) + @lock c.lock begin + foreach(empty!, (c.index, c.keys, c.vals, c.sizes, c.ref, c.live, c.free)) + c.hand = c.currentsize = 0 + end + return c +end + +# Entries in slot order. Call with the lock held. +function _pairs(c::LocalClockCache{K, V}) where {K, V} + return Pair{K, V}[c.keys[i].key::K => c.vals[i] for i in eachindex(c.keys) if c.live[i]] +end diff --git a/test/Project.toml b/test/Project.toml index 2d5162c..5b9d5b4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -3,6 +3,7 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Cached = "1b238080-9255-4fe9-b224-89eb24efe93b" ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc" Preferences = "21216c6a-2e73-6563-6e65-726566657250" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Tachikoma = "468859d6-42d8-48b7-8ad9-1d312e0e3b0a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" @@ -13,6 +14,7 @@ Cached = {path = ".."} [compat] Aqua = "0.8.9" ParallelTestRunner = "2.2.0" +Random = "1.10" Tachikoma = "2.7" Test = "1.10" TimerOutputs = "0.5, 1" diff --git a/test/test_concurrency.jl b/test/test_concurrency.jl new file mode 100644 index 0000000..ccafb10 --- /dev/null +++ b/test/test_concurrency.jl @@ -0,0 +1,117 @@ +using Test +using Random +using Cached +using Cached: cache_stats, LocalClockCache + +# Every value is a function of its key, so any value returned for the wrong key is detected. +expected(k::Int) = 3k +expected(k::NTuple{4, Int}) = (sum(k), k[1]) +expected(k::String) = length(k) +expected(k::Tuple{Vector{Int}, Symbol}) = (sum(k[1]), k[2]) + +function randkey(rng, n) + i = rand(rng, 1:n) + r = rand(rng) + return r < 0.4 ? i : r < 0.7 ? (i, i + 1, -i, 2i) : r < 0.85 ? string(i) : ([i, 1], :s) +end + +# Many tasks doing every operation at once, checking each returned value; returns the number +# of `get!` calls and whether all values were correct. +function stress!(c, ntasks, niter; nkeys = 500, maxsizes = (8, 64, 256)) + gets = Threads.Atomic{Int}(0) + ok = Threads.Atomic{Bool}(true) + @sync for t in 1:ntasks + Threads.@spawn begin + rng = Xoshiro(t) + n = 0 + for _ in 1:niter + k = randkey(rng, nkeys) + r = rand(rng) + if r < 0.8 + n += 1 + isequal(get!(() -> expected(k), c, k), expected(k)) || (ok[] = false) + elseif r < 0.9 + v = get(c, k, nothing) + v === nothing || isequal(v, expected(k)) || (ok[] = false) + elseif r < 0.95 + c[k] = expected(k) + elseif r < 0.99 + delete!(c, k) + elseif r < 0.995 + resize!(c; maxsize = rand(rng, maxsizes)) + elseif r < 0.997 + empty!(c) + else + all(((k, v),) -> isequal(v, expected(k)), c) || (ok[] = false) + end + end + Threads.atomic_add!(gets, n) + end + end + return gets[], ok[] +end + +const NTASKS = 4 * Threads.nthreads() + +@testset "$C: concurrent stress" for C in (LRU, ClockCache, LocalClockCache) + for by in (nothing, x -> 1 + (x isa Tuple ? 1 : 0)) + c = C{Any, Any}(; maxsize = 64, by) + gets, ok = stress!(c, NTASKS, 20_000) + @test ok + s = cache_stats(c) + @test s.hits + s.misses == gets + @test s.currentsize <= s.maxsize + @test length(c) == length(collect(c)) == s.length + @test all(((k, v),) -> isequal(v, expected(k)), c) + @test s.currentsize == (by === nothing ? length(c) : sum(by(v) for (k, v) in c; init = 0)) + end +end + +# Many readers of a few hot keys while a writer churns the rest of the cache, so that the +# lock-free table is rebuilt and entries are evicted under the readers. +@testset "ClockCache: hits during rebuilds" begin + c = ClockCache{Any, Any}(; maxsize = 2_000) + ok = Threads.Atomic{Bool}(true) + done = Threads.Atomic{Bool}(false) + @sync begin + for t in 1:max(1, Threads.nthreads() - 1) + Threads.@spawn while !done[] + k = mod(t, 8) + get!(() -> 3k, c, k) == 3k || (ok[] = false) + v = get(c, (k, k + 1, -k, 2k), nothing) + v === nothing || v == expected((k, k + 1, -k, 2k)) || (ok[] = false) + yield() # lets the writer run when there is a single thread + end + end + Threads.@spawn begin + rng = Xoshiro(0) + for i in 1:100_000 + k = (i, i + 1, -i, 2i) + get!(() -> expected(k), c, k) + i % 25_000 == 0 && empty!(c) + i % 7_919 == 0 && resize!(c; maxsize = rand(rng, 100:3_000)) + end + done[] = true + end + end + @test ok[] + @test all(((k, v),) -> isequal(v, expected(k)), c) +end + +@cached shared(x::Int) = x^2 +Cached.CacheStyle(::typeof(shared), ::Int) = GlobalCache{ClockCache}() +@cached tasklocal(x::Int) = x^2 +Cached.CacheStyle(::typeof(tasklocal), ::Int) = TaskLocalCache{ClockCache}() + +@testset "@cached with $f" for f in (shared, tasklocal) + @test @inferred(f(3)) == 9 + f(4) + @test (@allocated f(4)) == 0 +end + +@testset "containers of @cached" begin + c = only(cache_info(shared)).second + @test c isa ClockCache{Any, Any} + @test cache_stats(c).misses == 2 && cache_stats(c).hits >= 1 + @test Cached._container(tasklocal, TaskLocalCache{ClockCache}(), (1,), Int) isa LocalClockCache{Any, Any} +end diff --git a/test/test_containers.jl b/test/test_containers.jl index e83b854..d269017 100644 --- a/test/test_containers.jl +++ b/test/test_containers.jl @@ -1,8 +1,8 @@ using Test using Cached -using Cached: AbstractCache, cache_stats +using Cached: AbstractCache, cache_stats, LocalClockCache -const CACHETYPES = (LRU, ClockCache) +const CACHETYPES = (LRU, ClockCache, LocalClockCache) @testset "$C: basic interface" for C in CACHETYPES c = C{Int, String}(; maxsize = 3) @@ -56,8 +56,8 @@ end @test first.(collect(c)) == [4, 1, 3] # iteration is most to least recent end -@testset "ClockCache: second chance" begin - c = ClockCache{Int, Int}(; maxsize = 3) +@testset "$C: second chance" for C in (ClockCache, LocalClockCache) + c = C{Int, Int}(; maxsize = 3) for i in 1:3 c[i] = i end