diff --git a/README.md b/README.md index 3413abe..255ba6d 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,19 @@ Qwen3-Coder-30B's bf16 KV cache) and that model's measured cost per block: The compute-aware `random_compute` saves 10 points more prefill compute than LRU, at a lower hit ratio. + + + Compute saving ratio against cache size, from 16k to 256k blocks, for LRU, S3-FIFO, RandomCompute and the BeladyCompute oracle, on the freeinference and chutes traces + + +The gap is widest when the cache is tight. On freeinference, `random_compute`'s +lead over LRU shrinks from 10 points at 16k blocks to under 1 point from 64k +blocks on, where every policy saves over 91% of prefill compute. On chutes, +with 3.5 times as many distinct blocks, `random_compute` matches LRU from 32k +blocks on, S3-FIFO trails both, and the oracle stays ahead of LRU by 9 points +at 16k blocks and 2 points at 256k. The data and the script that plots it are +in [`doc/figures/`](doc/figures/). + ## Why PrefixSim - **The serving constraint, modelled.** A request needs *all* of its KV blocks @@ -169,7 +182,7 @@ zstd's 128 MiB decompression window, which does not grow with the trace. | `src/` | the simulator: trace readers, the serve loop, the lcp and JSONL writers, the CLI | | `tools/` | `traceConvLLM`, the JSONL ↔ lcp converter | | `test/` | tests of the serve loop and the lcp round trip | -| `doc/` | the documentation | +| `doc/` | the documentation, the logo, and the README's figure with its data and plotting script | | `libCacheSim/` | the [libCacheSim](https://github.com/1a1a11a/libCacheSim) submodule: the eviction algorithms and the lcp reader | libCacheSim's build options pass through, for example `-DENABLE_S4FIFO=ON`, and diff --git a/doc/figures/compute_saving-dark.svg b/doc/figures/compute_saving-dark.svg new file mode 100644 index 0000000..2fab336 --- /dev/null +++ b/doc/figures/compute_saving-dark.svg @@ -0,0 +1,1946 @@ + + + + + + + + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/figures/compute_saving.svg b/doc/figures/compute_saving.svg new file mode 100644 index 0000000..2762e7e --- /dev/null +++ b/doc/figures/compute_saving.svg @@ -0,0 +1,1946 @@ + + + + + + + + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/figures/compute_saving.txt b/doc/figures/compute_saving.txt new file mode 100644 index 0000000..39b1746 --- /dev/null +++ b/doc/figures/compute_saving.txt @@ -0,0 +1,40 @@ +RESULT trace=freeinference.lcp.zst algo=lru cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.610717 compute_saving_ratio=0.533338 n_eviction=255446604 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=s3fifo cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.620902 compute_saving_ratio=0.549143 n_eviction=271855146 n_self_eviction=23092206 n_unexpected_eviction=0 n_unobserved_eviction_round=271855146 +RESULT trace=freeinference.lcp.zst algo=random_compute cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.606811 compute_saving_ratio=0.637784 n_eviction=258015322 n_self_eviction=5364 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=belady_compute cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.682293 compute_saving_ratio=0.735341 n_eviction=208482972 n_self_eviction=6987 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=lru cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.838729 compute_saving_ratio=0.823051 n_eviction=105799683 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=s3fifo cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.835781 compute_saving_ratio=0.820035 n_eviction=118919232 n_self_eviction=11184700 n_unexpected_eviction=0 n_unobserved_eviction_round=118919232 +RESULT trace=freeinference.lcp.zst algo=random_compute cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.828944 compute_saving_ratio=0.868567 n_eviction=112220760 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=belady_compute cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.884081 compute_saving_ratio=0.911775 n_eviction=76037980 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=lru cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.917540 compute_saving_ratio=0.925862 n_eviction=54048024 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=s3fifo cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.912296 compute_saving_ratio=0.918161 n_eviction=62275276 n_self_eviction=4786178 n_unexpected_eviction=0 n_unobserved_eviction_round=62275276 +RESULT trace=freeinference.lcp.zst algo=random_compute cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.912510 compute_saving_ratio=0.933778 n_eviction=57348909 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=belady_compute cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.933162 compute_saving_ratio=0.946968 n_eviction=43796088 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=lru cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.932406 compute_saving_ratio=0.944083 n_eviction=44226904 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=s3fifo cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.930871 compute_saving_ratio=0.941506 n_eviction=46028777 n_self_eviction=794534 n_unexpected_eviction=0 n_unobserved_eviction_round=46028777 +RESULT trace=freeinference.lcp.zst algo=random_compute cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.930373 compute_saving_ratio=0.946146 n_eviction=45560841 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=belady_compute cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.938820 compute_saving_ratio=0.951056 n_eviction=40017479 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=lru cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.936234 compute_saving_ratio=0.948493 n_eviction=41583734 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=s3fifo cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.935633 compute_saving_ratio=0.947449 n_eviction=41978352 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=41978352 +RESULT trace=freeinference.lcp.zst algo=random_compute cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.934944 compute_saving_ratio=0.948877 n_eviction=42429914 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=freeinference.lcp.zst algo=belady_compute cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=327548 n_req_skipped=0 n_blocks=656240179 block_hit_ratio=0.939633 compute_saving_ratio=0.951409 n_eviction=39353263 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=lru cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.634644 compute_saving_ratio=0.599835 n_eviction=220527950 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=s3fifo cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.591440 compute_saving_ratio=0.548459 n_eviction=255705146 n_self_eviction=9097258 n_unexpected_eviction=0 n_unobserved_eviction_round=255705146 +RESULT trace=chutes.lcp.zst algo=random_compute cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.600043 compute_saving_ratio=0.608454 n_eviction=241415461 n_self_eviction=720 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=belady_compute cache_size=16384 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.712506 compute_saving_ratio=0.686683 n_eviction=173527664 n_self_eviction=808 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=lru cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.701898 compute_saving_ratio=0.668178 n_eviction=179914338 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=s3fifo cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.650384 compute_saving_ratio=0.607049 n_eviction=216884116 n_self_eviction=5873935 n_unexpected_eviction=0 n_unobserved_eviction_round=216884116 +RESULT trace=chutes.lcp.zst algo=random_compute cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.679880 compute_saving_ratio=0.666739 n_eviction=193205214 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=belady_compute cache_size=32768 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.746548 compute_saving_ratio=0.715279 n_eviction=152961286 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=lru cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.726936 compute_saving_ratio=0.693530 n_eviction=164767587 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=s3fifo cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.696152 compute_saving_ratio=0.653674 n_eviction=184419007 n_self_eviction=1068757 n_unexpected_eviction=0 n_unobserved_eviction_round=184419007 +RESULT trace=chutes.lcp.zst algo=random_compute cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.715450 compute_saving_ratio=0.692467 n_eviction=171700963 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=belady_compute cache_size=65536 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.759656 compute_saving_ratio=0.726546 n_eviction=145016299 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=lru cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.739949 compute_saving_ratio=0.706272 n_eviction=156846558 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=s3fifo cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.731584 compute_saving_ratio=0.692503 n_eviction=161910825 n_self_eviction=14497 n_unexpected_eviction=0 n_unobserved_eviction_round=161910825 +RESULT trace=chutes.lcp.zst algo=random_compute cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.733541 compute_saving_ratio=0.706114 n_eviction=160715116 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=belady_compute cache_size=131072 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.766055 compute_saving_ratio=0.731584 n_eviction=141087976 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=lru cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.748757 compute_saving_ratio=0.714960 n_eviction=151398698 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=s3fifo cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.746891 compute_saving_ratio=0.709313 n_eviction=152525240 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=152525240 +RESULT trace=chutes.lcp.zst algo=random_compute cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.744305 compute_saving_ratio=0.714755 n_eviction=154086090 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 +RESULT trace=chutes.lcp.zst algo=belady_compute cache_size=262144 cost_model=qwen3coder30b_blksz_16 block_id=prefix n_req=515817 n_req_skipped=0 n_blocks=603642254 block_hit_ratio=0.768453 compute_saving_ratio=0.733257 n_eviction=139509285 n_self_eviction=0 n_unexpected_eviction=0 n_unobserved_eviction_round=0 diff --git a/doc/figures/plot_compute_saving.py b/doc/figures/plot_compute_saving.py new file mode 100755 index 0000000..ed0190e --- /dev/null +++ b/doc/figures/plot_compute_saving.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Plot compute saving ratio against cache size, for the README. + +Reads the RESULT lines of prefixsim runs (compute_saving.txt, next to this +script) and writes compute_saving.svg and compute_saving-dark.svg, one for +each GitHub theme. Requires matplotlib; uses Inter when it is installed. + +To regenerate the data: 40 runs, with the traces from Hugging Face in +traces/. They take a few hours on one core, since every run streams a whole +trace, so run them in parallel if you can. --no-verify skips a self-check +after each request, which saves time and does not change the results. + + for t in freeinference chutes; do + for c in 16k 32k 64k 128k 256k; do + for a in lru s3fifo random_compute belady_compute; do + ./_build/bin/prefixsim --trace traces/$t.lcp.zst --cache-size $c \\ + --algo $a --cost-model qwen3coder30b_blksz_16 --no-verify + done + done + done | grep '^RESULT' | sed 's|trace=[^ ]*/|trace=|' > doc/figures/compute_saving.txt + + python3 doc/figures/plot_compute_saving.py +""" +import os +from collections import defaultdict + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib import font_manager # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# 16-token blocks of Qwen3-Coder-30B's bf16 KV cache: 1k blocks are 1.5 GiB. +GIB_PER_BLOCK = 1.5 / 1024 + +TRACES = [ + ("freeinference.lcp.zst", "freeinference", "coding and assistant agents, 7 days"), + ("chutes.lcp.zst", "chutes", "chat and roleplay, 131 days"), +] + +# (algo, label, light color, dark color, marker, linestyle). Categorical slots +# 1-3 of a palette checked for colour-vision deficiency; the offline oracle is +# drawn in neutral ink, dashed, as a bound rather than a competitor. +SERIES = [ + ("lru", "LRU", "#2a78d6", "#3987e5", "o", "-"), + ("s3fifo", "S3-FIFO", "#eb6834", "#d95926", "s", "-"), + ("random_compute", "RandomCompute", "#1baf7a", "#199e70", "D", "-"), + ("belady_compute", "BeladyCompute (offline oracle)", "#52514e", "#c3c2b7", "^", "--"), +] + +THEMES = { + "light": dict(ink="#0b0b0b", ink2="#52514e", muted="#6e6d68", grid="#e1e0d9", + axis="#c3c2b7", surface="#ffffff"), + "dark": dict(ink="#f0efec", ink2="#c3c2b7", muted="#9a988f", grid="#2c2c2a", + axis="#4a4a46", surface="#0d1117"), +} + + +def load(path): + data = defaultdict(dict) # (trace, algo) -> {cache_size: ratio} + with open(path) as f: + for line in f: + if not line.startswith("RESULT"): + continue + kv = dict(tok.split("=", 1) for tok in line.split()[1:]) + trace = os.path.basename(kv["trace"]) + data[(trace, kv["algo"])][int(kv["cache_size"])] = float(kv["compute_saving_ratio"]) + return data + + +def size_label(blocks): + return f"{blocks // 1024}k\n{blocks * GIB_PER_BLOCK:g} GiB" + + +def setup_font(): + for name in ("Inter", "Inter Display"): + for f in font_manager.findSystemFonts(): + if os.path.basename(f).startswith(name.replace(" ", "") + "-"): + font_manager.fontManager.addfont(f) + names = {f.name for f in font_manager.fontManager.ttflist} + plt.rcParams["font.family"] = "Inter" if "Inter" in names else "DejaVu Sans" + + +def plot(data, theme, out): + t = THEMES[theme] + plt.rcParams.update({ + "svg.fonttype": "path", # glyphs as outlines: renders without the font + "svg.hashsalt": "prefixsim", # stable ids, so regenerating gives the same file + "font.size": 11, + "text.color": t["ink"], + "axes.labelcolor": t["ink2"], + "xtick.color": t["muted"], + "ytick.color": t["muted"], + "xtick.labelcolor": t["ink2"], + "ytick.labelcolor": t["ink2"], + }) + fig, axes = plt.subplots(1, 2, figsize=(10, 4.1), sharey=True, layout="constrained") + fig.get_layout_engine().set(w_pad=0.1, h_pad=0.08, wspace=0.06) + fig.patch.set_alpha(0) + + ys = [v for d in data.values() for v in d.values()] + lo = max(0.0, (int(min(ys) * 10) / 10)) + hi = min(1.0, (int(max(ys) * 10) + 1) / 10) + + for ax, (trace, name, desc) in zip(axes, TRACES): + ax.set_facecolor("none") + for algo, label, light, dark, marker, ls in SERIES: + pts = sorted(data.get((trace, algo), {}).items()) + if not pts: + continue + xs, vs = zip(*pts) + color = light if theme == "light" else dark + ax.plot(xs, vs, ls, color=color, lw=2, marker=marker, ms=7, + mec=t["surface"], mew=1.5, label=label, solid_capstyle="round", + dash_capstyle="round", zorder=3) + xs = sorted({x for (tr, _), d in data.items() if tr == trace for x in d}) + if not xs: + continue + ax.set_xscale("log", base=2) + ax.set_xticks(xs, [size_label(x) for x in xs]) + ax.minorticks_off() + ax.set_xlim(xs[0] / 1.15, xs[-1] * 1.15) + ax.set_ylim(lo, hi) + ax.yaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(lambda v, _: f"{v:.0%}")) + ax.grid(axis="y", color=t["grid"], lw=0.8, zorder=0) + for side in ("top", "right", "left"): + ax.spines[side].set_visible(False) + ax.spines["bottom"].set_color(t["axis"]) + ax.tick_params(axis="both", length=0, pad=6) + ax.set_title(f"{name}", loc="left", fontsize=13, fontweight="semibold", + color=t["ink"], pad=22) + ax.text(0, 1.035, desc, transform=ax.transAxes, fontsize=10.5, color=t["ink2"]) + axes[0].set_ylabel("compute saving ratio", fontsize=10.5) + + handles, labels = axes[0].get_legend_handles_labels() + fig.legend(handles, labels, loc="outside upper center", ncol=len(labels), frameon=False, + fontsize=10.5, handlelength=2.4, columnspacing=1.8, labelcolor=t["ink2"]) + fig.supxlabel("cache size, in KV blocks and in GiB of Qwen3-Coder-30B's bf16 KV cache", + fontsize=10, color=t["ink2"]) + fig.savefig(out, format="svg", transparent=True, metadata={"Date": None}) + plt.close(fig) + + +def main(): + setup_font() + data = load(os.path.join(HERE, "compute_saving.txt")) + plot(data, "light", os.path.join(HERE, "compute_saving.svg")) + plot(data, "dark", os.path.join(HERE, "compute_saving-dark.svg")) + + +if __name__ == "__main__": + main()