From 7cc7dd02ca5fa96ca413406eba286c3f4fe28746 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Sun, 25 Oct 2026 17:27:03 +0100 Subject: [PATCH 1/2] [Engine] Packed prefill resolves automatically, for Huginn too ExecutionConfig.prefill_uva now defaults to None and resolves at engine construction: packed (variable-length) prefill when the attention backend packs prefill (Triton opts in, FlashAttention-4 natively), the KV layout is LAST_EXITED and prefill is not staged (wavefront chunking and mixed mode keep the per-row path their schedulers were written against). An explicit True/False always wins; --no-prefill-uva keeps per-row prefill. A CPU engine resolves to per-row prefill and records the decision. Huginn (boundary KV) now packs prefill too: prelude and coda already build packed metadata at depth 0. VLLM_RLT_HUGINN_PACKED_PREFILL=0 opts out. The GSM8K harness accepts Huginn checkpoints (mean_recurrence depth) for the gate. Packed bf16 prefill uses different arithmetic from per-row prefill, so a few greedy sequences change; it was gated on accuracy instead of identity. GH200, 2048-token prompts, async + graphs (Ouro) / sync + graphs (Huginn): Ouro-1.4B: 2.42x / 2.00x throughput at 8 / 32 requests, TTFT -61% at 32; 37/40 requests identical. Huginn-0125: 2.08x at 32 requests, median TTFT 41.9 -> 18.1 s; 512-token prompts 1.20x / 1.33x at 8 / 32, 36/41 identical. Ouro, 512-token prompts: within run-to-run noise of per-row prefill (0.98x / 1.19x at 8 / 32, two kept runs; a per-row rerun gave 0.98x / 1.10x). GSM8K gate (Huginn, greedy, the 70 questions both arms completed): per-row 20, packed 21 correct, 47/70 sequences identical. Signed-off-by: 0z5a Co-Authored-By: Claude Opus 5.5 --- benchmarks/gsm8k.py | 7 ++- benchmarks/gsm8k_backends.py | 9 +-- tests/test_engine.py | 32 +++++++++++ tests/test_gsm8k_accuracy.py | 1 + vllm_rlt/config.py | 10 +++- vllm_rlt/engine/stage_engine.py | 82 ++++++++++++++++++++++------ vllm_rlt/entrypoints/runtime_args.py | 10 +++- 7 files changed, 127 insertions(+), 24 deletions(-) diff --git a/benchmarks/gsm8k.py b/benchmarks/gsm8k.py index b753a55d..f06db051 100644 --- a/benchmarks/gsm8k.py +++ b/benchmarks/gsm8k.py @@ -123,7 +123,12 @@ def select_doc_ids(population, *, limit, seed, split): def checkpoint_loops(config): - """Full recurrent depth of an Ouro checkpoint, from its config.json.""" + """Full recurrent depth: Ouro ``total_ut_steps`` or Huginn ``mean_recurrence``.""" + if "RavenForCausalLM" in (config.get("architectures") or []): + loops = config.get("mean_recurrence") + if type(loops) is not int or loops < 1: + raise ValueError("A Huginn checkpoint needs an integer mean_recurrence >= 1") + return loops loops = config.get("total_ut_steps") if config.get("model_type") != "ouro" or type(loops) is not int or loops < 1: raise ValueError("This evaluation requires an Ouro checkpoint with total_ut_steps >= 1") diff --git a/benchmarks/gsm8k_backends.py b/benchmarks/gsm8k_backends.py index 2931f9d5..974b9cf5 100644 --- a/benchmarks/gsm8k_backends.py +++ b/benchmarks/gsm8k_backends.py @@ -28,10 +28,11 @@ def __call__(self, input_ids, scores): def check_depth(config, loops): - if config.total_ut_steps != loops: - raise ValueError( - f"Model has {config.total_ut_steps} loops but the protocol expects {loops}" - ) + depth = getattr(config, "total_ut_steps", None) + if depth is None: + depth = getattr(config, "mean_recurrence", None) # Huginn + if depth != loops: + raise ValueError(f"Model has {depth} loops but the protocol expects {loops}") class ReleaseExitHooks: diff --git a/tests/test_engine.py b/tests/test_engine.py index 1ad00979..66c66370 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -415,3 +415,35 @@ def test_finish_reason_preserves_public_string_values(reason): output = RequestOutput.from_request(request) assert output.finished and type(output.finish_reason) is str assert output.finish_reason == reason.value + + +def test_packed_prefill_resolves_automatically_when_unset(monkeypatch): + from vllm_rlt.config import ExecutionConfig + from vllm_rlt.engine.stage_engine import packed_prefill_eligible + + common = dict(layout="last_exited", boundary_kv=False, staged_prefill=False) + # Unset: on for CUDA, fully recurrent, last_exited, ordinary prefill; off otherwise. + assert packed_prefill_eligible(None, device_type="cuda", **common) + assert not packed_prefill_eligible(None, device_type="cpu", **common) + assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "layout": "shared"}) + # Huginn packs by default (GSM8K-87 paired gate: no regression); =0 keeps per-row prefill. + assert packed_prefill_eligible(None, device_type="cuda", **{**common, "boundary_kv": True}) + monkeypatch.setenv("VLLM_RLT_HUGINN_PACKED_PREFILL", "0") + assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "boundary_kv": True}) + monkeypatch.delenv("VLLM_RLT_HUGINN_PACKED_PREFILL") + assert not packed_prefill_eligible( + None, device_type="cuda", **{**common, "staged_prefill": True} + ) + # Explicit settings win regardless of eligibility. + assert not packed_prefill_eligible(False, device_type="cuda", **common) + assert packed_prefill_eligible(True, device_type="cuda", **{**common, "boundary_kv": True}) + assert ExecutionConfig().prefill_uva is None + with pytest.raises(ValueError, match="prefill_uva"): + ExecutionConfig(prefill_uva=1) + # A CPU engine resolves the unset value to the per-row path and records the decision. + engine = LLMEngine(tiny_model()) + assert engine.execution_config.prefill_uva is False + with pytest.raises(ValueError, match="prefill_uva requires CUDA"): + LLMEngine(tiny_model(), execution_config=ExecutionConfig(prefill_uva=True)) + engine = LLMEngine(tiny_model(), execution_config=ExecutionConfig(prefill_uva=False)) + assert engine.execution_config.prefill_uva is False diff --git a/tests/test_gsm8k_accuracy.py b/tests/test_gsm8k_accuracy.py index c7da98b0..8c90c3a6 100644 --- a/tests/test_gsm8k_accuracy.py +++ b/tests/test_gsm8k_accuracy.py @@ -522,6 +522,7 @@ def test_min_loops_is_validated_against_the_checkpoint_depth(): ({"model_type": "ouro", "total_ut_steps": 4}, 4), ({"model_type": "ouro", "total_ut_steps": 6}, 6), ({"model_type": "llama", "total_ut_steps": 4}, None), + ({"architectures": ["RavenForCausalLM"], "mean_recurrence": 32}, 32), ({"model_type": "ouro"}, None), ({"model_type": "ouro", "total_ut_steps": 0}, None), ({"model_type": "ouro", "total_ut_steps": 4.0}, None), diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 8872d018..dead3d9c 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -118,11 +118,17 @@ class ExecutionConfig: use_uva: bool = True # Select packed prefill metadata staging (and require packed attention). # The historical name is retained; use_uva controls its transfer mechanism. - prefill_uva: bool = False + # None resolves at engine construction: on when the attention backend packs + # prefill (Triton opts in, FlashAttention-4 natively) with last_exited KV and + # ordinary (non-staged) prefill, including Huginn unless + # VLLM_RLT_HUGINN_PACKED_PREFILL=0; off otherwise. Measured 2.0-2.4x on + # 2048-token prompts (Ouro and Huginn, refill mode). + prefill_uva: bool | None = None def __post_init__(self): + if self.prefill_uva is not None and type(self.prefill_uva) is not bool: + raise ValueError("prefill_uva must be a boolean or None") for name in ( - "prefill_uva", "use_uva", "async_scheduling", "multi_stream", diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 40a6c9a8..6e199276 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -29,6 +29,33 @@ class SignalHandle: row: int +def _huginn_packed_prefill_disabled() -> bool: + """Huginn packs prefill like Ouro; VLLM_RLT_HUGINN_PACKED_PREFILL=0 keeps per-row prefill. + + Measured on GH200 (Huginn-0125, 32 depths): 1.2-2.1x throughput and -39..-64% + TTFT at 512-2048-token prompts; GSM8K-87 (70 paired questions) 21 vs 20 correct + with 47/70 identical token sequences, i.e. no regression. + """ + import os + + return os.environ.get("VLLM_RLT_HUGINN_PACKED_PREFILL", "1") == "0" + + +def packed_prefill_eligible(prefill_uva, *, device_type, layout, boundary_kv, staged_prefill): + """Whether an unset ``prefill_uva`` may resolve to packed prefill. + + Explicit settings win. Packed prefill needs CUDA packed attention and + last_exited KV; staged prefill (wavefront chunking, mixed mode) keeps the + per-row path, which its schedulers were written against. Boundary-KV models + (Huginn) pack unless VLLM_RLT_HUGINN_PACKED_PREFILL=0. + """ + if prefill_uva is not None: + return bool(prefill_uva) + if boundary_kv and _huginn_packed_prefill_disabled(): + return False + return device_type == "cuda" and layout == "last_exited" and not staged_prefill + + class StageEngine: """Single-device loop-level engine with synchronous or pipelined scheduling.""" @@ -67,8 +94,12 @@ def __init__( raise ValueError( "Huginn does not support staged prefill required by this scheduler" ) - if self.execution_config.async_scheduling or self.execution_config.prefill_uva: - raise ValueError("Huginn requires synchronous scheduling without prefill UVA") + if self.execution_config.async_scheduling: + raise ValueError("Huginn requires synchronous scheduling") + if self.execution_config.prefill_uva is True and _huginn_packed_prefill_disabled(): + raise ValueError( + "Huginn packed prefill is disabled by VLLM_RLT_HUGINN_PACKED_PREFILL=0" + ) if speculative_config is not None: raise ValueError("Huginn speculative decoding is not yet supported") self.speculative_config = speculative_config @@ -122,21 +153,40 @@ def __init__( raise ValueError("prefill_uva requires CUDA packed prefill with last_exited KV") if staged_prefill and cache_config.layout != "last_exited": raise ValueError("staged prefill requires last_exited KV planes") - attention = create_backend( - attention_backend, - parameter.device, - parameter.dtype, - config.head_dim, - cache_config.block_size, - required_capabilities=BackendCapabilities( - packed_prefill=self.execution_config.prefill_uva, - cuda_graphs=self.execution_config.cuda_graphs, - async_scheduling=( - parameter.device.type == "cuda" and self.execution_config.async_scheduling - ), - device_context_lengths=self.async_speculative, - ), + auto_packed = packed_prefill_eligible( + self.execution_config.prefill_uva, + device_type=parameter.device.type, + layout=cache_config.layout, + boundary_kv=getattr(model, "requires_boundary_kv", False), + staged_prefill=staged_prefill, ) + + def backend(packed_prefill): + return create_backend( + attention_backend, + parameter.device, + parameter.dtype, + config.head_dim, + cache_config.block_size, + required_capabilities=BackendCapabilities( + packed_prefill=packed_prefill, + cuda_graphs=self.execution_config.cuda_graphs, + async_scheduling=( + parameter.device.type == "cuda" and self.execution_config.async_scheduling + ), + device_context_lengths=self.async_speculative, + ), + ) + + attention = backend(bool(self.execution_config.prefill_uva)) + if auto_packed: + # Keep the backend the user would get anyway; only Triton needs the + # explicit opt-in to pack prefill, FlashAttention-4 packs natively. + if attention.info["selected_backend"] == "triton": + attention = backend(True) + auto_packed = bool(attention.capabilities.packed_prefill) + if self.execution_config.prefill_uva is None: + self.execution_config = replace(self.execution_config, prefill_uva=auto_packed) num_blocks, self.memory_plan = plan_cache( model, cache_config, diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index a166705b..7d19882e 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -26,7 +26,15 @@ def add_runtime_args(parser): parser.add_argument( "--prefill-uva", action="store_true", - help="Use packed prefill metadata staging (requires packed attention)", + default=None, + help="Force packed prefill metadata staging (requires packed attention); " + "default resolves automatically (on for CUDA with last_exited KV)", + ) + parser.add_argument( + "--no-prefill-uva", + action="store_false", + dest="prefill_uva", + help="Keep per-row prefill even when packed attention is available", ) parser.add_argument( "--no-uva", From 60c601c02082815a0d3701001f5353338ed30c19 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Mon, 16 Nov 2026 07:54:32 +0000 Subject: [PATCH 2/2] refactor: clarify packed prefill policy and backend selection Rename prefill_uva to packed_prefill throughout configuration, CLI, runtime, tests and documentation. Remove the model-specific environment override and benchmark changes. Keep eligibility checks separate from attention backend creation, with optional packed support preserving backend priority. Validation: Ruff lint and format; 1080 CPU tests passed (716 skipped); GPU-enabled regression selection: 20 passed, 2 deselected. --- README.md | 4 ++ benchmarks/gsm8k.py | 7 +- benchmarks/gsm8k_backends.py | 9 ++- docs/dory.md | 2 +- docs/launching.md | 7 +- docs/runtime.md | 16 +++++ tests/test_backend_selection.py | 24 +++++++ tests/test_engine.py | 31 ++++---- tests/test_gsm8k_accuracy.py | 1 - tests/test_input_preparation.py | 8 +-- tests/test_prefix_growth.py | 6 +- tests/test_tensor_transfer.py | 15 ++-- vllm_rlt/attention/backend.py | 9 ++- vllm_rlt/config.py | 12 ++-- vllm_rlt/engine/packed_prefill.py | 23 ++++++ vllm_rlt/engine/stage_engine.py | 101 ++++++--------------------- vllm_rlt/entrypoints/runtime_args.py | 8 +-- vllm_rlt/worker/model_runner.py | 2 +- 18 files changed, 149 insertions(+), 136 deletions(-) create mode 100644 vllm_rlt/engine/packed_prefill.py diff --git a/README.md b/README.md index abe9d26d..59f0663a 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,10 @@ No PyPI publication is implied. - [Native CUDA build](docs/launching.md#automatic-native-cuda-build) - [Migration from v0.1.0](CHANGELOG.md) +Packed prefill uses `ExecutionConfig.packed_prefill` and the +`--packed-prefill` / `--no-packed-prefill` flags. Input transfer is controlled +independently by `use_uva`; see the [selection rules](docs/runtime.md#packed-prefill). + ## Model Support diff --git a/benchmarks/gsm8k.py b/benchmarks/gsm8k.py index f05cd8df..d1ecc7f0 100644 --- a/benchmarks/gsm8k.py +++ b/benchmarks/gsm8k.py @@ -123,12 +123,7 @@ def select_doc_ids(population, *, limit, seed, split): def checkpoint_loops(config): - """Full recurrent depth: Ouro ``total_ut_steps`` or Huginn ``mean_recurrence``.""" - if "RavenForCausalLM" in (config.get("architectures") or []): - loops = config.get("mean_recurrence") - if type(loops) is not int or loops < 1: - raise ValueError("A Huginn checkpoint needs an integer mean_recurrence >= 1") - return loops + """Full recurrent depth of an Ouro checkpoint, from its config.json.""" loops = config.get("total_ut_steps") if config.get("model_type") != "ouro" or type(loops) is not int or loops < 1: raise ValueError("This evaluation requires an Ouro checkpoint with total_ut_steps >= 1") diff --git a/benchmarks/gsm8k_backends.py b/benchmarks/gsm8k_backends.py index dd857669..15d57165 100644 --- a/benchmarks/gsm8k_backends.py +++ b/benchmarks/gsm8k_backends.py @@ -48,11 +48,10 @@ def __call__(self, input_ids, scores): def check_depth(config, loops): - depth = getattr(config, "total_ut_steps", None) - if depth is None: - depth = getattr(config, "mean_recurrence", None) # Huginn - if depth != loops: - raise ValueError(f"Model has {depth} loops but the protocol expects {loops}") + if config.total_ut_steps != loops: + raise ValueError( + f"Model has {config.total_ut_steps} loops but the protocol expects {loops}" + ) class ReleaseExitHooks: diff --git a/docs/dory.md b/docs/dory.md index d53dfe69..13f1f94a 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -94,7 +94,7 @@ layer id rather than their physical sublayer index. These fail fast rather than degrade silently: -- WCPB, mixed or staged prefill; packed prefill metadata (`prefill_uva`); PD. +- WCPB, mixed or staged prefill; packed prefill metadata (`packed_prefill`); PD. - Prefix caching. - Self-speculation combined with CUDA Graphs, asynchronous scheduling, preemption or `last_loop` KV, and sampled token-pipeline speculation. The diff --git a/docs/launching.md b/docs/launching.md index 19ebb747..604800ca 100644 --- a/docs/launching.md +++ b/docs/launching.md @@ -247,12 +247,17 @@ packages, and execution requirements. CPU uses Torch. For example, on an RTX 4090, the default 16-token block size rules out FA2's 256-token page requirement; auto selects Triton if it is installed. -GPU async scheduling and CUDA Graphs require a device backend. UVA packed +GPU async scheduling and CUDA Graphs require a device backend. Packed prefill requires declared packed-prefill support and LAST_EXITED KV. Auto reports an error if no implementation satisfies these requirements. Selection occurs before memory profiling, and the startup log reports the selected backend and why earlier candidates were skipped. +`--packed-prefill` forces packed prefill and `--no-packed-prefill` disables it; +omitting both selects it automatically when supported. `--no-uva` independently +selects copies instead of mapped host-memory reads. See the +[packed-prefill selection rules](runtime.md#packed-prefill). + Explicit choices (`torch`, `triton`, `flash_attn`, or `flash_attn_2/3/4`) retain their validation errors and never fall back. Existing defaults are unchanged: the inference CLI defaults to Torch and the local server defaults to Triton. diff --git a/docs/runtime.md b/docs/runtime.md index 460c5e9c..66f515e4 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -76,6 +76,22 @@ OMP_NUM_THREADS=1 python -m vllm_rlt.entrypoints.cli --toy --dtype float32 \ For a task-assigned GPU, set `CUDA_VISIBLE_DEVICES` and add `--device cuda --attention-backend triton`. Use `--single-stream` for the single-stream pipeline. `--num-blocks` overrides automatic sizing, or specify `--kv-cache-memory-bytes`; these two overrides are mutually exclusive. CPU auto mode keeps a 256-block default. `StageEngine.memory_plan` records the worker-local sizing decision; it is not an attribute of the public `LLM.engine` client. +## Packed prefill + +`ExecutionConfig.packed_prefill` controls packed prefill attention and metadata +staging. Its default, `None`, selects packed prefill when the selected backend +supports it on CUDA with `last_exited` KV and ordinary, non-staged prefill. +Mixed scheduling, wavefront chunked prefill and sliding-window attention keep +their existing paths. These selection rules apply to all models. + +Set `packed_prefill=True` / `--packed-prefill` to require packed prefill, or +`packed_prefill=False` / `--no-packed-prefill` to disable its staging path. +Explicit requests still undergo device, model and backend compatibility checks. + +`ExecutionConfig.use_uva` independently controls how input buffers are read. +`use_uva=False` / `--no-uva` copies them to the GPU and can be combined with +packed prefill. + ## Execution batches and mixed scheduling Stage composition, component ownership and Request handoff are described in the diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index e6179d0d..882b7d43 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -186,6 +186,30 @@ def test_triton_packed_prefill_is_per_selection_opt_in(packages, monkeypatch, re assert not later.capabilities.packed_prefill +@pytest.mark.parametrize( + "missing,expected,packed", + [ + ((), "flash_attn_4", True), + (("flash_attn.cute",), "flash_attn_3", False), + ( + ("flash_attn.cute", "flash_attn_3.flash_attn_interface", "flash_attn"), + "triton", + True, + ), + ], +) +def test_preferred_packed_prefill_preserves_backend_priority( + packages, monkeypatch, missing, expected, packed +): + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) + packages.difference_update(missing) + args = ("auto", torch.device("cuda"), torch.bfloat16, 128, 256) + baseline = create_backend(*args) + preferred = create_backend(*args, prefer_packed_prefill=True) + assert preferred.info["selected_backend"] == baseline.info["selected_backend"] == expected + assert preferred.capabilities.packed_prefill is packed + + def test_packed_prefill_requires_declared_support(packages, monkeypatch): monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) packages.remove("flash_attn.cute") diff --git a/tests/test_engine.py b/tests/test_engine.py index f67c2c64..e265904a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -417,34 +417,29 @@ def test_finish_reason_preserves_public_string_values(reason): assert output.finish_reason == reason.value -def test_packed_prefill_resolves_automatically_when_unset(monkeypatch): +def test_packed_prefill_resolves_automatically_when_unset(): from vllm_rlt.config import ExecutionConfig - from vllm_rlt.engine.stage_engine import packed_prefill_eligible + from vllm_rlt.engine.packed_prefill import packed_prefill_eligible - common = dict(layout="last_exited", boundary_kv=False, staged_prefill=False) - # Unset: on for CUDA, fully recurrent, last_exited, ordinary prefill; off otherwise. + common = dict(layout="last_exited", staged_prefill=False) + # Unset: on for CUDA, last_exited, ordinary prefill without sliding windows. assert packed_prefill_eligible(None, device_type="cuda", **common) assert not packed_prefill_eligible(None, device_type="cpu", **common) assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "layout": "shared"}) assert not packed_prefill_eligible(None, device_type="cuda", **common, sliding_window=True) - # Huginn packs by default (GSM8K-87 paired gate: no regression); =0 keeps per-row prefill. - assert packed_prefill_eligible(None, device_type="cuda", **{**common, "boundary_kv": True}) - monkeypatch.setenv("VLLM_RLT_HUGINN_PACKED_PREFILL", "0") - assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "boundary_kv": True}) - monkeypatch.delenv("VLLM_RLT_HUGINN_PACKED_PREFILL") assert not packed_prefill_eligible( None, device_type="cuda", **{**common, "staged_prefill": True} ) # Explicit settings win regardless of eligibility. assert not packed_prefill_eligible(False, device_type="cuda", **common) - assert packed_prefill_eligible(True, device_type="cuda", **{**common, "boundary_kv": True}) - assert ExecutionConfig().prefill_uva is None - with pytest.raises(ValueError, match="prefill_uva"): - ExecutionConfig(prefill_uva=1) + assert packed_prefill_eligible(True, device_type="cuda", **common) + assert ExecutionConfig().packed_prefill is None + with pytest.raises(ValueError, match="packed_prefill"): + ExecutionConfig(packed_prefill=1) # A CPU engine resolves the unset value to the per-row path and records the decision. engine = LLMEngine(tiny_model()) - assert engine.execution_config.prefill_uva is False - with pytest.raises(ValueError, match="prefill_uva requires CUDA"): - LLMEngine(tiny_model(), execution_config=ExecutionConfig(prefill_uva=True)) - engine = LLMEngine(tiny_model(), execution_config=ExecutionConfig(prefill_uva=False)) - assert engine.execution_config.prefill_uva is False + assert engine.execution_config.packed_prefill is False + with pytest.raises(ValueError, match="packed_prefill requires CUDA"): + LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=True)) + engine = LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=False)) + assert engine.execution_config.packed_prefill is False diff --git a/tests/test_gsm8k_accuracy.py b/tests/test_gsm8k_accuracy.py index 8c90c3a6..c7da98b0 100644 --- a/tests/test_gsm8k_accuracy.py +++ b/tests/test_gsm8k_accuracy.py @@ -522,7 +522,6 @@ def test_min_loops_is_validated_against_the_checkpoint_depth(): ({"model_type": "ouro", "total_ut_steps": 4}, 4), ({"model_type": "ouro", "total_ut_steps": 6}, 6), ({"model_type": "llama", "total_ut_steps": 4}, None), - ({"architectures": ["RavenForCausalLM"], "mean_recurrence": 32}, 32), ({"model_type": "ouro"}, None), ({"model_type": "ouro", "total_ut_steps": 0}, None), ({"model_type": "ouro", "total_ut_steps": 4.0}, None), diff --git a/tests/test_input_preparation.py b/tests/test_input_preparation.py index 6091a703..23e315ab 100644 --- a/tests/test_input_preparation.py +++ b/tests/test_input_preparation.py @@ -16,7 +16,7 @@ @pytest.mark.parametrize( - "device,layout,backend,uva", + "device,layout,backend,packed_prefill", [ ("cpu", "shared", "torch", False), ("cpu", "last_exited", "torch", False), @@ -28,7 +28,7 @@ ], ) def test_loop_preparation_reuses_growing_storage_and_only_saves_final_hidden( - monkeypatch, device, layout, backend, uva + monkeypatch, device, layout, backend, packed_prefill ): torch.manual_seed(123) config = ( @@ -43,7 +43,7 @@ def test_loop_preparation_reuses_growing_storage_and_only_saves_final_hidden( model, cache_config=CacheConfig(128, 16 if backend == "flash_attn_4" else 2, layout=layout), execution_config=ExecutionConfig( - static_buffers=True, pad_to_power_of_two=True, prefill_uva=uva + static_buffers=True, pad_to_power_of_two=True, packed_prefill=packed_prefill ), exit_config=ExitConfig("ouro"), scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=8), @@ -121,7 +121,7 @@ def core(*args, **kwargs): assert gates and not any(gates) assert outputs[0] == outputs[1] == outputs[2] == outputs[4] assert outputs[3] == outputs[5] - if uva: + if packed_prefill: previous = {} reused = grew = False for _, tensor, slot, count in snapshots: diff --git a/tests/test_prefix_growth.py b/tests/test_prefix_growth.py index c31d4aeb..9fe717c6 100644 --- a/tests/test_prefix_growth.py +++ b/tests/test_prefix_growth.py @@ -159,7 +159,7 @@ def test_lossless_preemption_preserves_looped_history(): @pytest.mark.gpu @pytest.mark.parametrize("graph", [False, True]) @pytest.mark.parametrize("use_uva", [False, True]) -def test_fa4_prefill_uva_prefix_growth_and_bank_reuse(graph, use_uva): +def test_fa4_packed_prefill_prefix_growth_and_bank_reuse(graph, use_uva): from vllm_rlt import ExecutionConfig, ExitConfig torch.manual_seed(21) @@ -181,7 +181,7 @@ def test_fa4_prefill_uva_prefix_growth_and_bank_reuse(graph, use_uva): exit_config=ExitConfig("trace", depths_by_request={"t": [4, 1, 2, 3, 4, 2, 1, 3]}), execution_config=ExecutionConfig( async_scheduling=True, - prefill_uva=enabled, + packed_prefill=enabled, use_uva=use_uva, static_buffers=graph, pad_to_power_of_two=graph, @@ -237,7 +237,7 @@ def test_priority_preempts_at_safe_boundary(): @pytest.mark.gpu @pytest.mark.parametrize("use_uva", [False, True]) -def test_prefill_uva_metadata_matches_reference_and_waits_for_consumer(use_uva): +def test_packed_prefill_metadata_matches_reference_and_waits_for_consumer(use_uva): from vllm_rlt.worker.buffers import PackedMetadataBuffer c = KVCacheManager( diff --git a/tests/test_tensor_transfer.py b/tests/test_tensor_transfer.py index 61dd0a90..787dd401 100644 --- a/tests/test_tensor_transfer.py +++ b/tests/test_tensor_transfer.py @@ -6,7 +6,7 @@ import torch from vllm_rlt import ExecutionConfig -from vllm_rlt.entrypoints.runtime_args import add_runtime_args +from vllm_rlt.entrypoints.runtime_args import add_runtime_args, runtime_configs from vllm_rlt.worker.tensors import ( StagedWriteTensor, TensorTransfer, @@ -379,15 +379,22 @@ def test_staged_inputs_preserve_slices_and_inflight_snapshots(device, use_uva): assert allocations[3] is allocations[5] -def test_uva_configuration_and_capability(monkeypatch): +@pytest.mark.parametrize( + "flags,packed_prefill", + [([], None), (["--packed-prefill"], True), (["--no-packed-prefill"], False)], +) +def test_uva_configuration_and_capability(monkeypatch, flags, packed_prefill): assert ExecutionConfig().use_uva with pytest.raises(ValueError, match="use_uva"): ExecutionConfig(use_uva=1) parser = argparse.ArgumentParser() add_runtime_args(parser) + parser.set_defaults(num_blocks=None, block_size=16) assert parser.parse_args([]).use_uva - args = parser.parse_args(["--no-uva", "--prefill-uva"]) - assert not args.use_uva and args.prefill_uva + args = parser.parse_args(["--no-uva", *flags]) + assert not args.use_uva and args.packed_prefill is packed_prefill + execution = runtime_configs(args)["execution_config"] + assert not execution.use_uva and execution.packed_prefill is packed_prefill monkeypatch.setattr(torch.version, "hip", None) assert use_uva_on("cuda") assert not use_uva_on("cuda", False) diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 7891b917..ce34d024 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -107,6 +107,7 @@ def create_backend( block_size: int, *, required_capabilities: BackendCapabilities = BackendCapabilities(), + prefer_packed_prefill: bool = False, ) -> AttentionBackend: """Explicit choices fail without fallback; auto tries FA4/3/2, Triton, then Torch. @@ -114,6 +115,8 @@ def create_backend( Only compatibility and import failures permit auto to try the next candidate; kernel execution errors never trigger a runtime fallback. Execution requirements filter candidates before memory profiling or device-buffer allocation. + Preferred packed prefill enables it where available without filtering or + reordering candidates; required packed prefill still rejects unsupported ones. """ validate_backend_name(backend) candidates = _AUTO_PRIORITY if backend == "auto" else (backend,) @@ -124,9 +127,11 @@ def create_backend( for candidate in candidates: try: implementation = _create_explicit(candidate, device, dtype, head_dim, block_size) - if candidate == "triton" and required_capabilities.packed_prefill: + if candidate == "triton" and ( + required_capabilities.packed_prefill or prefer_packed_prefill + ): # Packed BF16 prefill has different arithmetic from per-row decode. - # Opt in for prefill_uva without changing existing split/mixed or + # Opt in for packed_prefill without changing existing split/mixed or # Huginn boundary attention when that option is disabled. implementation.capabilities = replace( implementation.capabilities, packed_prefill=True diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 86092ece..b5b58cb6 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -149,17 +149,15 @@ class ExecutionConfig: # Shared switch for mapped pinned-memory reads; unsupported devices copy instead. use_uva: bool = True # Select packed prefill metadata staging (and require packed attention). - # The historical name is retained; use_uva controls its transfer mechanism. + # use_uva independently controls its transfer mechanism. # None resolves at engine construction: on when the attention backend packs # prefill (Triton opts in, FlashAttention-4 natively) with last_exited KV and - # ordinary (non-staged) prefill, including Huginn unless - # VLLM_RLT_HUGINN_PACKED_PREFILL=0; off otherwise. Measured 2.0-2.4x on - # 2048-token prompts (Ouro and Huginn, refill mode). - prefill_uva: bool | None = None + # ordinary (non-staged) prefill without sliding-window attention; off otherwise. + packed_prefill: bool | None = None def __post_init__(self): - if self.prefill_uva is not None and type(self.prefill_uva) is not bool: - raise ValueError("prefill_uva must be a boolean or None") + if self.packed_prefill is not None and type(self.packed_prefill) is not bool: + raise ValueError("packed_prefill must be a boolean or None") for name in ( "use_uva", "async_scheduling", diff --git a/vllm_rlt/engine/packed_prefill.py b/vllm_rlt/engine/packed_prefill.py new file mode 100644 index 00000000..49c49d14 --- /dev/null +++ b/vllm_rlt/engine/packed_prefill.py @@ -0,0 +1,23 @@ +"""Validate packed prefill requests and decide whether automatic packing is eligible.""" + + +def packed_prefill_eligible( + packed_prefill, *, device_type, layout, staged_prefill, sliding_window=False +): + """Validate explicit requests and return whether packing may be enabled. + + Staged prefill and sliding-window attention retain their existing metadata. + Backend support is checked separately by attention selection. + """ + if packed_prefill and sliding_window: + raise ValueError("packed_prefill does not support sliding-window attention") + if packed_prefill and (device_type != "cuda" or layout != "last_exited"): + raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") + if packed_prefill is not None: + return bool(packed_prefill) + return ( + device_type == "cuda" + and layout == "last_exited" + and not staged_prefill + and not sliding_window + ) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index c30925c8..1cae4096 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -15,6 +15,7 @@ from vllm_rlt.core.scheduler import Scheduler from vllm_rlt.core.scheduling_policy import requires_staged_prefill from vllm_rlt.engine.connector import StageConnector +from vllm_rlt.engine.packed_prefill import packed_prefill_eligible from vllm_rlt.engine.preemption import PreemptionManager from vllm_rlt.engine.stage_config import StageConfig from vllm_rlt.profiling import Profiler @@ -38,41 +39,6 @@ class SignalHandle: row: int -def _huginn_packed_prefill_disabled() -> bool: - """Huginn packs prefill like Ouro; VLLM_RLT_HUGINN_PACKED_PREFILL=0 keeps per-row prefill. - - Measured on GH200 (Huginn-0125, 32 depths): 1.2-2.1x throughput and -39..-64% - TTFT at 512-2048-token prompts; GSM8K-87 (70 paired questions) 21 vs 20 correct - with 47/70 identical token sequences, i.e. no regression. - """ - import os - - return os.environ.get("VLLM_RLT_HUGINN_PACKED_PREFILL", "1") == "0" - - -def packed_prefill_eligible( - prefill_uva, *, device_type, layout, boundary_kv, staged_prefill, sliding_window=False -): - """Whether an unset ``prefill_uva`` may resolve to packed prefill. - - Explicit settings win. Packed prefill needs CUDA packed attention and - last_exited KV; staged prefill (wavefront chunking, mixed mode) keeps the - per-row path, which its schedulers were written against. Boundary-KV models - (Huginn) pack unless VLLM_RLT_HUGINN_PACKED_PREFILL=0. Sliding-window - attention keeps its single-query-row metadata. - """ - if prefill_uva is not None: - return bool(prefill_uva) - if boundary_kv and _huginn_packed_prefill_disabled(): - return False - return ( - device_type == "cuda" - and layout == "last_exited" - and not staged_prefill - and not sliding_window - ) - - class StageEngine: """Single-device loop-level engine with synchronous or pipelined scheduling.""" @@ -118,18 +84,6 @@ def __init__( model, "supports_async_scheduling", False ): raise ValueError("boundary-KV models require synchronous scheduling") - if self.execution_config.prefill_uva and getattr( - model, "requires_sliding_window", False - ): - raise ValueError("prefill_uva does not support sliding-window attention") - if ( - self.execution_config.prefill_uva is True - and not getattr(model, "requires_sliding_window", False) - and _huginn_packed_prefill_disabled() - ): - raise ValueError( - "Huginn packed prefill is disabled by VLLM_RLT_HUGINN_PACKED_PREFILL=0" - ) if speculative_config is not None: if not getattr(model, "supports_self_speculation", False): raise ValueError("boundary-KV speculative decoding is not yet supported") @@ -218,48 +172,37 @@ def __init__( raise ValueError( "async scheduling requires ouro_delayed, random_lookahead or trace exit mode" ) - if self.execution_config.prefill_uva and ( - parameter.device.type != "cuda" or cache_config.layout != "last_exited" - ): - raise ValueError("prefill_uva requires CUDA packed prefill with last_exited KV") if staged_prefill and cache_config.layout != "last_exited": raise ValueError("staged prefill requires last_exited KV planes") auto_packed = packed_prefill_eligible( - self.execution_config.prefill_uva, + self.execution_config.packed_prefill, device_type=parameter.device.type, layout=cache_config.layout, - boundary_kv=getattr(model, "requires_boundary_kv", False), staged_prefill=staged_prefill, sliding_window=getattr(model, "requires_sliding_window", False), ) - - def backend(packed_prefill): - return create_backend( - attention_backend, - parameter.device, - parameter.dtype, - config.head_dim, - cache_config.block_size, - required_capabilities=BackendCapabilities( - packed_prefill=packed_prefill, - cuda_graphs=self.execution_config.cuda_graphs, - async_scheduling=( - parameter.device.type == "cuda" and self.execution_config.async_scheduling - ), - device_context_lengths=self.async_speculative, - sliding_window=getattr(model, "requires_sliding_window", False), + attention = create_backend( + attention_backend, + parameter.device, + parameter.dtype, + config.head_dim, + cache_config.block_size, + required_capabilities=BackendCapabilities( + packed_prefill=bool(self.execution_config.packed_prefill), + cuda_graphs=self.execution_config.cuda_graphs, + async_scheduling=( + parameter.device.type == "cuda" and self.execution_config.async_scheduling ), + device_context_lengths=self.async_speculative, + sliding_window=getattr(model, "requires_sliding_window", False), + ), + prefer_packed_prefill=auto_packed, + ) + if self.execution_config.packed_prefill is None: + self.execution_config = replace( + self.execution_config, + packed_prefill=auto_packed and attention.capabilities.packed_prefill, ) - - attention = backend(bool(self.execution_config.prefill_uva)) - if auto_packed: - # Keep the backend the user would get anyway; only Triton needs the - # explicit opt-in to pack prefill, FlashAttention-4 packs natively. - if attention.info["selected_backend"] == "triton": - attention = backend(True) - auto_packed = bool(attention.capabilities.packed_prefill) - if self.execution_config.prefill_uva is None: - self.execution_config = replace(self.execution_config, prefill_uva=auto_packed) num_blocks, self.memory_plan = plan_cache( model, cache_config, diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index a1f11a28..0995207f 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -25,16 +25,16 @@ def add_runtime_args(parser): parser.add_argument("--scheduling-policy", choices=["fcfs", "priority"], default="fcfs") parser.add_argument("--enable-preemption", action="store_true") parser.add_argument( - "--prefill-uva", + "--packed-prefill", action="store_true", default=None, help="Force packed prefill metadata staging (requires packed attention); " "default resolves automatically (on for CUDA with last_exited KV)", ) parser.add_argument( - "--no-prefill-uva", + "--no-packed-prefill", action="store_false", - dest="prefill_uva", + dest="packed_prefill", help="Keep per-row prefill even when packed attention is available", ) parser.add_argument( @@ -126,7 +126,7 @@ def runtime_configs(args): else None, ), execution_config=ExecutionConfig( - prefill_uva=args.prefill_uva, + packed_prefill=args.packed_prefill, use_uva=args.use_uva, cuda_graphs=args.cuda_graphs, cuda_graph_max_batch_size=args.cuda_graph_max_batch_size, diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 59634c3e..78f78a78 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -721,7 +721,7 @@ def _uses_packed_loop_inputs(self, batch): and batch.loop_count and self.cache_manager.layout != "shared" and self.cache_manager.attention_capabilities.packed_prefill - and self.execution_config.prefill_uva + and self.execution_config.packed_prefill ) def _prepare_loop_inputs(self, batch, packed_metadata=None):