diff --git a/README.md b/README.md index abe9d26d..59f0663a 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,10 @@ No PyPI publication is implied. - [Native CUDA build](docs/launching.md#automatic-native-cuda-build) - [Migration from v0.1.0](CHANGELOG.md) +Packed prefill uses `ExecutionConfig.packed_prefill` and the +`--packed-prefill` / `--no-packed-prefill` flags. Input transfer is controlled +independently by `use_uva`; see the [selection rules](docs/runtime.md#packed-prefill). + ## Model Support diff --git a/docs/dory.md b/docs/dory.md index d53dfe69..13f1f94a 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -94,7 +94,7 @@ layer id rather than their physical sublayer index. These fail fast rather than degrade silently: -- WCPB, mixed or staged prefill; packed prefill metadata (`prefill_uva`); PD. +- WCPB, mixed or staged prefill; packed prefill metadata (`packed_prefill`); PD. - Prefix caching. - Self-speculation combined with CUDA Graphs, asynchronous scheduling, preemption or `last_loop` KV, and sampled token-pipeline speculation. The diff --git a/docs/launching.md b/docs/launching.md index 19ebb747..604800ca 100644 --- a/docs/launching.md +++ b/docs/launching.md @@ -247,12 +247,17 @@ packages, and execution requirements. CPU uses Torch. For example, on an RTX 4090, the default 16-token block size rules out FA2's 256-token page requirement; auto selects Triton if it is installed. -GPU async scheduling and CUDA Graphs require a device backend. UVA packed +GPU async scheduling and CUDA Graphs require a device backend. Packed prefill requires declared packed-prefill support and LAST_EXITED KV. Auto reports an error if no implementation satisfies these requirements. Selection occurs before memory profiling, and the startup log reports the selected backend and why earlier candidates were skipped. +`--packed-prefill` forces packed prefill and `--no-packed-prefill` disables it; +omitting both selects it automatically when supported. `--no-uva` independently +selects copies instead of mapped host-memory reads. See the +[packed-prefill selection rules](runtime.md#packed-prefill). + Explicit choices (`torch`, `triton`, `flash_attn`, or `flash_attn_2/3/4`) retain their validation errors and never fall back. Existing defaults are unchanged: the inference CLI defaults to Torch and the local server defaults to Triton. diff --git a/docs/runtime.md b/docs/runtime.md index 460c5e9c..66f515e4 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -76,6 +76,22 @@ OMP_NUM_THREADS=1 python -m vllm_rlt.entrypoints.cli --toy --dtype float32 \ For a task-assigned GPU, set `CUDA_VISIBLE_DEVICES` and add `--device cuda --attention-backend triton`. Use `--single-stream` for the single-stream pipeline. `--num-blocks` overrides automatic sizing, or specify `--kv-cache-memory-bytes`; these two overrides are mutually exclusive. CPU auto mode keeps a 256-block default. `StageEngine.memory_plan` records the worker-local sizing decision; it is not an attribute of the public `LLM.engine` client. +## Packed prefill + +`ExecutionConfig.packed_prefill` controls packed prefill attention and metadata +staging. Its default, `None`, selects packed prefill when the selected backend +supports it on CUDA with `last_exited` KV and ordinary, non-staged prefill. +Mixed scheduling, wavefront chunked prefill and sliding-window attention keep +their existing paths. These selection rules apply to all models. + +Set `packed_prefill=True` / `--packed-prefill` to require packed prefill, or +`packed_prefill=False` / `--no-packed-prefill` to disable its staging path. +Explicit requests still undergo device, model and backend compatibility checks. + +`ExecutionConfig.use_uva` independently controls how input buffers are read. +`use_uva=False` / `--no-uva` copies them to the GPU and can be combined with +packed prefill. + ## Execution batches and mixed scheduling Stage composition, component ownership and Request handoff are described in the diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index e6179d0d..882b7d43 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -186,6 +186,30 @@ def test_triton_packed_prefill_is_per_selection_opt_in(packages, monkeypatch, re assert not later.capabilities.packed_prefill +@pytest.mark.parametrize( + "missing,expected,packed", + [ + ((), "flash_attn_4", True), + (("flash_attn.cute",), "flash_attn_3", False), + ( + ("flash_attn.cute", "flash_attn_3.flash_attn_interface", "flash_attn"), + "triton", + True, + ), + ], +) +def test_preferred_packed_prefill_preserves_backend_priority( + packages, monkeypatch, missing, expected, packed +): + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) + packages.difference_update(missing) + args = ("auto", torch.device("cuda"), torch.bfloat16, 128, 256) + baseline = create_backend(*args) + preferred = create_backend(*args, prefer_packed_prefill=True) + assert preferred.info["selected_backend"] == baseline.info["selected_backend"] == expected + assert preferred.capabilities.packed_prefill is packed + + def test_packed_prefill_requires_declared_support(packages, monkeypatch): monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) packages.remove("flash_attn.cute") diff --git a/tests/test_engine.py b/tests/test_engine.py index 1ad00979..e265904a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -415,3 +415,31 @@ def test_finish_reason_preserves_public_string_values(reason): output = RequestOutput.from_request(request) assert output.finished and type(output.finish_reason) is str assert output.finish_reason == reason.value + + +def test_packed_prefill_resolves_automatically_when_unset(): + from vllm_rlt.config import ExecutionConfig + from vllm_rlt.engine.packed_prefill import packed_prefill_eligible + + common = dict(layout="last_exited", staged_prefill=False) + # Unset: on for CUDA, last_exited, ordinary prefill without sliding windows. + assert packed_prefill_eligible(None, device_type="cuda", **common) + assert not packed_prefill_eligible(None, device_type="cpu", **common) + assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "layout": "shared"}) + assert not packed_prefill_eligible(None, device_type="cuda", **common, sliding_window=True) + assert not packed_prefill_eligible( + None, device_type="cuda", **{**common, "staged_prefill": True} + ) + # Explicit settings win regardless of eligibility. + assert not packed_prefill_eligible(False, device_type="cuda", **common) + assert packed_prefill_eligible(True, device_type="cuda", **common) + assert ExecutionConfig().packed_prefill is None + with pytest.raises(ValueError, match="packed_prefill"): + ExecutionConfig(packed_prefill=1) + # A CPU engine resolves the unset value to the per-row path and records the decision. + engine = LLMEngine(tiny_model()) + assert engine.execution_config.packed_prefill is False + with pytest.raises(ValueError, match="packed_prefill requires CUDA"): + LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=True)) + engine = LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=False)) + assert engine.execution_config.packed_prefill is False diff --git a/tests/test_input_preparation.py b/tests/test_input_preparation.py index 6091a703..23e315ab 100644 --- a/tests/test_input_preparation.py +++ b/tests/test_input_preparation.py @@ -16,7 +16,7 @@ @pytest.mark.parametrize( - "device,layout,backend,uva", + "device,layout,backend,packed_prefill", [ ("cpu", "shared", "torch", False), ("cpu", "last_exited", "torch", False), @@ -28,7 +28,7 @@ ], ) def test_loop_preparation_reuses_growing_storage_and_only_saves_final_hidden( - monkeypatch, device, layout, backend, uva + monkeypatch, device, layout, backend, packed_prefill ): torch.manual_seed(123) config = ( @@ -43,7 +43,7 @@ def test_loop_preparation_reuses_growing_storage_and_only_saves_final_hidden( model, cache_config=CacheConfig(128, 16 if backend == "flash_attn_4" else 2, layout=layout), execution_config=ExecutionConfig( - static_buffers=True, pad_to_power_of_two=True, prefill_uva=uva + static_buffers=True, pad_to_power_of_two=True, packed_prefill=packed_prefill ), exit_config=ExitConfig("ouro"), scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=8), @@ -121,7 +121,7 @@ def core(*args, **kwargs): assert gates and not any(gates) assert outputs[0] == outputs[1] == outputs[2] == outputs[4] assert outputs[3] == outputs[5] - if uva: + if packed_prefill: previous = {} reused = grew = False for _, tensor, slot, count in snapshots: diff --git a/tests/test_prefix_growth.py b/tests/test_prefix_growth.py index c31d4aeb..9fe717c6 100644 --- a/tests/test_prefix_growth.py +++ b/tests/test_prefix_growth.py @@ -159,7 +159,7 @@ def test_lossless_preemption_preserves_looped_history(): @pytest.mark.gpu @pytest.mark.parametrize("graph", [False, True]) @pytest.mark.parametrize("use_uva", [False, True]) -def test_fa4_prefill_uva_prefix_growth_and_bank_reuse(graph, use_uva): +def test_fa4_packed_prefill_prefix_growth_and_bank_reuse(graph, use_uva): from vllm_rlt import ExecutionConfig, ExitConfig torch.manual_seed(21) @@ -181,7 +181,7 @@ def test_fa4_prefill_uva_prefix_growth_and_bank_reuse(graph, use_uva): exit_config=ExitConfig("trace", depths_by_request={"t": [4, 1, 2, 3, 4, 2, 1, 3]}), execution_config=ExecutionConfig( async_scheduling=True, - prefill_uva=enabled, + packed_prefill=enabled, use_uva=use_uva, static_buffers=graph, pad_to_power_of_two=graph, @@ -237,7 +237,7 @@ def test_priority_preempts_at_safe_boundary(): @pytest.mark.gpu @pytest.mark.parametrize("use_uva", [False, True]) -def test_prefill_uva_metadata_matches_reference_and_waits_for_consumer(use_uva): +def test_packed_prefill_metadata_matches_reference_and_waits_for_consumer(use_uva): from vllm_rlt.worker.buffers import PackedMetadataBuffer c = KVCacheManager( diff --git a/tests/test_tensor_transfer.py b/tests/test_tensor_transfer.py index 61dd0a90..787dd401 100644 --- a/tests/test_tensor_transfer.py +++ b/tests/test_tensor_transfer.py @@ -6,7 +6,7 @@ import torch from vllm_rlt import ExecutionConfig -from vllm_rlt.entrypoints.runtime_args import add_runtime_args +from vllm_rlt.entrypoints.runtime_args import add_runtime_args, runtime_configs from vllm_rlt.worker.tensors import ( StagedWriteTensor, TensorTransfer, @@ -379,15 +379,22 @@ def test_staged_inputs_preserve_slices_and_inflight_snapshots(device, use_uva): assert allocations[3] is allocations[5] -def test_uva_configuration_and_capability(monkeypatch): +@pytest.mark.parametrize( + "flags,packed_prefill", + [([], None), (["--packed-prefill"], True), (["--no-packed-prefill"], False)], +) +def test_uva_configuration_and_capability(monkeypatch, flags, packed_prefill): assert ExecutionConfig().use_uva with pytest.raises(ValueError, match="use_uva"): ExecutionConfig(use_uva=1) parser = argparse.ArgumentParser() add_runtime_args(parser) + parser.set_defaults(num_blocks=None, block_size=16) assert parser.parse_args([]).use_uva - args = parser.parse_args(["--no-uva", "--prefill-uva"]) - assert not args.use_uva and args.prefill_uva + args = parser.parse_args(["--no-uva", *flags]) + assert not args.use_uva and args.packed_prefill is packed_prefill + execution = runtime_configs(args)["execution_config"] + assert not execution.use_uva and execution.packed_prefill is packed_prefill monkeypatch.setattr(torch.version, "hip", None) assert use_uva_on("cuda") assert not use_uva_on("cuda", False) diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 7891b917..ce34d024 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -107,6 +107,7 @@ def create_backend( block_size: int, *, required_capabilities: BackendCapabilities = BackendCapabilities(), + prefer_packed_prefill: bool = False, ) -> AttentionBackend: """Explicit choices fail without fallback; auto tries FA4/3/2, Triton, then Torch. @@ -114,6 +115,8 @@ def create_backend( Only compatibility and import failures permit auto to try the next candidate; kernel execution errors never trigger a runtime fallback. Execution requirements filter candidates before memory profiling or device-buffer allocation. + Preferred packed prefill enables it where available without filtering or + reordering candidates; required packed prefill still rejects unsupported ones. """ validate_backend_name(backend) candidates = _AUTO_PRIORITY if backend == "auto" else (backend,) @@ -124,9 +127,11 @@ def create_backend( for candidate in candidates: try: implementation = _create_explicit(candidate, device, dtype, head_dim, block_size) - if candidate == "triton" and required_capabilities.packed_prefill: + if candidate == "triton" and ( + required_capabilities.packed_prefill or prefer_packed_prefill + ): # Packed BF16 prefill has different arithmetic from per-row decode. - # Opt in for prefill_uva without changing existing split/mixed or + # Opt in for packed_prefill without changing existing split/mixed or # Huginn boundary attention when that option is disabled. implementation.capabilities = replace( implementation.capabilities, packed_prefill=True diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 648781d7..b5b58cb6 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -149,12 +149,16 @@ class ExecutionConfig: # Shared switch for mapped pinned-memory reads; unsupported devices copy instead. use_uva: bool = True # Select packed prefill metadata staging (and require packed attention). - # The historical name is retained; use_uva controls its transfer mechanism. - prefill_uva: bool = False + # use_uva independently controls its transfer mechanism. + # None resolves at engine construction: on when the attention backend packs + # prefill (Triton opts in, FlashAttention-4 natively) with last_exited KV and + # ordinary (non-staged) prefill without sliding-window attention; off otherwise. + packed_prefill: bool | None = None def __post_init__(self): + if self.packed_prefill is not None and type(self.packed_prefill) is not bool: + raise ValueError("packed_prefill must be a boolean or None") for name in ( - "prefill_uva", "use_uva", "async_scheduling", "multi_stream", diff --git a/vllm_rlt/engine/packed_prefill.py b/vllm_rlt/engine/packed_prefill.py new file mode 100644 index 00000000..49c49d14 --- /dev/null +++ b/vllm_rlt/engine/packed_prefill.py @@ -0,0 +1,23 @@ +"""Validate packed prefill requests and decide whether automatic packing is eligible.""" + + +def packed_prefill_eligible( + packed_prefill, *, device_type, layout, staged_prefill, sliding_window=False +): + """Validate explicit requests and return whether packing may be enabled. + + Staged prefill and sliding-window attention retain their existing metadata. + Backend support is checked separately by attention selection. + """ + if packed_prefill and sliding_window: + raise ValueError("packed_prefill does not support sliding-window attention") + if packed_prefill and (device_type != "cuda" or layout != "last_exited"): + raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") + if packed_prefill is not None: + return bool(packed_prefill) + return ( + device_type == "cuda" + and layout == "last_exited" + and not staged_prefill + and not sliding_window + ) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index e16f2e07..1cae4096 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -15,6 +15,7 @@ from vllm_rlt.core.scheduler import Scheduler from vllm_rlt.core.scheduling_policy import requires_staged_prefill from vllm_rlt.engine.connector import StageConnector +from vllm_rlt.engine.packed_prefill import packed_prefill_eligible from vllm_rlt.engine.preemption import PreemptionManager from vllm_rlt.engine.stage_config import StageConfig from vllm_rlt.profiling import Profiler @@ -79,13 +80,10 @@ def __init__( raise ValueError( "boundary-KV models do not support staged prefill required by this scheduler" ) - if self.execution_config.prefill_uva or ( - self.execution_config.async_scheduling - and not getattr(model, "supports_async_scheduling", False) + if self.execution_config.async_scheduling and not getattr( + model, "supports_async_scheduling", False ): - raise ValueError( - "boundary-KV models require synchronous scheduling without prefill UVA" - ) + raise ValueError("boundary-KV models require synchronous scheduling") if speculative_config is not None: if not getattr(model, "supports_self_speculation", False): raise ValueError("boundary-KV speculative decoding is not yet supported") @@ -174,12 +172,15 @@ def __init__( raise ValueError( "async scheduling requires ouro_delayed, random_lookahead or trace exit mode" ) - if self.execution_config.prefill_uva and ( - parameter.device.type != "cuda" or cache_config.layout != "last_exited" - ): - raise ValueError("prefill_uva requires CUDA packed prefill with last_exited KV") if staged_prefill and cache_config.layout != "last_exited": raise ValueError("staged prefill requires last_exited KV planes") + auto_packed = packed_prefill_eligible( + self.execution_config.packed_prefill, + device_type=parameter.device.type, + layout=cache_config.layout, + staged_prefill=staged_prefill, + sliding_window=getattr(model, "requires_sliding_window", False), + ) attention = create_backend( attention_backend, parameter.device, @@ -187,7 +188,7 @@ def __init__( config.head_dim, cache_config.block_size, required_capabilities=BackendCapabilities( - packed_prefill=self.execution_config.prefill_uva, + packed_prefill=bool(self.execution_config.packed_prefill), cuda_graphs=self.execution_config.cuda_graphs, async_scheduling=( parameter.device.type == "cuda" and self.execution_config.async_scheduling @@ -195,7 +196,13 @@ def __init__( device_context_lengths=self.async_speculative, sliding_window=getattr(model, "requires_sliding_window", False), ), + prefer_packed_prefill=auto_packed, ) + if self.execution_config.packed_prefill is None: + self.execution_config = replace( + self.execution_config, + packed_prefill=auto_packed and attention.capabilities.packed_prefill, + ) num_blocks, self.memory_plan = plan_cache( model, cache_config, diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 883d884b..0995207f 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -25,9 +25,17 @@ def add_runtime_args(parser): parser.add_argument("--scheduling-policy", choices=["fcfs", "priority"], default="fcfs") parser.add_argument("--enable-preemption", action="store_true") parser.add_argument( - "--prefill-uva", + "--packed-prefill", action="store_true", - help="Use packed prefill metadata staging (requires packed attention)", + default=None, + help="Force packed prefill metadata staging (requires packed attention); " + "default resolves automatically (on for CUDA with last_exited KV)", + ) + parser.add_argument( + "--no-packed-prefill", + action="store_false", + dest="packed_prefill", + help="Keep per-row prefill even when packed attention is available", ) parser.add_argument( "--no-uva", @@ -118,7 +126,7 @@ def runtime_configs(args): else None, ), execution_config=ExecutionConfig( - prefill_uva=args.prefill_uva, + packed_prefill=args.packed_prefill, use_uva=args.use_uva, cuda_graphs=args.cuda_graphs, cuda_graph_max_batch_size=args.cuda_graph_max_batch_size, diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 59634c3e..78f78a78 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -721,7 +721,7 @@ def _uses_packed_loop_inputs(self, batch): and batch.loop_count and self.cache_manager.layout != "shared" and self.cache_manager.attention_capabilities.packed_prefill - and self.execution_config.prefill_uva + and self.execution_config.packed_prefill ) def _prepare_loop_inputs(self, batch, packed_metadata=None):