diff --git a/recipe/simple_use_case/relax_demo.py b/recipe/simple_use_case/relax_demo.py index 6cb1766e..b6e8ce4e 100644 --- a/recipe/simple_use_case/relax_demo.py +++ b/recipe/simple_use_case/relax_demo.py @@ -324,7 +324,7 @@ def build_tq_config(config: DemoConfig): base = OmegaConf.load(resources.files("transfer_queue") / "config.yaml") override = OmegaConf.create( { - "controller": {"sampler": RankAwareSampler, "polling_mode": True}, + "controller": {"sampler": RankAwareSampler}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": {"num_data_storage_units": config.num_data_storage_units}, diff --git a/scripts/performance_test/perftest_config.yaml b/scripts/performance_test/perftest_config.yaml index bf2692fa..ca6cb792 100644 --- a/scripts/performance_test/perftest_config.yaml +++ b/scripts/performance_test/perftest_config.yaml @@ -10,8 +10,6 @@ metrics: controller: # User-defined sampler. User can pass sampler instance to overwrite this string config. sampler: SequentialSampler - # Whether return an empty BatchMeta to prevent request blocking when no enough data is available - polling_mode: False # ZMQ Server IP & Ports (automatically generated during init) zmq_info: null diff --git a/scripts/performance_test_ssd_offload/perftest_config.yaml b/scripts/performance_test_ssd_offload/perftest_config.yaml index 381c6b0d..a7c6a6c4 100644 --- a/scripts/performance_test_ssd_offload/perftest_config.yaml +++ b/scripts/performance_test_ssd_offload/perftest_config.yaml @@ -10,8 +10,6 @@ metrics: controller: # User-defined sampler. User can pass sampler instance to overwrite this string config. sampler: SequentialSampler - # Whether return an empty BatchMeta to prevent request blocking when no enough data is available - polling_mode: False # ZMQ Server IP & Ports (automatically generated during init) zmq_info: null diff --git a/tests/e2e/test_checkpoint_e2e.py b/tests/e2e/test_checkpoint_e2e.py index f2a8da92..de0baa88 100644 --- a/tests/e2e/test_checkpoint_e2e.py +++ b/tests/e2e/test_checkpoint_e2e.py @@ -34,7 +34,6 @@ _TQ_CONFIG = OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/e2e/test_data_dump_cross_topology_e2e.py b/tests/e2e/test_data_dump_cross_topology_e2e.py index 445bdbd3..1bf4d9f9 100644 --- a/tests/e2e/test_data_dump_cross_topology_e2e.py +++ b/tests/e2e/test_data_dump_cross_topology_e2e.py @@ -43,7 +43,6 @@ def _tq_config(num_storage_units: int) -> OmegaConf: return OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/e2e/test_data_dump_e2e.py b/tests/e2e/test_data_dump_e2e.py index 95816533..b849ac52 100644 --- a/tests/e2e/test_data_dump_e2e.py +++ b/tests/e2e/test_data_dump_e2e.py @@ -49,7 +49,6 @@ def _tq_config(num_storage_units: int) -> OmegaConf: return OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/tests/e2e/test_e2e_lifecycle_consistency.py b/tests/e2e/test_e2e_lifecycle_consistency.py index 97028878..a246311c 100644 --- a/tests/e2e/test_e2e_lifecycle_consistency.py +++ b/tests/e2e/test_e2e_lifecycle_consistency.py @@ -46,9 +46,6 @@ # Backend configurations for E2E tests BACKEND_CONFIGS = { "SimpleStorage": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { @@ -58,9 +55,6 @@ }, }, "MooncakeStore": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "MooncakeStore", "MooncakeStore": { @@ -74,9 +68,6 @@ }, }, "Yuanrong": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "Yuanrong", "Yuanrong": { @@ -90,9 +81,6 @@ # GPUDirect RDMA transfers directly from the registered CUDA staging buffer. # auto_init=true: TQ starts mooncake_master automatically on transfer_queue.init(). "MooncakeStore_GDR": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "MooncakeStore", "MooncakeStore": { diff --git a/tests/e2e/test_kv_global_lock_e2e.py b/tests/e2e/test_kv_global_lock_e2e.py index 661bf7fa..8623fb34 100644 --- a/tests/e2e/test_kv_global_lock_e2e.py +++ b/tests/e2e/test_kv_global_lock_e2e.py @@ -34,7 +34,6 @@ # Bound every wait so that a lock regression fails the test instead of hanging it. TIMEOUT_S = 60 CONF = { - "controller": {"polling_mode": True}, "backend": {"storage_backend": "SimpleStorage", "SimpleStorage": {"total_storage_size": 100}}, } diff --git a/tests/e2e/test_kv_interface_e2e.py b/tests/e2e/test_kv_interface_e2e.py index c3ec0c13..52b0d7dd 100644 --- a/tests/e2e/test_kv_interface_e2e.py +++ b/tests/e2e/test_kv_interface_e2e.py @@ -67,9 +67,6 @@ def tq_api(request): # Adjust values for GitHub CI environment (smaller memory footprint) BACKEND_CONFIGS = { "SimpleStorage": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { @@ -79,9 +76,6 @@ def tq_api(request): }, }, "MooncakeStore": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "MooncakeStore", "MooncakeStore": { @@ -96,9 +90,6 @@ def tq_api(request): }, }, "Yuanrong": { - "controller": { - "polling_mode": True, - }, "backend": { "storage_backend": "Yuanrong", "Yuanrong": { diff --git a/tests/e2e/test_kv_link_e2e.py b/tests/e2e/test_kv_link_e2e.py index d5ecc61f..74ec0e23 100644 --- a/tests/e2e/test_kv_link_e2e.py +++ b/tests/e2e/test_kv_link_e2e.py @@ -39,7 +39,6 @@ def tq_system(): tq.init( OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": {"total_storage_size": 200, "num_data_storage_units": 2}, diff --git a/tests/e2e/test_ssd_offload_e2e.py b/tests/e2e/test_ssd_offload_e2e.py index eb9060b6..86737798 100644 --- a/tests/e2e/test_ssd_offload_e2e.py +++ b/tests/e2e/test_ssd_offload_e2e.py @@ -52,7 +52,6 @@ def tq_system(ssd_root): config = OmegaConf.create( { - "controller": {"polling_mode": True}, "backend": { "storage_backend": "SimpleStorage", "SimpleStorage": { diff --git a/transfer_queue/config.yaml b/transfer_queue/config.yaml index e0ac6599..af35f2fd 100644 --- a/transfer_queue/config.yaml +++ b/transfer_queue/config.yaml @@ -10,8 +10,6 @@ metrics: controller: # User-defined sampler. User can pass sampler instance to overwrite this string config. sampler: SequentialSampler - # Whether return an empty BatchMeta to prevent request blocking when no enough data is available - polling_mode: False # ZMQ Server IP & Ports (automatically generated during init) zmq_info: null @@ -86,9 +84,6 @@ backend: # from ANY node's memory to this offload client via TCP/RDMA. # Ensure that `local_hostname` (above) is set to a cluster-reachable address so that # all nodes can transfer data to the offload client. - # - # TODO: Support multi-node distributed offload (one client per node) for better - # scalability in large clusters. offload: # Master switch for SSD offload. enabled: false diff --git a/transfer_queue/controller.py b/transfer_queue/controller.py index d500c027..db01b24e 100644 --- a/transfer_queue/controller.py +++ b/transfer_queue/controller.py @@ -17,6 +17,7 @@ import os import pickle import time +import warnings from collections import defaultdict, deque from collections.abc import Sequence from dataclasses import dataclass, field @@ -55,10 +56,6 @@ logger = get_logger(__name__) -TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1)) -TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5)) -# Above libzmq's default of 1023: the metrics exporter borrows this context to query -# storage units, so the budget covers a fleet rather than the two ROUTERs bound here. TQ_CONTROLLER_ZMQ_MAX_SOCKETS = int(os.environ.get("TQ_CONTROLLER_ZMQ_MAX_SOCKETS", 4096)) # Holds payloads that keys of other partitions read through kv_link, e.g. a prompt video @@ -1124,7 +1121,7 @@ class TransferQueueController: def __init__( self, sampler: BaseSampler | type[BaseSampler] = SequentialSampler, - polling_mode: bool = False, + polling_mode: bool | None = None, ) -> None: """Initialize the TransferQueue Controller. @@ -1135,11 +1132,17 @@ def __init__( - Defaults to SequentialSampler for simple sequential sampling - Example: sampler=GRPOGroupNSampler() (instance) - Example: sampler=SequentialSampler (class) - polling_mode: Whether to use polling mode for TransferQueue controller. - - If False, the controller will raise an error when no enough data is available. - - If True, the controller will return an empty BatchMeta when no enough data is available. - The user side is responsible for handling this empty case (retrying later). - """ + polling_mode: Deprecated. A metadata fetch returns an empty batch when no batch is available. + """ + if polling_mode is not None: + warnings.warn( + "The `polling_mode` argument is deprecated and ignored. It will be removed in the next release. " + "Remove it when creating the controller. " + "A metadata fetch returns an empty BatchMeta when no batch is available.", + FutureWarning, + stacklevel=2, + ) + if isinstance(sampler, BaseSampler): self.sampler = sampler elif isinstance(sampler, type) and issubclass(sampler, BaseSampler): @@ -1150,7 +1153,6 @@ def __init__( ) self.controller_id = f"TQ_CONTROLLER_{uuid4().hex[:8]}" - self.polling_mode = polling_mode self.tq_config = None # global config for TransferQueue system # Initialize ZMQ sockets for communication @@ -1417,10 +1419,8 @@ def get_metadata( **kwargs: Additional keyword arguments Returns: - BatchMeta object containing the requested metadata - - Raises: - TimeoutError: If waiting for sufficient data times out in fetch mode + BatchMeta containing the requested metadata. A fetch returns an empty batch + when no batch is available. """ if mode == "insert": @@ -1458,40 +1458,19 @@ def get_metadata( if batch_size is None: raise ValueError("must provide batch_size in fetch mode") - start_time = time.time() - while True: - # ready_for_consume_indexes: samples where all required fields are produced - # (production status is ready) and not yet consumed - ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name) - - if len(ready_for_consume_indexes) < batch_size: - if self.polling_mode: - # Return cached result if available - if self.sampler.has_cached_result(partition_id, task_name, sampling_config): - break - else: - logger.debug( - f"[{self.controller_id}]: Not enough data for task {task_name} in " - f"partition {partition_id}. Required: {batch_size}, " - f"Available: {len(ready_for_consume_indexes)}." - f" Returning None due to polling mode." - ) - return BatchMeta.empty() - else: - logger.warning( - f"[{self.controller_id}]: Insufficient data for task {task_name}. Required: {batch_size} " - f"samples with fields {data_fields} in partition {partition_id}, but only have " - f"{len(ready_for_consume_indexes)} samples meeting the criteria. " - f"Retrying in {TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL}s..." - ) - time.sleep(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL) - if time.time() - start_time > TQ_CONTROLLER_GET_METADATA_TIMEOUT: - raise TimeoutError( - f"Timeout while waiting for sufficient data for task {task_name}. " - f"Required: {batch_size}, Available: {len(ready_for_consume_indexes)}" - ) - else: - break + # ready_for_consume_indexes: samples where all required fields are produced + # (production status is ready) and not yet consumed + ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name) + + if len(ready_for_consume_indexes) < batch_size: + # Return cached result if available + if not self.sampler.has_cached_result(partition_id, task_name, sampling_config): + logger.debug( + f"[{self.controller_id}]: Insufficient data for task {task_name}. Required: {batch_size} " + f"samples with fields {data_fields} in partition {partition_id}, but only have " + f"{len(ready_for_consume_indexes)} samples meeting the criteria. " + ) + return BatchMeta.empty() batch_global_indexes, consumed_indexes = self.sampler( ready_for_consume_indexes, @@ -1505,14 +1484,14 @@ def get_metadata( # Some samplers (e.g. SeqlenBalancedSampler) may return variable-size # batches per DP rank, so we only check for empty results. if len(batch_global_indexes) == 0: - if self.polling_mode: - return BatchMeta.empty() - raise RuntimeError( + logger.debug( f"Sampler returned no samples. Please check the sampler logic. " f"Expected: {batch_size}, before sampling: {len(ready_for_consume_indexes)}, " f"after sampling: {len(batch_global_indexes)}" ) + return BatchMeta.empty() + # Mark samples as consumed if in fetch mode if consumed_indexes: partition = self.partitions[partition_id] @@ -1520,7 +1499,6 @@ def get_metadata( elif mode == "force_fetch": batch_global_indexes = self.index_manager.get_indexes_for_partition(partition_id) - consumed_indexes = [] # Package into metadata metadata = self.generate_batch_meta(partition_id, batch_global_indexes, data_fields, mode) diff --git a/transfer_queue/interface.py b/transfer_queue/interface.py index a9615c9b..7e05949d 100644 --- a/transfer_queue/interface.py +++ b/transfer_queue/interface.py @@ -33,6 +33,7 @@ import shutil import subprocess import time +import warnings from collections import Counter, defaultdict from importlib import resources from pathlib import Path @@ -41,7 +42,7 @@ import ray import torch -from omegaconf import DictConfig, OmegaConf +from omegaconf import DictConfig, OmegaConf, open_dict from tensordict import TensorDict from tensordict.tensorclass import NonTensorStack @@ -177,6 +178,17 @@ def init(conf: DictConfig | None = None) -> DictConfig | None: >>> metadata = tq.get_meta(...) >>> data = tq.get_data(metadata) """ + controller_conf = conf.get("controller") if conf is not None else None + has_legacy_polling_mode = controller_conf is not None and "polling_mode" in controller_conf.keys() + if has_legacy_polling_mode: + warnings.warn( + "`controller.polling_mode` is deprecated and ignored. It will be removed in the next release. " + "Remove this setting. " + "A metadata fetch returns an empty BatchMeta when no batch is available.", + FutureWarning, + stacklevel=2, + ) + if _init_from_existing(): return conf @@ -187,6 +199,9 @@ def init(conf: DictConfig | None = None) -> DictConfig | None: final_conf = OmegaConf.merge(final_conf, default_conf) if conf: final_conf = OmegaConf.merge(final_conf, conf) + if has_legacy_polling_mode: + with open_dict(final_conf.controller): + del final_conf.controller["polling_mode"] # TODO(hz): support load custom sampler class from external module. try: @@ -209,7 +224,7 @@ def init(conf: DictConfig | None = None) -> DictConfig | None: global _TQ_CONTROLLER, _TQ_IS_OWNER _TQ_CONTROLLER = TransferQueueController.options( # type: ignore[attr-defined] name="TransferQueueController", namespace=ray_namespace() - ).remote(sampler=sampler, polling_mode=final_conf.controller.polling_mode) + ).remote(sampler=sampler) _TQ_IS_OWNER = True logger.info("TransferQueueController has been created.") _start_lock_managers(lock_settings) diff --git a/transfer_queue/sampler/base.py b/transfer_queue/sampler/base.py index 173b19da..da19559b 100644 --- a/transfer_queue/sampler/base.py +++ b/transfer_queue/sampler/base.py @@ -83,9 +83,8 @@ def has_cached_result( ) -> bool: """Check whether the sampler has a cached sampling result for the given context. - This is used by the controller in polling mode to determine if a previously - computed sampling result is already available, so that it can skip waiting - for more data and directly proceed to return the cached result. + The controller checks this when there are too few unconsumed samples for a + new batch. A cache hit lets the sampler return its previous result. The check is based on the ``_states`` cache structure: ``_states[partition_id][task_name][dp_rank][batch_index]``. diff --git a/transfer_queue/sampler/seqlen_balanced_sampler.py b/transfer_queue/sampler/seqlen_balanced_sampler.py index 20c904d7..29d47fdd 100644 --- a/transfer_queue/sampler/seqlen_balanced_sampler.py +++ b/transfer_queue/sampler/seqlen_balanced_sampler.py @@ -169,10 +169,8 @@ def sample( # Cache the result self._balanced_cache[cache_key] = partitions - # Populate the inherited _states cache for ALL dp_ranks so that - # the controller's polling check (which looks at self.sampler._states) - # works correctly even when ready_indexes < batch_size for later ranks - # (because earlier ranks already consumed their portion). + # Cache each DP rank's assignment so later fetches can reuse it after + # earlier fetches reduce the ready sample count. if partition_id not in self._states: self._states[partition_id] = {} if task_name not in self._states[partition_id]: diff --git a/tutorial/06_streaming_dataloader.py b/tutorial/06_streaming_dataloader.py index c12186aa..b8906c2b 100644 --- a/tutorial/06_streaming_dataloader.py +++ b/tutorial/06_streaming_dataloader.py @@ -83,17 +83,12 @@ def setup_transfer_queue(): "Note: Using RankAwareSampler when each rank retrieves data independently. It guarantees that " "The same DP rank receives the same sample indices." ) - print( - "Note: When using streaming data retrieval, please set polling_mode=True when initializing " - "TransferQueueController. In polling_mode, the controller will return empty BatchMeta when " - "available data cannot meet the consumption requirements. User side need to retry later." - ) + print("StreamingDataset retries when a metadata fetch returns an empty batch.") config = OmegaConf.create( { "controller": { "sampler": RankAwareSampler, # RankAwareSampler enables consistent sampling for each DP rank - "polling_mode": True, # Enable polling mode for streaming data retrieval }, "backend": {"SimpleStorage": {"num_data_storage_units": 2}}, }, diff --git a/tutorial/basic.ipynb b/tutorial/basic.ipynb index 09f946d8..94c38ae2 100644 --- a/tutorial/basic.ipynb +++ b/tutorial/basic.ipynb @@ -77,7 +77,6 @@ "\n", "config = OmegaConf.create(\n", " {\n", - " \"controller\": {\"polling_mode\": True},\n", " \"backend\": {\n", " \"storage_backend\": \"SimpleStorage\",\n", " \"SimpleStorage\": {\n",