diff --git a/pathwaysutils/elastic/elastic.py b/pathwaysutils/elastic/elastic.py index f20aecd..223aec2 100644 --- a/pathwaysutils/elastic/elastic.py +++ b/pathwaysutils/elastic/elastic.py @@ -78,7 +78,8 @@ class DefaultSliceHealthChecker(SliceHealthChecker): that the devices are active and can perform computations. """ - def _plus_one(self, x: jax.Array) -> jax.Array: + @staticmethod + def _plus_one(x: jax.Array) -> jax.Array: return x + 1 def _simple_execution(self, devices: Sequence[jax.Device]) -> jax.Array: @@ -124,7 +125,6 @@ def validate(self) -> Set[int]: return frozenset(active_slice_indices) - def get_slice_to_devices( devices: Sequence[jax.Device], ) -> Mapping[int, Sequence[jax.Device]]: diff --git a/pathwaysutils/elastic/manager.py b/pathwaysutils/elastic/manager.py index 5473378..02119b6 100644 --- a/pathwaysutils/elastic/manager.py +++ b/pathwaysutils/elastic/manager.py @@ -100,11 +100,22 @@ class Manager: available_inactive_slices: Set[int] _stop_event: threading.Event | None _monitor_thread: threading.Thread | None - def __init__(self, devices: Sequence[jax.Device] | None = None) -> None: + def __init__( + self, + devices: Sequence[jax.Device] | None = None, + *, + minimum_slice_count: int | None = None, + poll_interval: float | int = 10, + timeout: float | int | None = None, + ) -> None: """Initializes the manager. Args: devices: The devices to use. If None, jax.devices() is used. + minimum_slice_count: Minimum active slices to wait for (< 1 = all). If + None, active slices are sampled once without waiting. + poll_interval: Seconds between slice health checks when waiting. + timeout: Maximum seconds to wait when waiting, or None for no timeout. """ if devices is None: devices = jax.devices() @@ -115,9 +126,18 @@ def __init__(self, devices: Sequence[jax.Device] | None = None) -> None: self._active_slice_indices = frozenset() self.available_inactive_slices = frozenset() - self.active_slice_indices = elastic.get_active_slice_indices( - slice_to_devices=self.slice_to_devices - ) + if minimum_slice_count is not None: + slice_count = None if minimum_slice_count < 1 else minimum_slice_count + self.active_slice_indices = elastic.wait_for_slices( + slice_count=slice_count, + poll_interval=poll_interval, + timeout=timeout, + slice_to_devices=self.slice_to_devices, + ) + else: + self.active_slice_indices = elastic.get_active_slice_indices( + slice_to_devices=self.slice_to_devices + ) self._stop_event = None self._monitor_thread = None