Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions pathwaysutils/elastic/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,22 @@ class Manager:
available_inactive_slices: Set[int]
_stop_event: threading.Event | None
_monitor_thread: threading.Thread | None
def __init__(self, devices: Sequence[jax.Device] | None = None) -> None:
def __init__(
self,
devices: Sequence[jax.Device] | None = None,
*,
minimum_slice_count: int | None = None,
poll_interval: float | int = 10,
timeout: float | int | None = None,
) -> None:
"""Initializes the manager.

Args:
devices: The devices to use. If None, jax.devices() is used.
minimum_slice_count: Minimum active slices to wait for (< 1 = all). If
None, active slices are sampled once without waiting.
poll_interval: Seconds between slice health checks when waiting.
timeout: Maximum seconds to wait when waiting, or None for no timeout.
"""
if devices is None:
devices = jax.devices()
Expand All @@ -115,9 +126,18 @@ def __init__(self, devices: Sequence[jax.Device] | None = None) -> None:
self._active_slice_indices = frozenset()
self.available_inactive_slices = frozenset()

self.active_slice_indices = elastic.get_active_slice_indices(
slice_to_devices=self.slice_to_devices
)
if minimum_slice_count is not None:
slice_count = None if minimum_slice_count < 1 else minimum_slice_count
self.active_slice_indices = elastic.wait_for_slices(
slice_count=slice_count,
poll_interval=poll_interval,
timeout=timeout,
slice_to_devices=self.slice_to_devices,
)
else:
self.active_slice_indices = elastic.get_active_slice_indices(
slice_to_devices=self.slice_to_devices
)

self._stop_event = None
self._monitor_thread = None
Expand Down
Loading