Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pathwaysutils/elastic/elastic.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ class DefaultSliceHealthChecker(SliceHealthChecker):
that the devices are active and can perform computations.
"""

def _plus_one(self, x: jax.Array) -> jax.Array:
@staticmethod
def _plus_one(x: jax.Array) -> jax.Array:
return x + 1

def _simple_execution(self, devices: Sequence[jax.Device]) -> jax.Array:
Expand Down Expand Up @@ -124,7 +125,6 @@ def validate(self) -> Set[int]:
return frozenset(active_slice_indices)



def get_slice_to_devices(
devices: Sequence[jax.Device],
) -> Mapping[int, Sequence[jax.Device]]:
Expand Down
28 changes: 24 additions & 4 deletions pathwaysutils/elastic/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,22 @@ class Manager:
available_inactive_slices: Set[int]
_stop_event: threading.Event | None
_monitor_thread: threading.Thread | None
def __init__(self, devices: Sequence[jax.Device] | None = None) -> None:
def __init__(
self,
devices: Sequence[jax.Device] | None = None,
*,
minimum_slice_count: int | None = None,
poll_interval: float | int = 10,
timeout: float | int | None = None,
) -> None:
"""Initializes the manager.

Args:
devices: The devices to use. If None, jax.devices() is used.
minimum_slice_count: Minimum active slices to wait for (< 1 = all). If
None, active slices are sampled once without waiting.
poll_interval: Seconds between slice health checks when waiting.
timeout: Maximum seconds to wait when waiting, or None for no timeout.
"""
if devices is None:
devices = jax.devices()
Expand All @@ -115,9 +126,18 @@ def __init__(self, devices: Sequence[jax.Device] | None = None) -> None:
self._active_slice_indices = frozenset()
self.available_inactive_slices = frozenset()

self.active_slice_indices = elastic.get_active_slice_indices(
slice_to_devices=self.slice_to_devices
)
if minimum_slice_count is not None:
slice_count = None if minimum_slice_count < 1 else minimum_slice_count
self.active_slice_indices = elastic.wait_for_slices(
slice_count=slice_count,
poll_interval=poll_interval,
timeout=timeout,
slice_to_devices=self.slice_to_devices,
)
else:
self.active_slice_indices = elastic.get_active_slice_indices(
slice_to_devices=self.slice_to_devices
)

self._stop_event = None
self._monitor_thread = None
Expand Down
Loading