Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion recml/core/training/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,13 @@ class JobInfo:

name: str
mode: Any
eval_names: tuple[str, ...] | None = None

def __post_init__(self):
if self.eval_names is not None and not isinstance(self.eval_names, tuple):
object.__setattr__(
self, "eval_names", tuple(str(n) for n in self.eval_names)
)

@classmethod
def from_string(cls, job_info: str) -> Self:
Expand All @@ -73,7 +80,10 @@ def from_string(cls, job_info: str) -> Self:

def to_string(self) -> str:
"""Converts the JobInfo to a string."""
return json.dumps(dataclasses.asdict(self))
payload = dataclasses.asdict(self)
if payload.get("eval_names") is None:
payload.pop("eval_names", None)
return json.dumps(payload)


class Trainer(abc.ABC, Generic[TaskT]):
Expand Down
141 changes: 93 additions & 48 deletions recml/core/training/keras_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,10 @@ def train_callbacks(self) -> list[keras.callbacks.Callback]:
),
]

@functools.cached_property
def eval_callbacks(self) -> list[keras.callbacks.Callback]:
"""Returns the evaluation callbacks."""
def _build_eval_callbacks_for_name(
self, eval_name: str
) -> list[keras.callbacks.Callback]:
"""Builds evaluation callbacks writing summaries under `logs/<eval_name>`."""
if keras.backend.backend() == "jax":
return [
*self._custom_eval_callbacks,
Expand All @@ -301,11 +302,7 @@ def eval_callbacks(self) -> list[keras.callbacks.Callback]:
steps_per_epoch=self._steps_per_loop,
write_steps_per_second=False,
# Allow eval jobs to have separate logging directories.
eval_subdir=(
self.job_info.name
if self.job_info is not None
else "validation"
),
eval_subdir=eval_name,
),
]
return [
Expand All @@ -316,12 +313,25 @@ def eval_callbacks(self) -> list[keras.callbacks.Callback]:
),
]

def _maybe_get_dataset_kws(self, task: KerasTask) -> Mapping[str, Any]:
@functools.cached_property
def eval_callbacks(self) -> list[keras.callbacks.Callback]:
"""Returns the evaluation callbacks."""
eval_name = (
self.job_info.name if self.job_info is not None else "validation"
)
return self._build_eval_callbacks_for_name(eval_name)

def _maybe_get_dataset_kws(
self, task: KerasTask, eval_name: str | None = None
) -> Mapping[str, Any]:
kws = {}
if py_utils.has_argument(task.create_dataset, "eval_name"):
kws["eval_name"] = (
self.job_info.name if self.job_info is not None else "validation"
)
if eval_name is not None:
kws["eval_name"] = eval_name
elif self.job_info is not None:
kws["eval_name"] = self.job_info.name
else:
kws["eval_name"] = "validation"
if py_utils.has_argument(task.create_dataset, "model_dir"):
kws["model_dir"] = self._model_dir

Expand Down Expand Up @@ -474,11 +484,38 @@ def evaluate_continuously(self, task: KerasTask) -> core.Logs | None:
"Continuous evaluation is only supported on the Jax backend."
)

eval_dataset = task.create_dataset(
training=False, **self._maybe_get_dataset_kws(task)
if self.job_info is not None and self.job_info.eval_names:
eval_names = self.job_info.eval_names
elif self.job_info is not None:
eval_names = (self.job_info.name,)
else:
eval_names = ("validation",)

default_eval_name = (
self.job_info.name if self.job_info is not None else "validation"
)
eval_suites = []
for idx, suite_name in enumerate(eval_names):
suite_dataset = task.create_dataset(
training=False,
**self._maybe_get_dataset_kws(task, eval_name=suite_name),
)
suite_callbacks = (
self.eval_callbacks
if len(eval_names) == 1 and suite_name == default_eval_name
else self._build_eval_callbacks_for_name(suite_name)
)
[suite_tb_cbk] = [
cbk
for cbk in suite_callbacks
if isinstance(cbk, keras_utils.EpochSummaryCallback)
]
eval_suites.append(
(idx, suite_name, suite_dataset, suite_callbacks, suite_tb_cbk)
)

model = task.create_model_for_eval(
**self._maybe_get_model_kws(task, eval_dataset),
**self._maybe_get_model_kws(task, eval_suites[0][2]),
)

def timeout_fn() -> bool:
Expand Down Expand Up @@ -526,43 +563,51 @@ def on_test_begin(self, logs: Mapping[str, Any] | None = None):
timeout_fn=timeout_fn,
):
restore_callback = _RestoreCallback(self._checkpoint_dir, epoch)
[tb_cbk] = [
cbk
for cbk in self.eval_callbacks
if isinstance(cbk, keras_utils.EpochSummaryCallback)
]
callbacks = [restore_callback, *self.eval_callbacks]
try:
logging.info(f"eval | epoch: {epoch: 6d} | {steps_msg}")
epoch_start_time = time.time()
for (
idx,
suite_name,
suite_dataset,
suite_callbacks,
suite_tb_cbk,
) in eval_suites:
callbacks = (
[restore_callback, *suite_callbacks]
if idx == 0
else list(suite_callbacks)
)
logging.info(
f"eval ({suite_name}) | epoch: {epoch: 6d} | {steps_msg}"
)
epoch_start_time = time.time()

logging.info(
"[Before] Memory usage:"
f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB"
)
history = model.evaluate(
eval_dataset,
steps=self._steps_per_eval,
callbacks=callbacks,
return_dict=True,
)
logging.info(
"[Before] Memory usage:"
f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB"
)
history = model.evaluate(
suite_dataset,
steps=self._steps_per_eval,
callbacks=callbacks,
return_dict=True,
)

logging.info(
"[After] Memory usage:"
f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB"
)
gc.collect()
logging.info(
"[After GC] Memory usage:"
f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB"
)
epoch_dt = time.time() - epoch_start_time
steps_per_second = self._steps_per_eval / epoch_dt # pyrefly: ignore[unsupported-operation]
logging.info(
"[After] Memory usage:"
f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB"
)
gc.collect()
logging.info(
"[After GC] Memory usage:"
f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB"
)
epoch_dt = time.time() - epoch_start_time
steps_per_second = self._steps_per_eval / epoch_dt # pyrefly: ignore[unsupported-operation]

tb_cbk.on_epoch_end(
epoch,
self._val_logs(history, self.eval_callbacks, steps_per_second),
)
suite_tb_cbk.on_epoch_end(
epoch,
self._val_logs(history, suite_callbacks, steps_per_second),
)
except FileNotFoundError:
logging.info("Checkpoint epoch: %s did not finish writing...", epoch)

Expand Down
69 changes: 69 additions & 0 deletions recml/core/training/keras_trainer_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,75 @@ def test_eval_name(self):
self.assertTrue(os.path.exists(expected_log_dir))
self.assertEqual(experiment.task.last_eval_name, eval_name)

def test_continuous_eval_multiplexes_eval_names_with_single_restore(self):
if keras.backend.backend() != "jax":
self.skipTest("`continuous_eval` is only supported on the Jax backend.")

model_dir = self.create_tempdir().full_path
seen_eval_names: list[str] = []

class _TrackingTask(_KerasTask):

def create_dataset(
self, training: bool, eval_name: str | None = None
) -> tf.data.Dataset:
if not training and eval_name is not None:
seen_eval_names.append(eval_name)
return super().create_dataset(training=training, eval_name=eval_name)

train_trainer = keras_trainer.KerasTrainer(
distribution=keras.distribution.DataParallel(),
train_steps=2,
steps_per_eval=1,
steps_per_loop=2,
model_dir=model_dir,
checkpoint_version="v3",
)
core.run_experiment(
core.Experiment(_TrackingTask(), train_trainer), core.Trainer.Mode.TRAIN
)
keras.backend.clear_session()

eval_names = (
"validation",
"validation_random_16",
"global_drop_15_validation_last_1",
)
eval_trainer = keras_trainer.KerasTrainer(
distribution=keras.distribution.DataParallel(),
steps_per_eval=1,
steps_per_loop=2,
model_dir=model_dir,
continuous_eval_timeout=1,
checkpoint_version="v3",
)
job_info_str = core.JobInfo(
name="validation",
mode=core.Trainer.Mode.CONTINUOUS_EVAL,
eval_names=eval_names,
).to_string()
eval_trainer.set_job_info(job_info_str)
self.assertEqual(eval_trainer.job_info.eval_names, eval_names)

with mock.patch.object(
keras_utils,
"restore_keras_checkpoint",
wraps=keras_utils.restore_keras_checkpoint,
) as mock_restore:
core.run_experiment(
core.Experiment(_TrackingTask(), eval_trainer),
core.Trainer.Mode.CONTINUOUS_EVAL,
)

self.assertEqual(seen_eval_names, list(eval_names))
# Checkpoint weights are restored once for the single checkpoint rather than
# once per multiplexed eval suite.
self.assertEqual(mock_restore.call_count, 1)
for suite_name in eval_names:
suite_log_dir = os.path.join(model_dir, "logs", suite_name)
self.assertTrue(os.path.isdir(suite_log_dir), msg=suite_log_dir)
self.assertNotEmpty(os.listdir(suite_log_dir), msg=suite_name)

def test_eval_with_create_model_for_eval_override(self) -> None:
if keras.backend.backend() == "jax":
distribution = keras.distribution.DataParallel()
Expand Down
Loading