From f2753685d479b6a040c5c82c8ad80e61a22d74e8 Mon Sep 17 00:00:00 2001 From: RecML authors Date: Sun, 15 Nov 2026 20:31:51 -0800 Subject: [PATCH] Internal change PiperOrigin-RevId: 1022467825 --- recml/core/training/core.py | 12 +- recml/core/training/keras_trainer.py | 141 ++++++++++++++-------- recml/core/training/keras_trainer_test.py | 69 +++++++++++ 3 files changed, 173 insertions(+), 49 deletions(-) diff --git a/recml/core/training/core.py b/recml/core/training/core.py index 75a4d2f..5f4b5c2 100644 --- a/recml/core/training/core.py +++ b/recml/core/training/core.py @@ -65,6 +65,13 @@ class JobInfo: name: str mode: Any + eval_names: tuple[str, ...] | None = None + + def __post_init__(self): + if self.eval_names is not None and not isinstance(self.eval_names, tuple): + object.__setattr__( + self, "eval_names", tuple(str(n) for n in self.eval_names) + ) @classmethod def from_string(cls, job_info: str) -> Self: @@ -73,7 +80,10 @@ def from_string(cls, job_info: str) -> Self: def to_string(self) -> str: """Converts the JobInfo to a string.""" - return json.dumps(dataclasses.asdict(self)) + payload = dataclasses.asdict(self) + if payload.get("eval_names") is None: + payload.pop("eval_names", None) + return json.dumps(payload) class Trainer(abc.ABC, Generic[TaskT]): diff --git a/recml/core/training/keras_trainer.py b/recml/core/training/keras_trainer.py index be865fc..60f10c6 100644 --- a/recml/core/training/keras_trainer.py +++ b/recml/core/training/keras_trainer.py @@ -290,9 +290,10 @@ def train_callbacks(self) -> list[keras.callbacks.Callback]: ), ] - @functools.cached_property - def eval_callbacks(self) -> list[keras.callbacks.Callback]: - """Returns the evaluation callbacks.""" + def _build_eval_callbacks_for_name( + self, eval_name: str + ) -> list[keras.callbacks.Callback]: + """Builds evaluation callbacks writing summaries under `logs/`.""" if keras.backend.backend() == "jax": return [ *self._custom_eval_callbacks, @@ -301,11 +302,7 @@ def eval_callbacks(self) -> list[keras.callbacks.Callback]: steps_per_epoch=self._steps_per_loop, write_steps_per_second=False, # Allow eval jobs to have separate logging directories. - eval_subdir=( - self.job_info.name - if self.job_info is not None - else "validation" - ), + eval_subdir=eval_name, ), ] return [ @@ -316,12 +313,25 @@ def eval_callbacks(self) -> list[keras.callbacks.Callback]: ), ] - def _maybe_get_dataset_kws(self, task: KerasTask) -> Mapping[str, Any]: + @functools.cached_property + def eval_callbacks(self) -> list[keras.callbacks.Callback]: + """Returns the evaluation callbacks.""" + eval_name = ( + self.job_info.name if self.job_info is not None else "validation" + ) + return self._build_eval_callbacks_for_name(eval_name) + + def _maybe_get_dataset_kws( + self, task: KerasTask, eval_name: str | None = None + ) -> Mapping[str, Any]: kws = {} if py_utils.has_argument(task.create_dataset, "eval_name"): - kws["eval_name"] = ( - self.job_info.name if self.job_info is not None else "validation" - ) + if eval_name is not None: + kws["eval_name"] = eval_name + elif self.job_info is not None: + kws["eval_name"] = self.job_info.name + else: + kws["eval_name"] = "validation" if py_utils.has_argument(task.create_dataset, "model_dir"): kws["model_dir"] = self._model_dir @@ -474,11 +484,38 @@ def evaluate_continuously(self, task: KerasTask) -> core.Logs | None: "Continuous evaluation is only supported on the Jax backend." ) - eval_dataset = task.create_dataset( - training=False, **self._maybe_get_dataset_kws(task) + if self.job_info is not None and self.job_info.eval_names: + eval_names = self.job_info.eval_names + elif self.job_info is not None: + eval_names = (self.job_info.name,) + else: + eval_names = ("validation",) + + default_eval_name = ( + self.job_info.name if self.job_info is not None else "validation" ) + eval_suites = [] + for idx, suite_name in enumerate(eval_names): + suite_dataset = task.create_dataset( + training=False, + **self._maybe_get_dataset_kws(task, eval_name=suite_name), + ) + suite_callbacks = ( + self.eval_callbacks + if len(eval_names) == 1 and suite_name == default_eval_name + else self._build_eval_callbacks_for_name(suite_name) + ) + [suite_tb_cbk] = [ + cbk + for cbk in suite_callbacks + if isinstance(cbk, keras_utils.EpochSummaryCallback) + ] + eval_suites.append( + (idx, suite_name, suite_dataset, suite_callbacks, suite_tb_cbk) + ) + model = task.create_model_for_eval( - **self._maybe_get_model_kws(task, eval_dataset), + **self._maybe_get_model_kws(task, eval_suites[0][2]), ) def timeout_fn() -> bool: @@ -526,43 +563,51 @@ def on_test_begin(self, logs: Mapping[str, Any] | None = None): timeout_fn=timeout_fn, ): restore_callback = _RestoreCallback(self._checkpoint_dir, epoch) - [tb_cbk] = [ - cbk - for cbk in self.eval_callbacks - if isinstance(cbk, keras_utils.EpochSummaryCallback) - ] - callbacks = [restore_callback, *self.eval_callbacks] try: - logging.info(f"eval | epoch: {epoch: 6d} | {steps_msg}") - epoch_start_time = time.time() + for ( + idx, + suite_name, + suite_dataset, + suite_callbacks, + suite_tb_cbk, + ) in eval_suites: + callbacks = ( + [restore_callback, *suite_callbacks] + if idx == 0 + else list(suite_callbacks) + ) + logging.info( + f"eval ({suite_name}) | epoch: {epoch: 6d} | {steps_msg}" + ) + epoch_start_time = time.time() - logging.info( - "[Before] Memory usage:" - f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB" - ) - history = model.evaluate( - eval_dataset, - steps=self._steps_per_eval, - callbacks=callbacks, - return_dict=True, - ) + logging.info( + "[Before] Memory usage:" + f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB" + ) + history = model.evaluate( + suite_dataset, + steps=self._steps_per_eval, + callbacks=callbacks, + return_dict=True, + ) - logging.info( - "[After] Memory usage:" - f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB" - ) - gc.collect() - logging.info( - "[After GC] Memory usage:" - f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB" - ) - epoch_dt = time.time() - epoch_start_time - steps_per_second = self._steps_per_eval / epoch_dt # pyrefly: ignore[unsupported-operation] + logging.info( + "[After] Memory usage:" + f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB" + ) + gc.collect() + logging.info( + "[After GC] Memory usage:" + f" {psutil.Process().memory_info().rss / 1024 ** 2:.1f} MB" + ) + epoch_dt = time.time() - epoch_start_time + steps_per_second = self._steps_per_eval / epoch_dt # pyrefly: ignore[unsupported-operation] - tb_cbk.on_epoch_end( - epoch, - self._val_logs(history, self.eval_callbacks, steps_per_second), - ) + suite_tb_cbk.on_epoch_end( + epoch, + self._val_logs(history, suite_callbacks, steps_per_second), + ) except FileNotFoundError: logging.info("Checkpoint epoch: %s did not finish writing...", epoch) diff --git a/recml/core/training/keras_trainer_test.py b/recml/core/training/keras_trainer_test.py index 2700fc6..43af05e 100644 --- a/recml/core/training/keras_trainer_test.py +++ b/recml/core/training/keras_trainer_test.py @@ -256,6 +256,75 @@ def test_eval_name(self): self.assertTrue(os.path.exists(expected_log_dir)) self.assertEqual(experiment.task.last_eval_name, eval_name) + def test_continuous_eval_multiplexes_eval_names_with_single_restore(self): + if keras.backend.backend() != "jax": + self.skipTest("`continuous_eval` is only supported on the Jax backend.") + + model_dir = self.create_tempdir().full_path + seen_eval_names: list[str] = [] + + class _TrackingTask(_KerasTask): + + def create_dataset( + self, training: bool, eval_name: str | None = None + ) -> tf.data.Dataset: + if not training and eval_name is not None: + seen_eval_names.append(eval_name) + return super().create_dataset(training=training, eval_name=eval_name) + + train_trainer = keras_trainer.KerasTrainer( + distribution=keras.distribution.DataParallel(), + train_steps=2, + steps_per_eval=1, + steps_per_loop=2, + model_dir=model_dir, + checkpoint_version="v3", + ) + core.run_experiment( + core.Experiment(_TrackingTask(), train_trainer), core.Trainer.Mode.TRAIN + ) + keras.backend.clear_session() + + eval_names = ( + "validation", + "validation_random_16", + "global_drop_15_validation_last_1", + ) + eval_trainer = keras_trainer.KerasTrainer( + distribution=keras.distribution.DataParallel(), + steps_per_eval=1, + steps_per_loop=2, + model_dir=model_dir, + continuous_eval_timeout=1, + checkpoint_version="v3", + ) + job_info_str = core.JobInfo( + name="validation", + mode=core.Trainer.Mode.CONTINUOUS_EVAL, + eval_names=eval_names, + ).to_string() + eval_trainer.set_job_info(job_info_str) + self.assertEqual(eval_trainer.job_info.eval_names, eval_names) + + with mock.patch.object( + keras_utils, + "restore_keras_checkpoint", + wraps=keras_utils.restore_keras_checkpoint, + ) as mock_restore: + core.run_experiment( + core.Experiment(_TrackingTask(), eval_trainer), + core.Trainer.Mode.CONTINUOUS_EVAL, + ) + + self.assertEqual(seen_eval_names, list(eval_names)) + # Checkpoint weights are restored once for the single checkpoint rather than + # once per multiplexed eval suite. + self.assertEqual(mock_restore.call_count, 1) + for suite_name in eval_names: + suite_log_dir = os.path.join(model_dir, "logs", suite_name) + self.assertTrue(os.path.isdir(suite_log_dir), msg=suite_log_dir) + self.assertNotEmpty(os.listdir(suite_log_dir), msg=suite_name) + def test_eval_with_create_model_for_eval_override(self) -> None: if keras.backend.backend() == "jax": distribution = keras.distribution.DataParallel()