From 25e6adfd8a77fd3a2ebcd9a5653a5e6411b9d096 Mon Sep 17 00:00:00 2001 From: Jeevan Chevula Date: Thu, 18 Sep 2025 16:33:09 +0530 Subject: [PATCH 1/8] Implement SAVED_CHECKPOINT event with proper APIs and documentation --- docs/source/handlers.rst | 31 ++++++++++++++++++++++ ignite/handlers/checkpoint.py | 15 ++++++++--- tests/ignite/handlers/test_checkpoint.py | 33 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 4c312be8b45b..94789a16bd8d 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -34,6 +34,37 @@ Complete list of generic handlers state_param_scheduler.StateParamScheduler +Checkpoint Events +----------------- + +.. versionadded:: 0.5.0 + +The Checkpoint handler provides a ``SAVED_CHECKPOINT`` event that fires after successful checkpoint saves. +This allows users to attach custom handlers that react to checkpoint operations without manual event registration. + +**Usage:** + +.. code-block:: python + + from ignite.handlers import Checkpoint + from ignite.engine import Engine, Events + + # Setup checkpoint handler + checkpoint_handler = Checkpoint( + {'model': model, 'optimizer': optimizer}, + save_dir, + n_saved=2 + ) + + # Attach handler to checkpoint event (no manual registration needed) + @trainer.on(Checkpoint.SAVED_CHECKPOINT) + def on_checkpoint_saved(engine): + print(f"Checkpoint saved at epoch {engine.state.epoch}") + # Add custom logic: notifications, logging, etc. + + trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler) + + Loggers -------- diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index aa408a478ef0..cf8839c6d5a6 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -21,10 +21,16 @@ import ignite.distributed as idist from ignite.base import Serializable -from ignite.engine import Engine, Events +from ignite.engine import Engine, Events, EventEnum from ignite.utils import _tree_apply2, _tree_map -__all__ = ["Checkpoint", "DiskSaver", "ModelCheckpoint", "BaseSaveHandler"] +__all__ = ["Checkpoint", "DiskSaver", "ModelCheckpoint", "BaseSaveHandler", "CheckpointEvents"] + + +class CheckpointEvents(EventEnum): + """Events fired by Checkpoint handler""" + + SAVED_CHECKPOINT = "saved_checkpoint" class BaseSaveHandler(metaclass=ABCMeta): @@ -276,6 +282,7 @@ class Checkpoint(Serializable): - `save_on_rank` saves objects on this rank in a distributed configuration. """ + SAVED_CHECKPOINT = CheckpointEvents.SAVED_CHECKPOINT Item = NamedTuple("Item", [("priority", int), ("filename", str)]) _state_dict_all_req_keys = ("_saved",) @@ -400,6 +407,8 @@ def _compare_fn(self, new: Union[int, float]) -> bool: return new > self._saved[0].priority def __call__(self, engine: Engine) -> None: + if not engine.has_registered_events(CheckpointEvents.SAVED_CHECKPOINT): + engine.register_events(CheckpointEvents.SAVED_CHECKPOINT) global_step = None if self.global_step_transform is not None: global_step = self.global_step_transform(engine, engine.last_event_name) @@ -460,11 +469,11 @@ def __call__(self, engine: Engine) -> None: if self.include_self: # Now that we've updated _saved, we can add our own state_dict. checkpoint["checkpointer"] = self.state_dict() - try: self.save_handler(checkpoint, filename, metadata) except TypeError: self.save_handler(checkpoint, filename) + engine.fire_event(CheckpointEvents.SAVED_CHECKPOINT) def _setup_checkpoint(self) -> Dict[str, Any]: if self.to_save is not None: diff --git a/tests/ignite/handlers/test_checkpoint.py b/tests/ignite/handlers/test_checkpoint.py index 445c84d7205b..762c1733abde 100644 --- a/tests/ignite/handlers/test_checkpoint.py +++ b/tests/ignite/handlers/test_checkpoint.py @@ -1850,6 +1850,39 @@ def test_load_single_object(obj_to_save, dirname): Checkpoint.load_objects(to_load=to_save, checkpoint=str(checkpoint_fp)) +def test_checkpoint_saved_event(): + """Test that SAVED_CHECKPOINT event is fired correctly.""" + save_handler = MagicMock(spec=BaseSaveHandler) + to_save = {"model": DummyModel()} + + checkpointer = Checkpoint(to_save, save_handler=save_handler, n_saved=2) + + trainer = Engine(lambda e, b: None) + trainer.state = State(epoch=0, iteration=0) + + # Track event firing + event_count = 0 + + # First, call the checkpoint handler to trigger automatic event registration + checkpointer(trainer) + + @trainer.on(Checkpoint.SAVED_CHECKPOINT) + def on_checkpoint_saved(engine): + nonlocal event_count + event_count += 1 + + # Verify the first checkpoint didn't trigger our handler (attached after) + assert event_count == 0 + + # Second checkpoint - should fire event and trigger our handler + trainer.state.iteration = 1 + checkpointer(trainer) + assert event_count == 1 + + # Verify save handler was called twice + assert save_handler.call_count == 2 + + @pytest.mark.distributed @pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") @pytest.mark.parametrize("atomic", [False, True]) From 354dd978080b3c6390aa851b6494d3df7d7cf461 Mon Sep 17 00:00:00 2001 From: Jeevan Chevula Date: Sat, 20 Sep 2025 22:16:12 +0530 Subject: [PATCH 2/8] Fix trailing whitespace in checkpoint.py --- docs/source/handlers.rst | 1 - ignite/handlers/checkpoint.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 94789a16bd8d..87ddcc9936ea 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -37,7 +37,6 @@ Complete list of generic handlers Checkpoint Events ----------------- -.. versionadded:: 0.5.0 The Checkpoint handler provides a ``SAVED_CHECKPOINT`` event that fires after successful checkpoint saves. This allows users to attach custom handlers that react to checkpoint operations without manual event registration. diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index cf8839c6d5a6..13dbfe8129ca 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -28,7 +28,10 @@ class CheckpointEvents(EventEnum): - """Events fired by Checkpoint handler""" + """Events fired by Checkpoint handler + + .. versionadded:: 0.5.3 + """ SAVED_CHECKPOINT = "saved_checkpoint" From ab9a9ed091997734935734666d198ad31d75f681 Mon Sep 17 00:00:00 2001 From: vfdev Date: Wed, 24 Sep 2025 08:36:42 +0100 Subject: [PATCH 3/8] Update ignite/handlers/checkpoint.py --- ignite/handlers/checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index 13dbfe8129ca..99d03f809ca3 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -411,7 +411,7 @@ def _compare_fn(self, new: Union[int, float]) -> bool: def __call__(self, engine: Engine) -> None: if not engine.has_registered_events(CheckpointEvents.SAVED_CHECKPOINT): - engine.register_events(CheckpointEvents.SAVED_CHECKPOINT) + engine.register_events(*CheckpointEvents) global_step = None if self.global_step_transform is not None: global_step = self.global_step_transform(engine, engine.last_event_name) From f8ecfab192dde17719c5c61331799189c70186fd Mon Sep 17 00:00:00 2001 From: vfdev Date: Wed, 24 Sep 2025 08:44:51 +0100 Subject: [PATCH 4/8] Add CheckpointEvents to handlers documentation --- docs/source/handlers.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 87ddcc9936ea..8299ca7f7bc1 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -11,6 +11,7 @@ Complete list of generic handlers :toctree: generated checkpoint.Checkpoint + checkpoint.CheckpointEvents DiskSaver checkpoint.ModelCheckpoint ema_handler.EMAHandler From a3ac2c1c0b55267e378e522004745bce76e3ec72 Mon Sep 17 00:00:00 2001 From: vfdev Date: Wed, 24 Sep 2025 08:45:58 +0100 Subject: [PATCH 5/8] Clarify docstring for CheckpointEvents class Updated docstring for CheckpointEvents class to clarify event trigger. --- ignite/handlers/checkpoint.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index 99d03f809ca3..2db22337cdf4 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -28,8 +28,10 @@ class CheckpointEvents(EventEnum): - """Events fired by Checkpoint handler - + """Events fired by :class:`~ignite.handlers.checkpoint.Checkpoint` + + - SAVED_CHECKPOINT : triggered when checkpoint handler has saved objects + .. versionadded:: 0.5.3 """ From ae0c03836117f884d05d0002c3711a55a5e8b4e7 Mon Sep 17 00:00:00 2001 From: vfdev Date: Wed, 24 Sep 2025 08:51:22 +0100 Subject: [PATCH 6/8] Fix formatting and spacing in checkpoint.py --- ignite/handlers/checkpoint.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index 2db22337cdf4..a62cd714eaf2 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -29,9 +29,9 @@ class CheckpointEvents(EventEnum): """Events fired by :class:`~ignite.handlers.checkpoint.Checkpoint` - + - SAVED_CHECKPOINT : triggered when checkpoint handler has saved objects - + .. versionadded:: 0.5.3 """ @@ -285,6 +285,8 @@ class Checkpoint(Serializable): - `score_name` can be used to define `score_function` automatically without providing `score_function`. - `save_handler` automatically saves to disk if path to directory is provided. - `save_on_rank` saves objects on this rank in a distributed configuration. + + .. """ SAVED_CHECKPOINT = CheckpointEvents.SAVED_CHECKPOINT From ac09cd197ec0a394d24a453e7277a13c86f1f419 Mon Sep 17 00:00:00 2001 From: Jeevan Chevula Date: Thu, 25 Sep 2025 13:27:52 +0530 Subject: [PATCH 7/8] Move checkpoint event example to Checkpoint docstring and document SAVED_CHECKPOINT attribute --- docs/source/handlers.rst | 30 ------------------------------ ignite/handlers/checkpoint.py | 25 ++++++++++++++++++++++++- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 8299ca7f7bc1..bb9b50b9fb82 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -35,36 +35,6 @@ Complete list of generic handlers state_param_scheduler.StateParamScheduler -Checkpoint Events ------------------ - - -The Checkpoint handler provides a ``SAVED_CHECKPOINT`` event that fires after successful checkpoint saves. -This allows users to attach custom handlers that react to checkpoint operations without manual event registration. - -**Usage:** - -.. code-block:: python - - from ignite.handlers import Checkpoint - from ignite.engine import Engine, Events - - # Setup checkpoint handler - checkpoint_handler = Checkpoint( - {'model': model, 'optimizer': optimizer}, - save_dir, - n_saved=2 - ) - - # Attach handler to checkpoint event (no manual registration needed) - @trainer.on(Checkpoint.SAVED_CHECKPOINT) - def on_checkpoint_saved(engine): - print(f"Checkpoint saved at epoch {engine.state.epoch}") - # Add custom logic: notifications, logging, etc. - - trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler) - - Loggers -------- diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index a62cd714eaf2..01e6a961147a 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -275,6 +275,29 @@ class Checkpoint(Serializable): to_save, save_handler=DiskSaver('/tmp/models', create_dir=True, **kwargs), n_saved=2 ) + Respond to checkpoint events: + + .. code-block:: python + + from ignite.handlers import Checkpoint + from ignite.engine import Engine, Events + + checkpoint_handler = Checkpoint( + {'model': model, 'optimizer': optimizer}, + save_dir, + n_saved=2 + ) + + @trainer.on(Checkpoint.SAVED_CHECKPOINT) + def on_checkpoint_saved(engine): + print(f"Checkpoint saved at epoch {engine.state.epoch}") + + trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler) + + Attributes: + SAVED_CHECKPOINT: Alias of ``SAVED_CHECKPOINT`` from + :class:`~ignite.handlers.checkpoint.CheckpointEvents`. + .. versionchanged:: 0.4.3 - Checkpoint can save model with same filename. @@ -286,7 +309,7 @@ class Checkpoint(Serializable): - `save_handler` automatically saves to disk if path to directory is provided. - `save_on_rank` saves objects on this rank in a distributed configuration. - .. + .. """ SAVED_CHECKPOINT = CheckpointEvents.SAVED_CHECKPOINT From 8dba9e8653ea4f18729f89ddc4d2c56c0531bc4c Mon Sep 17 00:00:00 2001 From: vfdev Date: Thu, 25 Sep 2025 10:01:15 +0100 Subject: [PATCH 8/8] Add SAVED_CHECKPOINT class attribute --- ignite/handlers/checkpoint.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index 01e6a961147a..f8648f783a06 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -309,7 +309,9 @@ def on_checkpoint_saved(engine): - `save_handler` automatically saves to disk if path to directory is provided. - `save_on_rank` saves objects on this rank in a distributed configuration. - .. + .. versionchanged:: 0.5.3 + + - Added ``SAVED_CHECKPOINT`` class attribute. """ SAVED_CHECKPOINT = CheckpointEvents.SAVED_CHECKPOINT