From 2fe06938a27486685799ad9453ffa96d245d4587 Mon Sep 17 00:00:00 2001 From: blanky Date: Sun, 8 Feb 2026 00:02:49 +0530 Subject: [PATCH 1/5] Enhance EarlyStopping with mode and min_delta_mode parameters for improved score evaluation --- ignite/handlers/early_stopping.py | 58 +++++++++---- pyproject.toml | 2 +- tests/ignite/handlers/test_early_stopping.py | 90 ++++++++++++++++++++ 3 files changed, 131 insertions(+), 19 deletions(-) diff --git a/ignite/handlers/early_stopping.py b/ignite/handlers/early_stopping.py index 3da94ceb3857..54038171c4e1 100644 --- a/ignite/handlers/early_stopping.py +++ b/ignite/handlers/early_stopping.py @@ -14,18 +14,23 @@ class EarlyStopping(Serializable): Args: patience: Number of events to wait if no improvement and then stop the training. score_function: It should be a function taking a single argument, an :class:`~ignite.engine.engine.Engine` - object, and return a score `float`. An improvement is considered if the score is higher. + object, and return a score `float`. An improvement is considered if the score is higher (for mode='max') + or lower (for mode='min'). trainer: Trainer engine to stop the run if no improvement. - min_delta: A minimum increase in the score to qualify as an improvement, - i.e. an increase of less than or equal to the minimum delta threshold (as determined by min_delta and min_delta_mode), will count as no improvement. - cumulative_delta: It True, `min_delta` defines an increase since the last `patience` reset, otherwise, - it defines an increase after the last event. Default value is False. - min_delta_mode: Determine whether `min_delta` is an absolute increase or a relative increase. - In 'abs' mode, the threshold is min_delta, - i.e. an increase of less than or equal to min_delta, will count as no improvement. - In 'rel' mode, the threshold is abs(best_score) * min_delta, - i.e. an increase of less than or equal to abs(best_score) * min_delta, will count as no improvement. + min_delta: A minimum change in the score to qualify as an improvement. For mode='max', it's a minimum + increase; for mode='min', it's a minimum decrease. An improvement is only considered if the change + exceeds the threshold determined by min_delta and min_delta_mode. + cumulative_delta: If True, `min_delta` defines the change since the last `patience` reset, otherwise, + it defines the change after the last event. Default value is False. + min_delta_mode: Determines whether `min_delta` is an absolute change or a relative change. + In 'abs' mode: + - For mode='max': improvement if score > best_score + min_delta + - For mode='min': improvement if score < best_score - min_delta + In 'rel' mode: + - For mode='max': improvement if score > best_score * (1 + min_delta) + - For mode='min': improvement if score < best_score * (1 - min_delta) Possible values are "abs" and "rel". Default value is "abs". + mode: Whether to maximize ('max') or minimize ('min') the score. Default is 'max'. Examples: .. code-block:: python @@ -41,6 +46,10 @@ def score_function(engine): # Note: the handler is attached to an *Evaluator* (runs one epoch on validation dataset). evaluator.add_event_handler(Events.COMPLETED, handler) + .. versionchanged:: 0.6.0 + Added ``mode`` parameter to support minimization in addition to maximization. + Added ``min_delta_mode`` parameter to support both absolute and relative improvements. + """ _state_dict_all_req_keys = ( @@ -56,6 +65,7 @@ def __init__( min_delta: float = 0.0, cumulative_delta: bool = False, min_delta_mode: Literal["abs", "rel"] = "abs", + mode: Literal["min", "max"] = "max", ): if not callable(score_function): raise TypeError("Argument score_function should be a function.") @@ -72,6 +82,9 @@ def __init__( if min_delta_mode not in ("abs", "rel"): raise ValueError("Argument min_delta_mode should be either 'abs' or 'rel'.") + if mode not in ("min", "max"): + raise ValueError("Argument mode should be either 'min' or 'max'.") + self.score_function = score_function self.patience = patience self.min_delta = min_delta @@ -81,6 +94,7 @@ def __init__( self.best_score: Optional[float] = None self.logger = setup_logger(__name__ + "." + self.__class__.__name__) self.min_delta_mode = min_delta_mode + self.mode = mode def __call__(self, engine: Engine) -> None: score = self.score_function(engine) @@ -88,14 +102,22 @@ def __call__(self, engine: Engine) -> None: if self.best_score is None: self.best_score = score return - upper_bound = ( - self.best_score + self.min_delta - if self.min_delta_mode == "abs" - else self.best_score + abs(self.best_score) * self.min_delta - ) - if score <= upper_bound: - if not self.cumulative_delta and score > self.best_score: - self.best_score = score + + if self.min_delta_mode == "abs": + improvement_threshold = ( + self.best_score + self.min_delta if self.mode == "max" else self.best_score - self.min_delta + ) + + else: + improvement_threshold = ( + self.best_score * (1 + self.min_delta) if self.mode == "max" else self.best_score * (1 - self.min_delta) + ) + + no_improvement = score <= improvement_threshold if self.mode == "max" else score >= improvement_threshold + + if no_improvement: + if not self.cumulative_delta: + self.best_score = max(score, self.best_score) if self.mode == "max" else min(score, self.best_score) self.counter += 1 self.logger.debug("EarlyStopping: %i / %i" % (self.counter, self.patience)) if self.counter >= self.patience: diff --git a/pyproject.toml b/pyproject.toml index 7bcc2a9a6cf2..5dfd773a7434 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ license-files = ["LICENSE"] classifiers = [ "Programming Language :: Python :: 3", ] -requires-python = ">=3.9,<=3.13" +requires-python = ">=3.9,<=3.14" dependencies = [ "torch>=1.10,<3", "packaging" diff --git a/tests/ignite/handlers/test_early_stopping.py b/tests/ignite/handlers/test_early_stopping.py index 0dce101331ce..3ea3a1f4d3aa 100644 --- a/tests/ignite/handlers/test_early_stopping.py +++ b/tests/ignite/handlers/test_early_stopping.py @@ -30,6 +30,9 @@ def test_args_validation(): with pytest.raises(ValueError, match=r"Argument min_delta_mode should be either 'abs' or 'rel'."): EarlyStopping(patience=2, min_delta_mode="invalid_mode", score_function=lambda engine: 0, trainer=trainer) + with pytest.raises(ValueError, match=r"Argument mode should be either 'min' or 'max'."): + EarlyStopping(patience=2, mode="invalid_mode", score_function=lambda engine: 0, trainer=trainer) + def test_simple_early_stopping(): scores = iter([1.0, 0.8, 0.88]) @@ -296,6 +299,93 @@ def evaluation(engine): assert trainer.state.epoch == 10 +def test_simple_early_stopping_min_mode(): + scores = iter([1.0, 1.2, 0.9]) + + def score_function(engine): + return next(scores) + + trainer = Engine(do_nothing_update_fn) + + h = EarlyStopping(patience=2, score_function=score_function, trainer=trainer, mode="min") + # Call 3 times and check if stopped + assert not trainer.should_terminate + h(None) # best_score=1.0 + assert not trainer.should_terminate + h(None) # score=1.2 (no improvement) + assert not trainer.should_terminate + h(None) # score=0.9 (improvement) + assert not trainer.should_terminate + + +def test_early_stopping_min_mode_with_delta(): + scores = iter([1.1, 0.95, 0.94, 0.93]) + + trainer = Engine(do_nothing_update_fn) + + h = EarlyStopping(patience=2, min_delta=0.1, score_function=lambda _: next(scores), trainer=trainer, mode="min") + + assert not trainer.should_terminate + h(None) # best_score=1.1 + assert not trainer.should_terminate + h(None) # score=0.95 (improvement: 0.95 < 1.1 - 0.1 = 1.0) + assert not trainer.should_terminate + h(None) # score=0.94 (no improvement: 0.94 >= 0.95 - 0.1 = 0.85) + assert not trainer.should_terminate + h(None) # score=0.93 (no improvement: 0.93 >= 0.95 - 0.1 = 0.85) + assert trainer.should_terminate + + +def test_early_stopping_min_mode_with_delta_cumulative(): + scores = iter([1.1, 0.95, 0.94, 0.93]) + + trainer = Engine(do_nothing_update_fn) + + h = EarlyStopping( + patience=2, + min_delta=0.1, + score_function=lambda _: next(scores), + trainer=trainer, + cumulative_delta=True, + mode="min", + ) + + assert not trainer.should_terminate + h(None) # best_score=1.1 + assert not trainer.should_terminate + h(None) # score=0.95 (improvement: 0.95 < 1.1 - 0.1 = 1.0) + assert not trainer.should_terminate + h(None) # score=0.94 (no improvement: 0.94 >= 0.95 - 0.1 = 0.85) + assert not trainer.should_terminate + h(None) # score=0.93 (no improvement: 0.93 >= 0.94 - 0.1 = 0.84) + assert trainer.should_terminate + + +def test_early_stopping_min_mode_rel_delta(): + scores = iter([1.0, 0.8, 0.79, 0.78]) + + trainer = Engine(do_nothing_update_fn) + + h = EarlyStopping( + patience=2, + min_delta=0.1, + min_delta_mode="rel", + score_function=lambda _: next(scores), + trainer=trainer, + mode="min", + ) + + assert not trainer.should_terminate + h(None) # best_score=1.0 + assert not trainer.should_terminate + h(None) # score=0.8 (improvement: 0.8 < 1.0 * (1 - 0.1) = 0.9) + assert not trainer.should_terminate + h(None) # score=0.79 (no improvement: 0.79 >= 0.8 * (1 - 0.1) = 0.72) + assert not trainer.should_terminate + h(None) # score=0.78 (no improvement) + assert trainer.should_terminate + + def _test_distrib_with_engine_early_stopping(device): if device is None: device = idist.device() From e5ddcae437a3385a607a6ca611c3e5930c28cd68 Mon Sep 17 00:00:00 2001 From: Aaishwarya Mishra Date: Sat, 7 Feb 2026 20:00:17 +0000 Subject: [PATCH 2/5] Improve documentation for min_delta_mode in EarlyStopping class --- ignite/handlers/early_stopping.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ignite/handlers/early_stopping.py b/ignite/handlers/early_stopping.py index 54038171c4e1..7e65ac684a9d 100644 --- a/ignite/handlers/early_stopping.py +++ b/ignite/handlers/early_stopping.py @@ -23,12 +23,17 @@ class EarlyStopping(Serializable): cumulative_delta: If True, `min_delta` defines the change since the last `patience` reset, otherwise, it defines the change after the last event. Default value is False. min_delta_mode: Determines whether `min_delta` is an absolute change or a relative change. - In 'abs' mode: - - For mode='max': improvement if score > best_score + min_delta - - For mode='min': improvement if score < best_score - min_delta - In 'rel' mode: - - For mode='max': improvement if score > best_score * (1 + min_delta) - - For mode='min': improvement if score < best_score * (1 - min_delta) + + - In 'abs' mode: + + - For mode='max': improvement if score > best_score + min_delta + - For mode='min': improvement if score < best_score - min_delta + + - In 'rel' mode: + + - For mode='max': improvement if score > best_score * (1 + min_delta) + - For mode='min': improvement if score < best_score * (1 - min_delta) + Possible values are "abs" and "rel". Default value is "abs". mode: Whether to maximize ('max') or minimize ('min') the score. Default is 'max'. From 63cb9a433b28d1ddf6050f29cc629df2304552ea Mon Sep 17 00:00:00 2001 From: blanky Date: Mon, 9 Feb 2026 15:38:00 +0530 Subject: [PATCH 3/5] Refactor EarlyStopping logic to simplify improvement threshold calculation based on min_delta_mode --- ignite/handlers/early_stopping.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ignite/handlers/early_stopping.py b/ignite/handlers/early_stopping.py index 7e65ac684a9d..15ae69b93b39 100644 --- a/ignite/handlers/early_stopping.py +++ b/ignite/handlers/early_stopping.py @@ -14,8 +14,8 @@ class EarlyStopping(Serializable): Args: patience: Number of events to wait if no improvement and then stop the training. score_function: It should be a function taking a single argument, an :class:`~ignite.engine.engine.Engine` - object, and return a score `float`. An improvement is considered if the score is higher (for mode='max') - or lower (for mode='min'). + object, and return a score `float`. An improvement is considered if the score is higher (for ``mode='max'``) + or lower (for ``mode='min'``). trainer: Trainer engine to stop the run if no improvement. min_delta: A minimum change in the score to qualify as an improvement. For mode='max', it's a minimum increase; for mode='min', it's a minimum decrease. An improvement is only considered if the change @@ -108,15 +108,11 @@ def __call__(self, engine: Engine) -> None: self.best_score = score return + min_delta = -self.min_delta if self.mode == "min" else self.min_delta if self.min_delta_mode == "abs": - improvement_threshold = ( - self.best_score + self.min_delta if self.mode == "max" else self.best_score - self.min_delta - ) - + improvement_threshold = self.best_score + min_delta else: - improvement_threshold = ( - self.best_score * (1 + self.min_delta) if self.mode == "max" else self.best_score * (1 - self.min_delta) - ) + improvement_threshold = self.best_score * (1 + min_delta) no_improvement = score <= improvement_threshold if self.mode == "max" else score >= improvement_threshold From 56e7cd5597b855605ab28a35081f34b8065fa267 Mon Sep 17 00:00:00 2001 From: Aaishwarya Mishra Date: Mon, 9 Feb 2026 17:26:34 +0530 Subject: [PATCH 4/5] Fix formatting of docstring in early_stopping.py --- ignite/handlers/early_stopping.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ignite/handlers/early_stopping.py b/ignite/handlers/early_stopping.py index 15ae69b93b39..2e0c844f7765 100644 --- a/ignite/handlers/early_stopping.py +++ b/ignite/handlers/early_stopping.py @@ -17,22 +17,22 @@ class EarlyStopping(Serializable): object, and return a score `float`. An improvement is considered if the score is higher (for ``mode='max'``) or lower (for ``mode='min'``). trainer: Trainer engine to stop the run if no improvement. - min_delta: A minimum change in the score to qualify as an improvement. For mode='max', it's a minimum - increase; for mode='min', it's a minimum decrease. An improvement is only considered if the change - exceeds the threshold determined by min_delta and min_delta_mode. + min_delta: A minimum change in the score to qualify as an improvement. For ``mode='max'``, it's a minimum + increase; for ``mode='min'``, it's a minimum decrease. An improvement is only considered if the change + exceeds the threshold determined by `min_delta` and `min_delta_mode`. cumulative_delta: If True, `min_delta` defines the change since the last `patience` reset, otherwise, it defines the change after the last event. Default value is False. min_delta_mode: Determines whether `min_delta` is an absolute change or a relative change. - In 'abs' mode: - - For mode='max': improvement if score > best_score + min_delta - - For mode='min': improvement if score < best_score - min_delta + - For ``mode='max'``: improvement if score > best_score + min_delta + - For ``mode='min'``: improvement if score < best_score - min_delta - In 'rel' mode: - - For mode='max': improvement if score > best_score * (1 + min_delta) - - For mode='min': improvement if score < best_score * (1 - min_delta) + - For ``mode='max'``: improvement if score > best_score * (1 + min_delta) + - For ``mode='min'``: improvement if score < best_score * (1 - min_delta) Possible values are "abs" and "rel". Default value is "abs". mode: Whether to maximize ('max') or minimize ('min') the score. Default is 'max'. From 3df739e20a8342aadcd649f7d930e9fc86061c0e Mon Sep 17 00:00:00 2001 From: Aaishwarya Mishra Date: Mon, 9 Feb 2026 17:28:40 +0530 Subject: [PATCH 5/5] Apply suggestions from code review --- ignite/handlers/early_stopping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ignite/handlers/early_stopping.py b/ignite/handlers/early_stopping.py index 2e0c844f7765..95e4d49cb571 100644 --- a/ignite/handlers/early_stopping.py +++ b/ignite/handlers/early_stopping.py @@ -52,8 +52,8 @@ def score_function(engine): evaluator.add_event_handler(Events.COMPLETED, handler) .. versionchanged:: 0.6.0 - Added ``mode`` parameter to support minimization in addition to maximization. - Added ``min_delta_mode`` parameter to support both absolute and relative improvements. + Added `mode` parameter to support minimization in addition to maximization. + Added `min_delta_mode` parameter to support both absolute and relative improvements. """