diff --git a/R-package/tests/testthat/test_lgb.Booster.R b/R-package/tests/testthat/test_lgb.Booster.R
index e6b0e8abda64..7bf0a1bf43d2 100644
--- a/R-package/tests/testthat/test_lgb.Booster.R
+++ b/R-package/tests/testthat/test_lgb.Booster.R
@@ -850,6 +850,7 @@ test_that("all parameters are stored correctly with save_model_to_string()", {
, "[extra_trees: 0]"
, "[extra_seed: 6642]"
, "[early_stopping_round: 0]"
+ , "[early_stopping_min_delta: 0]"
, "[first_metric_only: 0]"
, "[max_delta_step: 0]"
, "[lambda_l1: 0]"
diff --git a/docs/Parameters.rst b/docs/Parameters.rst
index 94f7e36d8ef2..02f01ae4408b 100644
--- a/docs/Parameters.rst
+++ b/docs/Parameters.rst
@@ -410,6 +410,10 @@ Learning Control Parameters
- can be used to speed up training
+- ``early_stopping_min_delta`` :raw-html:`🔗︎`, default = ``0.0``, type = double, constraints: ``early_stopping_min_delta >= 0.0``
+
+ - when early stopping is used (i.e. ``early_stopping_round > 0``), require the early stopping metric to improve by at least this delta to be considered an improvement
+
- ``first_metric_only`` :raw-html:`🔗︎`, default = ``false``, type = bool
- LightGBM allows you to provide multiple evaluation metrics. Set this to ``true``, if you want to use only the first metric for early stopping
diff --git a/include/LightGBM/config.h b/include/LightGBM/config.h
index a2f1a02370b7..b626e1b1bcc2 100644
--- a/include/LightGBM/config.h
+++ b/include/LightGBM/config.h
@@ -394,6 +394,10 @@ struct Config {
// desc = can be used to speed up training
int early_stopping_round = 0;
+ // check = >=0.0
+ // desc = when early stopping is used (i.e. ``early_stopping_round > 0``), require the early stopping metric to improve by at least this delta to be considered an improvement
+ double early_stopping_min_delta = 0.0;
+
// desc = LightGBM allows you to provide multiple evaluation metrics. Set this to ``true``, if you want to use only the first metric for early stopping
bool first_metric_only = false;
diff --git a/python-package/lightgbm/engine.py b/python-package/lightgbm/engine.py
index a19b29e7b584..4a4ab8b4fd13 100644
--- a/python-package/lightgbm/engine.py
+++ b/python-package/lightgbm/engine.py
@@ -241,6 +241,7 @@ def train(
callback.early_stopping(
stopping_rounds=params["early_stopping_round"], # type: ignore[arg-type]
first_metric_only=first_metric_only,
+ min_delta=params.get("early_stopping_min_delta", 0.0),
verbose=_choose_param_value(
main_param_name="verbosity",
params=params,
@@ -765,6 +766,7 @@ def cv(
callback.early_stopping(
stopping_rounds=params["early_stopping_round"], # type: ignore[arg-type]
first_metric_only=first_metric_only,
+ min_delta=params.get("early_stopping_min_delta", 0.0),
verbose=_choose_param_value(
main_param_name="verbosity",
params=params,
diff --git a/src/boosting/gbdt.cpp b/src/boosting/gbdt.cpp
index 5be3b9765bc4..86a8a5a3ca65 100644
--- a/src/boosting/gbdt.cpp
+++ b/src/boosting/gbdt.cpp
@@ -30,6 +30,7 @@ GBDT::GBDT()
config_(nullptr),
objective_function_(nullptr),
early_stopping_round_(0),
+ early_stopping_min_delta_(0.0),
es_first_metric_only_(false),
max_feature_idx_(0),
num_tree_per_iteration_(1),
@@ -65,6 +66,7 @@ void GBDT::Init(const Config* config, const Dataset* train_data, const Objective
num_class_ = config->num_class;
config_ = std::unique_ptr(new Config(*config));
early_stopping_round_ = config_->early_stopping_round;
+ early_stopping_min_delta_ = config->early_stopping_min_delta;
es_first_metric_only_ = config_->first_metric_only;
shrinkage_rate_ = config_->learning_rate;
@@ -576,7 +578,7 @@ std::string GBDT::OutputMetric(int iter) {
if (es_first_metric_only_ && j > 0) { continue; }
if (ret.empty() && early_stopping_round_ > 0) {
auto cur_score = valid_metrics_[i][j]->factor_to_bigger_better() * test_scores.back();
- if (cur_score > best_score_[i][j]) {
+ if (cur_score - best_score_[i][j] > early_stopping_min_delta_) {
best_score_[i][j] = cur_score;
best_iter_[i][j] = iter;
meet_early_stopping_pairs.emplace_back(i, j);
diff --git a/src/boosting/gbdt.h b/src/boosting/gbdt.h
index 28ebee446fad..4557830fa863 100644
--- a/src/boosting/gbdt.h
+++ b/src/boosting/gbdt.h
@@ -532,6 +532,8 @@ class GBDT : public GBDTBase {
std::vector> valid_metrics_;
/*! \brief Number of rounds for early stopping */
int early_stopping_round_;
+ /*! \brief Minimum improvement for early stopping */
+ double early_stopping_min_delta_;
/*! \brief Only use first metric for early stopping */
bool es_first_metric_only_;
/*! \brief Best iteration(s) for early stopping */
diff --git a/src/io/config_auto.cpp b/src/io/config_auto.cpp
index 394614af3f33..ca4fda1c3d4c 100644
--- a/src/io/config_auto.cpp
+++ b/src/io/config_auto.cpp
@@ -214,6 +214,7 @@ const std::unordered_set& Config::parameter_set() {
"extra_trees",
"extra_seed",
"early_stopping_round",
+ "early_stopping_min_delta",
"first_metric_only",
"max_delta_step",
"lambda_l1",
@@ -392,6 +393,9 @@ void Config::GetMembersFromString(const std::unordered_map>& Config::paramet
{"extra_trees", {"extra_tree"}},
{"extra_seed", {}},
{"early_stopping_round", {"early_stopping_rounds", "early_stopping", "n_iter_no_change"}},
+ {"early_stopping_min_delta", {}},
{"first_metric_only", {}},
{"max_delta_step", {"max_tree_output", "max_leaf_output"}},
{"lambda_l1", {"reg_alpha", "l1_regularization"}},
@@ -957,6 +963,7 @@ const std::unordered_map& Config::ParameterTypes() {
{"extra_trees", "bool"},
{"extra_seed", "int"},
{"early_stopping_round", "int"},
+ {"early_stopping_min_delta", "double"},
{"first_metric_only", "bool"},
{"max_delta_step", "double"},
{"lambda_l1", "double"},
diff --git a/tests/python_package_test/test_engine.py b/tests/python_package_test/test_engine.py
index 05c5792b1836..29210b94b4a1 100644
--- a/tests/python_package_test/test_engine.py
+++ b/tests/python_package_test/test_engine.py
@@ -1067,6 +1067,29 @@ def test_early_stopping_min_delta(first_only, single_metric, greater_is_better):
assert np.greater_equal(last_score, best_score - min_delta).any()
+@pytest.mark.parametrize("early_stopping_min_delta", [1e3, 0.0])
+def test_early_stopping_min_delta_via_global_params(early_stopping_min_delta):
+ X, y = load_breast_cancer(return_X_y=True)
+ num_trees = 5
+ params = {
+ "num_trees": num_trees,
+ "num_leaves": 5,
+ "objective": "binary",
+ "metric": "None",
+ "verbose": -1,
+ "early_stopping_round": 2,
+ "early_stopping_min_delta": early_stopping_min_delta,
+ }
+ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
+ lgb_train = lgb.Dataset(X_train, y_train)
+ lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
+ gbm = lgb.train(params, lgb_train, feval=decreasing_metric, valid_sets=lgb_eval)
+ if early_stopping_min_delta == 0:
+ assert gbm.best_iteration == num_trees
+ else:
+ assert gbm.best_iteration == 1
+
+
def test_early_stopping_can_be_triggered_via_custom_callback():
X, y = make_synthetic_regression()
@@ -1556,6 +1579,7 @@ def test_all_expected_params_are_written_out_to_model_text(tmp_path):
"[extra_trees: 0]",
"[extra_seed: 6642]",
"[early_stopping_round: 0]",
+ "[early_stopping_min_delta: 0]",
"[first_metric_only: 0]",
"[max_delta_step: 0]",
"[lambda_l1: 0]",