[Problem] tuning lgbm with early stop

[Nevermetyou65]

Hi, I tried to use flaml.tune to tune a user-defined function for lgbm
Here is my code

hyperparams = {
    "n_estimators" : tune.randint(100, 30_000),
    "objective": tune.choice(["poisson", "huber"]),
    "metric": "mape",
    "verbosity": -1,
    "boosting_type": "gbdt",
    "first_metric_only": True,
    "learning_rate": 0.05,
    "n_jobs": -1,
    "lambda_l1": tune.loguniform(lower=1e-8, upper=10.0),
    "lambda_l2": tune.loguniform(lower=1e-8, upper=10.0),
    "num_leaves": tune.lograndint(lower=10, upper=1000),
    "min_child_samples": tune.lograndint(lower=5, upper=50),
    "min_sum_hessian_in_leaf": tune.loguniform(lower=0.01, upper=1),
    "max_bin": tune.lograndint(lower=100, upper=500),
    "max_cat_threshold": tune.lograndint(lower=10, upper=500),
    "cat_l2": tune.randint(lower=0, upper=200),
    "cat_smooth": tune.randint(lower=0, upper=200),
}

def train_lgbm(config: dict) -> dict:
    params = LGBMEstimator(**config).params
    # train the model
    dtrain = lgb.Dataset(X_train, label=y_train)
    dvalid = lgb.Dataset(X_val, label=y_val)
    model = lgb.train(
        params,
        train_set=dtrain,
        valid_sets=dvalid,
        categorical_feature=list(
            data["train"][0].select_dtypes(include="category").columns
        ),
        callbacks=[lgb.early_stopping(500)]
    )
    # evaluate the model
    pred = model.predict(X_val)
    mape = mean_absolute_percentage_error(y_val, pred)
    # return eval results as a dictionary
    return {"mape": mape}

time = 60 * 5
analysis = tune.run(
    evaluation_function=train_lgbm,
    config=hyperparams,
    time_budget_s=time,  # total running time in seconds
    metric="mape",
    mode="min",
    search_alg="CFO",
)

While doing the train, I spotted that when I set an early stop callback in lgb.train(), the tunning process stops when the early stop is triggered, even if it is just trial 1.

What is happening here?

Thanks