import copyreg import io import pickle import re import warnings from unittest.mock import Mock import joblib import numpy as np import pytest from joblib.numpy_pickle import NumpyPickler from numpy.testing import assert_allclose, assert_array_equal import sklearn from sklearn._loss.loss import ( AbsoluteError, HalfBinomialLoss, HalfSquaredError, PinballLoss, ) from sklearn.base import BaseEstimator, TransformerMixin, clone, is_regressor from sklearn.compose import make_column_transformer from sklearn.datasets import make_classification, make_low_rank_matrix, make_regression from sklearn.dummy import DummyRegressor from sklearn.ensemble import ( HistGradientBoostingClassifier, HistGradientBoostingRegressor, ) from sklearn.ensemble._hist_gradient_boosting.binning import _BinMapper from sklearn.ensemble._hist_gradient_boosting.common import G_H_DTYPE from sklearn.ensemble._hist_gradient_boosting.grower import TreeGrower from sklearn.ensemble._hist_gradient_boosting.predictor import TreePredictor from sklearn.exceptions import NotFittedError from sklearn.metrics import get_scorer, mean_gamma_deviance, mean_poisson_deviance from sklearn.model_selection import cross_val_score, train_test_split from sklearn.pipeline import make_pipeline from sklearn.preprocessing import KBinsDiscretizer, MinMaxScaler, OneHotEncoder from sklearn.utils import _IS_32BIT, shuffle from sklearn.utils._openmp_helpers import _openmp_effective_n_threads from sklearn.utils._testing import _convert_container n_threads = _openmp_effective_n_threads() X_classification, y_classification = make_classification(random_state=0) X_regression, y_regression = make_regression(random_state=0) X_multi_classification, y_multi_classification = make_classification( n_classes=3, n_informative=3, random_state=0 ) def _make_dumb_dataset(n_samples): """Make a dumb dataset to test early stopping.""" rng = np.random.RandomState(42) X_dumb = rng.randn(n_samples, 1) y_dumb = (X_dumb[:, 0] > 0).astype("int64") return X_dumb, y_dumb @pytest.mark.parametrize( "GradientBoosting, X, y", [ (HistGradientBoostingClassifier, X_classification, y_classification), (HistGradientBoostingRegressor, X_regression, y_regression), ], ) @pytest.mark.parametrize( "params, err_msg", [ ( {"interaction_cst": [0, 1]}, "Interaction constraints must be a sequence of tuples or lists", ), ( {"interaction_cst": [{0, 9999}]}, r"Interaction constraints must consist of integer indices in \[0," r" n_features - 1\] = \[.*\], specifying the position of features,", ), ( {"interaction_cst": [{-1, 0}]}, r"Interaction constraints must consist of integer indices in \[0," r" n_features - 1\] = \[.*\], specifying the position of features,", ), ( {"interaction_cst": [{0.5}]}, r"Interaction constraints must consist of integer indices in \[0," r" n_features - 1\] = \[.*\], specifying the position of features,", ), ], ) def test_init_parameters_validation(GradientBoosting, X, y, params, err_msg): with pytest.raises(ValueError, match=err_msg): GradientBoosting(**params).fit(X, y) @pytest.mark.parametrize( "scoring, validation_fraction, early_stopping, n_iter_no_change, tol", [ ("neg_mean_squared_error", 0.1, True, 5, 1e-7), # use scorer ("neg_mean_squared_error", None, True, 5, 1e-1), # use scorer on train (None, 0.1, True, 5, 1e-7), # same with default scorer (None, None, True, 5, 1e-1), ("loss", 0.1, True, 5, 1e-7), # use loss ("loss", None, True, 5, 1e-1), # use loss on training data (None, None, False, 5, 0.0), # no early stopping ], ) def test_early_stopping_regression( scoring, validation_fraction, early_stopping, n_iter_no_change, tol ): max_iter = 200 X, y = make_regression(n_samples=50, random_state=0) gb = HistGradientBoostingRegressor( verbose=1, # just for coverage min_samples_leaf=5, # easier to overfit fast scoring=scoring, tol=tol, early_stopping=early_stopping, validation_fraction=validation_fraction, max_iter=max_iter, n_iter_no_change=n_iter_no_change, random_state=0, ) gb.fit(X, y) if early_stopping: assert n_iter_no_change <= gb.n_iter_ < max_iter else: assert gb.n_iter_ == max_iter @pytest.mark.parametrize( "data", ( make_classification(n_samples=30, random_state=0), make_classification( n_samples=30, n_classes=3, n_clusters_per_class=1, random_state=0 ), ), ) @pytest.mark.parametrize( "scoring, validation_fraction, early_stopping, n_iter_no_change, tol", [ ("accuracy", 0.1, True, 5, 1e-7), # use scorer ("accuracy", None, True, 5, 1e-1), # use scorer on training data (None, 0.1, True, 5, 1e-7), # same with default scorer (None, None, True, 5, 1e-1), ("loss", 0.1, True, 5, 1e-7), # use loss ("loss", None, True, 5, 1e-1), # use loss on training data (None, None, False, 5, 0.0), # no early stopping ], ) def test_early_stopping_classification( data, scoring, validation_fraction, early_stopping, n_iter_no_change, tol ): max_iter = 50 X, y = data gb = HistGradientBoostingClassifier( verbose=1, # just for coverage min_samples_leaf=5, # easier to overfit fast scoring=scoring, tol=tol, early_stopping=early_stopping, validation_fraction=validation_fraction, max_iter=max_iter, n_iter_no_change=n_iter_no_change, random_state=0, ) gb.fit(X, y) if early_stopping is True: assert n_iter_no_change <= gb.n_iter_ < max_iter else: assert gb.n_iter_ == max_iter @pytest.mark.parametrize( "GradientBoosting, X, y", [ (HistGradientBoostingClassifier, *_make_dumb_dataset(10000)), (HistGradientBoostingClassifier, *_make_dumb_dataset(10001)), (HistGradientBoostingRegressor, *_make_dumb_dataset(10000)), (HistGradientBoostingRegressor, *_make_dumb_dataset(10001)), ], ) def test_early_stopping_default(GradientBoosting, X, y): # Test that early stopping is enabled by default if and only if there # are more than 10000 samples gb = GradientBoosting(max_iter=10, n_iter_no_change=2, tol=1e-1) gb.fit(X, y) if X.shape[0] > 10000: assert gb.n_iter_ < gb.max_iter else: assert gb.n_iter_ == gb.max_iter @pytest.mark.parametrize( "scores, n_iter_no_change, tol, stopping", [ ([], 1, 0.001, False), # not enough iterations ([1, 1, 1], 5, 0.001, False), # not enough iterations ([1, 1, 1, 1, 1], 5, 0.001, False), # not enough iterations ([1, 2, 3, 4, 5, 6], 5, 0.001, False), # significant improvement ([1, 2, 3, 4, 5, 6], 5, 0.0, False), # significant improvement ([1, 2, 3, 4, 5, 6], 5, 0.999, False), # significant improvement ([1, 2, 3, 4, 5, 6], 5, 5 - 1e-5, False), # significant improvement ([1] * 6, 5, 0.0, True), # no significant improvement ([1] * 6, 5, 0.001, True), # no significant improvement ([1] * 6, 5, 5, True), # no significant improvement ], ) def test_should_stop(scores, n_iter_no_change, tol, stopping): gbdt = HistGradientBoostingClassifier(n_iter_no_change=n_iter_no_change, tol=tol) assert gbdt._should_stop(scores) == stopping def test_absolute_error(): # For coverage only. X, y = make_regression(n_samples=500, random_state=0) gbdt = HistGradientBoostingRegressor(loss="absolute_error", random_state=0) gbdt.fit(X, y) assert gbdt.score(X, y) > 0.9 def test_absolute_error_sample_weight(): # non regression test for issue #19400 # make sure no error is thrown during fit of # HistGradientBoostingRegressor with absolute_error loss function # and passing sample_weight rng = np.random.RandomState(0) n_samples = 100 X = rng.uniform(-1, 1, size=(n_samples, 2)) y = rng.uniform(-1, 1, size=n_samples) sample_weight = rng.uniform(0, 1, size=n_samples) gbdt = HistGradientBoostingRegressor(loss="absolute_error") gbdt.fit(X, y, sample_weight=sample_weight) @pytest.mark.parametrize("y", [([1.0, -2.0, 0.0]), ([0.0, 1.0, 2.0])]) def test_gamma_y_positive(y): # Test that ValueError is raised if any y_i <= 0. err_msg = r"loss='gamma' requires strictly positive y." gbdt = HistGradientBoostingRegressor(loss="gamma", random_state=0) with pytest.raises(ValueError, match=err_msg): gbdt.fit(np.zeros(shape=(len(y), 1)), y) def test_gamma(): # For a Gamma distributed target, we expect an HGBT trained with the Gamma deviance # (loss) to give better results than an HGBT with any other loss function, measured # in out-of-sample Gamma deviance as metric/score. # Note that squared error could potentially predict negative values which is # invalid (np.inf) for the Gamma deviance. A Poisson HGBT (having a log link) # does not have that defect. # Important note: It seems that a Poisson HGBT almost always has better # out-of-sample performance than the Gamma HGBT, measured in Gamma deviance. # LightGBM shows the same behaviour. Hence, we only compare to a squared error # HGBT, but not to a Poisson deviance HGBT. rng = np.random.RandomState(42) n_train, n_test, n_features = 500, 100, 20 X = make_low_rank_matrix( n_samples=n_train + n_test, n_features=n_features, random_state=rng, ) # We create a log-linear Gamma model. This gives y.min ~ 1e-2, y.max ~ 1e2 coef = rng.uniform(low=-10, high=20, size=n_features) # Numpy parametrizes gamma(shape=k, scale=theta) with mean = k * theta and # variance = k * theta^2. We parametrize it instead with mean = exp(X @ coef) # and variance = dispersion * mean^2 by setting k = 1 / dispersion, # theta = dispersion * mean. dispersion = 0.5 y = rng.gamma(shape=1 / dispersion, scale=dispersion * np.exp(X @ coef)) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=n_test, random_state=rng ) gbdt_gamma = HistGradientBoostingRegressor(loss="gamma", random_state=123) gbdt_mse = HistGradientBoostingRegressor(loss="squared_error", random_state=123) dummy = DummyRegressor(strategy="mean") for model in (gbdt_gamma, gbdt_mse, dummy): model.fit(X_train, y_train) for X, y in [(X_train, y_train), (X_test, y_test)]: loss_gbdt_gamma = mean_gamma_deviance(y, gbdt_gamma.predict(X)) # We restrict the squared error HGBT to predict at least the minimum seen y at # train time to make it strictly positive. loss_gbdt_mse = mean_gamma_deviance( y, np.maximum(np.min(y_train), gbdt_mse.predict(X)) ) loss_dummy = mean_gamma_deviance(y, dummy.predict(X)) assert loss_gbdt_gamma < loss_dummy assert loss_gbdt_gamma < loss_gbdt_mse @pytest.mark.parametrize("quantile", [0.2, 0.5, 0.8]) def test_quantile_asymmetric_error(quantile): """Test quantile regression for asymmetric distributed targets.""" n_samples = 10_000 rng = np.random.RandomState(42) # take care that X @ coef + intercept > 0 X = np.concatenate( ( np.abs(rng.randn(n_samples)[:, None]), -rng.randint(2, size=(n_samples, 1)), ), axis=1, ) intercept = 1.23 coef = np.array([0.5, -2]) # For an exponential distribution with rate lambda, e.g. exp(-lambda * x), # the quantile at level q is: # quantile(q) = - log(1 - q) / lambda # scale = 1/lambda = -quantile(q) / log(1-q) y = rng.exponential( scale=-(X @ coef + intercept) / np.log(1 - quantile), size=n_samples ) model = HistGradientBoostingRegressor( loss="quantile", quantile=quantile, max_iter=25, random_state=0, max_leaf_nodes=10, ).fit(X, y) assert_allclose(np.mean(model.predict(X) > y), quantile, rtol=1e-2) pinball_loss = PinballLoss(quantile=quantile) loss_true_quantile = pinball_loss(y, X @ coef + intercept) loss_pred_quantile = pinball_loss(y, model.predict(X)) # we are overfitting assert loss_pred_quantile <= loss_true_quantile @pytest.mark.parametrize("y", [([1.0, -2.0, 0.0]), ([0.0, 0.0, 0.0])]) def test_poisson_y_positive(y): # Test that ValueError is raised if either one y_i < 0 or sum(y_i) <= 0. err_msg = r"loss='poisson' requires non-negative y and sum\(y\) > 0." gbdt = HistGradientBoostingRegressor(loss="poisson", random_state=0) with pytest.raises(ValueError, match=err_msg): gbdt.fit(np.zeros(shape=(len(y), 1)), y) def test_poisson(): # For Poisson distributed target, Poisson loss should give better results # than least squares measured in Poisson deviance as metric. rng = np.random.RandomState(42) n_train, n_test, n_features = 500, 100, 100 X = make_low_rank_matrix( n_samples=n_train + n_test, n_features=n_features, random_state=rng ) # We create a log-linear Poisson model and downscale coef as it will get # exponentiated. coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0) y = rng.poisson(lam=np.exp(X @ coef)) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=n_test, random_state=rng ) gbdt_pois = HistGradientBoostingRegressor(loss="poisson", random_state=rng) gbdt_ls = HistGradientBoostingRegressor(loss="squared_error", random_state=rng) gbdt_pois.fit(X_train, y_train) gbdt_ls.fit(X_train, y_train) dummy = DummyRegressor(strategy="mean").fit(X_train, y_train) for X, y in [(X_train, y_train), (X_test, y_test)]: metric_pois = mean_poisson_deviance(y, gbdt_pois.predict(X)) # squared_error might produce non-positive predictions => clip metric_ls = mean_poisson_deviance(y, np.clip(gbdt_ls.predict(X), 1e-15, None)) metric_dummy = mean_poisson_deviance(y, dummy.predict(X)) assert metric_pois < metric_ls assert metric_pois < metric_dummy def test_binning_train_validation_are_separated(): # Make sure training and validation data are binned separately. # See issue 13926 rng = np.random.RandomState(0) validation_fraction = 0.2 gb = HistGradientBoostingClassifier( early_stopping=True, validation_fraction=validation_fraction, random_state=rng ) gb.fit(X_classification, y_classification) mapper_training_data = gb._bin_mapper # Note that since the data is small there is no subsampling and the # random_state doesn't matter mapper_whole_data = _BinMapper(random_state=0) mapper_whole_data.fit(X_classification) n_samples = X_classification.shape[0] assert np.all( mapper_training_data.n_bins_non_missing_ == int((1 - validation_fraction) * n_samples) ) assert np.all( mapper_training_data.n_bins_non_missing_ != mapper_whole_data.n_bins_non_missing_ ) def test_missing_values_trivial(): # sanity check for missing values support. With only one feature and # y == isnan(X), the gbdt is supposed to reach perfect accuracy on the # training set. n_samples = 100 n_features = 1 rng = np.random.RandomState(0) X = rng.normal(size=(n_samples, n_features)) mask = rng.binomial(1, 0.5, size=X.shape).astype(bool) X[mask] = np.nan y = mask.ravel() gb = HistGradientBoostingClassifier() gb.fit(X, y) assert gb.score(X, y) == pytest.approx(1) @pytest.mark.parametrize("problem", ("classification", "regression")) @pytest.mark.parametrize( ( "missing_proportion, expected_min_score_classification, " "expected_min_score_regression" ), [(0.1, 0.97, 0.89), (0.2, 0.93, 0.81), (0.5, 0.79, 0.52)], ) def test_missing_values_resilience( problem, missing_proportion, expected_min_score_classification, expected_min_score_regression, ): # Make sure the estimators can deal with missing values and still yield # decent predictions rng = np.random.RandomState(0) n_samples = 1000 n_features = 2 if problem == "regression": X, y = make_regression( n_samples=n_samples, n_features=n_features, n_informative=n_features, random_state=rng, ) gb = HistGradientBoostingRegressor() expected_min_score = expected_min_score_regression else: X, y = make_classification( n_samples=n_samples, n_features=n_features, n_informative=n_features, n_redundant=0, n_repeated=0, random_state=rng, ) gb = HistGradientBoostingClassifier() expected_min_score = expected_min_score_classification mask = rng.binomial(1, missing_proportion, size=X.shape).astype(bool) X[mask] = np.nan gb.fit(X, y) assert gb.score(X, y) > expected_min_score @pytest.mark.parametrize( "data", [ make_classification(random_state=0, n_classes=2), make_classification(random_state=0, n_classes=3, n_informative=3), ], ids=["binary_log_loss", "multiclass_log_loss"], ) def test_zero_division_hessians(data): # non regression test for issue #14018 # make sure we avoid zero division errors when computing the leaves values. # If the learning rate is too high, the raw predictions are bad and will # saturate the softmax (or sigmoid in binary classif). This leads to # probabilities being exactly 0 or 1, gradients being constant, and # hessians being zero. X, y = data gb = HistGradientBoostingClassifier(learning_rate=100, max_iter=10) gb.fit(X, y) def test_small_trainset(): # Make sure that the small trainset is stratified and has the expected # length (10k samples) n_samples = 20000 original_distrib = {0: 0.1, 1: 0.2, 2: 0.3, 3: 0.4} rng = np.random.RandomState(42) X = rng.randn(n_samples).reshape(n_samples, 1) y = [ [class_] * int(prop * n_samples) for (class_, prop) in original_distrib.items() ] y = shuffle(np.concatenate(y)) gb = HistGradientBoostingClassifier() # Compute the small training set X_small, y_small, *_ = gb._get_small_trainset( X, y, seed=42, sample_weight_train=None ) # Compute the class distribution in the small training set unique, counts = np.unique(y_small, return_counts=True) small_distrib = {class_: count / 10000 for (class_, count) in zip(unique, counts)} # Test that the small training set has the expected length assert X_small.shape[0] == 10000 assert y_small.shape[0] == 10000 # Test that the class distributions in the whole dataset and in the small # training set are identical assert small_distrib == pytest.approx(original_distrib) def test_missing_values_minmax_imputation(): # Compare the buit-in missing value handling of Histogram GBC with an # a-priori missing value imputation strategy that should yield the same # results in terms of decision function. # # Each feature (containing NaNs) is replaced by 2 features: # - one where the nans are replaced by min(feature) - 1 # - one where the nans are replaced by max(feature) + 1 # A split where nans go to the left has an equivalent split in the # first (min) feature, and a split where nans go to the right has an # equivalent split in the second (max) feature. # # Assuming the data is such that there is never a tie to select the best # feature to split on during training, the learned decision trees should be # strictly equivalent (learn a sequence of splits that encode the same # decision function). # # The MinMaxImputer transformer is meant to be a toy implementation of the # "Missing In Attributes" (MIA) missing value handling for decision trees # https://www.sciencedirect.com/science/article/abs/pii/S0167865508000305 # The implementation of MIA as an imputation transformer was suggested by # "Remark 3" in :arxiv:'<1902.06931>` class MinMaxImputer(TransformerMixin, BaseEstimator): def fit(self, X, y=None): mm = MinMaxScaler().fit(X) self.data_min_ = mm.data_min_ self.data_max_ = mm.data_max_ return self def transform(self, X): X_min, X_max = X.copy(), X.copy() for feature_idx in range(X.shape[1]): nan_mask = np.isnan(X[:, feature_idx]) X_min[nan_mask, feature_idx] = self.data_min_[feature_idx] - 1 X_max[nan_mask, feature_idx] = self.data_max_[feature_idx] + 1 return np.concatenate([X_min, X_max], axis=1) def make_missing_value_data(n_samples=int(1e4), seed=0): rng = np.random.RandomState(seed) X, y = make_regression(n_samples=n_samples, n_features=4, random_state=rng) # Pre-bin the data to ensure a deterministic handling by the 2 # strategies and also make it easier to insert np.nan in a structured # way: X = KBinsDiscretizer(n_bins=42, encode="ordinal").fit_transform(X) # First feature has missing values completely at random: rnd_mask = rng.rand(X.shape[0]) > 0.9 X[rnd_mask, 0] = np.nan # Second and third features have missing values for extreme values # (censoring missingness): low_mask = X[:, 1] == 0 X[low_mask, 1] = np.nan high_mask = X[:, 2] == X[:, 2].max() X[high_mask, 2] = np.nan # Make the last feature nan pattern very informative: y_max = np.percentile(y, 70) y_max_mask = y >= y_max y[y_max_mask] = y_max X[y_max_mask, 3] = np.nan # Check that there is at least one missing value in each feature: for feature_idx in range(X.shape[1]): assert any(np.isnan(X[:, feature_idx])) # Let's use a test set to check that the learned decision function is # the same as evaluated on unseen data. Otherwise it could just be the # case that we find two independent ways to overfit the training set. return train_test_split(X, y, random_state=rng) # n_samples need to be large enough to minimize the likelihood of having # several candidate splits with the same gain value in a given tree. X_train, X_test, y_train, y_test = make_missing_value_data( n_samples=int(1e4), seed=0 ) # Use a small number of leaf nodes and iterations so as to keep # under-fitting models to minimize the likelihood of ties when training the # model. gbm1 = HistGradientBoostingRegressor(max_iter=100, max_leaf_nodes=5, random_state=0) gbm1.fit(X_train, y_train) gbm2 = make_pipeline(MinMaxImputer(), clone(gbm1)) gbm2.fit(X_train, y_train) # Check that the model reach the same score: assert gbm1.score(X_train, y_train) == pytest.approx(gbm2.score(X_train, y_train)) assert gbm1.score(X_test, y_test) == pytest.approx(gbm2.score(X_test, y_test)) # Check the individual prediction match as a finer grained # decision function check. assert_allclose(gbm1.predict(X_train), gbm2.predict(X_train)) assert_allclose(gbm1.predict(X_test), gbm2.predict(X_test)) def test_infinite_values(): # Basic test for infinite values X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) y = np.array([0, 0, 1, 1]) gbdt = HistGradientBoostingRegressor(min_samples_leaf=1) gbdt.fit(X, y) np.testing.assert_allclose(gbdt.predict(X), y, atol=1e-4) def test_consistent_lengths(): X = np.array([-np.inf, 0, 1, np.inf]).reshape(-1, 1) y = np.array([0, 0, 1, 1]) sample_weight = np.array([0.1, 0.3, 0.1]) gbdt = HistGradientBoostingRegressor() with pytest.raises(ValueError, match=r"sample_weight.shape == \(3,\), expected"): gbdt.fit(X, y, sample_weight) with pytest.raises( ValueError, match="Found input variables with inconsistent number" ): gbdt.fit(X, y[1:]) def test_infinite_values_missing_values(): # High level test making sure that inf and nan values are properly handled # when both are present. This is similar to # test_split_on_nan_with_infinite_values() in test_grower.py, though we # cannot check the predictions for binned values here. X = np.asarray([-np.inf, 0, 1, np.inf, np.nan]).reshape(-1, 1) y_isnan = np.isnan(X.ravel()) y_isinf = X.ravel() == np.inf stump_clf = HistGradientBoostingClassifier( min_samples_leaf=1, max_iter=1, learning_rate=1, max_depth=2 ) assert stump_clf.fit(X, y_isinf).score(X, y_isinf) == 1 assert stump_clf.fit(X, y_isnan).score(X, y_isnan) == 1 @pytest.mark.parametrize("scoring", [None, "loss"]) def test_string_target_early_stopping(scoring): # Regression tests for #14709 where the targets need to be encoded before # to compute the score rng = np.random.RandomState(42) X = rng.randn(100, 10) y = np.array(["x"] * 50 + ["y"] * 50, dtype=object) gbrt = HistGradientBoostingClassifier(n_iter_no_change=10, scoring=scoring) gbrt.fit(X, y) def test_zero_sample_weights_regression(): # Make sure setting a SW to zero amounts to ignoring the corresponding # sample X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] gb = HistGradientBoostingRegressor(min_samples_leaf=1) gb.fit(X, y, sample_weight=sample_weight) assert gb.predict([[1, 0]])[0] > 0.5 def test_zero_sample_weights_classification(): # Make sure setting a SW to zero amounts to ignoring the corresponding # sample X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1) gb.fit(X, y, sample_weight=sample_weight) assert_array_equal(gb.predict([[1, 0]]), [1]) X = [[1, 0], [1, 0], [1, 0], [0, 1], [1, 1]] y = [0, 0, 1, 0, 2] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1, 1] gb = HistGradientBoostingClassifier(loss="log_loss", min_samples_leaf=1) gb.fit(X, y, sample_weight=sample_weight) assert_array_equal(gb.predict([[1, 0]]), [1]) @pytest.mark.parametrize( "problem", ("regression", "binary_classification", "multiclass_classification") ) @pytest.mark.parametrize("duplication", ("half", "all")) def test_sample_weight_effect(problem, duplication): # High level test to make sure that duplicating a sample is equivalent to # giving it weight of 2. # fails for n_samples > 255 because binning does not take sample weights # into account. Keeping n_samples <= 255 makes # sure only unique values are used so SW have no effect on binning. n_samples = 255 n_features = 2 if problem == "regression": X, y = make_regression( n_samples=n_samples, n_features=n_features, n_informative=n_features, random_state=0, ) Klass = HistGradientBoostingRegressor else: n_classes = 2 if problem == "binary_classification" else 3 X, y = make_classification( n_samples=n_samples, n_features=n_features, n_informative=n_features, n_redundant=0, n_clusters_per_class=1, n_classes=n_classes, random_state=0, ) Klass = HistGradientBoostingClassifier # This test can't pass if min_samples_leaf > 1 because that would force 2 # samples to be in the same node in est_sw, while these samples would be # free to be separate in est_dup: est_dup would just group together the # duplicated samples. est = Klass(min_samples_leaf=1) # Create dataset with duplicate and corresponding sample weights if duplication == "half": lim = n_samples // 2 else: lim = n_samples X_dup = np.r_[X, X[:lim]] y_dup = np.r_[y, y[:lim]] sample_weight = np.ones(shape=(n_samples)) sample_weight[:lim] = 2 est_sw = clone(est).fit(X, y, sample_weight=sample_weight) est_dup = clone(est).fit(X_dup, y_dup) # checking raw_predict is stricter than just predict for classification assert np.allclose(est_sw._raw_predict(X_dup), est_dup._raw_predict(X_dup)) @pytest.mark.parametrize("Loss", (HalfSquaredError, AbsoluteError)) def test_sum_hessians_are_sample_weight(Loss): # For losses with constant hessians, the sum_hessians field of the # histograms must be equal to the sum of the sample weight of samples at # the corresponding bin. rng = np.random.RandomState(0) n_samples = 1000 n_features = 2 X, y = make_regression(n_samples=n_samples, n_features=n_features, random_state=rng) bin_mapper = _BinMapper() X_binned = bin_mapper.fit_transform(X) # While sample weights are supposed to be positive, this still works. sample_weight = rng.normal(size=n_samples) loss = Loss(sample_weight=sample_weight) gradients, hessians = loss.init_gradient_and_hessian( n_samples=n_samples, dtype=G_H_DTYPE ) gradients, hessians = gradients.reshape((-1, 1)), hessians.reshape((-1, 1)) raw_predictions = rng.normal(size=(n_samples, 1)) loss.gradient_hessian( y_true=y, raw_prediction=raw_predictions, sample_weight=sample_weight, gradient_out=gradients, hessian_out=hessians, n_threads=n_threads, ) # build sum_sample_weight which contains the sum of the sample weights at # each bin (for each feature). This must be equal to the sum_hessians # field of the corresponding histogram sum_sw = np.zeros(shape=(n_features, bin_mapper.n_bins)) for feature_idx in range(n_features): for sample_idx in range(n_samples): sum_sw[feature_idx, X_binned[sample_idx, feature_idx]] += sample_weight[ sample_idx ] # Build histogram grower = TreeGrower( X_binned, gradients[:, 0], hessians[:, 0], n_bins=bin_mapper.n_bins ) histograms = grower.histogram_builder.compute_histograms_brute( grower.root.sample_indices ) for feature_idx in range(n_features): for bin_idx in range(bin_mapper.n_bins): assert histograms[feature_idx, bin_idx]["sum_hessians"] == ( pytest.approx(sum_sw[feature_idx, bin_idx], rel=1e-5) ) def test_max_depth_max_leaf_nodes(): # Non regression test for # https://github.com/scikit-learn/scikit-learn/issues/16179 # there was a bug when the max_depth and the max_leaf_nodes criteria were # met at the same time, which would lead to max_leaf_nodes not being # respected. X, y = make_classification(random_state=0) est = HistGradientBoostingClassifier(max_depth=2, max_leaf_nodes=3, max_iter=1).fit( X, y ) tree = est._predictors[0][0] assert tree.get_max_depth() == 2 assert tree.get_n_leaf_nodes() == 3 # would be 4 prior to bug fix def test_early_stopping_on_test_set_with_warm_start(): # Non regression test for #16661 where second fit fails with # warm_start=True, early_stopping is on, and no validation set X, y = make_classification(random_state=0) gb = HistGradientBoostingClassifier( max_iter=1, scoring="loss", warm_start=True, early_stopping=True, n_iter_no_change=1, validation_fraction=None, ) gb.fit(X, y) # does not raise on second call gb.set_params(max_iter=2) gb.fit(X, y) def test_early_stopping_with_sample_weights(monkeypatch): """Check that sample weights is passed in to the scorer and _raw_predict is not called.""" mock_scorer = Mock(side_effect=get_scorer("neg_median_absolute_error")) def mock_check_scoring(estimator, scoring): assert scoring == "neg_median_absolute_error" return mock_scorer monkeypatch.setattr( sklearn.ensemble._hist_gradient_boosting.gradient_boosting, "check_scoring", mock_check_scoring, ) X, y = make_regression(random_state=0) sample_weight = np.ones_like(y) hist = HistGradientBoostingRegressor( max_iter=2, early_stopping=True, random_state=0, scoring="neg_median_absolute_error", ) mock_raw_predict = Mock(side_effect=hist._raw_predict) hist._raw_predict = mock_raw_predict hist.fit(X, y, sample_weight=sample_weight) # _raw_predict should never be called with scoring as a string assert mock_raw_predict.call_count == 0 # For scorer is called twice (train and val) for the baseline score, and twice # per iteration (train and val) after that. So 6 times in total for `max_iter=2`. assert mock_scorer.call_count == 6 for arg_list in mock_scorer.call_args_list: assert "sample_weight" in arg_list[1] def test_raw_predict_is_called_with_custom_scorer(): """Custom scorer will still call _raw_predict.""" mock_scorer = Mock(side_effect=get_scorer("neg_median_absolute_error")) X, y = make_regression(random_state=0) hist = HistGradientBoostingRegressor( max_iter=2, early_stopping=True, random_state=0, scoring=mock_scorer, ) mock_raw_predict = Mock(side_effect=hist._raw_predict) hist._raw_predict = mock_raw_predict hist.fit(X, y) # `_raw_predict` and scorer is called twice (train and val) for the baseline score, # and twice per iteration (train and val) after that. So 6 times in total for # `max_iter=2`. assert mock_raw_predict.call_count == 6 assert mock_scorer.call_count == 6 @pytest.mark.parametrize( "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) ) def test_single_node_trees(Est): # Make sure it's still possible to build single-node trees. In that case # the value of the root is set to 0. That's a correct value: if the tree is # single-node that's because min_gain_to_split is not respected right from # the root, so we don't want the tree to have any impact on the # predictions. X, y = make_classification(random_state=0) y[:] = 1 # constant target will lead to a single root node est = Est(max_iter=20) est.fit(X, y) assert all(len(predictor[0].nodes) == 1 for predictor in est._predictors) assert all(predictor[0].nodes[0]["value"] == 0 for predictor in est._predictors) # Still gives correct predictions thanks to the baseline prediction assert_allclose(est.predict(X), y) @pytest.mark.parametrize( "Est, loss, X, y", [ ( HistGradientBoostingClassifier, HalfBinomialLoss(sample_weight=None), X_classification, y_classification, ), ( HistGradientBoostingRegressor, HalfSquaredError(sample_weight=None), X_regression, y_regression, ), ], ) def test_custom_loss(Est, loss, X, y): est = Est(loss=loss, max_iter=20) est.fit(X, y) @pytest.mark.parametrize( "HistGradientBoosting, X, y", [ (HistGradientBoostingClassifier, X_classification, y_classification), (HistGradientBoostingRegressor, X_regression, y_regression), ( HistGradientBoostingClassifier, X_multi_classification, y_multi_classification, ), ], ) def test_staged_predict(HistGradientBoosting, X, y): # Test whether staged predictor eventually gives # the same prediction. X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.5, random_state=0 ) gb = HistGradientBoosting(max_iter=10) # test raise NotFittedError if not fitted with pytest.raises(NotFittedError): next(gb.staged_predict(X_test)) gb.fit(X_train, y_train) # test if the staged predictions of each iteration # are equal to the corresponding predictions of the same estimator # trained from scratch. # this also test limit case when max_iter = 1 method_names = ( ["predict"] if is_regressor(gb) else ["predict", "predict_proba", "decision_function"] ) for method_name in method_names: staged_method = getattr(gb, "staged_" + method_name) staged_predictions = list(staged_method(X_test)) assert len(staged_predictions) == gb.n_iter_ for n_iter, staged_predictions in enumerate(staged_method(X_test), 1): aux = HistGradientBoosting(max_iter=n_iter) aux.fit(X_train, y_train) pred_aux = getattr(aux, method_name)(X_test) assert_allclose(staged_predictions, pred_aux) assert staged_predictions.shape == pred_aux.shape @pytest.mark.parametrize("insert_missing", [False, True]) @pytest.mark.parametrize( "Est", (HistGradientBoostingRegressor, HistGradientBoostingClassifier) ) @pytest.mark.parametrize("bool_categorical_parameter", [True, False]) @pytest.mark.parametrize("missing_value", [np.nan, -1]) def test_unknown_categories_nan( insert_missing, Est, bool_categorical_parameter, missing_value ): # Make sure no error is raised at predict if a category wasn't seen during # fit. We also make sure they're treated as nans. rng = np.random.RandomState(0) n_samples = 1000 f1 = rng.rand(n_samples) f2 = rng.randint(4, size=n_samples) X = np.c_[f1, f2] y = np.zeros(shape=n_samples) y[X[:, 1] % 2 == 0] = 1 if bool_categorical_parameter: categorical_features = [False, True] else: categorical_features = [1] if insert_missing: mask = rng.binomial(1, 0.01, size=X.shape).astype(bool) assert mask.sum() > 0 X[mask] = missing_value est = Est(max_iter=20, categorical_features=categorical_features).fit(X, y) assert_array_equal(est.is_categorical_, [False, True]) # Make sure no error is raised on unknown categories and nans # unknown categories will be treated as nans X_test = np.zeros((10, X.shape[1]), dtype=float) X_test[:5, 1] = 30 X_test[5:, 1] = missing_value assert len(np.unique(est.predict(X_test))) == 1 def test_categorical_encoding_strategies(): # Check native categorical handling vs different encoding strategies. We # make sure that native encoding needs only 1 split to achieve a perfect # prediction on a simple dataset. In contrast, OneHotEncoded data needs # more depth / splits, and treating categories as ordered (just using # OrdinalEncoder) requires even more depth. # dataset with one random continuous feature, and one categorical feature # with values in [0, 5], e.g. from an OrdinalEncoder. # class == 1 iff categorical value in {0, 2, 4} rng = np.random.RandomState(0) n_samples = 10_000 f1 = rng.rand(n_samples) f2 = rng.randint(6, size=n_samples) X = np.c_[f1, f2] y = np.zeros(shape=n_samples) y[X[:, 1] % 2 == 0] = 1 # make sure dataset is balanced so that the baseline_prediction doesn't # influence predictions too much with max_iter = 1 assert 0.49 < y.mean() < 0.51 native_cat_specs = [ [False, True], [1], ] try: import pandas as pd X = pd.DataFrame(X, columns=["f_0", "f_1"]) native_cat_specs.append(["f_1"]) except ImportError: pass for native_cat_spec in native_cat_specs: clf_cat = HistGradientBoostingClassifier( max_iter=1, max_depth=1, categorical_features=native_cat_spec ) clf_cat.fit(X, y) # Using native categorical encoding, we get perfect predictions with just # one split assert cross_val_score(clf_cat, X, y).mean() == 1 # quick sanity check for the bitset: 0, 2, 4 = 2**0 + 2**2 + 2**4 = 21 expected_left_bitset = [21, 0, 0, 0, 0, 0, 0, 0] left_bitset = clf_cat.fit(X, y)._predictors[0][0].raw_left_cat_bitsets[0] assert_array_equal(left_bitset, expected_left_bitset) # Treating categories as ordered, we need more depth / more splits to get # the same predictions clf_no_cat = HistGradientBoostingClassifier( max_iter=1, max_depth=4, categorical_features=None ) assert cross_val_score(clf_no_cat, X, y).mean() < 0.9 clf_no_cat.set_params(max_depth=5) assert cross_val_score(clf_no_cat, X, y).mean() == 1 # Using OHEd data, we need less splits than with pure OEd data, but we # still need more splits than with the native categorical splits ct = make_column_transformer( (OneHotEncoder(sparse_output=False), [1]), remainder="passthrough" ) X_ohe = ct.fit_transform(X) clf_no_cat.set_params(max_depth=2) assert cross_val_score(clf_no_cat, X_ohe, y).mean() < 0.9 clf_no_cat.set_params(max_depth=3) assert cross_val_score(clf_no_cat, X_ohe, y).mean() == 1 @pytest.mark.parametrize( "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) ) @pytest.mark.parametrize( "categorical_features, monotonic_cst, expected_msg", [ ( [b"hello", b"world"], None, re.escape( "categorical_features must be an array-like of bool, int or str, " "got: bytes40." ), ), ( np.array([b"hello", 1.3], dtype=object), None, re.escape( "categorical_features must be an array-like of bool, int or str, " "got: bytes, float." ), ), ( [0, -1], None, re.escape( "categorical_features set as integer indices must be in " "[0, n_features - 1]" ), ), ( [True, True, False, False, True], None, re.escape( "categorical_features set as a boolean mask must have shape " "(n_features,)" ), ), ( [True, True, False, False], [0, -1, 0, 1], "Categorical features cannot have monotonic constraints", ), ], ) def test_categorical_spec_errors( Est, categorical_features, monotonic_cst, expected_msg ): # Test errors when categories are specified incorrectly n_samples = 100 X, y = make_classification(random_state=0, n_features=4, n_samples=n_samples) rng = np.random.RandomState(0) X[:, 0] = rng.randint(0, 10, size=n_samples) X[:, 1] = rng.randint(0, 10, size=n_samples) est = Est(categorical_features=categorical_features, monotonic_cst=monotonic_cst) with pytest.raises(ValueError, match=expected_msg): est.fit(X, y) @pytest.mark.parametrize( "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) ) def test_categorical_spec_errors_with_feature_names(Est): pd = pytest.importorskip("pandas") n_samples = 10 X = pd.DataFrame( { "f0": range(n_samples), "f1": range(n_samples), "f2": [1.0] * n_samples, } ) y = [0, 1] * (n_samples // 2) est = Est(categorical_features=["f0", "f1", "f3"]) expected_msg = re.escape( "categorical_features has a item value 'f3' which is not a valid " "feature name of the training data." ) with pytest.raises(ValueError, match=expected_msg): est.fit(X, y) est = Est(categorical_features=["f0", "f1"]) expected_msg = re.escape( "categorical_features should be passed as an array of integers or " "as a boolean mask when the model is fitted on data without feature " "names." ) with pytest.raises(ValueError, match=expected_msg): est.fit(X.to_numpy(), y) @pytest.mark.parametrize( "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) ) @pytest.mark.parametrize("categorical_features", ([False, False], [])) @pytest.mark.parametrize("as_array", (True, False)) def test_categorical_spec_no_categories(Est, categorical_features, as_array): # Make sure we can properly detect that no categorical features are present # even if the categorical_features parameter is not None X = np.arange(10).reshape(5, 2) y = np.arange(5) if as_array: categorical_features = np.asarray(categorical_features) est = Est(categorical_features=categorical_features).fit(X, y) assert est.is_categorical_ is None @pytest.mark.parametrize( "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) ) @pytest.mark.parametrize( "use_pandas, feature_name", [(False, "at index 0"), (True, "'f0'")] ) def test_categorical_bad_encoding_errors(Est, use_pandas, feature_name): # Test errors when categories are encoded incorrectly gb = Est(categorical_features=[True], max_bins=2) if use_pandas: pd = pytest.importorskip("pandas") X = pd.DataFrame({"f0": [0, 1, 2]}) else: X = np.array([[0, 1, 2]]).T y = np.arange(3) msg = ( f"Categorical feature {feature_name} is expected to have a " "cardinality <= 2 but actually has a cardinality of 3." ) with pytest.raises(ValueError, match=msg): gb.fit(X, y) # nans are ignored in the counts X = np.array([[0, 1, np.nan]]).T y = np.arange(3) gb.fit(X, y) @pytest.mark.parametrize( "Est", (HistGradientBoostingClassifier, HistGradientBoostingRegressor) ) def test_uint8_predict(Est): # Non regression test for # https://github.com/scikit-learn/scikit-learn/issues/18408 # Make sure X can be of dtype uint8 (i.e. X_BINNED_DTYPE) in predict. It # will be converted to X_DTYPE. rng = np.random.RandomState(0) X = rng.randint(0, 100, size=(10, 2)).astype(np.uint8) y = rng.randint(0, 2, size=10).astype(np.uint8) est = Est() est.fit(X, y) est.predict(X) @pytest.mark.parametrize( "interaction_cst, n_features, result", [ (None, 931, None), ([{0, 1}], 2, [{0, 1}]), ("pairwise", 2, [{0, 1}]), ("pairwise", 4, [{0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}]), ("no_interactions", 2, [{0}, {1}]), ("no_interactions", 4, [{0}, {1}, {2}, {3}]), ([(1, 0), [5, 1]], 6, [{0, 1}, {1, 5}, {2, 3, 4}]), ], ) def test_check_interaction_cst(interaction_cst, n_features, result): """Check that _check_interaction_cst returns the expected list of sets""" est = HistGradientBoostingRegressor() est.set_params(interaction_cst=interaction_cst) assert est._check_interaction_cst(n_features) == result def test_interaction_cst_numerically(): """Check that interaction constraints have no forbidden interactions.""" rng = np.random.RandomState(42) n_samples = 1000 X = rng.uniform(size=(n_samples, 2)) # Construct y with a strong interaction term # y = x0 + x1 + 5 * x0 * x1 y = np.hstack((X, 5 * X[:, [0]] * X[:, [1]])).sum(axis=1) est = HistGradientBoostingRegressor(random_state=42) est.fit(X, y) est_no_interactions = HistGradientBoostingRegressor( interaction_cst=[{0}, {1}], random_state=42 ) est_no_interactions.fit(X, y) delta = 0.25 # Make sure we do not extrapolate out of the training set as tree-based estimators # are very bad in doing so. X_test = X[(X[:, 0] < 1 - delta) & (X[:, 1] < 1 - delta)] X_delta_d_0 = X_test + [delta, 0] X_delta_0_d = X_test + [0, delta] X_delta_d_d = X_test + [delta, delta] # Note: For the y from above as a function of x0 and x1, we have # y(x0+d, x1+d) = y(x0, x1) + 5 * d * (2/5 + x0 + x1) + 5 * d**2 # y(x0+d, x1) = y(x0, x1) + 5 * d * (1/5 + x1) # y(x0, x1+d) = y(x0, x1) + 5 * d * (1/5 + x0) # Without interaction constraints, we would expect a result of 5 * d**2 for the # following expression, but zero with constraints in place. assert_allclose( est_no_interactions.predict(X_delta_d_d) + est_no_interactions.predict(X_test) - est_no_interactions.predict(X_delta_d_0) - est_no_interactions.predict(X_delta_0_d), 0, atol=1e-12, ) # Correct result of the expressions is 5 * delta**2. But this is hard to achieve by # a fitted tree-based model. However, with 100 iterations the expression should # at least be positive! assert np.all( est.predict(X_delta_d_d) + est.predict(X_test) - est.predict(X_delta_d_0) - est.predict(X_delta_0_d) > 0.01 ) def test_no_user_warning_with_scoring(): """Check that no UserWarning is raised when scoring is set. Non-regression test for #22907. """ pd = pytest.importorskip("pandas") X, y = make_regression(n_samples=50, random_state=0) X_df = pd.DataFrame(X, columns=[f"col{i}" for i in range(X.shape[1])]) est = HistGradientBoostingRegressor( random_state=0, scoring="neg_mean_absolute_error", early_stopping=True ) with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) est.fit(X_df, y) def test_class_weights(): """High level test to check class_weights.""" n_samples = 255 n_features = 2 X, y = make_classification( n_samples=n_samples, n_features=n_features, n_informative=n_features, n_redundant=0, n_clusters_per_class=1, n_classes=2, random_state=0, ) y_is_1 = y == 1 # class_weight is the same as sample weights with the corresponding class clf = HistGradientBoostingClassifier( min_samples_leaf=2, random_state=0, max_depth=2 ) sample_weight = np.ones(shape=(n_samples)) sample_weight[y_is_1] = 3.0 clf.fit(X, y, sample_weight=sample_weight) class_weight = {0: 1.0, 1: 3.0} clf_class_weighted = clone(clf).set_params(class_weight=class_weight) clf_class_weighted.fit(X, y) assert_allclose(clf.decision_function(X), clf_class_weighted.decision_function(X)) # Check that sample_weight and class_weight are multiplicative clf.fit(X, y, sample_weight=sample_weight**2) clf_class_weighted.fit(X, y, sample_weight=sample_weight) assert_allclose(clf.decision_function(X), clf_class_weighted.decision_function(X)) # Make imbalanced dataset X_imb = np.concatenate((X[~y_is_1], X[y_is_1][:10])) y_imb = np.concatenate((y[~y_is_1], y[y_is_1][:10])) # class_weight="balanced" is the same as sample_weights to be # inversely proportional to n_samples / (n_classes * np.bincount(y)) clf_balanced = clone(clf).set_params(class_weight="balanced") clf_balanced.fit(X_imb, y_imb) class_weight = y_imb.shape[0] / (2 * np.bincount(y_imb)) sample_weight = class_weight[y_imb] clf_sample_weight = clone(clf).set_params(class_weight=None) clf_sample_weight.fit(X_imb, y_imb, sample_weight=sample_weight) assert_allclose( clf_balanced.decision_function(X_imb), clf_sample_weight.decision_function(X_imb), ) def test_unknown_category_that_are_negative(): """Check that unknown categories that are negative does not error. Non-regression test for #24274. """ rng = np.random.RandomState(42) n_samples = 1000 X = np.c_[rng.rand(n_samples), rng.randint(4, size=n_samples)] y = np.zeros(shape=n_samples) y[X[:, 1] % 2 == 0] = 1 hist = HistGradientBoostingRegressor( random_state=0, categorical_features=[False, True], max_iter=10, ).fit(X, y) # Check that negative values from the second column are treated like a # missing category X_test_neg = np.asarray([[1, -2], [3, -4]]) X_test_nan = np.asarray([[1, np.nan], [3, np.nan]]) assert_allclose(hist.predict(X_test_neg), hist.predict(X_test_nan)) @pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) @pytest.mark.parametrize( "HistGradientBoosting", [HistGradientBoostingClassifier, HistGradientBoostingRegressor], ) def test_dataframe_categorical_results_same_as_ndarray( dataframe_lib, HistGradientBoosting ): """Check that pandas categorical give the same results as ndarray.""" pytest.importorskip(dataframe_lib) rng = np.random.RandomState(42) n_samples = 5_000 n_cardinality = 50 max_bins = 100 f_num = rng.rand(n_samples) f_cat = rng.randint(n_cardinality, size=n_samples) # Make f_cat an informative feature y = (f_cat % 3 == 0) & (f_num > 0.2) X = np.c_[f_num, f_cat] f_cat = [f"cat{c:0>3}" for c in f_cat] X_df = _convert_container( np.asarray([f_num, f_cat]).T, dataframe_lib, ["f_num", "f_cat"], categorical_feature_names=["f_cat"], ) X_train, X_test, X_train_df, X_test_df, y_train, y_test = train_test_split( X, X_df, y, random_state=0 ) hist_kwargs = dict(max_iter=10, max_bins=max_bins, random_state=0) hist_np = HistGradientBoosting(categorical_features=[False, True], **hist_kwargs) hist_np.fit(X_train, y_train) hist_pd = HistGradientBoosting(categorical_features="from_dtype", **hist_kwargs) hist_pd.fit(X_train_df, y_train) # Check categories are correct and sorted categories = hist_pd._preprocessor.named_transformers_["encoder"].categories_[0] assert_array_equal(categories, np.unique(f_cat)) assert len(hist_np._predictors) == len(hist_pd._predictors) for predictor_1, predictor_2 in zip(hist_np._predictors, hist_pd._predictors): assert len(predictor_1[0].nodes) == len(predictor_2[0].nodes) score_np = hist_np.score(X_test, y_test) score_pd = hist_pd.score(X_test_df, y_test) assert score_np == pytest.approx(score_pd) assert_allclose(hist_np.predict(X_test), hist_pd.predict(X_test_df)) @pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) @pytest.mark.parametrize( "HistGradientBoosting", [HistGradientBoostingClassifier, HistGradientBoostingRegressor], ) def test_dataframe_categorical_errors(dataframe_lib, HistGradientBoosting): """Check error cases for pandas categorical feature.""" pytest.importorskip(dataframe_lib) msg = "Categorical feature 'f_cat' is expected to have a cardinality <= 16" hist = HistGradientBoosting(categorical_features="from_dtype", max_bins=16) rng = np.random.RandomState(42) f_cat = rng.randint(0, high=100, size=100).astype(str) X_df = _convert_container( f_cat[:, None], dataframe_lib, ["f_cat"], categorical_feature_names=["f_cat"] ) y = rng.randint(0, high=2, size=100) with pytest.raises(ValueError, match=msg): hist.fit(X_df, y) @pytest.mark.parametrize("dataframe_lib", ["pandas", "polars"]) def test_categorical_different_order_same_model(dataframe_lib): """Check that the order of the categorical gives same model.""" pytest.importorskip(dataframe_lib) rng = np.random.RandomState(42) n_samples = 1_000 f_ints = rng.randint(low=0, high=2, size=n_samples) # Construct a target with some noise y = f_ints.copy() flipped = rng.choice([True, False], size=n_samples, p=[0.1, 0.9]) y[flipped] = 1 - y[flipped] # Construct categorical where 0 -> A and 1 -> B and 1 -> A and 0 -> B f_cat_a_b = np.asarray(["A", "B"])[f_ints] f_cat_b_a = np.asarray(["B", "A"])[f_ints] df_a_b = _convert_container( f_cat_a_b[:, None], dataframe_lib, ["f_cat"], categorical_feature_names=["f_cat"], ) df_b_a = _convert_container( f_cat_b_a[:, None], dataframe_lib, ["f_cat"], categorical_feature_names=["f_cat"], ) hist_a_b = HistGradientBoostingClassifier( categorical_features="from_dtype", random_state=0 ) hist_b_a = HistGradientBoostingClassifier( categorical_features="from_dtype", random_state=0 ) hist_a_b.fit(df_a_b, y) hist_b_a.fit(df_b_a, y) assert len(hist_a_b._predictors) == len(hist_b_a._predictors) for predictor_1, predictor_2 in zip(hist_a_b._predictors, hist_b_a._predictors): assert len(predictor_1[0].nodes) == len(predictor_2[0].nodes) # TODO(1.6): Remove warning and change default in 1.6 def test_categorical_features_warn(): """Raise warning when there are categorical features in the input DataFrame. This is not tested for polars because polars categories must always be strings and strings can only be handled as categories. Therefore the situation in which a categorical column is currently being treated as numbers and in the future will be treated as categories cannot occur with polars. """ pd = pytest.importorskip("pandas") X = pd.DataFrame({"a": pd.Series([1, 2, 3], dtype="category"), "b": [4, 5, 6]}) y = [0, 1, 0] hist = HistGradientBoostingClassifier(random_state=0) msg = "The categorical_features parameter will change to 'from_dtype' in v1.6" with pytest.warns(FutureWarning, match=msg): hist.fit(X, y) def get_different_bitness_node_ndarray(node_ndarray): new_dtype_for_indexing_fields = np.int64 if _IS_32BIT else np.int32 # field names in Node struct with np.intp types (see # sklearn/ensemble/_hist_gradient_boosting/common.pyx) indexing_field_names = ["feature_idx"] new_dtype_dict = { name: dtype for name, (dtype, _) in node_ndarray.dtype.fields.items() } for name in indexing_field_names: new_dtype_dict[name] = new_dtype_for_indexing_fields new_dtype = np.dtype( {"names": list(new_dtype_dict.keys()), "formats": list(new_dtype_dict.values())} ) return node_ndarray.astype(new_dtype, casting="same_kind") def reduce_predictor_with_different_bitness(predictor): cls, args, state = predictor.__reduce__() new_state = state.copy() new_state["nodes"] = get_different_bitness_node_ndarray(new_state["nodes"]) return (cls, args, new_state) def test_different_bitness_pickle(): X, y = make_classification(random_state=0) clf = HistGradientBoostingClassifier(random_state=0, max_depth=3) clf.fit(X, y) score = clf.score(X, y) def pickle_dump_with_different_bitness(): f = io.BytesIO() p = pickle.Pickler(f) p.dispatch_table = copyreg.dispatch_table.copy() p.dispatch_table[TreePredictor] = reduce_predictor_with_different_bitness p.dump(clf) f.seek(0) return f # Simulate loading a pickle of the same model trained on a platform with different # bitness that than the platform it will be used to make predictions on: new_clf = pickle.load(pickle_dump_with_different_bitness()) new_score = new_clf.score(X, y) assert score == pytest.approx(new_score) def test_different_bitness_joblib_pickle(): # Make sure that a platform specific pickle generated on a 64 bit # platform can be converted at pickle load time into an estimator # with Cython code that works with the host's native integer precision # to index nodes in the tree data structure when the host is a 32 bit # platform (and vice versa). # # This is in particular useful to be able to train a model on a 64 bit Linux # server and deploy the model as part of a (32 bit) WASM in-browser # application using pyodide. X, y = make_classification(random_state=0) clf = HistGradientBoostingClassifier(random_state=0, max_depth=3) clf.fit(X, y) score = clf.score(X, y) def joblib_dump_with_different_bitness(): f = io.BytesIO() p = NumpyPickler(f) p.dispatch_table = copyreg.dispatch_table.copy() p.dispatch_table[TreePredictor] = reduce_predictor_with_different_bitness p.dump(clf) f.seek(0) return f new_clf = joblib.load(joblib_dump_with_different_bitness()) new_score = new_clf.score(X, y) assert score == pytest.approx(new_score) def test_pandas_nullable_dtype(): # Non regression test for https://github.com/scikit-learn/scikit-learn/issues/28317 pd = pytest.importorskip("pandas") rng = np.random.default_rng(0) X = pd.DataFrame({"a": rng.integers(10, size=100)}).astype(pd.Int64Dtype()) y = rng.integers(2, size=100) clf = HistGradientBoostingClassifier() clf.fit(X, y)