Skip to content

Commit ec2658c

Browse files
qiagurasbt
qiagu
authored and
rasbt
committed
Add random_state parameter to stacking cv estimators (#523)
* add random_state parameter to stacking cv estimators * update changelog and jupyter docs * update changelog again * minor change * default stacking cv random_state to None
1 parent c338a1f commit ec2658c

File tree

7 files changed

+261
-221
lines changed

7 files changed

+261
-221
lines changed

‎docs/sources/CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ The CHANGELOG for the current development version is available at
1717

1818
##### New Features
1919

20+
- `StackingCVClassifier` and `StackingCVRegressor` now support `random_state` parameter, which, together with `shuffle`, controls the randomness in the cv splitting. ([#523](https://github.com/rasbt/mlxtend/pull/523) via [Qiang Gu](https://github.com/qiaguhttps://github.com/qiagu))
2021
- Other stacking estimators, including `StackingClassifier`, `StackingCVClassifier` and `StackingRegressor`, support grid search over the `regressors` and even a single base regressor. ([#522](https://github.com/rasbt/mlxtend/pull/522) via [Qiang Gu](https://github.com/qiaguhttps://github.com/qiagu))
2122
- Adds multiprocessing support to `StackingCVClassifier`. ([#522](https://github.com/rasbt/mlxtend/pull/522) via [Qiang Gu](https://github.com/qiaguhttps://github.com/qiagu))
2223
- Adds multiprocessing support to `StackingCVRegressor`. ([#512](https://github.com/rasbt/mlxtend/pull/512) via [Qiang Gu](https://github.com/qiaguhttps://github.com/qiagu))

‎docs/sources/user_guide/classifier/StackingCVClassifier.ipynb

+81-70
Large diffs are not rendered by default.

‎docs/sources/user_guide/regressor/StackingCVRegressor.ipynb

+87-65
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@
8787
"R^2 Score: 0.46 (+/- 0.29) [SVM]\n",
8888
"R^2 Score: 0.43 (+/- 0.14) [Lasso]\n",
8989
"R^2 Score: 0.53 (+/- 0.28) [Random Forest]\n",
90-
"R^2 Score: 0.58 (+/- 0.23) [StackingCVRegressor]\n"
90+
"R^2 Score: 0.57 (+/- 0.24) [StackingCVRegressor]\n"
9191
]
9292
}
9393
],
@@ -109,13 +109,11 @@
109109
"rf = RandomForestRegressor(n_estimators=5, \n",
110110
" random_state=RANDOM_SEED)\n",
111111
"\n",
112-
"# The StackingCVRegressor uses scikit-learn's check_cv\n",
113-
"# internally, which doesn't support a random seed. Thus\n",
114-
"# NumPy's random seed need to be specified explicitely for\n",
115-
"# deterministic behavior\n",
116-
"np.random.seed(RANDOM_SEED)\n",
112+
"# Starting from v0.16.0, StackingCVRegressor supports\n",
113+
"# `random_state` to get deterministic result.\n",
117114
"stack = StackingCVRegressor(regressors=(svr, lasso, rf),\n",
118-
" meta_regressor=lasso)\n",
115+
" meta_regressor=lasso,\n",
116+
" random_state=RANDOM_SEED)\n",
119117
"\n",
120118
"print('5-fold cross validation scores:\\n')\n",
121119
"\n",
@@ -141,16 +139,11 @@
141139
"Neg. MSE Score: -33.34 (+/- 22.36) [SVM]\n",
142140
"Neg. MSE Score: -35.53 (+/- 16.99) [Lasso]\n",
143141
"Neg. MSE Score: -27.25 (+/- 16.76) [Random Forest]\n",
144-
"Neg. MSE Score: -25.56 (+/- 18.22) [StackingCVRegressor]\n"
142+
"Neg. MSE Score: -25.82 (+/- 18.10) [StackingCVRegressor]\n"
145143
]
146144
}
147145
],
148146
"source": [
149-
"# The StackingCVRegressor uses scikit-learn's check_cv\n",
150-
"# internally, which doesn't support a random seed. Thus\n",
151-
"# NumPy's random seed need to be specified explicitely for\n",
152-
"# deterministic behavior\n",
153-
"np.random.seed(RANDOM_SEED)\n",
154147
"stack = StackingCVRegressor(regressors=(svr, lasso, rf),\n",
155148
" meta_regressor=lasso)\n",
156149
"\n",
@@ -186,18 +179,18 @@
186179
"metadata": {},
187180
"outputs": [
188181
{
189-
"name": "stderr",
182+
"name": "stdout",
190183
"output_type": "stream",
191184
"text": [
192-
"/Users/guq/miniconda3/envs/python3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
193-
" DeprecationWarning)\n"
185+
"Best: 0.679576 using {'lasso__alpha': 1.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4}\n"
194186
]
195187
},
196188
{
197-
"name": "stdout",
189+
"name": "stderr",
198190
"output_type": "stream",
199191
"text": [
200-
"Best: 0.674237 using {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.2}\n"
192+
"/Users/guq/miniconda3/envs/python3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
193+
" DeprecationWarning)\n"
201194
]
202195
}
203196
],
@@ -215,13 +208,9 @@
215208
"lasso = Lasso(random_state=RANDOM_SEED)\n",
216209
"rf = RandomForestRegressor(random_state=RANDOM_SEED)\n",
217210
"\n",
218-
"# The StackingCVRegressor uses scikit-learn's check_cv\n",
219-
"# internally, which doesn't support a random seed. Thus\n",
220-
"# NumPy's random seed need to be specified explicitely for\n",
221-
"# deterministic behavior\n",
222-
"np.random.seed(RANDOM_SEED)\n",
223211
"stack = StackingCVRegressor(regressors=(lasso, ridge),\n",
224212
" meta_regressor=rf, \n",
213+
" random_state=RANDOM_SEED,\n",
225214
" use_features_in_secondary=True)\n",
226215
"\n",
227216
"params = {'lasso__alpha': [0.1, 1.0, 10.0],\n",
@@ -252,21 +241,21 @@
252241
"name": "stdout",
253242
"output_type": "stream",
254243
"text": [
255-
"0.616 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.05}\n",
244+
"0.637 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.05}\n",
256245
"0.656 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.1}\n",
257-
"0.653 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.15}\n",
258-
"0.669 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.2}\n",
259-
"0.632 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.25}\n",
260-
"0.664 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.3}\n",
261-
"0.632 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.35}\n",
262-
"0.642 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4}\n",
263-
"0.653 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.45}\n",
264-
"0.657 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.05}\n",
265-
"0.650 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.1}\n",
266-
"0.648 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.15}\n",
246+
"0.635 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.15}\n",
247+
"0.647 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.2}\n",
248+
"0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.25}\n",
249+
"0.628 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.3}\n",
250+
"0.639 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.35}\n",
251+
"0.641 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4}\n",
252+
"0.653 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.45}\n",
253+
"0.644 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.05}\n",
254+
"0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.1}\n",
255+
"0.646 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.15}\n",
267256
"...\n",
268-
"Best parameters: {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.2}\n",
269-
"Accuracy: 0.67\n"
257+
"Best parameters: {'lasso__alpha': 1.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4}\n",
258+
"Accuracy: 0.68\n"
270259
]
271260
}
272261
],
@@ -318,21 +307,12 @@
318307
"text": [
319308
"## StackingCVRegressor\n",
320309
"\n",
321-
"*StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, use_features_in_secondary=False, store_train_meta_features=False, refit=True)*\n",
310+
"*StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2*n_jobs')*\n",
322311
"\n",
323312
"A 'Stacking Cross-Validation' regressor for scikit-learn estimators.\n",
324313
"\n",
325314
"New in mlxtend v0.7.0\n",
326315
"\n",
327-
"**Notes**\n",
328-
"\n",
329-
"The StackingCVRegressor uses scikit-learn's check_cv\n",
330-
"internally, which doesn't support a random seed. Thus\n",
331-
"NumPy's random seed need to be specified explicitely for\n",
332-
"deterministic behavior, for instance, by setting\n",
333-
"np.random.seed(RANDOM_SEED)\n",
334-
"prior to fitting the StackingCVRegressor\n",
335-
"\n",
336316
"**Parameters**\n",
337317
"\n",
338318
"- `regressors` : array-like, shape = [n_regressors]\n",
@@ -357,28 +337,21 @@
357337
" - An iterable yielding train, test splits.\n",
358338
" For integer/None inputs, it will use `KFold` cross-validation\n",
359339
"\n",
360-
"- `use_features_in_secondary` : bool (default: False)\n",
361-
"\n",
362-
" If True, the meta-regressor will be trained both on\n",
363-
" the predictions of the original regressors and the\n",
364-
" original dataset.\n",
365-
" If False, the meta-regressor will be trained only on\n",
366-
" the predictions of the original regressors.\n",
367-
"\n",
368340
"- `shuffle` : bool (default: True)\n",
369341
"\n",
370342
" If True, and the `cv` argument is integer, the training data will\n",
371343
" be shuffled at fitting stage prior to cross-validation. If the `cv`\n",
372344
" argument is a specific cross validation technique, this argument is\n",
373345
" omitted.\n",
374346
"\n",
375-
"- `store_train_meta_features` : bool (default: False)\n",
347+
"- `random_state` : int, RandomState instance or None, optional (default: None)\n",
376348
"\n",
377-
" If True, the meta-features computed from the training data\n",
378-
" used for fitting the\n",
379-
" meta-regressor stored in the `self.train_meta_features_` array,\n",
380-
" which can be\n",
381-
" accessed after calling `fit`.\n",
349+
" Constrols the randomness of the cv splitter. Used when `cv` is\n",
350+
" integer and `shuffle=True`. New in v0.16.0.\n",
351+
"\n",
352+
"- `verbose` : int, optional (default=0)\n",
353+
"\n",
354+
" Controls the verbosity of the building process. New in v0.16.0\n",
382355
"\n",
383356
"- `refit` : bool (default: True)\n",
384357
"\n",
@@ -389,6 +362,45 @@
389362
" the scikit-learn fit/predict API interface but are not compatible\n",
390363
" to scikit-learn's `clone` function.\n",
391364
"\n",
365+
"- `use_features_in_secondary` : bool (default: False)\n",
366+
"\n",
367+
" If True, the meta-regressor will be trained both on\n",
368+
" the predictions of the original regressors and the\n",
369+
" original dataset.\n",
370+
" If False, the meta-regressor will be trained only on\n",
371+
" the predictions of the original regressors.\n",
372+
"\n",
373+
"- `store_train_meta_features` : bool (default: False)\n",
374+
"\n",
375+
" If True, the meta-features computed from the training data\n",
376+
" used for fitting the\n",
377+
" meta-regressor stored in the `self.train_meta_features_` array,\n",
378+
" which can be\n",
379+
" accessed after calling `fit`.\n",
380+
"\n",
381+
"- `n_jobs` : int or None, optional (default=None)\n",
382+
"\n",
383+
" The number of CPUs to use to do the computation.\n",
384+
" ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.\n",
385+
" ``-1`` means using all processors. See :term:`Glossary <n_jobs>`\n",
386+
" for more details. New in v0.16.0.\n",
387+
"\n",
388+
"- `pre_dispatch` : int, or string, optional\n",
389+
"\n",
390+
" Controls the number of jobs that get dispatched during parallel\n",
391+
" execution. Reducing this number can be useful to avoid an\n",
392+
" explosion of memory consumption when more jobs get dispatched\n",
393+
" than CPUs can process. This parameter can be:\n",
394+
" - None, in which case all the jobs are immediately\n",
395+
" created and spawned. Use this for lightweight and\n",
396+
" fast-running jobs, to avoid delays due to on-demand\n",
397+
" spawning of the jobs\n",
398+
" - An int, giving the exact number of total jobs that are\n",
399+
" spawned\n",
400+
" - A string, giving an expression as a function of n_jobs,\n",
401+
" as in '2*n_jobs'\n",
402+
" New in v0.16.0.\n",
403+
"\n",
392404
"**Attributes**\n",
393405
"\n",
394406
"- `train_meta_features` : numpy array, shape = [n_samples, n_regressors]\n",
@@ -546,7 +558,10 @@
546558
"\n",
547559
"- `X` : array-like, shape = (n_samples, n_features)\n",
548560
"\n",
549-
" Test samples.\n",
561+
" Test samples. For some estimators this may be a\n",
562+
" precomputed kernel matrix instead, shape = (n_samples,\n",
563+
" n_samples_fitted], where n_samples_fitted is the number of\n",
564+
" samples used in the fitting for the estimator.\n",
550565
"\n",
551566
"\n",
552567
"- `y` : array-like, shape = (n_samples) or (n_samples, n_outputs)\n",
@@ -570,15 +585,22 @@
570585
"\n",
571586
"Set the parameters of this estimator.\n",
572587
"\n",
573-
"The method works on simple estimators as well as on nested objects\n",
574-
"(such as pipelines). The latter have parameters of the form\n",
575-
"``<component>__<parameter>`` so that it's possible to update each\n",
576-
"component of a nested object.\n",
588+
"Valid parameter keys can be listed with ``get_params()``.\n",
577589
"\n",
578590
"**Returns**\n",
579591
"\n",
580592
"self\n",
581593
"\n",
594+
"### Properties\n",
595+
"\n",
596+
"<hr>\n",
597+
"\n",
598+
"*named_regressors*\n",
599+
"\n",
600+
"**Returns**\n",
601+
"\n",
602+
"List of named estimator tuples, like [('svc', SVC(...))]\n",
603+
"\n",
582604
"\n"
583605
]
584606
}

‎mlxtend/classifier/stacking_cv_classification.py

+21-26
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,6 @@ class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
2929
3030
New in mlxtend v0.4.3
3131
32-
Notes
33-
-------
34-
The StackingCVClassifier uses scikit-learn's check_cv
35-
internally, which doesn't support a random seed. Thus
36-
NumPy's random seed need to be specified explicitely for
37-
deterministic behavior, for instance, by setting
38-
np.random.seed(RANDOM_SEED)
39-
prior to fitting the StackingCVClassifier
40-
4132
Parameters
4233
----------
4334
classifiers : array-like, shape = [n_classifiers]
@@ -61,20 +52,18 @@ class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
6152
For integer/None inputs, it will use either a `KFold` or
6253
`StratifiedKFold` cross validation depending the value of `stratify`
6354
argument.
64-
use_features_in_secondary : bool (default: False)
65-
If True, the meta-classifier will be trained both on the predictions
66-
of the original classifiers and the original dataset.
67-
If False, the meta-classifier will be trained only on the predictions
68-
of the original classifiers.
69-
stratify : bool (default: True)
70-
If True, and the `cv` argument is integer it will follow a stratified
71-
K-Fold cross validation technique. If the `cv` argument is a specific
72-
cross validation technique, this argument is omitted.
7355
shuffle : bool (default: True)
7456
If True, and the `cv` argument is integer, the training data will be
7557
shuffled at fitting stage prior to cross-validation. If the `cv`
7658
argument is a specific cross validation technique, this argument is
7759
omitted.
60+
random_state : int, RandomState instance or None, optional (default: None)
61+
Constrols the randomness of the cv splitter. Used when `cv` is
62+
integer and `shuffle=True`. New in v0.16.0.
63+
stratify : bool (default: True)
64+
If True, and the `cv` argument is integer it will follow a stratified
65+
K-Fold cross validation technique. If the `cv` argument is a specific
66+
cross validation technique, this argument is omitted.
7867
verbose : int, optional (default=0)
7968
Controls the verbosity of the building process.
8069
- `verbose=0` (default): Prints nothing
@@ -84,6 +73,11 @@ class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
8473
regressor being fitted
8574
- `verbose>2`: Changes `verbose` param of the underlying regressor to
8675
self.verbose - 2
76+
use_features_in_secondary : bool (default: False)
77+
If True, the meta-classifier will be trained both on the predictions
78+
of the original classifiers and the original dataset.
79+
If False, the meta-classifier will be trained only on the predictions
80+
of the original classifiers.
8781
store_train_meta_features : bool (default: False)
8882
If True, the meta-features computed from the training data used
8983
for fitting the meta-classifier stored in the
@@ -103,7 +97,7 @@ class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
10397
The number of CPUs to use to do the computation.
10498
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
10599
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
106-
for more details.
100+
for more details. New in v0.16.0.
107101
pre_dispatch : int, or string, optional
108102
Controls the number of jobs that get dispatched during parallel
109103
execution. Reducing this number can be useful to avoid an
@@ -117,7 +111,7 @@ class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
117111
spawned
118112
- A string, giving an expression as a function of n_jobs,
119113
as in '2*n_jobs'
120-
114+
New in v0.16.0.
121115
122116
Attributes
123117
----------
@@ -137,22 +131,22 @@ class StackingCVClassifier(_BaseXComposition, ClassifierMixin,
137131
138132
"""
139133
def __init__(self, classifiers, meta_classifier,
140-
use_probas=False, cv=2,
134+
use_probas=False, cv=2, shuffle=True,
135+
random_state=None, stratify=True, verbose=0,
141136
use_features_in_secondary=False,
142-
stratify=True,
143-
shuffle=True, verbose=0,
144137
store_train_meta_features=False,
145138
use_clones=True, n_jobs=None,
146139
pre_dispatch='2*n_jobs'):
147140

148141
self.classifiers = classifiers
149142
self.meta_classifier = meta_classifier
150143
self.use_probas = use_probas
151-
self.verbose = verbose
152144
self.cv = cv
153-
self.use_features_in_secondary = use_features_in_secondary
154-
self.stratify = stratify
155145
self.shuffle = shuffle
146+
self.random_state = random_state
147+
self.stratify = stratify
148+
self.verbose = verbose
149+
self.use_features_in_secondary = use_features_in_secondary
156150
self.store_train_meta_features = store_train_meta_features
157151
self.use_clones = use_clones
158152
self.n_jobs = n_jobs
@@ -203,6 +197,7 @@ def fit(self, X, y, groups=None, sample_weight=None):
203197
# Override shuffle parameter in case of self generated
204198
# cross-validation strategy
205199
final_cv.shuffle = self.shuffle
200+
final_cv.random_state = self.random_state
206201

207202
# Input validation.
208203
X, y = check_X_y(X, y, accept_sparse=['csc', 'csr'])

0 commit comments

Comments
 (0)