import os
from math import inf
import numpy as np
import warnings
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.model_selection import train_test_split
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import RidgeCV, ElasticNet, LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor, RandomForestClassifier
from sklearn.metrics import mean_absolute_error, accuracy_score, roc_curve, roc_auc_score, log_loss, mean_squared_error
from spotpython.hyperparameters.values import set_control_key_value
from spotpython.plot.validation import plot_confusion_matrix, plot_roc
from spotpython.spot import Spot
from spotpython.hyperparameters.values import (get_one_core_model_from_X, add_core_model_to_fun_control, get_default_hyperparameters_as_array)
from spotpython.hyperdict.sklearn_hyper_dict import SklearnHyperDict
from spotpython.hyperparameters.values import (modify_hyper_parameter_bounds, modify_hyper_parameter_levels, get_default_hyperparameters_as_array)
from spotpython.fun.hypersklearn import HyperSklearn
from spotpython.utils.eda import print_res_table
from spotpython.utils.init import (fun_control_init, design_control_init, surrogate_control_init, get_tensorboard_path, get_experiment_name)
from spotpython.utils.file import load_result
if not os.path.exists('./figures'):
'./figures')
os.makedirs("ignore") warnings.filterwarnings(
32 HPT: sklearn SVC on Moons Data
This chapter is a tutorial for the Hyperparameter Tuning (HPT) of a sklearn
SVC model on the Moons dataset.
32.1 Packages used in this Chapter
32.2 Step 1: Setup
Before we consider the detailed experimental setup, we select the parameters that affect run time, initial design size and the device that is used.
- MAX_TIME is set to one minute for demonstration purposes. For real experiments, this should be increased to at least 1 hour.
= 1
MAX_TIME = "401_sklearn_classification" PREFIX
32.3 Step 2: Initialization of the Empty fun_control
Dictionary
spotpython
supports the visualization of the hyperparameter tuning process with TensorBoard. The following example shows how to use TensorBoard with spotpython
. The fun_control
dictionary is the central data structure that is used to control the optimization process. It is initialized as follows:
= fun_control_init(
fun_control =PREFIX,
PREFIX=True,
TENSORBOARD_CLEAN=MAX_TIME,
max_time=inf,
fun_evals= np.sqrt(np.spacing(1))) tolerance_x
Moving TENSORBOARD_PATH: runs/ to TENSORBOARD_PATH_OLD: runs_OLD/runs_2025_09_13_21_45_10_0
- Since the
spot_tensorboard_path
argument is notNone
, which is the default,spotpython
will log the optimization process in the TensorBoard folder. - The
TENSORBOARD_CLEAN
argument is set toTrue
to archive the TensorBoard folder if it already exists. This is useful if you want to start a hyperparameter tuning process from scratch. If you want to continue a hyperparameter tuning process, setTENSORBOARD_CLEAN
toFalse
. Then the TensorBoard folder will not be archived and the old and new TensorBoard files will shown in the TensorBoard dashboard.
32.4 Step 3: SKlearn Load Data (Classification)
Randomly generate classification data.
= 2
n_features = 500
n_samples = "y"
target_column = make_moons(n_samples, noise=0.5, random_state=0)
ds = ds
X, y = train_test_split(
X_train, X_test, y_train, y_test =0.3, random_state=42
X, y, test_size
)= pd.DataFrame(np.hstack((X_train, y_train.reshape(-1, 1))))
train = pd.DataFrame(np.hstack((X_test, y_test.reshape(-1, 1))))
test = [f"x{i}" for i in range(1, n_features+1)] + [target_column]
train.columns = [f"x{i}" for i in range(1, n_features+1)] + [target_column]
test.columns train.head()
x1 | x2 | y | |
---|---|---|---|
0 | 1.960101 | 0.383172 | 0.0 |
1 | 2.354420 | -0.536942 | 1.0 |
2 | 1.682186 | -0.332108 | 0.0 |
3 | 1.856507 | 0.687220 | 1.0 |
4 | 1.925524 | 0.427413 | 1.0 |
Figure 32.1 shows the moons data used for the hyperparameter tuning example.
= X[:, 0].min() - 0.5, X[:, 0].max() + 0.5
x_min, x_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5
y_min, y_max = plt.cm.RdBu
cm = ListedColormap(["#FF0000", "#0000FF"])
cm_bright = plt.subplot(1, 1, 1)
ax "Input data")
ax.set_title(# Plot the training points
0], X_train[:, 1], c=y_train, cmap=cm_bright, edgecolors="k")
ax.scatter(X_train[:, # Plot the testing points
ax.scatter(0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6, edgecolors="k"
X_test[:,
)
ax.set_xlim(x_min, x_max)
ax.set_ylim(y_min, y_max)
ax.set_xticks(())
ax.set_yticks(())
plt.tight_layout() plt.show()

= len(train)
n_samples # add the dataset to the fun_control
"data": None, # dataset,
fun_control.update({"train": train,
"test": test,
"n_samples": n_samples,
"target_column": target_column})
32.5 Step 4: Specification of the Preprocessing Model
Data preprocessing can be very simple, e.g., you can ignore it. Then you would choose the prep_model
“None”:
= None
prep_model "prep_model": prep_model}) fun_control.update({
A default approach for numerical data is the StandardScaler
(mean 0, variance 1). This can be selected as follows:
= StandardScaler
prep_model "prep_model": prep_model}) fun_control.update({
Even more complicated pre-processing steps are possible, e.g., the following pipeline:
categorical_columns = []
one_hot_encoder = OneHotEncoder(handle_unknown="ignore", sparse_output=False)
prep_model = ColumnTransformer(
transformers=[
("categorical", one_hot_encoder, categorical_columns),
],
remainder=StandardScaler,
)
32.6 Step 5: Select Model (algorithm
) and core_model_hyper_dict
The selection of the algorithm (ML model) that should be tuned is done by specifying the its name from the sklearn
implementation. For example, the SVC
support vector machine classifier is selected as follows:
=SVC,
add_core_model_to_fun_control(core_model=fun_control,
fun_control=SklearnHyperDict,
hyper_dict=None) filename
Now fun_control
has the information from the JSON file. The corresponding entries for the core_model
class are shown below.
'core_model_hyper_dict'] fun_control[
{'C': {'type': 'float',
'default': 1.0,
'transform': 'None',
'lower': 0.1,
'upper': 10.0},
'kernel': {'levels': ['linear', 'poly', 'rbf', 'sigmoid'],
'type': 'factor',
'default': 'rbf',
'transform': 'None',
'core_model_parameter_type': 'str',
'lower': 0,
'upper': 3},
'degree': {'type': 'int',
'default': 3,
'transform': 'None',
'lower': 3,
'upper': 3},
'gamma': {'levels': ['scale', 'auto'],
'type': 'factor',
'default': 'scale',
'transform': 'None',
'core_model_parameter_type': 'str',
'lower': 0,
'upper': 1},
'coef0': {'type': 'float',
'default': 0.0,
'transform': 'None',
'lower': 0.0,
'upper': 0.0},
'shrinking': {'levels': [0, 1],
'type': 'factor',
'default': 0,
'transform': 'None',
'core_model_parameter_type': 'bool',
'lower': 0,
'upper': 1},
'probability': {'levels': [0, 1],
'type': 'factor',
'default': 0,
'transform': 'None',
'core_model_parameter_type': 'bool',
'lower': 0,
'upper': 1},
'tol': {'type': 'float',
'default': 0.001,
'transform': 'None',
'lower': 0.0001,
'upper': 0.01},
'cache_size': {'type': 'float',
'default': 200,
'transform': 'None',
'lower': 100,
'upper': 400},
'break_ties': {'levels': [0, 1],
'type': 'factor',
'default': 0,
'transform': 'None',
'core_model_parameter_type': 'bool',
'lower': 0,
'upper': 1}}
sklearn Model
Selection
The following sklearn
models are supported by default:
- RidgeCV
- RandomForestClassifier
- SVC
- LogisticRegression
- KNeighborsClassifier
- GradientBoostingClassifier
- GradientBoostingRegressor
- ElasticNet
They can be imported as follows:
from sklearn.linear_model import RidgeCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import ElasticNet
32.7 Step 6: Modify hyper_dict
Hyperparameters for the Selected Algorithm aka core_model
spotpython
provides functions for modifying the hyperparameters, their bounds and factors as well as for activating and de-activating hyperparameters without re-compilation of the Python source code. These functions were described in Section 12.19.1.
32.7.1 Modify hyperparameter of type numeric and integer (boolean)
Numeric and boolean values can be modified using the modify_hyper_parameter_bounds
method.
sklearn Model
Hyperparameters
The hyperparameters of the sklearn
SVC
model are described in the sklearn documentation.
- For example, to change the
tol
hyperparameter of theSVC
model to the interval [1e-5, 1e-3], the following code can be used:
"tol", bounds=[1e-5, 1e-3])
modify_hyper_parameter_bounds(fun_control, "probability", bounds=[0, 0])
modify_hyper_parameter_bounds(fun_control, "core_model_hyper_dict"]["tol"] fun_control[
{'type': 'float',
'default': 0.001,
'transform': 'None',
'lower': 1e-05,
'upper': 0.001}
32.7.2 Modify hyperparameter of type factor
Factors can be modified with the modify_hyper_parameter_levels
function. For example, to exclude the sigmoid
kernel from the tuning, the kernel
hyperparameter of the SVC
model can be modified as follows:
"kernel", ["poly", "rbf"])
modify_hyper_parameter_levels(fun_control, "core_model_hyper_dict"]["kernel"] fun_control[
{'levels': ['poly', 'rbf'],
'type': 'factor',
'default': 'rbf',
'transform': 'None',
'core_model_parameter_type': 'str',
'lower': 0,
'upper': 1}
32.7.3 Optimizers
Optimizers are described in Section 15.2.
32.8 Step 7: Selection of the Objective (Loss) Function
There are two metrics:
metric_river
is used for the river based evaluation viaeval_oml_iter_progressive
.metric_sklearn
is used for the sklearn based evaluation.
fun_control.update({"metric_sklearn": log_loss,
"weights": 1.0,
})
metric_sklearn
: Minimization and Maximization
- Because the
metric_sklearn
is used for the sklearn based evaluation, it is important to know whether the metric should be minimized or maximized. - The
weights
parameter is used to indicate whether the metric should be minimized or maximized. - If
weights
is set to-1.0
, the metric is maximized. - If
weights
is set to1.0
, the metric is minimized, e.g.,weights = 1.0
formean_absolute_error
, orweights = -1.0
forroc_auc_score
.
32.8.1 Predict Classes or Class Probabilities
If the key "predict_proba"
is set to True
, the class probabilities are predicted. False
is the default, i.e., the classes are predicted.
fun_control.update({"predict_proba": False,
})
32.9 Step 8: Calling the SPOT Function
32.9.1 The Objective Function
The objective function is selected next. It implements an interface from sklearn
’s training, validation, and testing methods to spotpython
.
= HyperSklearn().fun_sklearn fun
The following code snippet shows how to get the default hyperparameters as an array, so that they can be passed to the Spot
function.
= get_default_hyperparameters_as_array(fun_control) X_start
32.9.2 Run the Spot
Optimizer
The class Spot
[SOURCE] is the hyperparameter tuning workhorse. It is initialized with the following parameters:
fun
: the objective functionfun_control
: the dictionary with the control parameters for the objective functiondesign
: the experimental designdesign_control
: the dictionary with the control parameters for the experimental designsurrogate
: the surrogate modelsurrogate_control
: the dictionary with the control parameters for the surrogate modeloptimizer
: the optimizeroptimizer_control
: the dictionary with the control parameters for the optimizer
The total run time may exceed the specified max_time
, because the initial design (here: init_size
= INIT_SIZE as specified above) is always evaluated, even if this takes longer than max_time
.
= design_control_init()
design_control
= surrogate_control_init(method="regression")
surrogate_control
= Spot(fun=fun,
spot_tuner =fun_control,
fun_control=design_control,
design_control=surrogate_control)
surrogate_control=X_start) spot_tuner.run(X_start
spotpython tuning: 6.436366676628063 [----------] 2.38%
spotpython tuning: 6.436366676628063 [----------] 4.33%
spotpython tuning: 6.436366676628063 [#---------] 5.80%
spotpython tuning: 6.436366676628063 [#---------] 7.20%
spotpython tuning: 6.436366676628063 [#---------] 8.67%
spotpython tuning: 6.436366676628063 [#---------] 10.08%
spotpython tuning: 6.436366676628063 [#---------] 11.68%
spotpython tuning: 6.436366676628063 [#---------] 13.22%
spotpython tuning: 6.436366676628063 [##--------] 15.08%
spotpython tuning: 6.436366676628063 [##--------] 16.71%
spotpython tuning: 6.436366676628063 [##--------] 18.41%
spotpython tuning: 6.436366676628063 [##--------] 19.96%
spotpython tuning: 6.436366676628063 [##--------] 21.59%
spotpython tuning: 6.436366676628063 [##--------] 23.37%
spotpython tuning: 6.436366676628063 [###-------] 25.05%
spotpython tuning: 6.436366676628063 [###-------] 26.99%
spotpython tuning: 6.436366676628063 [###-------] 28.80%
spotpython tuning: 6.436366676628063 [###-------] 30.64%
spotpython tuning: 6.436366676628063 [###-------] 32.73%
spotpython tuning: 6.436366676628063 [###-------] 34.85%
spotpython tuning: 6.436366676628063 [####------] 36.94%
spotpython tuning: 6.436366676628063 [####------] 39.02%
spotpython tuning: 6.436366676628063 [####------] 41.19%
spotpython tuning: 6.436366676628063 [####------] 43.02%
spotpython tuning: 6.436366676628063 [####------] 44.89%
spotpython tuning: 6.436366676628063 [#####-----] 46.96%
spotpython tuning: 6.436366676628063 [#####-----] 48.81%
spotpython tuning: 6.436366676628063 [#####-----] 50.92%
spotpython tuning: 6.436366676628063 [#####-----] 52.78%
spotpython tuning: 6.436366676628063 [#####-----] 54.93%
spotpython tuning: 6.436366676628063 [######----] 57.15%
spotpython tuning: 6.436366676628063 [######----] 59.09%
spotpython tuning: 6.436366676628063 [######----] 61.16%
spotpython tuning: 6.436366676628063 [######----] 63.43%
spotpython tuning: 6.436366676628063 [#######---] 65.48%
spotpython tuning: 6.436366676628063 [#######---] 67.28%
spotpython tuning: 6.436366676628063 [#######---] 69.34%
spotpython tuning: 6.436366676628063 [#######---] 71.29%
spotpython tuning: 6.436366676628063 [#######---] 73.30%
spotpython tuning: 6.436366676628063 [########--] 75.41%
spotpython tuning: 6.436366676628063 [########--] 77.83%
spotpython tuning: 6.436366676628063 [########--] 79.98%
spotpython tuning: 6.436366676628063 [########--] 82.24%
spotpython tuning: 6.436366676628063 [########--] 84.35%
spotpython tuning: 6.436366676628063 [#########-] 86.46%
spotpython tuning: 6.436366676628063 [#########-] 88.55%
spotpython tuning: 6.436366676628063 [#########-] 90.87%
spotpython tuning: 6.436366676628063 [#########-] 93.02%
spotpython tuning: 6.436366676628063 [##########] 95.29%
spotpython tuning: 6.436366676628063 [##########] 97.47%
spotpython tuning: 6.436366676628063 [##########] 99.71%
spotpython tuning: 6.436366676628063 [##########] 100.00% Done...
Experiment saved to 401_sklearn_classification_res.pkl
<spotpython.spot.spot.Spot at 0x15a2af980>
32.9.3 TensorBoard
Now we can start TensorBoard in the background with the following command, where ./runs
is the default directory for the TensorBoard log files:
tensorboard --logdir="./runs"
The TensorBoard path can be printed with the following command:
get_tensorboard_path(fun_control)
'runs/'
We can access the TensorBoard web server with the following URL:
http://localhost:6006/
The TensorBoard plot illustrates how spotpython
can be used as a microscope for the internal mechanisms of the surrogate-based optimization process. Here, one important parameter, the learning rate \(\theta\) of the Kriging surrogate [SOURCE] is plotted against the number of optimization steps.
32.10 Step 9: Results
After the hyperparameter tuning run is finished, the results can be saved and reloaded with the following commands:
= load_result(PREFIX=PREFIX) spot_tuner
Loaded experiment from 401_sklearn_classification_res.pkl
After the hyperparameter tuning run is finished, the progress of the hyperparameter tuning can be visualized. The black points represent the performance values (score or metric) of hyperparameter configurations from the initial design, whereas the red points represents the hyperparameter configurations found by the surrogate model based optimization.
=True) spot_tuner.plot_progress(log_y
Results can also be printed in tabular form.
print_res_table(spot_tuner)
| name | type | default | lower | upper | tuned | transform | importance | stars |
|-------------|--------|-----------|---------|---------|-----------------------|-------------|--------------|---------|
| C | float | 1.0 | 0.1 | 10.0 | 1.3459476182876375 | None | 0.11 | . |
| kernel | factor | rbf | 0.0 | 1.0 | rbf | None | 100.00 | *** |
| degree | int | 3 | 3.0 | 3.0 | 3.0 | None | 0.00 | |
| gamma | factor | scale | 0.0 | 1.0 | scale | None | 0.29 | . |
| coef0 | float | 0.0 | 0.0 | 0.0 | 0.0 | None | 0.00 | |
| shrinking | factor | 0 | 0.0 | 1.0 | 1 | None | 0.00 | |
| probability | factor | 0 | 0.0 | 0.0 | 0 | None | 0.00 | |
| tol | float | 0.001 | 1e-05 | 0.001 | 2.988661226661179e-05 | None | 0.09 | |
| cache_size | float | 200.0 | 100.0 | 400.0 | 174.45504889441855 | None | 0.00 | |
| break_ties | factor | 0 | 0.0 | 1.0 | 0 | None | 0.00 | |
A histogram can be used to visualize the most important hyperparameters.
=0.0025) spot_tuner.plot_importance(threshold
32.11 Get Default Hyperparameters
The default hyperparameters, whihc will be used for a comparion with the tuned hyperparameters, can be obtained with the following commands:
= get_default_hyperparameters_as_array(fun_control)
X_start = get_one_core_model_from_X(X_start, fun_control, default=True)
model_default model_default
SVC(cache_size=200.0, shrinking=False)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
SVC(cache_size=200.0, shrinking=False)
32.12 Get SPOT Results
In a similar way, we can obtain the hyperparameters found by spotpython
.
= spot_tuner.to_all_dim(spot_tuner.min_X.reshape(1,-1))
X = get_one_core_model_from_X(X, fun_control) model_spot
32.12.1 Plot: Compare Predictions
=[model_default, model_spot], fun_control= fun_control, model_names=["Default", "Spot"]) plot_roc(model_list
=model_default, fun_control=fun_control, title = "Default") plot_confusion_matrix(model
# label: 401_plot_confusion_matrix
=model_spot, fun_control=fun_control, title="SPOT") plot_confusion_matrix(model
min(spot_tuner.y), max(spot_tuner.y)
(np.float64(6.436366676628063), np.float64(10.813096016735146))
32.12.2 Detailed Hyperparameter Plots
=3) spot_tuner.plot_important_hyperparameter_contour(max_imp
C: 0.10763841054107509
kernel: 100.0
gamma: 0.28576007483433935
shrinking: 0.0027414361177529214
tol: 0.09357778553602186
cache_size: 0.0027414361177529214
break_ties: 0.0027414361177529214
32.12.3 Parallel Coordinates Plot
spot_tuner.parallel_plot()
32.12.4 Plot all Combinations of Hyperparameters
- Warning: this may take a while.
= False
PLOT_ALL if PLOT_ALL:
= spot_tuner.k
n for i in range(n-1):
for j in range(i+1, n):
=i, j=j, min_z=min_z, max_z = max_z) spot_tuner.plot_contour(i