import numpy as np
from math import inf
from spotPython.fun.objectivefunctions import analytical
from spotPython.spot import spot
from scipy.optimize import shgo
from scipy.optimize import direct
from scipy.optimize import differential_evolution
import matplotlib.pyplot as pltAppendix D — Documentation of the Sequential Parameter Optimization
This document describes the Spot features. The official spotPython documentation can be found here: https://sequential-parameter-optimization.github.io/spotPython/.
D.1 An Initial Example
The spotPython package provides several classes of objective functions. We will use an analytical objective function, i.e., a function that can be described by a (closed) formula: \[
f(x) = x^2.
\]
fun = analytical().fun_spherex = np.linspace(-1,1,100).reshape(-1,1)
y = fun(x)
plt.figure()
plt.plot(x,y, "k")
plt.show()
from spotPython.utils.init import fun_control_init, design_control_init, surrogate_control_init, optimizer_control_init
spot_1 = spot.Spot(fun=fun,
fun_control=fun_control_init(
lower = np.array([-10]),
upper = np.array([100]),
fun_evals = 7,
fun_repeats = 1,
max_time = inf,
noise = False,
tolerance_x = np.sqrt(np.spacing(1)),
var_type=["num"],
infill_criterion = "y",
n_points = 1,
seed=123,
log_level = 50),
design_control=design_control_init(
init_size=5,
repeats=1),
surrogate_control=surrogate_control_init(
noise=False,
min_theta=-4,
max_theta=3,
n_theta=1,
model_optimizer=differential_evolution,
model_fun_evals=10000))
spot_1.run()spotPython tuning: 2.0106521524877827 [#########-] 85.71%
spotPython tuning: 0.01033163973935242 [##########] 100.00% Done...
{'CHECKPOINT_PATH': 'runs/saved_models/',
'DATASET_PATH': 'data/',
'PREFIX': None,
'RESULTS_PATH': 'results/',
'TENSORBOARD_PATH': 'runs/',
'_L_in': None,
'_L_out': None,
'_torchmetric': None,
'accelerator': 'auto',
'converters': None,
'core_model': None,
'core_model_name': None,
'counter': 7,
'data': None,
'data_dir': './data',
'data_module': None,
'data_set': None,
'data_set_name': None,
'db_dict_name': None,
'design': None,
'device': None,
'devices': 1,
'enable_progress_bar': False,
'eval': None,
'fun_evals': 7,
'fun_repeats': 1,
'horizon': None,
'infill_criterion': 'y',
'k_folds': 3,
'log_graph': False,
'log_level': 50,
'loss_function': None,
'lower': array([-10]),
'max_surrogate_points': 30,
'max_time': inf,
'metric_params': {},
'metric_river': None,
'metric_sklearn': None,
'metric_sklearn_name': None,
'metric_torch': None,
'model_dict': {},
'n_points': 1,
'n_samples': None,
'n_total': None,
'noise': False,
'num_workers': 0,
'ocba_delta': 0,
'oml_grace_period': None,
'optimizer': None,
'path': None,
'prep_model': None,
'prep_model_name': None,
'progress_file': None,
'save_model': False,
'scenario': None,
'seed': 123,
'show_batch_interval': 1000000,
'show_models': False,
'show_progress': True,
'shuffle': None,
'sigma': 0.0,
'spot_tensorboard_path': None,
'spot_writer': None,
'target_column': None,
'target_type': None,
'task': None,
'test': None,
'test_seed': 1234,
'test_size': 0.4,
'tolerance_x': 1.4901161193847656e-08,
'train': None,
'upper': array([100]),
'var_name': None,
'var_type': ['num'],
'verbosity': 0,
'weight_coeff': 0.0,
'weights': 1.0,
'weights_entry': None}<spotPython.spot.spot.Spot at 0x3d41cab90>D.2 Organization
Spot organizes the surrogate based optimization process in four steps:
- Selection of the objective function:
fun. - Selection of the initial design:
design. - Selection of the optimization algorithm:
optimizer. - Selection of the surrogate model:
surrogate.
For each of these steps, the user can specify an object:
from spotPython.fun.objectivefunctions import analytical
fun = analytical().fun_sphere
from spotPython.design.spacefilling import spacefilling
design = spacefilling(2)
from scipy.optimize import differential_evolution
optimizer = differential_evolution
from spotPython.build.kriging import Kriging
surrogate = Kriging()For each of these steps, the user can specify a dictionary of control parameters.
fun_controldesign_controloptimizer_controlsurrogate_control
Each of these dictionaries has an initialzaion method, e.g., fun_control_init(). The initialization methods set the default values for the control parameters.
Important:
- The specification of an lower bound in
fun_controlis mandatory.
from spotPython.utils.init import fun_control_init, design_control_init, optimizer_control_init, surrogate_control_init
fun_control=fun_control_init(lower=np.array([-1, -1]),
upper=np.array([1, 1]))
design_control=design_control_init()
optimizer_control=optimizer_control_init()
surrogate_control=surrogate_control_init()D.3 The Spot Object
Based on the definition of the fun, design, optimizer, and surrogate objects, and their corresponding control parameter dictionaries, fun_control, design_control, optimizer_control, and surrogate_control, the spot object can be build as follows:
from spotPython.spot import spot
spot_tuner = spot.Spot(fun=fun,
fun_control=fun_control,
design_control=design_control,
optimizer_control=optimizer_control,
surrogate_control=surrogate_control)D.4 Run
spot_tuner.run()spotPython tuning: 1.801603872454505e-05 [#######---] 73.33%
spotPython tuning: 1.801603872454505e-05 [########--] 80.00%
spotPython tuning: 1.801603872454505e-05 [#########-] 86.67%
spotPython tuning: 1.801603872454505e-05 [#########-] 93.33%
spotPython tuning: 1.801603872454505e-05 [##########] 100.00% Done...
{'CHECKPOINT_PATH': 'runs/saved_models/',
'DATASET_PATH': 'data/',
'PREFIX': None,
'RESULTS_PATH': 'results/',
'TENSORBOARD_PATH': 'runs/',
'_L_in': None,
'_L_out': None,
'_torchmetric': None,
'accelerator': 'auto',
'converters': None,
'core_model': None,
'core_model_name': None,
'counter': 15,
'data': None,
'data_dir': './data',
'data_module': None,
'data_set': None,
'data_set_name': None,
'db_dict_name': None,
'design': None,
'device': None,
'devices': 1,
'enable_progress_bar': False,
'eval': None,
'fun_evals': 15,
'fun_repeats': 1,
'horizon': None,
'infill_criterion': 'y',
'k_folds': 3,
'log_graph': False,
'log_level': 50,
'loss_function': None,
'lower': array([-1, -1]),
'max_surrogate_points': 30,
'max_time': 1,
'metric_params': {},
'metric_river': None,
'metric_sklearn': None,
'metric_sklearn_name': None,
'metric_torch': None,
'model_dict': {},
'n_points': 1,
'n_samples': None,
'n_total': None,
'noise': False,
'num_workers': 0,
'ocba_delta': 0,
'oml_grace_period': None,
'optimizer': None,
'path': None,
'prep_model': None,
'prep_model_name': None,
'progress_file': None,
'save_model': False,
'scenario': None,
'seed': 123,
'show_batch_interval': 1000000,
'show_models': False,
'show_progress': True,
'shuffle': None,
'sigma': 0.0,
'spot_tensorboard_path': None,
'spot_writer': None,
'target_column': None,
'target_type': None,
'task': None,
'test': None,
'test_seed': 1234,
'test_size': 0.4,
'tolerance_x': 0,
'train': None,
'upper': array([1, 1]),
'var_name': None,
'var_type': ['num'],
'verbosity': 0,
'weight_coeff': 0.0,
'weights': 1.0,
'weights_entry': None}<spotPython.spot.spot.Spot at 0x1063feb90>D.5 Print the Results
spot_tuner.print_results()min y: 1.801603872454505e-05
x0: 0.0019077911677074135
x1: 0.003791618596979743[['x0', 0.0019077911677074135], ['x1', 0.003791618596979743]]D.6 Show the Progress
spot_tuner.plot_progress()
D.7 Visualize the Surrogate
- The plot method of the
krigingsurrogate is used. - Note: the plot uses the interval defined by the ranges of the natural variables.
spot_tuner.surrogate.plot()
D.8 Run With a Specific Start Design
To pass a specific start design, use the X_start argument of the run method.
spot_x0 = spot.Spot(fun=fun,
fun_control=fun_control_init(
lower = np.array([-10]),
upper = np.array([100]),
fun_evals = 7,
fun_repeats = 1,
max_time = inf,
noise = False,
tolerance_x = np.sqrt(np.spacing(1)),
var_type=["num"],
infill_criterion = "y",
n_points = 1,
seed=123,
log_level = 50),
design_control=design_control_init(
init_size=5,
repeats=1),
surrogate_control=surrogate_control_init(
noise=False,
min_theta=-4,
max_theta=3,
n_theta=1,
model_optimizer=differential_evolution,
model_fun_evals=10000))
spot_x0.run(X_start=np.array([0.5, -0.5]))
spot_x0.plot_progress()spotPython tuning: 2.0106521524877827 [#########-] 85.71%
spotPython tuning: 0.01033163973935242 [##########] 100.00% Done...
{'CHECKPOINT_PATH': 'runs/saved_models/',
'DATASET_PATH': 'data/',
'PREFIX': None,
'RESULTS_PATH': 'results/',
'TENSORBOARD_PATH': 'runs/',
'_L_in': None,
'_L_out': None,
'_torchmetric': None,
'accelerator': 'auto',
'converters': None,
'core_model': None,
'core_model_name': None,
'counter': 7,
'data': None,
'data_dir': './data',
'data_module': None,
'data_set': None,
'data_set_name': None,
'db_dict_name': None,
'design': None,
'device': None,
'devices': 1,
'enable_progress_bar': False,
'eval': None,
'fun_evals': 7,
'fun_repeats': 1,
'horizon': None,
'infill_criterion': 'y',
'k_folds': 3,
'log_graph': False,
'log_level': 50,
'loss_function': None,
'lower': array([-10]),
'max_surrogate_points': 30,
'max_time': inf,
'metric_params': {},
'metric_river': None,
'metric_sklearn': None,
'metric_sklearn_name': None,
'metric_torch': None,
'model_dict': {},
'n_points': 1,
'n_samples': None,
'n_total': None,
'noise': False,
'num_workers': 0,
'ocba_delta': 0,
'oml_grace_period': None,
'optimizer': None,
'path': None,
'prep_model': None,
'prep_model_name': None,
'progress_file': None,
'save_model': False,
'scenario': None,
'seed': 123,
'show_batch_interval': 1000000,
'show_models': False,
'show_progress': True,
'shuffle': None,
'sigma': 0.0,
'spot_tensorboard_path': None,
'spot_writer': None,
'target_column': None,
'target_type': None,
'task': None,
'test': None,
'test_seed': 1234,
'test_size': 0.4,
'tolerance_x': 1.4901161193847656e-08,
'train': None,
'upper': array([100]),
'var_name': None,
'var_type': ['num'],
'verbosity': 0,
'weight_coeff': 0.0,
'weights': 1.0,
'weights_entry': None}
D.9 Init: Build Initial Design
from spotPython.design.spacefilling import spacefilling
from spotPython.build.kriging import Kriging
from spotPython.fun.objectivefunctions import analytical
gen = spacefilling(2)
rng = np.random.RandomState(1)
lower = np.array([-5,-0])
upper = np.array([10,15])
fun = analytical().fun_branin
fun_control = {"sigma": 0,
"seed": 123}
X = gen.scipy_lhd(10, lower=lower, upper = upper)
print(X)
y = fun(X, fun_control=fun_control)
print(y)[[ 8.97647221 13.41926847]
[ 0.66946019 1.22344228]
[ 5.23614115 13.78185824]
[ 5.6149825 11.5851384 ]
[-1.72963184 1.66516096]
[-4.26945568 7.1325531 ]
[ 1.26363761 10.17935555]
[ 2.88779942 8.05508969]
[-3.39111089 4.15213772]
[ 7.30131231 5.22275244]]
[128.95676449 31.73474356 172.89678121 126.71295908 64.34349975
70.16178611 48.71407916 31.77322887 76.91788181 30.69410529]D.10 Replicability
Seed
gen = spacefilling(2, seed=123)
X0 = gen.scipy_lhd(3)
gen = spacefilling(2, seed=345)
X1 = gen.scipy_lhd(3)
X2 = gen.scipy_lhd(3)
gen = spacefilling(2, seed=123)
X3 = gen.scipy_lhd(3)
X0, X1, X2, X3(array([[0.77254938, 0.31539299],
[0.59321338, 0.93854273],
[0.27469803, 0.3959685 ]]),
array([[0.78373509, 0.86811887],
[0.06692621, 0.6058029 ],
[0.41374778, 0.00525456]]),
array([[0.121357 , 0.69043832],
[0.41906219, 0.32838498],
[0.86742658, 0.52910374]]),
array([[0.77254938, 0.31539299],
[0.59321338, 0.93854273],
[0.27469803, 0.3959685 ]]))D.11 Surrogates
D.11.1 A Simple Predictor
The code below shows how to use a simple model for prediction. Assume that only two (very costly) measurements are available:
- f(0) = 0.5
- f(2) = 2.5
We are interested in the value at \(x_0 = 1\), i.e., \(f(x_0 = 1)\), but cannot run an additional, third experiment.
from sklearn import linear_model
X = np.array([[0], [2]])
y = np.array([0.5, 2.5])
S_lm = linear_model.LinearRegression()
S_lm = S_lm.fit(X, y)
X0 = np.array([[1]])
y0 = S_lm.predict(X0)
print(y0)[1.5]Central Idea: Evaluation of the surrogate model S_lm is much cheaper (or / and much faster) than running the real-world experiment \(f\).
D.12 Demo/Test: Objective Function Fails
SPOT expects np.nan values from failed objective function values. These are handled. Note: SPOT’s counter considers only successful executions of the objective function.
import numpy as np
from spotPython.fun.objectivefunctions import analytical
from spotPython.spot import spot
import numpy as np
from math import inf
# number of initial points:
ni = 20
# number of points
n = 30
fun = analytical().fun_random_error
fun_control=fun_control_init(
lower = np.array([-1]),
upper= np.array([1]),
fun_evals = n,
show_progress=False)
design_control=design_control_init(init_size=ni)
spot_1 = spot.Spot(fun=fun,
fun_control=fun_control,
design_control=design_control)
spot_1.run()
# To check whether the run was successfully completed,
# we compare the number of evaluated points to the specified
# number of points.
assert spot_1.y.shape[0] == n[ nan nan -0.02203599 -0.21843718 0.78240941 nan
-0.3923345 0.67234256 0.31802454 -0.68898927 -0.75129705 0.97550354
0.41757584 nan 0.82585329 nan -0.49274073 nan
-0.17991251 0.1481835 ]
[-1.]
[nan]
[-0.14624037]
[0.166475]
[nan]
[-0.3352401]
[-0.47259301]
[0.95541987]
[0.17335968]
[-0.58552368]
[-0.20126111]
[-0.60100809]
[-0.97897336]
[-0.2748985]
[0.8359486]
[0.99035591]
[0.01641232]
[0.5629346]
{'CHECKPOINT_PATH': 'runs/saved_models/',
'DATASET_PATH': 'data/',
'PREFIX': None,
'RESULTS_PATH': 'results/',
'TENSORBOARD_PATH': 'runs/',
'_L_in': None,
'_L_out': None,
'_torchmetric': None,
'accelerator': 'auto',
'converters': None,
'core_model': None,
'core_model_name': None,
'counter': 30,
'data': None,
'data_dir': './data',
'data_module': None,
'data_set': None,
'data_set_name': None,
'db_dict_name': None,
'design': None,
'device': None,
'devices': 1,
'enable_progress_bar': False,
'eval': None,
'fun_evals': 30,
'fun_repeats': 1,
'horizon': None,
'infill_criterion': 'y',
'k_folds': 3,
'log_graph': False,
'log_level': 50,
'loss_function': None,
'lower': array([-1]),
'max_surrogate_points': 30,
'max_time': 1,
'metric_params': {},
'metric_river': None,
'metric_sklearn': None,
'metric_sklearn_name': None,
'metric_torch': None,
'model_dict': {},
'n_points': 1,
'n_samples': None,
'n_total': None,
'noise': False,
'num_workers': 0,
'ocba_delta': 0,
'oml_grace_period': None,
'optimizer': None,
'path': None,
'prep_model': None,
'prep_model_name': None,
'progress_file': None,
'save_model': False,
'scenario': None,
'seed': 123,
'show_batch_interval': 1000000,
'show_models': False,
'show_progress': False,
'shuffle': None,
'sigma': 0.0,
'spot_tensorboard_path': None,
'spot_writer': None,
'target_column': None,
'target_type': None,
'task': None,
'test': None,
'test_seed': 1234,
'test_size': 0.4,
'tolerance_x': 0,
'train': None,
'upper': array([1]),
'var_name': None,
'var_type': ['num'],
'verbosity': 0,
'weight_coeff': 0.0,
'weights': 1.0,
'weights_entry': None}