import numpy as np
from math import inf
from spotpython.fun.objectivefunctions import Analytical
from spotpython.spot import spot
from spotpython.utils.init import fun_control_init, surrogate_control_init, design_control_init
import matplotlib.pyplot as plt
12 Expected Improvement
This chapter describes, analyzes, and compares different infill criterion. An infill criterion defines how the next point \(x_{n+1}\) is selected from the surrogate model \(S\). Expected improvement is a popular infill criterion in Bayesian optimization.
12.1 Example: Spot
and the 1-dim Sphere Function
12.1.1 The Objective Function: 1-dim Sphere
- The
spotpython
package provides several classes of objective functions. - We will use an analytical objective function, i.e., a function that can be described by a (closed) formula: \[f(x) = x^2 \]
= Analytical().fun_sphere fun
- The size of the
lower
bound vector determines the problem dimension. - Here we will use
np.array([-1])
, i.e., a one-dim function.
Similar to the one-dimensional case, which was introduced in Section Section 7.5, we can use TensorBoard to monitor the progress of the optimization. We will use the same code, only the prefix is different:
from spotpython.utils.init import fun_control_init
= "07_Y"
PREFIX = fun_control_init(
fun_control =PREFIX,
PREFIX= 25,
fun_evals = np.array([-1]),
lower = np.array([1]),
upper = np.sqrt(np.spacing(1)),)
tolerance_x = design_control_init(init_size=10) design_control
= spot.Spot(
spot_1 =fun,
fun=fun_control,
fun_control=design_control)
design_control spot_1.run()
spotpython tuning: 4.960293502265715e-09 [####------] 44.00%
spotpython tuning: 4.960293502265715e-09 [#####-----] 48.00%
spotpython tuning: 3.91132389612039e-09 [#####-----] 52.00%
spotpython tuning: 3.91132389612039e-09 [######----] 56.00%
spotpython tuning: 2.8792582932753584e-09 [######----] 60.00%
spotpython tuning: 2.8792582932753584e-09 [######----] 64.00%
spotpython tuning: 2.8792582932753584e-09 [#######---] 68.00%
spotpython tuning: 2.8792582932753584e-09 [#######---] 72.00%
spotpython tuning: 2.8792582932753584e-09 [########--] 76.00%
spotpython tuning: 2.8792582932753584e-09 [########--] 80.00%
spotpython tuning: 5.121302337041985e-10 [########--] 84.00%
spotpython tuning: 5.121302337041985e-10 [#########-] 88.00%
spotpython tuning: 9.915881842777748e-12 [#########-] 92.00%
spotpython tuning: 9.915881842777748e-12 [##########] 96.00%
spotpython tuning: 9.915881842777748e-12 [##########] 100.00% Done...
<spotpython.spot.spot.Spot at 0x14a2b2f30>
12.1.2 Results
spot_1.print_results()
min y: 9.915881842777748e-12
x0: -3.1489493236280808e-06
[['x0', np.float64(-3.1489493236280808e-06)]]
=True) spot_1.plot_progress(log_y
12.2 Same, but with EI as infill_criterion
= "07_EI_ISO"
PREFIX = fun_control_init(
fun_control =PREFIX,
PREFIX= np.array([-1]),
lower = np.array([1]),
upper = 25,
fun_evals = np.sqrt(np.spacing(1)),
tolerance_x = "ei") infill_criterion
= spot.Spot(fun=fun,
spot_1_ei =fun_control)
fun_control spot_1_ei.run()
spotpython tuning: 1.0205727057090308e-08 [####------] 44.00%
spotpython tuning: 1.0205727057090308e-08 [#####-----] 48.00%
spotpython tuning: 1.0205727057090308e-08 [#####-----] 52.00%
spotpython tuning: 1.0205727057090308e-08 [######----] 56.00%
spotpython tuning: 2.2781224716456335e-14 [######----] 60.00%
spotpython tuning: 2.2781224716456335e-14 [######----] 64.00%
spotpython tuning: 2.2781224716456335e-14 [#######---] 68.00%
spotpython tuning: 2.2781224716456335e-14 [#######---] 72.00%
spotpython tuning: 2.2781224716456335e-14 [########--] 76.00%
spotpython tuning: 2.2781224716456335e-14 [########--] 80.00%
spotpython tuning: 2.2781224716456335e-14 [########--] 84.00%
spotpython tuning: 2.2781224716456335e-14 [#########-] 88.00%
spotpython tuning: 2.2781224716456335e-14 [#########-] 92.00%
spotpython tuning: 2.2781224716456335e-14 [##########] 96.00%
spotpython tuning: 2.2781224716456335e-14 [##########] 100.00% Done...
<spotpython.spot.spot.Spot at 0x14a5870e0>
=True) spot_1_ei.plot_progress(log_y
spot_1_ei.print_results()
min y: 2.2781224716456335e-14
x0: 1.5093450472458687e-07
[['x0', np.float64(1.5093450472458687e-07)]]
12.3 Non-isotropic Kriging
= "07_EI_NONISO"
PREFIX = fun_control_init(
fun_control =PREFIX,
PREFIX= np.array([-1, -1]),
lower = np.array([1, 1]),
upper = 25,
fun_evals = np.sqrt(np.spacing(1)),
tolerance_x = "ei")
infill_criterion = surrogate_control_init(
surrogate_control =2,
n_theta=False,
noise )
= spot.Spot(fun=fun,
spot_2_ei_noniso =fun_control,
fun_control=surrogate_control)
surrogate_control spot_2_ei_noniso.run()
spotpython tuning: 1.885355036033269e-05 [####------] 44.00%
spotpython tuning: 1.885355036033269e-05 [#####-----] 48.00%
spotpython tuning: 1.885355036033269e-05 [#####-----] 52.00%
spotpython tuning: 1.885355036033269e-05 [######----] 56.00%
spotpython tuning: 1.885355036033269e-05 [######----] 60.00%
spotpython tuning: 1.885355036033269e-05 [######----] 64.00%
spotpython tuning: 1.885355036033269e-05 [#######---] 68.00%
spotpython tuning: 1.885355036033269e-05 [#######---] 72.00%
spotpython tuning: 9.720933590577543e-06 [########--] 76.00%
spotpython tuning: 8.679050871673044e-06 [########--] 80.00%
spotpython tuning: 8.679050871673044e-06 [########--] 84.00%
spotpython tuning: 8.679050871673044e-06 [#########-] 88.00%
spotpython tuning: 8.679050871673044e-06 [#########-] 92.00%
spotpython tuning: 8.679050871673044e-06 [##########] 96.00%
spotpython tuning: 8.679050871673044e-06 [##########] 100.00% Done...
<spotpython.spot.spot.Spot at 0x14a6974a0>
=True) spot_2_ei_noniso.plot_progress(log_y
spot_2_ei_noniso.print_results()
min y: 8.679050871673044e-06
x0: -0.0029115430445807067
x1: -0.000449408468129777
[['x0', np.float64(-0.0029115430445807067)],
['x1', np.float64(-0.000449408468129777)]]
spot_2_ei_noniso.surrogate.plot()
12.4 Using sklearn
Surrogates
12.4.1 The spot Loop
The spot
loop consists of the following steps:
- Init: Build initial design \(X\)
- Evaluate initial design on real objective \(f\): \(y = f(X)\)
- Build surrogate: \(S = S(X,y)\)
- Optimize on surrogate: \(X_0 = \text{optimize}(S)\)
- Evaluate on real objective: \(y_0 = f(X_0)\)
- Impute (Infill) new points: \(X = X \cup X_0\), \(y = y \cup y_0\).
- Got 3.
The spot
loop is implemented in R
as follows:
12.4.2 spot: The Initial Model
12.4.2.1 Example: Modifying the initial design size
This is the “Example: Modifying the initial design size” from Chapter 4.5.1 in [bart21i].
= spot.Spot(fun=fun,
spot_ei =fun_control_init(
fun_control= np.array([-1,-1]),
lower = np.array([1,1])),
upper= design_control_init(init_size=5))
design_control spot_ei.run()
spotpython tuning: 0.13771720107579405 [####------] 40.00%
spotpython tuning: 0.008747581912500914 [#####-----] 46.67%
spotpython tuning: 0.002833855020194859 [#####-----] 53.33%
spotpython tuning: 0.0008113730206162874 [######----] 60.00%
spotpython tuning: 0.0003658334488102912 [#######---] 66.67%
spotpython tuning: 0.000357376362957228 [#######---] 73.33%
spotpython tuning: 0.000357376362957228 [########--] 80.00%
spotpython tuning: 0.00032569308461158667 [#########-] 86.67%
spotpython tuning: 0.000272252838237925 [#########-] 93.33%
spotpython tuning: 0.0001495137205828153 [##########] 100.00% Done...
<spotpython.spot.spot.Spot at 0x14a60e6f0>
spot_ei.plot_progress()
min(spot_1.y), np.min(spot_ei.y) np.
(np.float64(9.915881842777748e-12), np.float64(0.0001495137205828153))
12.4.3 Init: Build Initial Design
from spotpython.design.spacefilling import SpaceFilling
from spotpython.build.kriging import Kriging
from spotpython.fun.objectivefunctions import Analytical
= SpaceFilling(2)
gen = np.random.RandomState(1)
rng = np.array([-5,-0])
lower = np.array([10,15])
upper = Analytical().fun_branin
fun
= gen.scipy_lhd(10, lower=lower, upper = upper)
X print(X)
= fun(X, fun_control=fun_control)
y print(y)
[[ 8.97647221 13.41926847]
[ 0.66946019 1.22344228]
[ 5.23614115 13.78185824]
[ 5.6149825 11.5851384 ]
[-1.72963184 1.66516096]
[-4.26945568 7.1325531 ]
[ 1.26363761 10.17935555]
[ 2.88779942 8.05508969]
[-3.39111089 4.15213772]
[ 7.30131231 5.22275244]]
[128.95676449 31.73474356 172.89678121 126.71295908 64.34349975
70.16178611 48.71407916 31.77322887 76.91788181 30.69410529]
= Kriging(name='kriging', seed=123)
S
S.fit(X, y) S.plot()
= SpaceFilling(2, seed=123)
gen = gen.scipy_lhd(3)
X0 = SpaceFilling(2, seed=345)
gen = gen.scipy_lhd(3)
X1 = gen.scipy_lhd(3)
X2 = SpaceFilling(2, seed=123)
gen = gen.scipy_lhd(3)
X3 X0, X1, X2, X3
(array([[0.77254938, 0.31539299],
[0.59321338, 0.93854273],
[0.27469803, 0.3959685 ]]),
array([[0.78373509, 0.86811887],
[0.06692621, 0.6058029 ],
[0.41374778, 0.00525456]]),
array([[0.121357 , 0.69043832],
[0.41906219, 0.32838498],
[0.86742658, 0.52910374]]),
array([[0.77254938, 0.31539299],
[0.59321338, 0.93854273],
[0.27469803, 0.3959685 ]]))
12.4.4 Evaluate
12.4.5 Build Surrogate
12.4.6 A Simple Predictor
The code below shows how to use a simple model for prediction.
Assume that only two (very costly) measurements are available:
- f(0) = 0.5
- f(2) = 2.5
We are interested in the value at \(x_0 = 1\), i.e., \(f(x_0 = 1)\), but cannot run an additional, third experiment.
from sklearn import linear_model
= np.array([[0], [2]])
X = np.array([0.5, 2.5])
y = linear_model.LinearRegression()
S_lm = S_lm.fit(X, y)
S_lm = np.array([[1]])
X0 = S_lm.predict(X0)
y0 print(y0)
[1.5]
- Central Idea:
- Evaluation of the surrogate model
S_lm
is much cheaper (or / and much faster) than running the real-world experiment \(f\).
- Evaluation of the surrogate model
12.5 Gaussian Processes regression: basic introductory example
This example was taken from scikit-learn. After fitting our model, we see that the hyperparameters of the kernel have been optimized. Now, we will use our kernel to compute the mean prediction of the full dataset and plot the 95% confidence interval.
import numpy as np
import matplotlib.pyplot as plt
import math as m
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF
= np.linspace(start=0, stop=10, num=1_000).reshape(-1, 1)
X = np.squeeze(X * np.sin(X))
y = np.random.RandomState(1)
rng = rng.choice(np.arange(y.size), size=6, replace=False)
training_indices = X[training_indices], y[training_indices]
X_train, y_train
= 1 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e2))
kernel = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
gaussian_process
gaussian_process.fit(X_train, y_train)
gaussian_process.kernel_
= gaussian_process.predict(X, return_std=True)
mean_prediction, std_prediction
=r"$f(x) = x \sin(x)$", linestyle="dotted")
plt.plot(X, y, label="Observations")
plt.scatter(X_train, y_train, label="Mean prediction")
plt.plot(X, mean_prediction, label
plt.fill_between(
X.ravel(),- 1.96 * std_prediction,
mean_prediction + 1.96 * std_prediction,
mean_prediction =0.5,
alpha=r"95% confidence interval",
label
)
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("sk-learn Version: Gaussian process regression on noise-free dataset") _
from spotpython.build.kriging import Kriging
import numpy as np
import matplotlib.pyplot as plt
= np.random.RandomState(1)
rng = np.linspace(start=0, stop=10, num=1_000).reshape(-1, 1)
X = np.squeeze(X * np.sin(X))
y = rng.choice(np.arange(y.size), size=6, replace=False)
training_indices = X[training_indices], y[training_indices]
X_train, y_train
= Kriging(name='kriging', seed=123, log_level=50, cod_type="norm")
S
S.fit(X_train, y_train)
= S.predict(X, return_val="all")
mean_prediction, std_prediction, ei
std_prediction
=r"$f(x) = x \sin(x)$", linestyle="dotted")
plt.plot(X, y, label="Observations")
plt.scatter(X_train, y_train, label="Mean prediction")
plt.plot(X, mean_prediction, label
plt.fill_between(
X.ravel(),- 1.96 * std_prediction,
mean_prediction + 1.96 * std_prediction,
mean_prediction =0.5,
alpha=r"95% confidence interval",
label
)
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("spotpython Version: Gaussian process regression on noise-free dataset") _
12.6 The Surrogate: Using scikit-learn models
Default is the internal kriging
surrogate.
= Kriging(name='kriging', seed=123) S_0
Models from scikit-learn
can be selected, e.g., Gaussian Process:
# Needed for the sklearn surrogates:
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn import linear_model
from sklearn import tree
import pandas as pd
= 1 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e2))
kernel = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) S_GP
- and many more:
= DecisionTreeRegressor(random_state=0)
S_Tree = linear_model.LinearRegression()
S_LM = linear_model.Ridge()
S_Ridge = RandomForestRegressor(max_depth=2, random_state=0) S_RF
- The scikit-learn GP model
S_GP
is selected.
= S_GP S
isinstance(S, GaussianProcessRegressor)
True
from spotpython.fun.objectivefunctions import Analytical
= Analytical().fun_branin
fun = fun_control_init(
fun_control = np.array([-5,-0]),
lower = np.array([10,15]),
upper = 15)
fun_evals = design_control_init(init_size=5)
design_control = spot.Spot(fun=fun,
spot_GP =fun_control,
fun_control=S,
surrogate=design_control)
design_control spot_GP.run()
spotpython tuning: 24.51465459019188 [####------] 40.00%
spotpython tuning: 11.003092545432404 [#####-----] 46.67%
spotpython tuning: 11.003092545432404 [#####-----] 53.33%
spotpython tuning: 7.281405479109784 [######----] 60.00%
spotpython tuning: 7.281405479109784 [#######---] 66.67%
spotpython tuning: 7.281405479109784 [#######---] 73.33%
spotpython tuning: 2.9520033012954237 [########--] 80.00%
spotpython tuning: 2.9520033012954237 [#########-] 86.67%
spotpython tuning: 2.1049818033904044 [#########-] 93.33%
spotpython tuning: 1.9431597967021723 [##########] 100.00% Done...
<spotpython.spot.spot.Spot at 0x14a7b33e0>
spot_GP.y
array([ 69.32459936, 152.38491454, 107.92560483, 24.51465459,
76.73500031, 86.30426863, 11.00309255, 16.11758333,
7.28140548, 21.82343562, 10.96088904, 2.9520033 ,
3.02912616, 2.1049818 , 1.9431598 ])
spot_GP.plot_progress()
spot_GP.print_results()
min y: 1.9431597967021723
x0: 10.0
x1: 2.99858238342458
[['x0', np.float64(10.0)], ['x1', np.float64(2.99858238342458)]]
12.7 Additional Examples
# Needed for the sklearn surrogates:
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn import linear_model
from sklearn import tree
import pandas as pd
= 1 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e2))
kernel = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) S_GP
from spotpython.build.kriging import Kriging
import numpy as np
import spotpython
from spotpython.fun.objectivefunctions import Analytical
from spotpython.spot import spot
= Kriging(name='kriging',
S_K =123,
seed=50,
log_level= "y",
infill_criterion =1,
n_theta=False,
noise="norm")
cod_type= Analytical().fun_sphere
fun
= fun_control_init(
fun_control = np.array([-1,-1]),
lower = np.array([1,1]),
upper = 25)
fun_evals
= spot.Spot(fun=fun,
spot_S_K =fun_control,
fun_control=S_K,
surrogate=design_control,
design_control=surrogate_control)
surrogate_control spot_S_K.run()
spotpython tuning: 0.13771716894083716 [##--------] 24.00%
spotpython tuning: 0.008764900158613986 [###-------] 28.00%
spotpython tuning: 0.002831737294424899 [###-------] 32.00%
spotpython tuning: 0.0008144336474759649 [####------] 36.00%
spotpython tuning: 0.000363982666025624 [####------] 40.00%
spotpython tuning: 0.0003615841465166041 [####------] 44.00%
spotpython tuning: 0.0003590011672749327 [#####-----] 48.00%
spotpython tuning: 0.00032913641643097483 [#####-----] 52.00%
spotpython tuning: 0.0002791331313588125 [######----] 56.00%
spotpython tuning: 0.00016536102611694684 [######----] 60.00%
spotpython tuning: 1.979364042364845e-05 [######----] 64.00%
spotpython tuning: 2.328711577671373e-06 [#######---] 68.00%
spotpython tuning: 5.408003176528451e-07 [#######---] 72.00%
spotpython tuning: 4.501997119079259e-07 [########--] 76.00%
spotpython tuning: 3.902062597093855e-07 [########--] 80.00%
spotpython tuning: 1.9521044693395355e-07 [########--] 84.00%
spotpython tuning: 1.684568701593145e-07 [#########-] 88.00%
spotpython tuning: 1.684568701593145e-07 [#########-] 92.00%
spotpython tuning: 1.684568701593145e-07 [##########] 96.00%
spotpython tuning: 1.684568701593145e-07 [##########] 100.00% Done...
<spotpython.spot.spot.Spot at 0x158c630b0>
=True) spot_S_K.plot_progress(log_y
spot_S_K.surrogate.plot()
spot_S_K.print_results()
min y: 1.684568701593145e-07
x0: 0.0003249898265724749
x1: 0.0002506760514762174
[['x0', np.float64(0.0003249898265724749)],
['x1', np.float64(0.0002506760514762174)]]
12.7.1 Optimize on Surrogate
12.7.2 Evaluate on Real Objective
12.7.3 Impute / Infill new Points
12.8 Tests
import numpy as np
from spotpython.spot import spot
from spotpython.fun.objectivefunctions import Analytical
= Analytical().fun_sphere
fun_sphere
= fun_control_init(
fun_control =np.array([-1, -1]),
lower=np.array([1, 1]),
upper= 2)
n_points = spot.Spot(
spot_1 =fun_sphere,
fun=fun_control,
fun_control
)
# (S-2) Initial Design:
= spot_1.design.scipy_lhd(
spot_1.X "init_size"], lower=spot_1.lower, upper=spot_1.upper
spot_1.design_control[
)print(spot_1.X)
# (S-3): Eval initial design:
= spot_1.fun(spot_1.X)
spot_1.y print(spot_1.y)
spot_1.fit_surrogate()= spot_1.suggest_new_X()
X0 print(X0)
assert X0.size == spot_1.n_points * spot_1.k
[[ 0.86352963 0.7892358 ]
[-0.24407197 -0.83687436]
[ 0.36481882 0.8375811 ]
[ 0.415331 0.54468512]
[-0.56395091 -0.77797854]
[-0.90259409 -0.04899292]
[-0.16484832 0.35724741]
[ 0.05170659 0.07401196]
[-0.78548145 -0.44638164]
[ 0.64017497 -0.30363301]]
[1.36857656 0.75992983 0.83463487 0.46918172 0.92329124 0.8170764
0.15480068 0.00815134 0.81623768 0.502017 ]
[[0.00160545 0.00421075]
[0.00171842 0.00407727]]
12.9 EI: The Famous Schonlau Example
= np.array([1, 2, 3, 4, 12]).reshape(-1,1)
X_train0 = np.linspace(start=0, stop=10, num=5).reshape(-1, 1) X_train
from spotpython.build.kriging import Kriging
import numpy as np
import matplotlib.pyplot as plt
= np.array([1., 2., 3., 4., 12.]).reshape(-1,1)
X_train = np.array([0., -1.75, -2, -0.5, 5.])
y_train
= Kriging(name='kriging', seed=123, log_level=50, n_theta=1, noise=False, cod_type="norm")
S
S.fit(X_train, y_train)
= np.linspace(start=0, stop=13, num=1000).reshape(-1, 1)
X = S.predict(X, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label="Mean prediction")
plt.plot(X, mean_prediction, labelif True:
plt.fill_between(
X.ravel(),- 2 * std_prediction,
mean_prediction + 2 * std_prediction,
mean_prediction =0.5,
alpha=r"95% confidence interval",
label
)
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression on noise-free dataset") _
#plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted")
# plt.scatter(X_train, y_train, label="Observations")
-ei, label="Expected Improvement")
plt.plot(X,
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression on noise-free dataset") _
S.log
{'negLnLike': array([1.20788205]),
'theta': array([-0.99002508]),
'p': [],
'Lambda': []}
12.10 EI: The Forrester Example
from spotpython.build.kriging import Kriging
import numpy as np
import matplotlib.pyplot as plt
import spotpython
from spotpython.fun.objectivefunctions import Analytical
from spotpython.spot import spot
# exact x locations are unknown:
= np.array([0.0, 0.175, 0.225, 0.3, 0.35, 0.375, 0.5,1]).reshape(-1,1)
X_train
= Analytical().fun_forrester
fun = fun_control_init(
fun_control ="07_EI_FORRESTER",
PREFIX=1.0,
sigma=123,)
seed= fun(X_train, fun_control=fun_control)
y_train
= Kriging(name='kriging', seed=123, log_level=50, n_theta=1, noise=False, cod_type="norm")
S
S.fit(X_train, y_train)
= np.linspace(start=0, stop=1, num=1000).reshape(-1, 1)
X = S.predict(X, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label="Mean prediction")
plt.plot(X, mean_prediction, labelif True:
plt.fill_between(
X.ravel(),- 2 * std_prediction,
mean_prediction + 2 * std_prediction,
mean_prediction =0.5,
alpha=r"95% confidence interval",
label
)
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression on noise-free dataset") _
#plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted")
# plt.scatter(X_train, y_train, label="Observations")
-ei, label="Expected Improvement")
plt.plot(X,
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression on noise-free dataset") _
12.11 Noise
import numpy as np
import spotpython
from spotpython.fun.objectivefunctions import Analytical
from spotpython.spot import spot
from spotpython.design.spacefilling import SpaceFilling
from spotpython.build.kriging import Kriging
import matplotlib.pyplot as plt
= SpaceFilling(1)
gen = np.random.RandomState(1)
rng = np.array([-10])
lower = np.array([10])
upper = Analytical().fun_sphere
fun = fun_control_init(
fun_control ="07_Y",
PREFIX=2.0,
sigma=123,)
seed= gen.scipy_lhd(10, lower=lower, upper = upper)
X print(X)
= fun(X, fun_control=fun_control)
y print(y)
y.shape= X.reshape(-1,1)
X_train = y
y_train
= Kriging(name='kriging',
S =123,
seed=50,
log_level=1,
n_theta=False)
noise
S.fit(X_train, y_train)
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
#plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted")
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Sphere: Gaussian process regression on noisy dataset") _
[[ 0.63529627]
[-4.10764204]
[-0.44071975]
[ 9.63125638]
[-8.3518118 ]
[-3.62418901]
[ 4.15331 ]
[ 3.4468512 ]
[ 6.36049088]
[-7.77978539]]
[-1.57464135 16.13714981 2.77008442 93.14904827 71.59322218 14.28895359
15.9770567 12.96468767 39.82265329 59.88028242]
S.log
{'negLnLike': array([26.18505386]),
'theta': array([-1.10547472]),
'p': [],
'Lambda': []}
= Kriging(name='kriging',
S =123,
seed=50,
log_level=1,
n_theta=True)
noise
S.fit(X_train, y_train)
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
#plt.plot(X, y, label=r"$f(x) = x \sin(x)$", linestyle="dotted")
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Sphere: Gaussian process regression with nugget on noisy dataset") _
S.log
{'negLnLike': array([21.82276721]),
'theta': array([-2.94197609]),
'p': [],
'Lambda': array([4.89634062e-05])}
12.12 Cubic Function
import numpy as np
import spotpython
from spotpython.fun.objectivefunctions import Analytical
from spotpython.spot import spot
from spotpython.design.spacefilling import SpaceFilling
from spotpython.build.kriging import Kriging
import matplotlib.pyplot as plt
= SpaceFilling(1)
gen = np.random.RandomState(1)
rng = np.array([-10])
lower = np.array([10])
upper = Analytical().fun_cubed
fun = fun_control_init(
fun_control ="07_Y",
PREFIX=10.0,
sigma=123,)
seed
= gen.scipy_lhd(10, lower=lower, upper = upper)
X print(X)
= fun(X, fun_control=fun_control)
y print(y)
y.shape= X.reshape(-1,1)
X_train = y
y_train
= Kriging(name='kriging', seed=123, log_level=50, n_theta=1, noise=False)
S
S.fit(X_train, y_train)
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Cubed: Gaussian process regression on noisy dataset") _
[[ 0.63529627]
[-4.10764204]
[-0.44071975]
[ 9.63125638]
[-8.3518118 ]
[-3.62418901]
[ 4.15331 ]
[ 3.4468512 ]
[ 6.36049088]
[-7.77978539]]
[ -9.63480707 -72.98497325 12.7936499 895.34567477 -573.35961837
-41.83176425 65.27989461 46.37081417 254.1530734 -474.09587355]
= Kriging(name='kriging', seed=123, log_level=0, n_theta=1, noise=True)
S
S.fit(X_train, y_train)
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Cubed: Gaussian process with nugget regression on noisy dataset") _
import numpy as np
import spotpython
from spotpython.fun.objectivefunctions import Analytical
from spotpython.spot import spot
from spotpython.design.spacefilling import SpaceFilling
from spotpython.build.kriging import Kriging
import matplotlib.pyplot as plt
= SpaceFilling(1)
gen = np.random.RandomState(1)
rng = np.array([-10])
lower = np.array([10])
upper = Analytical().fun_runge
fun = fun_control_init(
fun_control ="07_Y",
PREFIX=0.25,
sigma=123,)
seed
= gen.scipy_lhd(10, lower=lower, upper = upper)
X print(X)
= fun(X, fun_control=fun_control)
y print(y)
y.shape= X.reshape(-1,1)
X_train = y
y_train
= Kriging(name='kriging', seed=123, log_level=50, n_theta=1, noise=False)
S
S.fit(X_train, y_train)
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression on noisy dataset") _
[[ 0.63529627]
[-4.10764204]
[-0.44071975]
[ 9.63125638]
[-8.3518118 ]
[-3.62418901]
[ 4.15331 ]
[ 3.4468512 ]
[ 6.36049088]
[-7.77978539]]
[ 0.46517267 -0.03599548 1.15933822 0.05915901 0.24419145 0.21502359
-0.10432134 0.21312309 -0.05502681 -0.06434374]
= Kriging(name='kriging',
S =123,
seed=50,
log_level=1,
n_theta=True)
noise
S.fit(X_train, y_train)
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression with nugget on noisy dataset") _
12.13 Modifying Lambda Search Space
= Kriging(name='kriging',
S =123,
seed=50,
log_level=1,
n_theta=True,
noise=0.1,
min_Lambda=10)
max_Lambda
S.fit(X_train, y_train)
print(f"Lambda: {S.Lambda}")
Lambda: 0.8567558302695025
= np.linspace(start=-13, stop=13, num=1000).reshape(-1, 1)
X_axis = S.predict(X_axis, return_val="all")
mean_prediction, std_prediction, ei
="Observations")
plt.scatter(X_train, y_train, label#plt.plot(X, ei, label="Expected Improvement")
="mue")
plt.plot(X_axis, mean_prediction, label
plt.legend()"$x$")
plt.xlabel("$f(x)$")
plt.ylabel(= plt.title("Gaussian process regression with nugget on noisy dataset. Modified Lambda search space.") _