41  Hyperparameter Tuning with spotpython and PyTorch Lightning Using a CondNet Model

• We use the Diabetes dataset to illustrate the hyperparameter tuning process of a CondNet model using the spotpython package.
• The CondNet model is a conditional neural network that can be used to model conditional distributions [LINK].

from spotpython.data.diabetes import Diabetes
from spotpython.hyperdict.light_hyper_dict import LightHyperDict
from spotpython.fun.hyperlight import HyperLight
from spotpython.utils.init import (fun_control_init, surrogate_control_init, design_control_init)
from spotpython.utils.eda import gen_design_table
from spotpython.spot import spot
from spotpython.utils.file import get_experiment_filename
from math import inf
from spotpython.hyperparameters.values import set_hyperparameter

PREFIX="CondNet_01"

data_set = Diabetes()
input_dim = 10
output_dim = 1
cond_dim = 2

fun_control = fun_control_init(
    PREFIX=PREFIX,
    fun_evals=inf,
    max_time=1,
    data_set = data_set,
    core_model_name="light.regression.NNCondNetRegressor",
    hyperdict=LightHyperDict,
    _L_in=input_dim - cond_dim,
    _L_out=1,
    _L_cond=cond_dim,)

fun = HyperLight().fun


set_hyperparameter(fun_control, "optimizer", [ "Adadelta", "Adam", "Adamax"])
set_hyperparameter(fun_control, "l1", [3,4])
set_hyperparameter(fun_control, "epochs", [3,7])
set_hyperparameter(fun_control, "batch_size", [4,5])
set_hyperparameter(fun_control, "dropout_prob", [0.0, 0.025])
set_hyperparameter(fun_control, "patience", [2,3])
set_hyperparameter(fun_control, "lr_mult", [0.1, 20.0])

design_control = design_control_init(init_size=10)

print(gen_design_table(fun_control))

module_name: light
submodule_name: regression
model_name: NNCondNetRegressor
| name           | type   | default   |   lower |   upper | transform             |
|----------------|--------|-----------|---------|---------|-----------------------|
| l1             | int    | 3         |     3   |   4     | transform_power_2_int |
| epochs         | int    | 4         |     3   |   7     | transform_power_2_int |
| batch_size     | int    | 4         |     4   |   5     | transform_power_2_int |
| act_fn         | factor | ReLU      |     0   |   5     | None                  |
| optimizer      | factor | SGD       |     0   |   2     | None                  |
| dropout_prob   | float  | 0.01      |     0   |   0.025 | None                  |
| lr_mult        | float  | 1.0       |     0.1 |  20     | None                  |
| patience       | int    | 2         |     2   |   3     | transform_power_2_int |
| batch_norm     | factor | 0         |     0   |   1     | None                  |
| initialization | factor | Default   |     0   |   4     | None                  |

spot_tuner = spot.Spot(fun=fun,fun_control=fun_control, design_control=design_control)
res = spot_tuner.run()

In fun(): config:
{'act_fn': Sigmoid(),
 'batch_norm': False,
 'batch_size': 32,
 'dropout_prob': np.float64(0.010469763733360567),
 'epochs': 16,
 'initialization': 'xavier_uniform',
 'l1': 16,
 'lr_mult': np.float64(8.212543453926155),
 'optimizer': 'Adam',
 'patience': 4}

train_model result: {'val_loss': 24158.83203125, 'hp_metric': 24158.83203125}

In fun(): config:
{'act_fn': ReLU(),
 'batch_norm': False,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0184251494885258),
 'epochs': 32,
 'initialization': 'kaiming_normal',
 'l1': 8,
 'lr_mult': np.float64(6.2144595555349165),
 'optimizer': 'Adadelta',
 'patience': 8}

train_model result: {'val_loss': 23447.546875, 'hp_metric': 23447.546875}

In fun(): config:
{'act_fn': ELU(),
 'batch_norm': True,
 'batch_size': 32,
 'dropout_prob': np.float64(0.00996276270809942),
 'epochs': 64,
 'initialization': 'Default',
 'l1': 16,
 'lr_mult': np.float64(17.072444874507983),
 'optimizer': 'Adadelta',
 'patience': 4}

train_model result: {'val_loss': 7307.08984375, 'hp_metric': 7307.08984375}

In fun(): config:
{'act_fn': LeakyReLU(),
 'batch_norm': True,
 'batch_size': 32,
 'dropout_prob': np.float64(0.004305336774252681),
 'epochs': 8,
 'initialization': 'kaiming_normal',
 'l1': 8,
 'lr_mult': np.float64(0.5038833291649055),
 'optimizer': 'Adamax',
 'patience': 4}

train_model result: {'val_loss': 23786.861328125, 'hp_metric': 23786.861328125}

In fun(): config:
{'act_fn': Tanh(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.021718144359373085),
 'epochs': 32,
 'initialization': 'kaiming_uniform',
 'l1': 16,
 'lr_mult': np.float64(15.991195791187765),
 'optimizer': 'Adam',
 'patience': 8}

train_model result: {'val_loss': 22770.90625, 'hp_metric': 22770.90625}

In fun(): config:
{'act_fn': LeakyReLU(),
 'batch_norm': False,
 'batch_size': 16,
 'dropout_prob': np.float64(0.023931753071792624),
 'epochs': 16,
 'initialization': 'xavier_normal',
 'l1': 16,
 'lr_mult': np.float64(2.4181463288559466),
 'optimizer': 'Adamax',
 'patience': 8}

train_model result: {'val_loss': 23846.84375, 'hp_metric': 23846.84375}

In fun(): config:
{'act_fn': ELU(),
 'batch_norm': False,
 'batch_size': 32,
 'dropout_prob': np.float64(0.0074444117802003025),
 'epochs': 8,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(19.065991931545753),
 'optimizer': 'Adam',
 'patience': 8}

train_model result: {'val_loss': 23610.291015625, 'hp_metric': 23610.291015625}

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0012790404219919403),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(4.855811791679552),
 'optimizer': 'Adadelta',
 'patience': 4}

train_model result: {'val_loss': 4652.201171875, 'hp_metric': 4652.201171875}

In fun(): config:
{'act_fn': Tanh(),
 'batch_norm': False,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0153979445945591),
 'epochs': 32,
 'initialization': 'xavier_uniform',
 'l1': 8,
 'lr_mult': np.float64(12.138553034122333),
 'optimizer': 'Adamax',
 'patience': 4}

train_model result: {'val_loss': 22452.896484375, 'hp_metric': 22452.896484375}

In fun(): config:
{'act_fn': ReLU(),
 'batch_norm': True,
 'batch_size': 32,
 'dropout_prob': np.float64(0.013939072152682473),
 'epochs': 64,
 'initialization': 'xavier_uniform',
 'l1': 16,
 'lr_mult': np.float64(11.738437881491578),
 'optimizer': 'Adam',
 'patience': 8}
train_model result: {'val_loss': 22722.826171875, 'hp_metric': 22722.826171875}

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0033682803176620702),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(8.795451519544661),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 3762.646728515625, 'hp_metric': 3762.646728515625}
spotpython tuning: 3762.646728515625 [----------] 4.64% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.001483504871229586),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(8.965542571854463),
 'optimizer': 'Adadelta',
 'patience': 4}

train_model result: {'val_loss': 4189.49609375, 'hp_metric': 4189.49609375}
spotpython tuning: 3762.646728515625 [#---------] 8.04% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.025),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(7.029758527728445),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 3598.41845703125, 'hp_metric': 3598.41845703125}
spotpython tuning: 3598.41845703125 [#---------] 12.40% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(7.059808792052545),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 4432.4228515625, 'hp_metric': 4432.4228515625}
spotpython tuning: 3598.41845703125 [##--------] 17.87% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.025),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(8.172224667427638),
 'optimizer': 'Adadelta',
 'patience': 4}

train_model result: {'val_loss': 17434.79296875, 'hp_metric': 17434.79296875}
spotpython tuning: 3598.41845703125 [#####-----] 47.62% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.025),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(6.863815152944327),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 4271.0537109375, 'hp_metric': 4271.0537109375}
spotpython tuning: 3598.41845703125 [#####-----] 52.20% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(8.850235720504392),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 9081.8525390625, 'hp_metric': 9081.8525390625}
spotpython tuning: 3598.41845703125 [#####-----] 54.95% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0009178232930834657),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(8.754200791960354),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 8019.37744140625, 'hp_metric': 8019.37744140625}
spotpython tuning: 3598.41845703125 [######----] 58.49% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.0024940202257315303),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(8.986811004365546),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 3723.706298828125, 'hp_metric': 3723.706298828125}
spotpython tuning: 3598.41845703125 [#######---] 65.93% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.021028581178670843),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(7.028166774184535),
 'optimizer': 'Adadelta',
 'patience': 4}

train_model result: {'val_loss': 18928.12109375, 'hp_metric': 18928.12109375}
spotpython tuning: 3598.41845703125 [##########] 96.55% 

In fun(): config:
{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.025),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(7.101842452512002),
 'optimizer': 'Adadelta',
 'patience': 4}
train_model result: {'val_loss': 4800.939453125, 'hp_metric': 4800.939453125}
spotpython tuning: 3598.41845703125 [##########] 100.00% Done...

41.1 Looking at the Results

41.1.1 Tuning Progress

After the hyperparameter tuning run is finished, the progress of the hyperparameter tuning can be visualized with spotpython’s method plot_progress. The black points represent the performace values (score or metric) of hyperparameter configurations from the initial design, whereas the red points represents the hyperparameter configurations found by the surrogate model based optimization.

spot_tuner.plot_progress()

41.1.2 Tuned Hyperparameters and Their Importance

Results can be printed in tabular form.

from spotpython.utils.eda import gen_design_table
print(gen_design_table(fun_control=fun_control, spot=spot_tuner))

| name           | type   | default   |   lower |   upper | tuned             | transform             |   importance | stars   |
|----------------|--------|-----------|---------|---------|-------------------|-----------------------|--------------|---------|
| l1             | int    | 3         |     3.0 |     4.0 | 3.0               | transform_power_2_int |        23.91 | *       |
| epochs         | int    | 4         |     3.0 |     7.0 | 7.0               | transform_power_2_int |         1.87 | *       |
| batch_size     | int    | 4         |     4.0 |     5.0 | 4.0               | transform_power_2_int |         0.17 | .       |
| act_fn         | factor | ReLU      |     0.0 |     5.0 | Swish             | None                  |         0.19 | .       |
| optimizer      | factor | SGD       |     0.0 |     2.0 | Adadelta          | None                  |         2.54 | *       |
| dropout_prob   | float  | 0.01      |     0.0 |   0.025 | 0.025             | None                  |       100.00 | ***     |
| lr_mult        | float  | 1.0       |     0.1 |    20.0 | 7.029758527728445 | None                  |       100.00 | ***     |
| patience       | int    | 2         |     2.0 |     3.0 | 2.0               | transform_power_2_int |         0.09 |         |
| batch_norm     | factor | 0         |     0.0 |     1.0 | 1                 | None                  |        12.05 | *       |
| initialization | factor | Default   |     0.0 |     4.0 | kaiming_uniform   | None                  |         0.00 |         |

A histogram can be used to visualize the most important hyperparameters.

spot_tuner.plot_importance(threshold=1.0)

spot_tuner.plot_important_hyperparameter_contour(max_imp=3)

l1:  23.905541394452918
epochs:  1.8673607015801097
batch_size:  0.17416714976924005
act_fn:  0.19060411498209742
optimizer:  2.5444315773597572
dropout_prob:  100.0
lr_mult:  100.0
patience:  0.09336005323992341
batch_norm:  12.049903681595016
initialization:  0.0016963375212337038

41.1.3 Get the Tuned Architecture

import pprint
from spotpython.hyperparameters.values import get_tuned_architecture
config = get_tuned_architecture(spot_tuner, fun_control)
pprint.pprint(config)

{'act_fn': Swish(),
 'batch_norm': True,
 'batch_size': 16,
 'dropout_prob': np.float64(0.025),
 'epochs': 128,
 'initialization': 'kaiming_uniform',
 'l1': 8,
 'lr_mult': np.float64(7.029758527728445),
 'optimizer': 'Adadelta',
 'patience': 4}