encoderblock

EncoderBlock

Bases: Module

Source code in spotpython/light/transformer/encoderblock.py

class EncoderBlock(nn.Module):
    def __init__(self, input_dim, num_heads, dim_feedforward, dropout=0.0) -> None:
        """
        Initializes the EncoderBlock object.

        Args:
            input_dim (int): The dimensionality of the input.
            num_heads (int): The number of heads to use in the attention block.
            dim_feedforward (int): The dimensionality of the hidden layer in the MLP.
            dropout (float): The dropout probability to use in the dropout layers.

        Returns:
            None
        """
        super().__init__()

        # Attention layer
        self.self_attn = MultiheadAttention(input_dim, input_dim, num_heads)

        # Two-layer MLP
        self.linear_net = nn.Sequential(
            nn.Linear(input_dim, dim_feedforward),
            nn.Dropout(dropout),
            nn.ReLU(inplace=True),
            nn.Linear(dim_feedforward, input_dim),
        )

        # Layers to apply in between the main layers
        self.norm1 = nn.LayerNorm(input_dim)
        self.norm2 = nn.LayerNorm(input_dim)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, mask=None):
        # Attention part
        attn_out = self.self_attn(x, mask=mask)
        x = x + self.dropout(attn_out)
        x = self.norm1(x)

        # MLP part
        linear_out = self.linear_net(x)
        x = x + self.dropout(linear_out)
        x = self.norm2(x)

        return x

__init__(input_dim, num_heads, dim_feedforward, dropout=0.0)

Initializes the EncoderBlock object.

Parameters:

Name	Type	Description	Default
input_dim	int	The dimensionality of the input.	required
num_heads	int	The number of heads to use in the attention block.	required
dim_feedforward	int	The dimensionality of the hidden layer in the MLP.	required
dropout	float	The dropout probability to use in the dropout layers.	0.0

Returns:

Type	Description
None	None

Source code in spotpython/light/transformer/encoderblock.py

def __init__(self, input_dim, num_heads, dim_feedforward, dropout=0.0) -> None:
    """
    Initializes the EncoderBlock object.

    Args:
        input_dim (int): The dimensionality of the input.
        num_heads (int): The number of heads to use in the attention block.
        dim_feedforward (int): The dimensionality of the hidden layer in the MLP.
        dropout (float): The dropout probability to use in the dropout layers.

    Returns:
        None
    """
    super().__init__()

    # Attention layer
    self.self_attn = MultiheadAttention(input_dim, input_dim, num_heads)

    # Two-layer MLP
    self.linear_net = nn.Sequential(
        nn.Linear(input_dim, dim_feedforward),
        nn.Dropout(dropout),
        nn.ReLU(inplace=True),
        nn.Linear(dim_feedforward, input_dim),
    )

    # Layers to apply in between the main layers
    self.norm1 = nn.LayerNorm(input_dim)
    self.norm2 = nn.LayerNorm(input_dim)
    self.dropout = nn.Dropout(dropout)