attention

scaled_dot_product(q, k, v, mask=None)

Compute scaled dot product attention.
Args:
    q: Queries
    k: Keys
    v: Values
    mask: Mask to apply to the attention logits

Returns:
    Tuple of (Values, Attention weights)

Examples:
>>> from spotpython.light.transformer.attention import scaled_dot_product
    seq_len, d_k = 1, 2
    pl.seed_everything(42)
    q = torch.randn(seq_len, d_k)
    k = torch.randn(seq_len, d_k)
    v = torch.randn(seq_len, d_k)
    values, attention = scaled_dot_product(q, k, v)
    print("Q

”, q) print(“K “, k) print(“V “, v) print(“Values “, values) print(“Attention “, attention)

Source code in spotpython/light/transformer/attention.py

def scaled_dot_product(q, k, v, mask=None):
    """
    Compute scaled dot product attention.
    Args:
        q: Queries
        k: Keys
        v: Values
        mask: Mask to apply to the attention logits

    Returns:
        Tuple of (Values, Attention weights)

    Examples:
    >>> from spotpython.light.transformer.attention import scaled_dot_product
        seq_len, d_k = 1, 2
        pl.seed_everything(42)
        q = torch.randn(seq_len, d_k)
        k = torch.randn(seq_len, d_k)
        v = torch.randn(seq_len, d_k)
        values, attention = scaled_dot_product(q, k, v)
        print("Q\n", q)
        print("K\n", k)
        print("V\n", v)
        print("Values\n", values)
        print("Attention\n", attention)
    """
    d_k = q.size()[-1]
    attn_logits = torch.matmul(q, k.transpose(-2, -1))
    attn_logits = attn_logits / math.sqrt(d_k)
    if mask is not None:
        attn_logits = attn_logits.masked_fill(mask == 0, -9e15)
    attention = F.softmax(attn_logits, dim=-1)
    values = torch.matmul(attention, v)
    return values, attention