Natural Language Processing

tensoris.backend.components.blocks.nlp

Natural Language Processing Transformer Encoder Blocks.

Classes

TransformerEncoderBlock

Bases: Module

Transformer Encoder Block with Multi-Head Self Attention and FeedForward network.

References

Vaswani, A. et al. (2017). Attention Is All You Need. NIPS 2017. arXiv: https://arxiv.org/abs/1706.03762

Source code in src/tensoris/backend/components/blocks/nlp.py

class TransformerEncoderBlock(nn.Module):
    """Transformer Encoder Block with Multi-Head Self Attention and FeedForward network.

    References:
        Vaswani, A. et al. (2017). Attention Is All You Need. NIPS 2017.
        arXiv: https://arxiv.org/abs/1706.03762
    """

    def __init__(
        self,
        d_model: int = 256,
        nhead: int = 4,
        dim_feedforward: int = 512,
        dropout: float = 0.1,
    ) -> None:
        """Initialize TransformerEncoderBlock parameters.

        Args:
            d_model: Feature embedding dimension.
            nhead: Number of parallel attention heads.
            dim_feedforward: Hidden dimension of feedforward network.
            dropout: Dropout probability.
        """
        super().__init__()
        self.encoder_layer = nn.TransformerEncoderLayer(
            d_model=d_model,
            nhead=nhead,
            dim_feedforward=dim_feedforward,
            dropout=dropout,
            batch_first=True,
        )

    def forward(
        self, x: torch.Tensor, mask: torch.Tensor | None = None
    ) -> torch.Tensor:
        """Execute TransformerEncoderBlock forward pass."""
        return self.encoder_layer(x, src_key_padding_mask=mask)

Methods:

__init__

__init__(
    d_model=256, nhead=4, dim_feedforward=512, dropout=0.1
)

Initialize TransformerEncoderBlock parameters.

Parameters:

Name	Type	Description	Default
d_model	int	Feature embedding dimension.	256
nhead	int	Number of parallel attention heads.	4
dim_feedforward	int	Hidden dimension of feedforward network.	512
dropout	float	Dropout probability.	0.1

Source code in src/tensoris/backend/components/blocks/nlp.py

def __init__(
    self,
    d_model: int = 256,
    nhead: int = 4,
    dim_feedforward: int = 512,
    dropout: float = 0.1,
) -> None:
    """Initialize TransformerEncoderBlock parameters.

    Args:
        d_model: Feature embedding dimension.
        nhead: Number of parallel attention heads.
        dim_feedforward: Hidden dimension of feedforward network.
        dropout: Dropout probability.
    """
    super().__init__()
    self.encoder_layer = nn.TransformerEncoderLayer(
        d_model=d_model,
        nhead=nhead,
        dim_feedforward=dim_feedforward,
        dropout=dropout,
        batch_first=True,
    )

forward

forward(x, mask=None)

Execute TransformerEncoderBlock forward pass.

Source code in src/tensoris/backend/components/blocks/nlp.py

def forward(
    self, x: torch.Tensor, mask: torch.Tensor | None = None
) -> torch.Tensor:
    """Execute TransformerEncoderBlock forward pass."""
    return self.encoder_layer(x, src_key_padding_mask=mask)