% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/nnf-activation.R
\name{nnf_multi_head_attention_forward}
\alias{nnf_multi_head_attention_forward}
\title{Multi head attention forward}
\usage{
nnf_multi_head_attention_forward(
  query,
  key,
  value,
  embed_dim_to_check,
  num_heads,
  in_proj_weight,
  in_proj_bias,
  bias_k,
  bias_v,
  add_zero_attn,
  dropout_p,
  out_proj_weight,
  out_proj_bias,
  training = TRUE,
  key_padding_mask = NULL,
  need_weights = TRUE,
  attn_mask = NULL,
  avg_weights = TRUE,
  use_separate_proj_weight = FALSE,
  q_proj_weight = NULL,
  k_proj_weight = NULL,
  v_proj_weight = NULL,
  static_k = NULL,
  static_v = NULL,
  batch_first = FALSE
)
}
\arguments{
\item{query}{\eqn{(L, N, E)} where L is the target sequence length, N is the batch size, E is
the embedding dimension. If batch_first is TRUE, the first two dimensions are transposed.}

\item{key}{\eqn{(S, N, E)}, where S is the source sequence length, N is the batch size, E is
the embedding dimension. If batch_first is TRUE, the first two dimensions are transposed.}

\item{value}{\eqn{(S, N, E)} where S is the source sequence length, N is the batch size, E is
the embedding dimension. If batch_first is TRUE, the first two dimensions are transposed.}

\item{embed_dim_to_check}{total dimension of the model.}

\item{num_heads}{parallel attention heads.}

\item{in_proj_weight}{input projection weight and bias.}

\item{in_proj_bias}{currently undocumented.}

\item{bias_k}{bias of the key and value sequences to be added at dim=0.}

\item{bias_v}{currently undocumented.}

\item{add_zero_attn}{add a new batch of zeros to the key and
value sequences at dim=1.}

\item{dropout_p}{probability of an element to be zeroed.}

\item{out_proj_weight}{the output projection weight and bias.}

\item{out_proj_bias}{currently undocumented.}

\item{training}{apply dropout if is \code{TRUE}.}

\item{key_padding_mask}{\eqn{(N, S)} where N is the batch size, S is the source sequence length.
If a ByteTensor is provided, the non-zero positions will be ignored while the position
with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the
value of \code{True} will be ignored while the position with the value of \code{False} will be unchanged.}

\item{need_weights}{output attn_output_weights.}

\item{attn_mask}{2D mask \eqn{(L, S)} where L is the target sequence length, S is the source sequence length.
3D mask \eqn{(N*num_heads, L, S)} where N is the batch size, L is the target sequence length,
S is the source sequence length. attn_mask ensure that position i is allowed to attend the unmasked
positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend
while the zero positions will be unchanged. If a BoolTensor is provided, positions with \code{True}
is not allowed to attend while \code{False} values will be unchanged. If a FloatTensor
is provided, it will be added to the attention weight.}

\item{avg_weights}{Logical; whether to average attn_output_weights over the
attention heads before outputting them. This doesn't change the returned
value of attn_output; it only affects the returned attention weight matrix.}

\item{use_separate_proj_weight}{the function accept the proj. weights for
query, key, and value in different forms. If false, in_proj_weight will be used,
which is a combination of q_proj_weight, k_proj_weight, v_proj_weight.}

\item{q_proj_weight}{input projection weight and bias.}

\item{k_proj_weight}{currently undocumented.}

\item{v_proj_weight}{currently undocumented.}

\item{static_k}{static key and value used for attention operators.}

\item{static_v}{currently undocumented.}

\item{batch_first}{Logical; whether to expect query, key, and value to have
batch as their first parameter, and to return output with batch first.}
}
\description{
Allows the model to jointly attend to information from different representation
subspaces. See reference: Attention Is All You Need
}
