api: YAML JSON TOON Database
version: 1.0.0
format: json
dataset:
  id: 63
  slug: attention-mechanisms
  title: Attention Mechanisms in Deep Learning
  description: "Attention architectures: self-attention, multi-head attention, cross-attention, scaled dot-product, sparse attention, and efficient attention variants (Linformer, Performer, FlashAttention)."
  summary: "Attention mechanisms in deep learning with formula, complexity, description, uses and notes: scaled dot-product, multi-head, cross-attention, causal masking, sparse attention, FlashAttention and linear attention variants."
  use_when[2]: You need the definition and cost of an attention variant.,You are comparing attention approaches for long sequences.
  caveats[2]: "Standard attention cost grows quadratically with sequence length, which motivates sparse and linear variants.",Approximations can trade quality for speed; benchmark on your task.
  related[4]: ml-model-architectures,neural-network-types,llm-glossary,deep-learning-optimizers
  category: AI & ML Terms
  category_slug: ai-ml-terms
  tags: "attention,self-attention,transformer,multi-head,cross-attention,sparse-attention,flash-attention"
  view_count: 0
  created_at: 1778695229
  updated_at: 1778695229
  entry_count: 8
  fields[9]{section,field,type,example}:
    mechanisms,name,string,Scaled Dot-Product Attention
    mechanisms,formula,string,"Attention(Q,K,V) = softmax(QK^T / √d_k) V"
    mechanisms,complexity,string,O(n² * d)
    mechanisms,description,string,"Core attention mechanism: queries attend to key…"
    mechanisms,use_cases,array,"[3 items]"
    mechanisms,notes,string,Scaling by √d_k prevents softmax saturation for…
    mechanisms,mechanism,string,"Q from one sequence, K,V from another"
    mechanisms,variants,array,"[3 items]"
    mechanisms,connection,string,Attention weights can be interpreted as soft al…
data:
  mechanisms[8]:
    - name: Scaled Dot-Product Attention
      formula: "Attention(Q,K,V) = softmax(QK^T / √d_k) V"
      complexity: O(n² * d)
      description: "Core attention mechanism: queries attend to keys, weighted values are summed"
      use_cases[3]: Foundation of all Transformer models,Sequence-to-sequence tasks,Any pairwise interaction modeling
      notes: Scaling by √d_k prevents softmax saturation for large d_k; Q=K=V gives self-attention
    - name: Multi-Head Attention
      formula: Concatenate h heads of attention with different learned projections
      complexity: "O(n² * d) (same as single head, parallelized)"
      description: Multiple attention heads capture different representation subspaces
      use_cases[3]: All Transformer architectures,Cross-modal attention (vision-language),Multi-modal fusion
      notes: "Each head: Q_i = XW_i^Q, K_i = XW_i^K, V_i = XW_i^V; typical h=8,12,16"
    - name: Cross-Attention
      mechanism: "Q from one sequence, K,V from another"
      complexity: O(n * m * d)
      description: Attention between two different sequences; decoder attends to encoder output
      use_cases[3]: "Encoder-decoder (translation, summarization)",Image captioning (text attends to image features),Multimodal models
      notes: "Q projects decoder state; K,V project encoder output; different sequence lengths allowed"
    - name: Causal (Masked) Attention
      mechanism: "Upper triangular mask on attention scores: M[i,j] = -∞ for j > i"
      complexity: O(n² * d) but only n(n+1)/2 actual computations
      description: Prevents positions from attending to future positions; essential for autoregressive generation
      use_cases[3]: GPT-style language models,Autoregressive generation,Any causal sequence modeling
      notes: "Enables efficient KV cache during inference: only compute for new token, reuse cached K,V"
    - name: Sparse Attention
      variants[3]: Fixed pattern (local/windowed),Strided,Learned sparse patterns
      complexity: O(n * w * d) for window size w (linear in n)
      description: Restrict attention to subset of positions; reduces quadratic complexity
      use_cases[3]: Long document processing,Genomic sequences,When full attention is too expensive
      notes: "Longformer, BigBird, Reformer use different sparsity patterns; trade-off between expressiveness and efficiency"
    - name: FlashAttention
      mechanism: Tiling + recomputation to minimize HBM (GPU memory) reads/writes
      complexity: O(n² * d) FLOPs but O(n²) memory I/O (vs O(n²d) for standard)
      description: Exact attention with optimal memory access patterns; 2-4x faster than standard attention
      use_cases[3]: Training large language models,Long-context models,Any attention-heavy workload
      notes: No approximation; same mathematical result as standard attention; now default in most training frameworks
    - name: "Linear Attention (Performer, Linformer)"
      mechanism: Approximate softmax kernel with random features or low-rank projection
      complexity: O(n * d²) or O(n * log n)
      description: Approximate attention with linear complexity in sequence length
      use_cases[3]: Very long sequences (10K+ tokens),When quadratic attention is infeasible,Real-time inference on long inputs
      notes: "Performer uses FAVOR+ (Fast Attention via positive Orthogonal Random Features); Linformer projects K,V to lower dimension"
    - name: Cross-Entropy Loss with Attention
      connection: Attention weights can be interpreted as soft alignment probabilities
      description: "Attention provides interpretable alignment: which input tokens influenced each output token"
      use_cases[3]: Interpretable NLP models,Attention visualization,Debugging model behavior
      notes: "Attention ≠ explanation (caution: attention weights don't always indicate feature importance)"