{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 63,
        "slug": "attention-mechanisms",
        "title": "Attention Mechanisms in Deep Learning",
        "description": "Attention architectures: self-attention, multi-head attention, cross-attention, scaled dot-product, sparse attention, and efficient attention variants (Linformer, Performer, FlashAttention).",
        "summary": "Attention mechanisms in deep learning with formula, complexity, description, uses and notes: scaled dot-product, multi-head, cross-attention, causal masking, sparse attention, FlashAttention and linear attention variants.",
        "use_when": [
            "You need the definition and cost of an attention variant.",
            "You are comparing attention approaches for long sequences."
        ],
        "caveats": [
            "Standard attention cost grows quadratically with sequence length, which motivates sparse and linear variants.",
            "Approximations can trade quality for speed; benchmark on your task."
        ],
        "related": [
            "ml-model-architectures",
            "neural-network-types",
            "llm-glossary",
            "deep-learning-optimizers"
        ],
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "attention,self-attention,transformer,multi-head,cross-attention,sparse-attention,flash-attention",
        "view_count": 0,
        "created_at": 1778695229,
        "updated_at": 1778695229,
        "entry_count": 8,
        "fields": [
            {
                "section": "mechanisms",
                "field": "name",
                "type": "string",
                "example": "Scaled Dot-Product Attention"
            },
            {
                "section": "mechanisms",
                "field": "formula",
                "type": "string",
                "example": "Attention(Q,K,V) = softmax(QK^T / √d_k) V"
            },
            {
                "section": "mechanisms",
                "field": "complexity",
                "type": "string",
                "example": "O(n² * d)"
            },
            {
                "section": "mechanisms",
                "field": "description",
                "type": "string",
                "example": "Core attention mechanism: queries attend to key…"
            },
            {
                "section": "mechanisms",
                "field": "use_cases",
                "type": "array",
                "example": "[3 items]"
            },
            {
                "section": "mechanisms",
                "field": "notes",
                "type": "string",
                "example": "Scaling by √d_k prevents softmax saturation for…"
            },
            {
                "section": "mechanisms",
                "field": "mechanism",
                "type": "string",
                "example": "Q from one sequence, K,V from another"
            },
            {
                "section": "mechanisms",
                "field": "variants",
                "type": "array",
                "example": "[3 items]"
            },
            {
                "section": "mechanisms",
                "field": "connection",
                "type": "string",
                "example": "Attention weights can be interpreted as soft al…"
            }
        ]
    },
    "data": {
        "mechanisms": [
            {
                "name": "Scaled Dot-Product Attention",
                "formula": "Attention(Q,K,V) = softmax(QK^T / √d_k) V",
                "complexity": "O(n² * d)",
                "description": "Core attention mechanism: queries attend to keys, weighted values are summed",
                "use_cases": [
                    "Foundation of all Transformer models",
                    "Sequence-to-sequence tasks",
                    "Any pairwise interaction modeling"
                ],
                "notes": "Scaling by √d_k prevents softmax saturation for large d_k; Q=K=V gives self-attention"
            },
            {
                "name": "Multi-Head Attention",
                "formula": "Concatenate h heads of attention with different learned projections",
                "complexity": "O(n² * d) (same as single head, parallelized)",
                "description": "Multiple attention heads capture different representation subspaces",
                "use_cases": [
                    "All Transformer architectures",
                    "Cross-modal attention (vision-language)",
                    "Multi-modal fusion"
                ],
                "notes": "Each head: Q_i = XW_i^Q, K_i = XW_i^K, V_i = XW_i^V; typical h=8,12,16"
            },
            {
                "name": "Cross-Attention",
                "mechanism": "Q from one sequence, K,V from another",
                "complexity": "O(n * m * d)",
                "description": "Attention between two different sequences; decoder attends to encoder output",
                "use_cases": [
                    "Encoder-decoder (translation, summarization)",
                    "Image captioning (text attends to image features)",
                    "Multimodal models"
                ],
                "notes": "Q projects decoder state; K,V project encoder output; different sequence lengths allowed"
            },
            {
                "name": "Causal (Masked) Attention",
                "mechanism": "Upper triangular mask on attention scores: M[i,j] = -∞ for j > i",
                "complexity": "O(n² * d) but only n(n+1)/2 actual computations",
                "description": "Prevents positions from attending to future positions; essential for autoregressive generation",
                "use_cases": [
                    "GPT-style language models",
                    "Autoregressive generation",
                    "Any causal sequence modeling"
                ],
                "notes": "Enables efficient KV cache during inference: only compute for new token, reuse cached K,V"
            },
            {
                "name": "Sparse Attention",
                "variants": [
                    "Fixed pattern (local/windowed)",
                    "Strided",
                    "Learned sparse patterns"
                ],
                "complexity": "O(n * w * d) for window size w (linear in n)",
                "description": "Restrict attention to subset of positions; reduces quadratic complexity",
                "use_cases": [
                    "Long document processing",
                    "Genomic sequences",
                    "When full attention is too expensive"
                ],
                "notes": "Longformer, BigBird, Reformer use different sparsity patterns; trade-off between expressiveness and efficiency"
            },
            {
                "name": "FlashAttention",
                "mechanism": "Tiling + recomputation to minimize HBM (GPU memory) reads/writes",
                "complexity": "O(n² * d) FLOPs but O(n²) memory I/O (vs O(n²d) for standard)",
                "description": "Exact attention with optimal memory access patterns; 2-4x faster than standard attention",
                "use_cases": [
                    "Training large language models",
                    "Long-context models",
                    "Any attention-heavy workload"
                ],
                "notes": "No approximation; same mathematical result as standard attention; now default in most training frameworks"
            },
            {
                "name": "Linear Attention (Performer, Linformer)",
                "mechanism": "Approximate softmax kernel with random features or low-rank projection",
                "complexity": "O(n * d²) or O(n * log n)",
                "description": "Approximate attention with linear complexity in sequence length",
                "use_cases": [
                    "Very long sequences (10K+ tokens)",
                    "When quadratic attention is infeasible",
                    "Real-time inference on long inputs"
                ],
                "notes": "Performer uses FAVOR+ (Fast Attention via positive Orthogonal Random Features); Linformer projects K,V to lower dimension"
            },
            {
                "name": "Cross-Entropy Loss with Attention",
                "connection": "Attention weights can be interpreted as soft alignment probabilities",
                "description": "Attention provides interpretable alignment: which input tokens influenced each output token",
                "use_cases": [
                    "Interpretable NLP models",
                    "Attention visualization",
                    "Debugging model behavior"
                ],
                "notes": "Attention ≠ explanation (caution: attention weights don't always indicate feature importance)"
            }
        ]
    }
}