{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 28,
        "slug": "ml-training-algorithms",
        "title": "Machine Learning Training Algorithms",
        "description": "Core ML training algorithms: gradient descent variants, backpropagation, optimization methods (SGD, Adam, RMSprop), and when to use each.",
        "summary": "Training algorithms for machine learning, including gradient descent in its batch, stochastic and mini-batch forms, momentum, RMSprop, Adam, backpropagation and learning-rate schedules, each with type, mechanism, update rule, pros, cons and when to use it, plus practical tips.",
        "use_when": [
            "You need to understand or choose a training procedure.",
            "You are debugging unstable training."
        ],
        "caveats": [
            "Training behavior depends on initialization, batch size and learning rate; defaults do not transfer across all problems.",
            "Backpropagation computes gradients; the optimizer decides how to use them."
        ],
        "related": [
            "deep-learning-optimizers",
            "regularization-techniques",
            "transfer-learning",
            "ml-evaluation-metrics"
        ],
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "machine-learning,training,optimization,gradient-descent,backpropagation",
        "view_count": 0,
        "created_at": 1777673262,
        "updated_at": 1777673262,
        "entry_count": 13,
        "fields": [
            {
                "section": "algorithms",
                "field": "name",
                "type": "string",
                "example": "Gradient Descent (Batch)"
            },
            {
                "section": "algorithms",
                "field": "type",
                "type": "string",
                "example": "First-order optimization"
            },
            {
                "section": "algorithms",
                "field": "how_it_works",
                "type": "string",
                "example": "Computes gradient over entire training set, upd…"
            },
            {
                "section": "algorithms",
                "field": "update_rule",
                "type": "string",
                "example": "θ = θ - α × ∇J(θ)"
            },
            {
                "section": "algorithms",
                "field": "pros",
                "type": "array",
                "example": "[2 items]"
            },
            {
                "section": "algorithms",
                "field": "cons",
                "type": "array",
                "example": "[2 items]"
            },
            {
                "section": "algorithms",
                "field": "use_when",
                "type": "string",
                "example": "Small-to-medium datasets, convex optimization, …"
            },
            {
                "section": "algorithms",
                "field": "hyperparameters",
                "type": "object",
                "example": "{γ (momentum),α (learning rate)}"
            },
            {
                "section": "algorithms",
                "field": "phases",
                "type": "array",
                "example": "[3 items]"
            },
            {
                "section": "algorithms",
                "field": "key_formula",
                "type": "string",
                "example": "δ = ∇ₐL ⊙ σ'(z); ∇ₐL = Wᵀδ (for layer)"
            },
            {
                "section": "algorithms",
                "field": "notes",
                "type": "string",
                "example": "Not an optimizer itself — computes gradients fo…"
            },
            {
                "section": "algorithms",
                "field": "variants",
                "type": "array",
                "example": "[3 items]"
            },
            {
                "section": "algorithms",
                "field": "strategies",
                "type": "object",
                "example": "{step_decay,exponential_decay,1/t_decay,cosine_annealing}"
            },
            {
                "section": "algorithms",
                "field": "purpose",
                "type": "string",
                "example": "Prevent oscillation near minimum, allow large i…"
            },
            {
                "section": "practical_tips",
                "field": "*",
                "type": "array",
                "example": "[5 items]"
            }
        ]
    },
    "data": {
        "algorithms": [
            {
                "name": "Gradient Descent (Batch)",
                "type": "First-order optimization",
                "how_it_works": "Computes gradient over entire training set, updates parameters in opposite direction of gradient.",
                "update_rule": "θ = θ - α × ∇J(θ)",
                "pros": [
                    "Converges to global minimum for convex functions",
                    "Stable convergence"
                ],
                "cons": [
                    "Slow for large datasets (full pass each iteration)",
                    "Memory intensive"
                ],
                "use_when": "Small-to-medium datasets, convex optimization, need exact gradient"
            },
            {
                "name": "Stochastic Gradient Descent (SGD)",
                "type": "First-order optimization",
                "how_it_works": "Updates parameters per training example; gradient estimate noisy but cheap.",
                "update_rule": "For each example: θ = θ - α × ∇J(θ; xᵢ, yᵢ)",
                "pros": [
                    "Fast per update",
                    "Can escape shallow local minima (noise)",
                    "Works with massive datasets"
                ],
                "cons": [
                    "Noisy convergence",
                    "May never converge exactly",
                    "Requires learning rate schedule"
                ],
                "use_when": "Large-scale learning, online learning, deep neural networks"
            },
            {
                "name": "Mini-batch Gradient Descent",
                "type": "First-order optimization",
                "how_it_works": "Update per mini-batch (typically 32-512 examples); balance between SGD and batch.",
                "update_rule": "For each batch B: θ = θ - α × (1/|B|) Σ ∇J(θ; xᵢ, yᵢ)",
                "pros": [
                    "More stable than SGD",
                    "Vectorized operations (GPU-friendly)",
                    "Good convergence speed"
                ],
                "cons": [
                    "Need to tune batch size",
                    "Still some noise"
                ],
                "use_when": "Default choice for most deep learning; works with GPU acceleration"
            },
            {
                "name": "Momentum",
                "type": "Gradient descent with velocity",
                "how_it_works": "Accumulates velocity in consistent gradient directions; dampens oscillations.",
                "update_rule": "v = γv + α∇J(θ); θ = θ - v",
                "hyperparameters": {
                    "γ (momentum)": "0.9 typical",
                    "α (learning rate)": "same as SGD"
                },
                "pros": [
                    "Faster convergence on ravines",
                    "Reduces oscillations",
                    "Can escape shallow local minima"
                ],
                "cons": [
                    "May overshoot if momentum too high",
                    "Requires tuning γ"
                ],
                "use_when": "Training deep nets, ill-conditioned problems"
            },
            {
                "name": "RMSprop",
                "type": "Adaptive learning rate",
                "how_it_works": "Divides learning rate by running average of squared gradients; adapt per-parameter.",
                "update_rule": "E[g²] = βE[g²] + (1-β)g²; θ = θ - α / √(E[g²] + ε) × g",
                "hyperparameters": {
                    "β": "0.9 typical",
                    "α": "0.001 common",
                    "ε": "1e-8 numerical stability"
                },
                "pros": [
                    "Adapts learning rate per parameter",
                    "Handles sparse gradients well",
                    "Works for non-convex"
                ],
                "cons": [
                    "May still need schedule",
                    "Biases early iterations"
                ],
                "use_when": "RNNs, sparse data, non-stationary objectives"
            },
            {
                "name": "Adam (Adaptive Moment Estimation)",
                "type": "Adaptive + momentum",
                "how_it_works": "Combines momentum (first moment) and RMSprop (second moment) with bias correction.",
                "update_rule": "m = β₁m + (1-β₁)g; v = β₂v + (1-β₂)g²; m̂ = m/(1-β₁ᵗ); v̂ = v/(1-β₂ᵗ); θ = θ - α × m̂ / (√v̂ + ε)",
                "hyperparameters": {
                    "β₁": "0.9 (momentum decay)",
                    "β₂": "0.999 (RMSprop decay)",
                    "α": "0.001 default",
                    "ε": "1e-8"
                },
                "pros": [
                    "Robust default",
                    "Fast convergence",
                    "Low memory",
                    "Works for most problems"
                ],
                "cons": [
                    "May generalize worse than SGD with momentum (some papers)",
                    "Can diverge if α too large"
                ],
                "use_when": "Default for most deep learning; quick prototyping"
            },
            {
                "name": "Backpropagation",
                "type": "Gradient computation algorithm",
                "how_it_works": "Applies chain rule efficiently through computational graph; forward pass then backward pass.",
                "phases": [
                    "Forward: compute loss",
                    "Backward: compute ∂loss/∂params via chain rule",
                    "Update: optimizer step"
                ],
                "key_formula": "δ = ∇ₐL ⊙ σ'(z); ∇ₐL = Wᵀδ (for layer)",
                "notes": "Not an optimizer itself — computes gradients for any optimizer to use. Works with any differentiable model.",
                "variants": [
                    "Standard backprop (DAG)",
                    "Backprop through time (RNN)",
                    "Reverse-mode autodiff (frameworks)"
                ]
            },
            {
                "name": "Learning Rate Schedules",
                "type": "Meta-strategy",
                "strategies": {
                    "step_decay": "Drop LR by factor every N epochs",
                    "exponential_decay": "α = α₀ × e^{-kt}",
                    "1/t_decay": "α = α₀ / (1 + kt)",
                    "cosine_annealing": "Cosine curve from α₀ to 0",
                    "warmup": "Gradually increase LR from 0 to α₀ in first few epochs"
                },
                "purpose": "Prevent oscillation near minimum, allow large initial steps, fine-tune later",
                "notes": "Adam often doesn't need schedule, but warmup helps early training stability"
            }
        ],
        "practical_tips": [
            "Start with Adam (α=0.001) for quick results; switch to SGD+momentum (α=0.1) for final tuning if generalization suffers",
            "Normalize inputs (zero mean, unit variance) for stable gradient descent",
            "Use gradient clipping (norm or value) to prevent exploding gradients (especially RNNs)",
            "Monitor training/validation loss curves to detect divergence or plateaus",
            "Batch normalization helps stabilize gradients in deep networks"
        ]
    }
}