{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 2851,
        "slug": "ml-training-algorithms",
        "title": "Machine Learning Training Algorithms",
        "description": "Core ML training algorithms: gradient descent variants, backpropagation, optimization methods (SGD, Adam, RMSprop), and when to use each.",
        "summary": "Training algorithms for machine learning, including gradient descent in its batch, stochastic and mini-batch forms, momentum, RMSprop, Adam, backpropagation and learning-rate schedules, each with type, mechanism, update rule, pros, cons and when to use it, plus practical tips.",
        "use_when": [
            "You need to understand or choose a training procedure.",
            "You are debugging unstable training."
        ],
        "caveats": [
            "Training behavior depends on initialization, batch size and learning rate; defaults do not transfer across all problems.",
            "Backpropagation computes gradients; the optimizer decides how to use them."
        ],
        "related": [
            "deep-learning-optimizers",
            "regularization-techniques",
            "transfer-learning",
            "ml-evaluation-metrics"
        ],
        "entry_count": 13,
        "fields": [
            {
                "section": "algorithms",
                "field": "name",
                "type": "string",
                "example": "Gradient Descent (Batch)"
            },
            {
                "section": "algorithms",
                "field": "type",
                "type": "string",
                "example": "First-order optimization"
            },
            {
                "section": "algorithms",
                "field": "how_it_works",
                "type": "string",
                "example": "Computes gradient over entire training set, upd\u2026"
            },
            {
                "section": "algorithms",
                "field": "update_rule",
                "type": "string",
                "example": "\u03b8 = \u03b8 - \u03b1 \u00d7 \u2207J(\u03b8)"
            },
            {
                "section": "algorithms",
                "field": "pros",
                "type": "array",
                "example": "[2 items]"
            },
            {
                "section": "algorithms",
                "field": "cons",
                "type": "array",
                "example": "[2 items]"
            },
            {
                "section": "algorithms",
                "field": "use_when",
                "type": "string",
                "example": "Small-to-medium datasets, convex optimization, \u2026"
            },
            {
                "section": "algorithms",
                "field": "hyperparameters",
                "type": "object",
                "example": "{\u03b3 (momentum),\u03b1 (learning rate)}"
            },
            {
                "section": "algorithms",
                "field": "phases",
                "type": "array",
                "example": "[3 items]"
            },
            {
                "section": "algorithms",
                "field": "key_formula",
                "type": "string",
                "example": "\u03b4 = \u2207\u2090L \u2299 \u03c3'(z); \u2207\u2090L = W\u1d40\u03b4 (for layer)"
            },
            {
                "section": "algorithms",
                "field": "notes",
                "type": "string",
                "example": "Not an optimizer itself \u2014 computes gradients fo\u2026"
            },
            {
                "section": "algorithms",
                "field": "variants",
                "type": "array",
                "example": "[3 items]"
            },
            {
                "section": "algorithms",
                "field": "strategies",
                "type": "object",
                "example": "{step_decay,exponential_decay,1\/t_decay,cosine_annealing}"
            },
            {
                "section": "algorithms",
                "field": "purpose",
                "type": "string",
                "example": "Prevent oscillation near minimum, allow large i\u2026"
            },
            {
                "section": "practical_tips",
                "field": "*",
                "type": "array",
                "example": "[5 items]"
            }
        ],
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "machine-learning,training,optimization,gradient-descent,backpropagation",
        "view_count": 47,
        "created_at": 1777673262,
        "updated_at": 1777673262
    },
    "data": {
        "algorithms": [
            {
                "name": "Gradient Descent (Batch)",
                "type": "First-order optimization",
                "how_it_works": "Computes gradient over entire training set, updates parameters in opposite direction of gradient.",
                "update_rule": "\u03b8 = \u03b8 - \u03b1 \u00d7 \u2207J(\u03b8)",
                "pros": [
                    "Converges to global minimum for convex functions",
                    "Stable convergence"
                ],
                "cons": [
                    "Slow for large datasets (full pass each iteration)",
                    "Memory intensive"
                ],
                "use_when": "Small-to-medium datasets, convex optimization, need exact gradient"
            },
            {
                "name": "Stochastic Gradient Descent (SGD)",
                "type": "First-order optimization",
                "how_it_works": "Updates parameters per training example; gradient estimate noisy but cheap.",
                "update_rule": "For each example: \u03b8 = \u03b8 - \u03b1 \u00d7 \u2207J(\u03b8; x\u1d62, y\u1d62)",
                "pros": [
                    "Fast per update",
                    "Can escape shallow local minima (noise)",
                    "Works with massive datasets"
                ],
                "cons": [
                    "Noisy convergence",
                    "May never converge exactly",
                    "Requires learning rate schedule"
                ],
                "use_when": "Large-scale learning, online learning, deep neural networks"
            },
            {
                "name": "Mini-batch Gradient Descent",
                "type": "First-order optimization",
                "how_it_works": "Update per mini-batch (typically 32-512 examples); balance between SGD and batch.",
                "update_rule": "For each batch B: \u03b8 = \u03b8 - \u03b1 \u00d7 (1\/|B|) \u03a3 \u2207J(\u03b8; x\u1d62, y\u1d62)",
                "pros": [
                    "More stable than SGD",
                    "Vectorized operations (GPU-friendly)",
                    "Good convergence speed"
                ],
                "cons": [
                    "Need to tune batch size",
                    "Still some noise"
                ],
                "use_when": "Default choice for most deep learning; works with GPU acceleration"
            },
            {
                "name": "Momentum",
                "type": "Gradient descent with velocity",
                "how_it_works": "Accumulates velocity in consistent gradient directions; dampens oscillations.",
                "update_rule": "v = \u03b3v + \u03b1\u2207J(\u03b8); \u03b8 = \u03b8 - v",
                "hyperparameters": {
                    "\u03b3 (momentum)": "0.9 typical",
                    "\u03b1 (learning rate)": "same as SGD"
                },
                "pros": [
                    "Faster convergence on ravines",
                    "Reduces oscillations",
                    "Can escape shallow local minima"
                ],
                "cons": [
                    "May overshoot if momentum too high",
                    "Requires tuning \u03b3"
                ],
                "use_when": "Training deep nets, ill-conditioned problems"
            },
            {
                "name": "RMSprop",
                "type": "Adaptive learning rate",
                "how_it_works": "Divides learning rate by running average of squared gradients; adapt per-parameter.",
                "update_rule": "E[g\u00b2] = \u03b2E[g\u00b2] + (1-\u03b2)g\u00b2; \u03b8 = \u03b8 - \u03b1 \/ \u221a(E[g\u00b2] + \u03b5) \u00d7 g",
                "hyperparameters": {
                    "\u03b2": "0.9 typical",
                    "\u03b1": "0.001 common",
                    "\u03b5": "1e-8 numerical stability"
                },
                "pros": [
                    "Adapts learning rate per parameter",
                    "Handles sparse gradients well",
                    "Works for non-convex"
                ],
                "cons": [
                    "May still need schedule",
                    "Biases early iterations"
                ],
                "use_when": "RNNs, sparse data, non-stationary objectives"
            },
            {
                "name": "Adam (Adaptive Moment Estimation)",
                "type": "Adaptive + momentum",
                "how_it_works": "Combines momentum (first moment) and RMSprop (second moment) with bias correction.",
                "update_rule": "m = \u03b2\u2081m + (1-\u03b2\u2081)g; v = \u03b2\u2082v + (1-\u03b2\u2082)g\u00b2; m\u0302 = m\/(1-\u03b2\u2081\u1d57); v\u0302 = v\/(1-\u03b2\u2082\u1d57); \u03b8 = \u03b8 - \u03b1 \u00d7 m\u0302 \/ (\u221av\u0302 + \u03b5)",
                "hyperparameters": {
                    "\u03b2\u2081": "0.9 (momentum decay)",
                    "\u03b2\u2082": "0.999 (RMSprop decay)",
                    "\u03b1": "0.001 default",
                    "\u03b5": "1e-8"
                },
                "pros": [
                    "Robust default",
                    "Fast convergence",
                    "Low memory",
                    "Works for most problems"
                ],
                "cons": [
                    "May generalize worse than SGD with momentum (some papers)",
                    "Can diverge if \u03b1 too large"
                ],
                "use_when": "Default for most deep learning; quick prototyping"
            },
            {
                "name": "Backpropagation",
                "type": "Gradient computation algorithm",
                "how_it_works": "Applies chain rule efficiently through computational graph; forward pass then backward pass.",
                "phases": [
                    "Forward: compute loss",
                    "Backward: compute \u2202loss\/\u2202params via chain rule",
                    "Update: optimizer step"
                ],
                "key_formula": "\u03b4 = \u2207\u2090L \u2299 \u03c3'(z); \u2207\u2090L = W\u1d40\u03b4 (for layer)",
                "notes": "Not an optimizer itself \u2014 computes gradients for any optimizer to use. Works with any differentiable model.",
                "variants": [
                    "Standard backprop (DAG)",
                    "Backprop through time (RNN)",
                    "Reverse-mode autodiff (frameworks)"
                ]
            },
            {
                "name": "Learning Rate Schedules",
                "type": "Meta-strategy",
                "strategies": {
                    "step_decay": "Drop LR by factor every N epochs",
                    "exponential_decay": "\u03b1 = \u03b1\u2080 \u00d7 e^{-kt}",
                    "1\/t_decay": "\u03b1 = \u03b1\u2080 \/ (1 + kt)",
                    "cosine_annealing": "Cosine curve from \u03b1\u2080 to 0",
                    "warmup": "Gradually increase LR from 0 to \u03b1\u2080 in first few epochs"
                },
                "purpose": "Prevent oscillation near minimum, allow large initial steps, fine-tune later",
                "notes": "Adam often doesn't need schedule, but warmup helps early training stability"
            }
        ],
        "practical_tips": [
            "Start with Adam (\u03b1=0.001) for quick results; switch to SGD+momentum (\u03b1=0.1) for final tuning if generalization suffers",
            "Normalize inputs (zero mean, unit variance) for stable gradient descent",
            "Use gradient clipping (norm or value) to prevent exploding gradients (especially RNNs)",
            "Monitor training\/validation loss curves to detect divergence or plateaus",
            "Batch normalization helps stabilize gradients in deep networks"
        ]
    }
}