{ "api": "YAML JSON TOON Database", "version": "1.0.0", "format": "json", "dataset": { "id": 64, "slug": "transfer-learning", "title": "Transfer Learning and Fine-Tuning", "description": "Transfer learning strategies: feature extraction, fine-tuning, domain adaptation, multi-task learning, prompt tuning, LoRA, and pretraining paradigms.", "summary": "Strategies for adapting pretrained models: feature extraction, full fine-tuning, layer-wise learning-rate decay, progressive unfreezing, domain adaptation, LoRA, prompt and prefix tuning and multi-task learning, with approach, when to use, pros, cons and examples.", "use_when": [ "You have limited data or compute and want to start from a pretrained model.", "You are choosing how much of a model to update." ], "caveats": [ "Pretrained models carry biases and limitations from their original data.", "More parameter updates give more flexibility and more risk of forgetting or overfitting." ], "related": [ "llm-glossary", "ml-model-architectures", "regularization-techniques", "deep-learning-optimizers" ], "category": "AI & ML Terms", "category_slug": "ai-ml-terms", "tags": "transfer-learning,fine-tuning,feature-extraction,domain-adaptation,lora,prompt-tuning,pretraining", "view_count": 0, "created_at": 1778695229, "updated_at": 1778695229, "entry_count": 8, "fields": [ { "section": "strategies", "field": "name", "type": "string", "example": "Feature Extraction (Frozen Backbone)" }, { "section": "strategies", "field": "approach", "type": "string", "example": "Use pretrained model as fixed feature extractor…" }, { "section": "strategies", "field": "when_to_use", "type": "array", "example": "[3 items]" }, { "section": "strategies", "field": "pros", "type": "array", "example": "[3 items]" }, { "section": "strategies", "field": "cons", "type": "array", "example": "[2 items]" }, { "section": "strategies", "field": "example", "type": "string", "example": "ResNet-50 pretrained on ImageNet → freeze all l…" }, { "section": "strategies", "field": "notes", "type": "string", "example": "Used for most LLM fine-tuning (Alpaca, Vicuna, …" } ] }, "data": { "strategies": [ { "name": "Feature Extraction (Frozen Backbone)", "approach": "Use pretrained model as fixed feature extractor; train only new classification head", "when_to_use": [ "Small target dataset", "Target domain similar to source", "Limited compute resources" ], "pros": [ "Fast training", "No catastrophic forgetting", "Minimal overfitting risk" ], "cons": [ "Limited adaptation to target domain", "Features may not be optimal for new task" ], "example": "ResNet-50 pretrained on ImageNet → freeze all layers → train only final FC layer on custom dataset" }, { "name": "Fine-Tuning (Full Model)", "approach": "Unfreeze all layers; continue training entire model on target data with lower learning rate", "when_to_use": [ "Large target dataset", "Target domain differs from source", "Maximum performance needed" ], "pros": [ "Full adaptation to target domain", "Best possible performance" ], "cons": [ "Risk of catastrophic forgetting", "Requires more data", "Computationally expensive" ], "example": "BERT pretrained → fine-tune all 110M parameters on SQuAD QA dataset with LR=2e-5" }, { "name": "Layer-wise Learning Rate Decay", "approach": "Assign lower LR to earlier layers, higher LR to later layers during fine-tuning", "when_to_use": [ "Medium-sized target dataset", "Want to preserve generic low-level features" ], "pros": [ "Balances preservation and adaptation", "Reduces forgetting of early features" ], "cons": [ "Adds hyperparameter (decay rate)", "Requires tuning" ], "example": "LR(layer_i) = base_lr × decay^(depth - i); typical decay=0.9-0.95" }, { "name": "Progressive Unfreezing", "approach": "Start with frozen backbone + trainable head; gradually unfreeze layers from top to bottom", "when_to_use": [ "Small target dataset", "Want controlled adaptation", "Avoid catastrophic forgetting" ], "pros": [ "Gradual adaptation reduces shock to pretrained weights", "Better stability" ], "cons": [ "Slower training process", "More complex training loop" ], "example": "Phase 1: train head only. Phase 2: unfreeze last block + head. Phase 3: unfreeze all." }, { "name": "Domain Adaptation", "approach": "Align feature distributions between source and target domains (e.g., adversarial training, MMD loss)", "when_to_use": [ "Source and target domains differ significantly", "Labeled target data scarce", "Domain shift is known issue" ], "pros": [ "Explicitly addresses domain gap", "Works with unlabeled target data" ], "cons": [ "More complex training", "Requires domain-specific techniques" ], "example": "DANN (Domain-Adversarial Neural Network): gradient reversal layer makes features domain-invariant" }, { "name": "LoRA (Low-Rank Adaptation)", "approach": "Inject trainable low-rank decomposition matrices into attention/FFN layers; freeze pretrained weights", "when_to_use": [ "Fine-tuning large models (LLMs, ViT)", "Memory-constrained environments", "Multiple downstream tasks from same base model" ], "pros": [ "Dramatically fewer trainable parameters (0.1-1% of original)", "No inference latency overhead (merged into weights)", "Multiple LoRA adapters can share same base model" ], "cons": [ "Slight accuracy drop vs full fine-tuning", "Rank hyperparameter to tune" ], "example": "LLaMA 7B: full fine-tuning = 7B params; LoRA (r=8) = 4.7M params (0.07%)", "notes": "Used for most LLM fine-tuning (Alpaca, Vicuna, etc.)" }, { "name": "Prompt Tuning / Prefix Tuning", "approach": "Learn soft prompts (continuous vectors) prepended to input; freeze entire model", "when_to_use": [ "Very large models where even LoRA is expensive", "Multiple tasks from single model", "When prompt engineering is natural fit" ], "pros": [ "Minimal trainable parameters (thousands vs billions)", "No model modification needed", "Easy to switch between tasks" ], "cons": [ "Lower performance than full fine-tuning", "Only works well for very large models (10B+)" ], "example": "T5-11B: learn 100-token soft prompt (128-dim each) = 1.28M params vs 11B full model" }, { "name": "Multi-Task Learning", "approach": "Train single model on multiple related tasks simultaneously with shared backbone", "when_to_use": [ "Multiple related tasks available", "Data-scarce individual tasks", "Want shared representations" ], "pros": [ "Improved generalization via inductive bias", "Single model for multiple tasks", "Data augmentation across tasks" ], "cons": [ "Task interference possible", "Complex loss weighting", "Harder to debug" ], "example": "BERT pretraining: MLM + NSP simultaneously. MT-DNN: shared BERT + task-specific heads." } ] } }