Model Debugging

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve
 
def diagnose_bias_variance(model, X, y, cv=5):
    """
    Generate learning curves to diagnose bias vs variance.
    
    Learning curves show how training and validation performance
    evolve as training set size increases.
    """
    train_sizes, train_scores, val_scores = learning_curve(
        model, X, y,
        train_sizes=np.linspace(0.1, 1.0, 10),
        cv=cv,
        scoring='accuracy',
        n_jobs=-1
    )
    
    train_mean = train_scores.mean(axis=1)
    train_std = train_scores.std(axis=1)
    val_mean = val_scores.mean(axis=1)
    val_std = val_scores.std(axis=1)
    
    # Diagnosis logic
    final_train = train_mean[-1]
    final_val = val_mean[-1]
    gap = final_train - final_val
    
    if final_train < 0.7:  # Arbitrary threshold
        diagnosis = "HIGH BIAS: Model underfits. Increase complexity."
    elif gap > 0.15:
        diagnosis = "HIGH VARIANCE: Model overfits. Regularize or get more data."
    elif final_val < 0.85:
        diagnosis = "BOTH: High bias and variance. Consider different architecture."
    else:
        diagnosis = "GOOD FIT: Model generalizes well."
    
    print(f"Train accuracy: {final_train:.3f}")
    print(f"Val accuracy: {final_val:.3f}")
    print(f"Gap: {gap:.3f}")
    print(f"Diagnosis: {diagnosis}")
    
    return {
        'train_sizes': train_sizes,
        'train_scores': train_mean,
        'val_scores': val_mean,
        'diagnosis': diagnosis
    }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

import pandas as pd
import numpy as np
from collections import defaultdict
 
class ErrorAnalyzer:
    """Systematic error analysis for classification models."""
    
    def __init__(self, model, X_test, y_test, feature_names=None):
        self.model = model
        self.X_test = X_test
        self.y_test = y_test
        self.feature_names = feature_names
        self.predictions = model.predict(X_test)
        self.probabilities = model.predict_proba(X_test)
        
    def get_errors(self):
        """Get all misclassified examples."""
        error_mask = self.predictions != self.y_test
        return {
            'indices': np.where(error_mask)[0],
            'X': self.X_test[error_mask],
            'y_true': self.y_test[error_mask],
            'y_pred': self.predictions[error_mask],
            'confidence': self.probabilities[error_mask].max(axis=1)
        }
    
    def analyze_by_confidence(self):
        """Stratify errors by prediction confidence."""
        errors = self.get_errors()
        confidence_buckets = pd.cut(
            errors['confidence'], 
            bins=[0, 0.5, 0.7, 0.9, 1.0],
            labels=['very_low', 'low', 'medium', 'high']
        )
        return pd.Series(confidence_buckets).value_counts()
    
    def analyze_confusion_patterns(self):
        """Find which class pairs are most confused."""
        errors = self.get_errors()
        confusion_pairs = defaultdict(int)
        for true, pred in zip(errors['y_true'], errors['y_pred']):
            confusion_pairs[(true, pred)] += 1
        return dict(sorted(
            confusion_pairs.items(), 
            key=lambda x: -x[1]
        )[:10])
    
    def slice_analysis(self, slice_fn, slice_name):
        """Analyze performance on a specific data slice."""
        slice_mask = slice_fn(self.X_test)
        slice_preds = self.predictions[slice_mask]
        slice_true = self.y_test[slice_mask]
        
        accuracy = (slice_preds == slice_true).mean()
        error_rate = 1 - accuracy
        
        return {
            'slice_name': slice_name,
            'n_samples': slice_mask.sum(),
            'accuracy': accuracy,
            'error_rate': error_rate,
            'contribution_to_total_error': (
                (slice_preds != slice_true).sum() / 
                (self.predictions != self.y_test).sum()
            )
        }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

import torch
import torch.nn as nn
 
def check_dead_relu(model, sample_input):
    """Detect ReLU layers with dead neurons."""
    dead_neuron_report = {}
    
    def hook_fn(name):
        def hook(module, input, output):
            if isinstance(output, torch.Tensor):
                # Fraction of neurons that never activate
                dead_fraction = (output <= 0).float().mean().item()
                if dead_fraction > 0.5:
                    dead_neuron_report[name] = dead_fraction
        return hook
    
    hooks = []
    for name, module in model.named_modules():
        if isinstance(module, nn.ReLU):
            hooks.append(module.register_forward_hook(hook_fn(name)))
    
    model(sample_input)
    
    for hook in hooks:
        hook.remove()
    
    if dead_neuron_report:
        print("⚠️ DEAD ReLU NEURONS DETECTED:")
        for name, frac in dead_neuron_report.items():
            print(f"  {name}: {frac:.1%} dead")
    else:
        print("✓ No dead neuron issues detected")
    
    return dead_neuron_report
 
def verify_gradient_flow(model, sample_input, sample_target, loss_fn):
    """Verify gradients flow to all parameters."""
    model.zero_grad()
    output = model(sample_input)
    loss = loss_fn(output, sample_target)
    loss.backward()
    
    no_grad_params = []
    for name, param in model.named_parameters():
        if param.grad is None or param.grad.abs().sum() == 0:
            no_grad_params.append(name)
    
    if no_grad_params:
        print("⚠️ PARAMETERS WITH NO GRADIENT:")
        for name in no_grad_params:
            print(f"  {name}")
    else:
        print("✓ All parameters receive gradients")
    
    return no_grad_params