Accuracy-Interpretability Tradeoff

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

import numpy as np
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
from interpret.glassbox import ExplainableBoostingClassifier
import scipy.stats as stats
 
def assess_accuracy_interpretability_tradeoff(X, y, metric='roc_auc', cv=5, random_state=42):
    """
    Empirically assess the accuracy-interpretability tradeoff for a specific dataset.
    
    Returns gap between interpretable and complex models with confidence intervals.
    """
    cv_strategy = StratifiedKFold(n_splits=cv, shuffle=True, random_state=random_state)
    
    # Define interpretable models
    interpretable_models = {
        'Logistic Regression': LogisticRegression(max_iter=1000, random_state=random_state),
        'EBM (GAM)': ExplainableBoostingClassifier(random_state=random_state)
    }
    
    # Define complex models
    complex_models = {
        'Gradient Boosting': GradientBoostingClassifier(
            n_estimators=200, max_depth=6, random_state=random_state
        ),
        # Add XGBoost, LightGBM, Neural Network as needed
    }
    
    results = {}
    
    print("📊 Accuracy-Interpretability Tradeoff Assessment")
    print("="*60)
    
    # Evaluate all models
    for name, model in {**interpretable_models, **complex_models}.items():
        scores = cross_val_score(model, X, y, cv=cv_strategy, scoring=metric)
        results[name] = {
            'mean': scores.mean(),
            'std': scores.std(),
            'scores': scores,
            'ci_95': stats.t.interval(
                0.95, len(scores)-1, 
                loc=scores.mean(), 
                scale=stats.sem(scores)
            )
        }
        print(f"  {name}: {scores.mean():.4f} ± {scores.std():.4f}")
        print(f"    95% CI: [{results[name]['ci_95'][0]:.4f}, {results[name]['ci_95'][1]:.4f}]")
    
    # Compute gaps
    print("
📊 Tradeoff Analysis")
    print("="*60)
    
    best_interpretable = max(
        [(name, r) for name, r in results.items() if name in interpretable_models],
        key=lambda x: x[1]['mean']
    )
    
    best_complex = max(
        [(name, r) for name, r in results.items() if name in complex_models],
        key=lambda x: x[1]['mean']
    )
    
    gap = best_complex[1]['mean'] - best_interpretable[1]['mean']
    
    print(f"  Best interpretable: {best_interpretable[0]} ({best_interpretable[1]['mean']:.4f})")
    print(f"  Best complex: {best_complex[0]} ({best_complex[1]['mean']:.4f})")
    print(f"  Accuracy gap: {gap:+.4f} ({gap*100:+.2f}%)")
    
    # Statistical significance test
    t_stat, p_value = stats.ttest_ind(
        best_interpretable[1]['scores'],
        best_complex[1]['scores']
    )
    
    print(f"
  Statistical significance: p-value = {p_value:.4f}")
    if p_value < 0.05:
        if gap > 0:
            print("  ✓ Complex model is significantly better")
        else:
            print("  ✓ Interpretable model is significantly better!")
    else:
        print("  → Difference is NOT statistically significant")
        print("  → Consider using interpretable model for transparency")
    
    # Recommendation
    print("
📊 Recommendation")
    print("="*60)
    
    if gap < 0.01:
        print("  STRONG: Use interpretable model. Essentially no accuracy cost.")
    elif gap < 0.03:
        print("  MODERATE: Consider interpretable model. Gap is small.")
        print("  For high-stakes decisions, interpretability likely worth 1-3% accuracy.")
    elif gap < 0.10:
        print("  WEAKER: Complex model has meaningful advantage.")
        print("  Consider: domain requirements, regulatory environment, post-hoc explainability.")
    else:
        print("  WEAK: Complex model has substantial advantage.")
        print("  Use complex model with robust post-hoc interpretability pipeline.")
    
    return results, gap, p_value
 
# Usage
results, gap, p_value = assess_accuracy_interpretability_tradeoff(X, y)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

import numpy as np
from interpret.glassbox import ExplainableBoostingClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
 
# ============================================
# Technique 1: Explainable Boosting Machines
# ============================================
print("📊 EBM: Narrowing the Gap")
print("="*60)
 
# Standard linear model (baseline interpretable)
lr = LogisticRegression(max_iter=1000)
lr_scores = cross_val_score(lr, X, y, cv=5, scoring='roc_auc')
 
# Gradient boosting (complex benchmark)
gb = GradientBoostingClassifier(n_estimators=200, max_depth=5)
gb_scores = cross_val_score(gb, X, y, cv=5, scoring='roc_auc')
 
# EBM (interpretable but powerful)
ebm = ExplainableBoostingClassifier()
ebm_scores = cross_val_score(ebm, X, y, cv=5, scoring='roc_auc')
 
print(f"  Logistic Regression: {lr_scores.mean():.4f}")
print(f"  Gradient Boosting:   {gb_scores.mean():.4f}")
print(f"  EBM (Interpretable): {ebm_scores.mean():.4f}")
 
gap_lr_gb = gb_scores.mean() - lr_scores.mean()
gap_ebm_gb = gb_scores.mean() - ebm_scores.mean()
gap_closed = 1 - (gap_ebm_gb / gap_lr_gb) if gap_lr_gb > 0 else 1.0
 
print(f"
  Gap (LR to GB): {gap_lr_gb:.4f}")
print(f"  Gap (EBM to GB): {gap_ebm_gb:.4f}")
print(f"  Gap closed by EBM: {gap_closed:.1%}")
 
# ============================================
# Technique 2: Knowledge Distillation
# ============================================
print("
📊 Knowledge Distillation")
print("="*60)
 
# Train teacher (complex model)
teacher = GradientBoostingClassifier(n_estimators=200, max_depth=6)
teacher.fit(X, y)
 
# Get soft labels (teacher predictions)
soft_labels = teacher.predict_proba(X)[:, 1]
 
# Train student on soft labels
# For classification, we'll use regression to match probabilities
from sklearn.linear_model import Ridge
 
student_features = X
student = Ridge(alpha=1.0)
student.fit(student_features, soft_labels)
 
# Evaluate: student predicting teacher's outputs
student_preds = student.predict(X_test)
teacher_preds = teacher.predict_proba(X_test)[:, 1]
correlation = np.corrcoef(student_preds, teacher_preds)[0, 1]
 
print(f"  Student-Teacher correlation: {correlation:.4f}")
print(f"  Student is interpretable linear model mimicking GB")
 
# ============================================
# Technique 3: Monotonicity Constraints
# ============================================
print("
📊 Monotonicity Constraints (XGBoost)")
print("="*60)
 
try:
    import xgboost as xgb
    
    # Define monotonic constraints
    # +1: feature must increase output, -1: must decrease, 0: no constraint
    # Example: higher income (+1), higher debt_ratio (-1) for credit
    monotone_constraints = [0] * X.shape[1]  # Start with no constraints
    # monotone_constraints[income_idx] = 1   # Income increases approval
    # monotone_constraints[debt_idx] = -1    # Debt decreases approval
    
    # Train constrained model
    model_constrained = xgb.XGBClassifier(
        n_estimators=100,
        max_depth=5,
        monotone_constraints=tuple(monotone_constraints)
    )
    constrained_scores = cross_val_score(model_constrained, X, y, cv=5, scoring='roc_auc')
    
    # Train unconstrained model
    model_unconstrained = xgb.XGBClassifier(n_estimators=100, max_depth=5)
    unconstrained_scores = cross_val_score(model_unconstrained, X, y, cv=5, scoring='roc_auc')
    
    print(f"  Unconstrained XGBoost: {unconstrained_scores.mean():.4f}")
    print(f"  Monotonic XGBoost:     {constrained_scores.mean():.4f}")
    print(f"  Accuracy cost:         {unconstrained_scores.mean() - constrained_scores.mean():.4f}")
    print("
  Benefit: Monotonic constraints make model behavior predictable")
    print("  Example: 'Higher income ALWAYS helps approval' is guaranteed")
 
except ImportError:
    print("  (XGBoost not installed - skipping monotonicity example)")