Cost-Sensitive Evaluation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

import numpy as np
from sklearn.metrics import confusion_matrix
 
def compute_expected_cost(y_true, y_pred, cost_matrix):
    """
    Compute total expected cost given predictions and a cost matrix.
    
    Parameters
    ----------
    y_true : array-like
        True labels (0 or 1)
    y_pred : array-like
        Predicted labels (0 or 1)
    cost_matrix : dict
        Dictionary with keys 'tp', 'fp', 'fn', 'tn' representing costs
        
    Returns
    -------
    dict
        Dictionary containing total_cost, cost_per_sample, and breakdown
    """
    # Compute confusion matrix elements
    # Note: sklearn's confusion_matrix returns [[TN, FP], [FN, TP]]
    cm = confusion_matrix(y_true, y_pred)
    tn, fp, fn, tp = cm.ravel()
    
    # Compute cost contributions
    cost_tp = tp * cost_matrix.get('tp', 0)
    cost_fp = fp * cost_matrix.get('fp', 0)
    cost_fn = fn * cost_matrix.get('fn', 0)
    cost_tn = tn * cost_matrix.get('tn', 0)
    
    total_cost = cost_tp + cost_fp + cost_fn + cost_tn
    n_samples = len(y_true)
    
    return {
        'total_cost': total_cost,
        'cost_per_sample': total_cost / n_samples,
        'breakdown': {
            'tp_contribution': cost_tp,
            'fp_contribution': cost_fp,
            'fn_contribution': cost_fn,
            'tn_contribution': cost_tn,
        },
        'confusion_matrix': {
            'tp': tp, 'fp': fp, 'fn': fn, 'tn': tn
        }
    }
 
 
# Example: Fraud detection cost matrix
fraud_cost_matrix = {
    'tp': 5,    # Investigation cost for caught fraud
    'fp': 7,    # Investigation + customer friction
    'fn': 150,  # Chargeback and losses from missed fraud
    'tn': 0     # No cost for correctly passing legitimate
}
 
# Simulated predictions
np.random.seed(42)
n_samples = 10000
fraud_rate = 0.01
 
y_true = np.random.binomial(1, fraud_rate, n_samples)
 
# Model A: Conservative (fewer false negatives, more false positives)
y_pred_conservative = y_true.copy()
# Add some false positives
false_positive_mask = np.random.random(n_samples) < 0.08
y_pred_conservative[false_positive_mask & (y_true == 0)] = 1
# Miss some frauds (10% false negative rate)
false_negative_mask = np.random.random(n_samples) < 0.10
y_pred_conservative[false_negative_mask & (y_true == 1)] = 0
 
# Model B: Aggressive (more false negatives, fewer false positives)
y_pred_aggressive = y_true.copy()
# Fewer false positives (2%)
false_positive_mask = np.random.random(n_samples) < 0.02
y_pred_aggressive[false_positive_mask & (y_true == 0)] = 1
# More missed frauds (30% false negative rate)
false_negative_mask = np.random.random(n_samples) < 0.30
y_pred_aggressive[false_negative_mask & (y_true == 1)] = 0
 
# Evaluate both models
result_conservative = compute_expected_cost(y_true, y_pred_conservative, fraud_cost_matrix)
result_aggressive = compute_expected_cost(y_true, y_pred_aggressive, fraud_cost_matrix)
 
print("=== Model Comparison (Cost-Sensitive Evaluation) ===
")
print("Conservative Model (favors catching fraud):")
print(f"  Total Cost: ${result_conservative['total_cost']:,.2f}")
print(f"  Cost per Transaction: ${result_conservative['cost_per_sample']:.4f}")
print(f"  Confusion Matrix: {result_conservative['confusion_matrix']}")
print(f"  Cost Breakdown: {result_conservative['breakdown']}
")
 
print("Aggressive Model (favors fewer investigations):")
print(f"  Total Cost: ${result_aggressive['total_cost']:,.2f}")
print(f"  Cost per Transaction: ${result_aggressive['cost_per_sample']:.4f}")
print(f"  Confusion Matrix: {result_aggressive['confusion_matrix']}")
print(f"  Cost Breakdown: {result_aggressive['breakdown']}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

import numpy as np
from sklearn.metrics import confusion_matrix
 
def cost_sensitive_metrics(y_true, y_pred, cost_matrix):
    """
    Compute comprehensive cost-sensitive evaluation metrics.
    
    Returns multiple perspectives on cost-sensitive performance.
    """
    cm = confusion_matrix(y_true, y_pred)
    tn, fp, fn, tp = cm.ravel()
    
    n = len(y_true)
    n_pos = sum(y_true)
    n_neg = n - n_pos
    
    # Extract costs (with defaults)
    c_tp = cost_matrix.get('tp', 0)
    c_fp = cost_matrix.get('fp', 1)
    c_fn = cost_matrix.get('fn', 1)
    c_tn = cost_matrix.get('tn', 0)
    
    # 1. Total and per-sample cost
    total_cost = tp * c_tp + fp * c_fp + fn * c_fn + tn * c_tn
    cost_per_sample = total_cost / n
    
    # 2. Baseline cost (always predict majority class)
    if n_pos > n_neg:
        # Majority is positive: predict all positive
        baseline_cost = n_pos * c_tp + n_neg * c_fp
    else:
        # Majority is negative: predict all negative
        baseline_cost = n_pos * c_fn + n_neg * c_tn
    
    # 3. Cost Reduction Ratio
    crr = 1 - (total_cost / baseline_cost) if baseline_cost > 0 else 0
    
    # 4. Expected Cost Per Action (per positive prediction)
    n_pred_pos = tp + fp
    ecpa = (tp * c_tp + fp * c_fp) / n_pred_pos if n_pred_pos > 0 else 0
    
    # 5. Weighted Accuracy (weights = inverse of costs, normalized)
    # Higher weight for cells with lower cost
    max_cost = max(c_tp, c_fp, c_fn, c_tn) + 1  # +1 to avoid division by zero
    w_tp = (max_cost - c_tp)
    w_fp = (max_cost - c_fp) 
    w_fn = (max_cost - c_fn)
    w_tn = (max_cost - c_tn)
    
    weighted_acc = (w_tp * tp + w_tn * tn) / (w_tp * tp + w_tn * tn + w_fp * fp + w_fn * fn)
    
    # 6. Normalized Expected Cost
    # Best case: perfect predictions -> cost = c_tp * n_pos + c_tn * n_neg
    best_cost = n_pos * c_tp + n_neg * c_tn
    # Worst case: all wrong predictions
    worst_cost = n_pos * c_fn + n_neg * c_fp
    
    nec = (total_cost - best_cost) / (worst_cost - best_cost) if worst_cost != best_cost else 0
    
    return {
        'total_cost': total_cost,
        'cost_per_sample': cost_per_sample,
        'baseline_cost': baseline_cost,
        'cost_reduction_ratio': crr,
        'expected_cost_per_action': ecpa,
        'weighted_accuracy': weighted_acc,
        'normalized_expected_cost': nec,
        'standard_metrics': {
            'accuracy': (tp + tn) / n,
            'precision': tp / (tp + fp) if (tp + fp) > 0 else 0,
            'recall': tp / (tp + fn) if (tp + fn) > 0 else 0,
        }
    }
 
 
# Example usage
np.random.seed(42)
y_true = np.array([1]*100 + [0]*9900)  # 1% positive rate
np.random.shuffle(y_true)
 
# Simulate model predictions (80% recall, 10% precision)
y_pred = np.zeros_like(y_true)
y_pred[(y_true == 1)] = np.random.binomial(1, 0.80, sum(y_true))  # 80% TP rate
y_pred[(y_true == 0) & (np.random.random(len(y_true)) < 0.08)] = 1  # ~8% FP rate
 
cost_matrix = {'tp': 5, 'fp': 7, 'fn': 150, 'tn': 0}
 
metrics = cost_sensitive_metrics(y_true, y_pred, cost_matrix)
 
print("Cost-Sensitive Evaluation Metrics")
print("=" * 50)
print(f"Total Cost: ${metrics['total_cost']:,.2f}")
print(f"Cost per Sample: ${metrics['cost_per_sample']:.4f}")
print(f"Baseline Cost: ${metrics['baseline_cost']:,.2f}")
print(f"Cost Reduction Ratio: {metrics['cost_reduction_ratio']:.2%}")
print(f"Expected Cost per Action: ${metrics['expected_cost_per_action']:.2f}")
print(f"Weighted Accuracy: {metrics['weighted_accuracy']:.4f}")
print(f"Normalized Expected Cost: {metrics['normalized_expected_cost']:.4f}")
print(f"
Standard Metrics for Reference:")
for k, v in metrics['standard_metrics'].items():
    print(f"  {k.capitalize()}: {v:.4f}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

import numpy as np
import pandas as pd
 
def compute_instance_dependent_cost(y_true, y_pred, cost_fn_false_neg, cost_fp_false_pos):
    """
    Compute total cost with instance-dependent costs.
    
    Parameters
    ----------
    y_true : array-like
        True labels
    y_pred : array-like
        Predicted labels  
    cost_fn_false_neg : array-like
        Per-instance cost of false negative
    cost_fp_false_pos : array-like
        Per-instance cost of false positive
        
    Returns
    -------
    dict
        Cost breakdown and totals
    """
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    cost_fn = np.array(cost_fn_false_neg)
    cost_fp = np.array(cost_fp_false_pos)
    
    # Identify error types
    false_negatives = (y_true == 1) & (y_pred == 0)
    false_positives = (y_true == 0) & (y_pred == 1)
    
    # Compute costs
    fn_cost = cost_fn[false_negatives].sum()
    fp_cost = cost_fp[false_positives].sum()
    total_cost = fn_cost + fp_cost
    
    return {
        'total_cost': total_cost,
        'fn_cost': fn_cost,
        'fp_cost': fp_cost,
        'n_false_negatives': false_negatives.sum(),
        'n_false_positives': false_positives.sum(),
        'avg_fn_cost': cost_fn[false_negatives].mean() if false_negatives.any() else 0,
        'avg_fp_cost': cost_fp[false_positives].mean() if false_positives.any() else 0,
    }
 
 
# Create a realistic fraud detection dataset
np.random.seed(42)
n_transactions = 10000
 
# Generate transaction amounts (log-normal distribution)
amounts = np.random.lognormal(mean=4, sigma=1.5, size=n_transactions)
amounts = np.clip(amounts, 10, 50000)  # $10 to $50,000 range
 
# Generate fraud labels (higher amounts slightly more likely to be fraud)
fraud_prob = 0.005 + 0.0001 * (amounts / 1000)  # Base 0.5% + amount factor
is_fraud = np.random.random(n_transactions) < fraud_prob
 
# Instance-dependent costs
# FN cost: amount * 1.5 (chargeback + admin + reputation)
cost_false_negative = amounts * 1.5
 
# FP cost: fixed investigation cost + small customer friction
cost_false_positive = np.full(n_transactions, 5.0) + amounts * 0.001
 
print(f"Dataset Statistics:")
print(f"  Total transactions: {n_transactions:,}")
print(f"  Fraud rate: {is_fraud.mean():.2%}")
print(f"  Average transaction: ${amounts.mean():,.2f}")
print(f"  Average FN cost: ${cost_false_negative.mean():,.2f}")
print(f"  Average FP cost: ${cost_false_positive.mean():.2f}")
print()
 
# Simulate two models
# Model A: High recall, lower precision
pred_a = is_fraud.copy()
pred_a[np.random.random(n_transactions) < 0.05] = 1  # 5% extra FP
pred_a[(is_fraud) & (np.random.random(n_transactions) < 0.10)] = 0  # 10% FN
 
# Model B: Lower recall, higher precision  
pred_b = is_fraud.copy()
pred_b[np.random.random(n_transactions) < 0.01] = 1  # 1% extra FP
pred_b[(is_fraud) & (np.random.random(n_transactions) < 0.25)] = 0  # 25% FN
 
# Evaluate with instance-dependent costs
result_a = compute_instance_dependent_cost(is_fraud, pred_a, cost_false_negative, cost_false_positive)
result_b = compute_instance_dependent_cost(is_fraud, pred_b, cost_false_negative, cost_false_positive)
 
print("Model A (High Recall):")
print(f"  Total Cost: ${result_a['total_cost']:,.2f}")
print(f"  FN Cost: ${result_a['fn_cost']:,.2f} ({result_a['n_false_negatives']} missed)")
print(f"  FP Cost: ${result_a['fp_cost']:,.2f} ({result_a['n_false_positives']} false alarms)")
print()
 
print("Model B (High Precision):")
print(f"  Total Cost: ${result_b['total_cost']:,.2f}")
print(f"  FN Cost: ${result_b['fn_cost']:,.2f} ({result_b['n_false_negatives']} missed)")
print(f"  FP Cost: ${result_b['fp_cost']:,.2f} ({result_b['n_false_positives']} false alarms)")
print()
 
print("Winner:", "Model A" if result_a['total_cost'] < result_b['total_cost'] else "Model B")
print(f"Cost Difference: ${abs(result_a['total_cost'] - result_b['total_cost']):,.2f}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
 
def compute_sample_weights_from_costs(y, cost_fp, cost_fn):
    """
    Compute sample weights from cost matrix.
    
    Weight samples by the cost of misclassifying them.
    """
    weights = np.ones(len(y), dtype=float)
    
    # Positive samples: weight by cost of false negative
    weights[y == 1] = cost_fn
    
    # Negative samples: weight by cost of false positive
    weights[y == 0] = cost_fp
    
    # Normalize weights to sum to len(y) for stability
    weights = weights * len(y) / weights.sum()
    
    return weights
 
 
def train_and_evaluate_cost_sensitive(X, y, cost_matrix, test_size=0.3):
    """
    Compare standard vs cost-sensitive training.
    """
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, stratify=y, random_state=42
    )
    
    cost_fp = cost_matrix['fp']
    cost_fn = cost_matrix['fn']
    
    # Compute sample weights for training
    sample_weights = compute_sample_weights_from_costs(y_train, cost_fp, cost_fn)
    
    # Standard training (no weights)
    model_standard = RandomForestClassifier(n_estimators=100, random_state=42)
    model_standard.fit(X_train, y_train)
    
    # Cost-sensitive training (with weights)
    model_weighted = RandomForestClassifier(n_estimators=100, random_state=42)
    model_weighted.fit(X_train, y_train, sample_weight=sample_weights)
    
    # Also try class_weight parameter
    # Compute class weights from cost ratio
    cost_ratio = cost_fn / cost_fp
    model_class_weight = RandomForestClassifier(
        n_estimators=100, 
        class_weight={0: 1, 1: cost_ratio},
        random_state=42
    )
    model_class_weight.fit(X_train, y_train)
    
    # Evaluate all models
    results = {}
    
    for name, model in [
        ('Standard', model_standard),
        ('Sample Weighted', model_weighted),
        ('Class Weighted', model_class_weight)
    ]:
        y_pred = model.predict(X_test)
        tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()
        
        total_cost = fp * cost_fp + fn * cost_fn
        
        results[name] = {
            'confusion_matrix': {'tp': tp, 'fp': fp, 'fn': fn, 'tn': tn},
            'total_cost': total_cost,
            'accuracy': (tp + tn) / len(y_test),
            'recall': tp / (tp + fn) if (tp + fn) > 0 else 0,
            'precision': tp / (tp + fp) if (tp + fp) > 0 else 0,
        }
    
    return results
 
 
# Generate synthetic imbalanced data
from sklearn.datasets import make_classification
 
X, y = make_classification(
    n_samples=10000,
    n_features=20,
    n_informative=10,
    n_redundant=5,
    n_clusters_per_class=2,
    weights=[0.95, 0.05],  # 5% positive class
    random_state=42
)
 
# Define cost matrix with high asymmetry
cost_matrix = {'tp': 0, 'fp': 10, 'fn': 100, 'tn': 0}  # 10:1 cost ratio
 
results = train_and_evaluate_cost_sensitive(X, y, cost_matrix)
 
print("Cost-Sensitive Training Comparison")
print("=" * 60)
print(f"Cost Matrix: FP=${cost_matrix['fp']}, FN=${cost_matrix['fn']}")
print()
 
for name, metrics in results.items():
    print(f"{name}:")
    print(f"  Total Cost: ${metrics['total_cost']:,.2f}")
    print(f"  Accuracy: {metrics['accuracy']:.4f}")
    print(f"  Recall: {metrics['recall']:.4f}")
    print(f"  Precision: {metrics['precision']:.4f}")
    cm = metrics['confusion_matrix']
    print(f"  FP: {cm['fp']}, FN: {cm['fn']}")
    print()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

import numpy as np
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import brier_score_loss
 
def cost_optimal_threshold(cost_fp, cost_fn):
    """Compute the cost-optimal decision threshold."""
    return cost_fp / (cost_fp + cost_fn)
 
def cost_optimal_predictions(y_proba, cost_fp, cost_fn):
    """
    Make predictions using the cost-optimal threshold.
    
    Parameters
    ----------
    y_proba : array-like
        Predicted probabilities for positive class
    cost_fp, cost_fn : float
        Costs of false positive and false negative
        
    Returns
    -------
    array
        Binary predictions using cost-optimal threshold
    """
    threshold = cost_optimal_threshold(cost_fp, cost_fn)
    return (y_proba >= threshold).astype(int)
 
 
def evaluate_calibration_for_costs(X, y, cost_matrix):
    """
    Compare calibrated vs uncalibrated models for cost-sensitive decisions.
    """
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.3, stratify=y, random_state=42
    )
    
    cost_fp = cost_matrix['fp']
    cost_fn = cost_matrix['fn']
    optimal_threshold = cost_optimal_threshold(cost_fp, cost_fn)
    
    # Uncalibrated model
    model_raw = RandomForestClassifier(n_estimators=100, random_state=42)
    model_raw.fit(X_train, y_train)
    
    # Calibrated model (Platt scaling)
    model_platt = CalibratedClassifierCV(
        RandomForestClassifier(n_estimators=100, random_state=42),
        method='sigmoid',
        cv=5
    )
    model_platt.fit(X_train, y_train)
    
    # Calibrated model (Isotonic)
    model_isotonic = CalibratedClassifierCV(
        RandomForestClassifier(n_estimators=100, random_state=42),
        method='isotonic',
        cv=5
    )
    model_isotonic.fit(X_train, y_train)
    
    results = {}
    
    for name, model in [
        ('Uncalibrated', model_raw),
        ('Platt Calibrated', model_platt),
        ('Isotonic Calibrated', model_isotonic)
    ]:
        # Get probabilities
        y_proba = model.predict_proba(X_test)[:, 1]
        
        # Make cost-optimal predictions
        y_pred = cost_optimal_predictions(y_proba, cost_fp, cost_fn)
        
        # Compute confusion matrix
        from sklearn.metrics import confusion_matrix
        tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()
        
        # Compute costs
        total_cost = fp * cost_fp + fn * cost_fn
        
        # Compute calibration metrics
        brier = brier_score_loss(y_test, y_proba)
        
        # Reliability diagram data
        prob_true, prob_pred = calibration_curve(y_test, y_proba, n_bins=10)
        calibration_error = np.mean(np.abs(prob_true - prob_pred))
        
        results[name] = {
            'total_cost': total_cost,
            'brier_score': brier,
            'mean_calibration_error': calibration_error,
            'confusion': {'tp': tp, 'fp': fp, 'fn': fn, 'tn': tn},
            'threshold_used': optimal_threshold,
        }
    
    return results
 
 
# Generate imbalanced data
from sklearn.datasets import make_classification
 
X, y = make_classification(
    n_samples=5000,
    n_features=15,
    n_informative=8,
    n_redundant=4,
    weights=[0.95, 0.05],
    random_state=42
)
 
cost_matrix = {'fp': 10, 'fn': 150}
 
print(f"Cost-Optimal Threshold: {cost_optimal_threshold(cost_matrix['fp'], cost_matrix['fn']):.4f}")
print(f"(Compare to default 0.5)
")
 
results = evaluate_calibration_for_costs(X, y, cost_matrix)
 
print("Calibration Impact on Cost-Sensitive Decisions")
print("=" * 60)
 
for name, metrics in results.items():
    print(f"
{name}:")
    print(f"  Total Cost: ${metrics['total_cost']:,.2f}")
    print(f"  Brier Score: {metrics['brier_score']:.4f}")
    print(f"  Mean Calibration Error: {metrics['mean_calibration_error']:.4f}")
    cm = metrics['confusion']
    print(f"  Confusion: TP={cm['tp']}, FP={cm['fp']}, FN={cm['fn']}, TN={cm['tn']}")