F1 and F-beta Scores

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

import numpy as np
from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix
 
def compute_f1_from_components(y_true, y_pred):
    """
    Calculate F1 score from precision and recall, and directly from cm.
    """
    cm = confusion_matrix(y_true, y_pred)
    TN, FP, FN, TP = cm.ravel()
    
    precision = TP / (TP + FP) if (TP + FP) > 0 else 0
    recall = TP / (TP + FN) if (TP + FN) > 0 else 0
    
    # Method 1: Harmonic mean formula
    if precision + recall > 0:
        f1_harmonic = 2 * precision * recall / (precision + recall)
    else:
        f1_harmonic = 0
    
    # Method 2: Direct formula from confusion matrix
    f1_direct = 2 * TP / (2 * TP + FP + FN) if (2 * TP + FP + FN) > 0 else 0
    
    # Method 3: sklearn
    f1_sklearn = f1_score(y_true, y_pred)
    
    return {
        'precision': precision,
        'recall': recall,
        'f1_harmonic': f1_harmonic,
        'f1_direct': f1_direct,
        'f1_sklearn': f1_sklearn,
        'TP': TP, 'FP': FP, 'FN': FN, 'TN': TN,
    }
 
# Example
y_true = [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
y_pred = [1, 1, 1, 0, 0, 0, 0, 0, 1, 1]
 
result = compute_f1_from_components(y_true, y_pred)
 
print("F1 Score Calculation Demonstration")
print("=" * 50)
print(f"
Confusion Matrix: TP={result['TP']}, FP={result['FP']}, FN={result['FN']}, TN={result['TN']}")
print(f"Precision: {result['precision']:.4f}")
print(f"Recall: {result['recall']:.4f}")
print(f"
F1 Score (harmonic mean): {result['f1_harmonic']:.4f}")
print(f"F1 Score (direct formula): {result['f1_direct']:.4f}")
print(f"F1 Score (sklearn):        {result['f1_sklearn']:.4f}")
print(f"
All three methods give identical results: {np.allclose([result['f1_harmonic']], [result['f1_direct']])}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

import numpy as np
import matplotlib.pyplot as plt
 
def compare_means():
    """
    Visualize how different mean types behave with varying precision/recall.
    """
    # Fixed recall = 0.5, varying precision
    recall_fixed = 0.5
    precisions = np.linspace(0.01, 1, 100)
    
    arithmetic = (precisions + recall_fixed) / 2
    geometric = np.sqrt(precisions * recall_fixed)
    harmonic = 2 * precisions * recall_fixed / (precisions + recall_fixed)
    
    plt.figure(figsize=(12, 5))
    
    # Left plot: Different means
    plt.subplot(1, 2, 1)
    plt.plot(precisions, arithmetic, 'b-', linewidth=2, label='Arithmetic Mean')
    plt.plot(precisions, geometric, 'g-', linewidth=2, label='Geometric Mean')
    plt.plot(precisions, harmonic, 'r-', linewidth=2, label='Harmonic Mean (F1)')
    plt.axhline(y=recall_fixed, color='gray', linestyle='--', alpha=0.5, label=f'Recall = {recall_fixed}')
    plt.xlabel('Precision', fontsize=12)
    plt.ylabel('Combined Score', fontsize=12)
    plt.title('Different Means with Fixed Recall = 0.5', fontsize=14)
    plt.legend()
    plt.grid(True, alpha=0.3)
    
    # Right plot: F1 contours
    plt.subplot(1, 2, 2)
    p_grid, r_grid = np.meshgrid(np.linspace(0.01, 1, 100), np.linspace(0.01, 1, 100))
    f1_grid = 2 * p_grid * r_grid / (p_grid + r_grid)
    
    contour = plt.contourf(p_grid, r_grid, f1_grid, levels=20, cmap='viridis')
    plt.colorbar(contour, label='F1 Score')
    
    # Add iso-F1 curves
    for f1_target in [0.2, 0.4, 0.6, 0.8]:
        plt.contour(p_grid, r_grid, f1_grid, levels=[f1_target], colors='white', linestyles='--')
    
    plt.xlabel('Precision', fontsize=12)
    plt.ylabel('Recall', fontsize=12)
    plt.title('F1 Score Contours (Iso-F1 curves in white)', fontsize=14)
    
    plt.tight_layout()
    plt.savefig('harmonic_mean_properties.png', dpi=150)
    plt.show()
    
    # Numerical examples
    print("
Numerical Comparison of Means")
    print("=" * 55)
    print(f"{'Precision':>10} {'Recall':>8} {'Arith':>8} {'Geom':>8} {'Harm(F1)':>10}")
    print("-" * 55)
    
    test_cases = [(0.9, 0.9), (0.9, 0.5), (0.9, 0.1), (0.5, 0.5), (0.1, 0.1)]
    for p, r in test_cases:
        a = (p + r) / 2
        g = np.sqrt(p * r)
        h = 2 * p * r / (p + r)
        print(f"{p:>10.1f} {r:>8.1f} {a:>8.2f} {g:>8.2f} {h:>10.2f}")
 
compare_means()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

import numpy as np
from sklearn.metrics import fbeta_score, precision_score, recall_score
import matplotlib.pyplot as plt
 
def explore_fbeta(y_true, y_pred):
    """
    Explore how F-beta changes with different beta values.
    """
    precision = precision_score(y_true, y_pred)
    recall = recall_score(y_true, y_pred)
    
    print("F-beta Score Exploration")
    print("=" * 60)
    print(f"Precision: {precision:.3f}")
    print(f"Recall:    {recall:.3f}")
    print(f"
{'Beta':>6} {'F-beta':>10} {'Interpretation':<30}")
    print("-" * 60)
    
    betas = [0.25, 0.5, 1.0, 2.0, 4.0]
    for beta in betas:
        f_beta = fbeta_score(y_true, y_pred, beta=beta)
        if beta < 1:
            interp = f"Precision {1/beta:.1f}x more important"
        elif beta > 1:
            interp = f"Recall {beta:.1f}x more important"
        else:
            interp = "Equal weight (F1)"
        print(f"{beta:>6.2f} {f_beta:>10.4f} {interp:<30}")
    
    return precision, recall
 
def visualize_fbeta_curves():
    """
    Visualize iso-F-beta curves for different beta values.
    """
    fig, axes = plt.subplots(1, 3, figsize=(15, 4))
    betas = [0.5, 1.0, 2.0]
    titles = ['F0.5 (Precision-focused)', 'F1 (Balanced)', 'F2 (Recall-focused)']
    
    for ax, beta, title in zip(axes, betas, titles):
        p_grid, r_grid = np.meshgrid(np.linspace(0.01, 1, 100), np.linspace(0.01, 1, 100))
        f_grid = (1 + beta**2) * p_grid * r_grid / (beta**2 * p_grid + r_grid)
        
        contour = ax.contourf(p_grid, r_grid, f_grid, levels=20, cmap='viridis')
        plt.colorbar(contour, ax=ax, label=f'F{beta}')
        
        # Mark the optimal point for fixed P+R=1 (trade-off line)
        ax.plot([0, 1], [1, 0], 'w--', alpha=0.5, label='P + R = 1')
        
        ax.set_xlabel('Precision', fontsize=12)
        ax.set_ylabel('Recall', fontsize=12)
        ax.set_title(title, fontsize=14)
    
    plt.tight_layout()
    plt.savefig('fbeta_contours.png', dpi=150)
    plt.show()
 
# Example: Model with high precision, lower recall
y_true = [1]*100 + [0]*900
y_pred = [1]*60 + [0]*40 + [0]*880 + [1]*20  # 60/100 recall, 60/80 precision
 
precision, recall = explore_fbeta(y_true, y_pred)
 
print(f"
Key Insight:")
print(f"  With Precision={precision:.2%} and Recall={recall:.2%}:")
print(f"  - F0.5 rewards the higher precision")
print(f"  - F2 would reward higher recall (this model scores lower)")
 
visualize_fbeta_curves()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

import numpy as np
from sklearn.metrics import f1_score
import warnings
 
def explore_f1_edge_cases():
    """
    Demonstrate edge cases and boundary behavior of F1 score.
    """
    print("F1 Score Edge Cases and Boundary Behavior")
    print("=" * 55)
    
    # Case 1: Perfect classifier
    print("
Case 1: Perfect classifier")
    y_true = [1, 1, 1, 0, 0, 0]
    y_pred = [1, 1, 1, 0, 0, 0]
    print(f"  F1 = {f1_score(y_true, y_pred):.4f} (perfect)")
    
    # Case 2: Zero recall (no true positives found)
    print("
Case 2: Zero recall (all positives missed)")
    y_true = [1, 1, 1, 0, 0, 0]
    y_pred = [0, 0, 0, 0, 0, 0]
    print(f"  F1 = {f1_score(y_true, y_pred):.4f} (zero - recall is 0)")
    
    # Case 3: Zero precision (all predictions wrong)
    print("
Case 3: Zero precision (no correct positive predictions)")
    y_true = [0, 0, 0, 0, 0, 0]
    y_pred = [1, 1, 1, 0, 0, 0]
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        print(f"  F1 = {f1_score(y_true, y_pred, zero_division=0):.4f} (zero - precision is 0)")
    
    # Case 4: No predictions and no positives (undefined)
    print("
Case 4: No positive predictions or actual positives")
    y_true = [0, 0, 0]
    y_pred = [0, 0, 0]
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        print(f"  F1 = {f1_score(y_true, y_pred, zero_division=0):.4f} (undefined → set to 0)")
        print(f"  F1 = {f1_score(y_true, y_pred, zero_division=1):.4f} (undefined → set to 1)")
    
    # Case 5: Single correct prediction
    print("
Case 5: Single true positive")
    y_true = [1]*100 + [0]*900
    y_pred = [1]*1 + [0]*99 + [0]*900  # One correct positive prediction
    f1 = f1_score(y_true, y_pred)
    print(f"  Precision = 1/1 = 100%")
    print(f"  Recall = 1/100 = 1%")
    print(f"  F1 = {f1:.4f} (low due to terrible recall)")
    
    # Case 6: Effect of threshold
    print("
Case 6: Threshold effects")
    y_true = [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
    y_scores = [0.9, 0.8, 0.7, 0.6, 0.4, 0.5, 0.3, 0.2, 0.1, 0.05]
    
    print(f"  {'Threshold':>10} {'Precision':>10} {'Recall':>8} {'F1':>8}")
    for thresh in [0.3, 0.5, 0.7, 0.9]:
        y_pred = [1 if s >= thresh else 0 for s in y_scores]
        from sklearn.metrics import precision_score, recall_score
        p = precision_score(y_true, y_pred, zero_division=0)
        r = recall_score(y_true, y_pred, zero_division=0)
        f = f1_score(y_true, y_pred, zero_division=0)
        print(f"  {thresh:>10.1f} {p:>10.2%} {r:>8.0%} {f:>8.4f}")
 
explore_f1_edge_cases()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

import numpy as np
from sklearn.metrics import f1_score, precision_recall_fscore_support, confusion_matrix
 
def multiclass_fscore_analysis():
    """
    Demonstrate multi-class F-score calculation and aggregation strategies.
    """
    # Imbalanced 4-class problem
    class_names = ['A', 'B', 'C', 'D']
    y_true = [0]*100 + [1]*50 + [2]*30 + [3]*20  # 100, 50, 30, 20
    
    # Predictions with varying per-class performance
    y_pred = (
        [0]*85 + [1]*10 + [2]*3 + [3]*2 +   # Class A: 85/100 = 0.85 recall
        [1]*40 + [0]*5 + [2]*3 + [3]*2 +    # Class B: 40/50 = 0.80 recall
        [2]*20 + [0]*5 + [1]*3 + [3]*2 +    # Class C: 20/30 = 0.67 recall
        [3]*10 + [0]*5 + [1]*3 + [2]*2      # Class D: 10/20 = 0.50 recall
    )
    
    cm = confusion_matrix(y_true, y_pred)
    
    print("Multi-Class F-score Analysis")
    print("=" * 60)
    print(f"
Class distribution: A={100}, B={50}, C={30}, D={20}")
    print(f"
Confusion Matrix:")
    print(f"           Predicted")
    print(f"           A     B     C     D")
    for i, cls in enumerate(class_names):
        print(f"{cls:>8}  {cm[i, 0]:3d}   {cm[i, 1]:3d}   {cm[i, 2]:3d}   {cm[i, 3]:3d}")
    
    # Per-class F1 scores
    precision, recall, f1, support = precision_recall_fscore_support(y_true, y_pred)
    
    print(f"
Per-Class Metrics:")
    print(f"{'Class':>8} {'n':>6} {'Precision':>10} {'Recall':>10} {'F1':>10}")
    print("-" * 50)
    for i, cls in enumerate(class_names):
        print(f"{cls:>8} {support[i]:>6} {precision[i]:>10.3f} {recall[i]:>10.3f} {f1[i]:>10.3f}")
    
    # Aggregated F1 scores
    f1_macro = f1_score(y_true, y_pred, average='macro')
    f1_weighted = f1_score(y_true, y_pred, average='weighted')
    f1_micro = f1_score(y_true, y_pred, average='micro')
    
    print(f"
Aggregated F1 Scores:")
    print(f"  Macro F1:    {f1_macro:.4f} (simple average of per-class F1)")
    print(f"  Weighted F1: {f1_weighted:.4f} (weighted by class frequency)")
    print(f"  Micro F1:    {f1_micro:.4f} (global TP/FP/FN aggregation)")
    
    # Manual verification of macro
    manual_macro = np.mean(f1)
    print(f"
Verification: Manual macro = {manual_macro:.4f}")
    
    # Manual verification of weighted
    weights = support / support.sum()
    manual_weighted = np.sum(weights * f1)
    print(f"Verification: Manual weighted = {manual_weighted:.4f}")
 
multiclass_fscore_analysis()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_curve, f1_score
 
def optimize_threshold_for_f1(y_true, y_scores):
    """
    Find the optimal classification threshold that maximizes F1 score.
    """
    # Get precision, recall at all thresholds
    precision, recall, thresholds = precision_recall_curve(y_true, y_scores)
    
    # Calculate F1 at each threshold
    # Note: precision_recall_curve returns n+1 precision/recall values
    # for n threshold values, so we need to handle the length mismatch
    f1_scores = []
    for i in range(len(thresholds)):
        if precision[i] + recall[i] > 0:
            f1 = 2 * precision[i] * recall[i] / (precision[i] + recall[i])
        else:
            f1 = 0
        f1_scores.append(f1)
    
    f1_scores = np.array(f1_scores)
    
    # Find optimal threshold
    optimal_idx = np.argmax(f1_scores)
    optimal_threshold = thresholds[optimal_idx]
    optimal_f1 = f1_scores[optimal_idx]
    
    return {
        'thresholds': thresholds,
        'f1_scores': f1_scores,
        'precision': precision[:-1],
        'recall': recall[:-1],
        'optimal_threshold': optimal_threshold,
        'optimal_f1': optimal_f1,
        'optimal_precision': precision[optimal_idx],
        'optimal_recall': recall[optimal_idx],
    }
 
# Generate synthetic example
np.random.seed(42)
n_pos, n_neg = 100, 400  # Imbalanced: 20% positive
 
# Positive class has higher scores
scores_pos = np.clip(np.random.normal(0.65, 0.2, n_pos), 0, 1)
scores_neg = np.clip(np.random.normal(0.35, 0.2, n_neg), 0, 1)
 
y_true = np.array([1]*n_pos + [0]*n_neg)
y_scores = np.concatenate([scores_pos, scores_neg])
 
# Find optimal threshold
result = optimize_threshold_for_f1(y_true, y_scores)
 
print("Threshold Optimization for F1 Score")
print("=" * 50)
print(f"
Dataset: {n_pos} positives, {n_neg} negatives ({n_pos/(n_pos+n_neg):.1%} positive)")
 
# Compare default vs optimal threshold
default_preds = (y_scores >= 0.5).astype(int)
optimal_preds = (y_scores >= result['optimal_threshold']).astype(int)
 
f1_default = f1_score(y_true, default_preds)
f1_optimal = result['optimal_f1']
 
print(f"
Default threshold (0.5):")
print(f"  F1 = {f1_default:.4f}")
 
print(f"
Optimal threshold ({result['optimal_threshold']:.3f}):")
print(f"  F1 = {f1_optimal:.4f}")
print(f"  Precision = {result['optimal_precision']:.3f}")
print(f"  Recall = {result['optimal_recall']:.3f}")
 
print(f"
Improvement: {(f1_optimal - f1_default) / f1_default * 100:.1f}%")
 
# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
 
# Left: F1 vs threshold
axes[0].plot(result['thresholds'], result['f1_scores'], 'b-', linewidth=2)
axes[0].axvline(x=0.5, color='gray', linestyle='--', label='Default (0.5)')
axes[0].axvline(x=result['optimal_threshold'], color='red', linestyle='-', 
                label=f'Optimal ({result["optimal_threshold"]:.2f})')
axes[0].scatter([result['optimal_threshold']], [result['optimal_f1']], 
                color='red', s=100, zorder=5)
axes[0].set_xlabel('Decision Threshold', fontsize=12)
axes[0].set_ylabel('F1 Score', fontsize=12)
axes[0].set_title('F1 Score vs Classification Threshold', fontsize=14)
axes[0].legend()
axes[0].grid(True, alpha=0.3)
 
# Right: Precision, Recall, F1 vs threshold
axes[1].plot(result['thresholds'], result['precision'], 'g-', linewidth=2, label='Precision')
axes[1].plot(result['thresholds'], result['recall'], 'r-', linewidth=2, label='Recall')
axes[1].plot(result['thresholds'], result['f1_scores'], 'b-', linewidth=2, label='F1')
axes[1].axvline(x=result['optimal_threshold'], color='blue', linestyle='--', alpha=0.5)
axes[1].set_xlabel('Decision Threshold', fontsize=12)
axes[1].set_ylabel('Score', fontsize=12)
axes[1].set_title('All Metrics vs Threshold', fontsize=14)
axes[1].legend()
axes[1].grid(True, alpha=0.3)
 
plt.tight_layout()
plt.savefig('threshold_optimization.png', dpi=150)
plt.show()