AUC Interpretation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

import numpy as np
 
def auc_trapezoidal(fpr, tpr):
    """
    Compute AUC using the trapezoidal rule.
    
    Parameters:
    -----------
    fpr : array-like
        False positive rates (sorted in ascending order)
    tpr : array-like
        Corresponding true positive rates
    
    Returns:
    --------
    float : Area under the ROC curve
    """
    fpr = np.array(fpr)
    tpr = np.array(tpr)
    
    # Ensure proper ordering
    order = np.argsort(fpr)
    fpr = fpr[order]
    tpr = tpr[order]
    
    # Trapezoidal integration
    # Area = sum of trapezoid areas between consecutive points
    # Trapezoid area = (base) × (average of heights)
    #                = (fpr[i+1] - fpr[i]) × (tpr[i+1] + tpr[i]) / 2
    
    auc = 0.0
    for i in range(len(fpr) - 1):
        width = fpr[i + 1] - fpr[i]
        height = (tpr[i] + tpr[i + 1]) / 2
        auc += width * height
    
    return auc
 
# Equivalent one-liner with numpy:
# auc = np.trapz(tpr, fpr)
 
# Example usage
fpr_example = [0.0, 0.1, 0.2, 0.5, 1.0]
tpr_example = [0.0, 0.4, 0.6, 0.9, 1.0]
print(f"AUC = {auc_trapezoidal(fpr_example, tpr_example):.4f}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

import numpy as np
 
def auc_mann_whitney(scores, labels):
    """
    Compute AUC using the Mann-Whitney U statistic interpretation.
    
    This directly computes the probability that a random positive
    is scored higher than a random negative.
    
    Time Complexity: O(P × N) - can be slow for large datasets
    """
    scores = np.array(scores)
    labels = np.array(labels)
    
    pos_scores = scores[labels == 1]
    neg_scores = scores[labels == 0]
    
    P = len(pos_scores)
    N = len(neg_scores)
    
    if P == 0 or N == 0:
        return 0.5  # Undefined, return random baseline
    
    # Count pairs where positive > negative
    correct = 0
    ties = 0
    
    for pos_score in pos_scores:
        for neg_score in neg_scores:
            if pos_score > neg_score:
                correct += 1
            elif pos_score == neg_score:
                ties += 1
    
    # AUC = (correct + 0.5 * ties) / (P * N)
    return (correct + 0.5 * ties) / (P * N)
 
# Efficient O(n log n) version using sorting
def auc_efficient(scores, labels):
    """
    Compute AUC efficiently using sorting and ranking.
    
    Time Complexity: O(n log n)
    """
    from scipy import stats
    
    scores = np.array(scores)
    labels = np.array(labels)
    
    # Use scipy's rankdata for efficient ranking
    ranks = stats.rankdata(scores)
    
    pos_ranks = ranks[labels == 1]
    P = len(pos_ranks)
    N = len(labels) - P
    
    if P == 0 or N == 0:
        return 0.5
    
    # Mann-Whitney U statistic formula
    # Sum of ranks of positives minus expected sum if random
    rank_sum = np.sum(pos_ranks)
    expected_if_random = P * (P + 1) / 2  # Minimum possible sum
    
    U = rank_sum - expected_if_random
    auc = U / (P * N)
    
    return auc

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

import numpy as np
from scipy import stats
 
def delong_variance(scores, labels):
    """
    Compute DeLong variance estimate for AUC.
    
    Based on: DeLong, DeLong, Clarke-Pearson (1988)
    "Comparing the Areas under Two or More Correlated
    Receiver Operating Characteristic Curves"
    """
    scores = np.array(scores)
    labels = np.array(labels)
    
    pos_scores = scores[labels == 1]
    neg_scores = scores[labels == 0]
    
    P = len(pos_scores)
    N = len(neg_scores)
    
    # Placement values: for each positive, what fraction of 
    # negatives are scored lower?
    V10 = np.zeros(P)  # Placements for positives
    for i, ps in enumerate(pos_scores):
        V10[i] = np.mean(ps > neg_scores) + 0.5 * np.mean(ps == neg_scores)
    
    # For each negative, what fraction of positives score higher?
    V01 = np.zeros(N)
    for j, ns in enumerate(neg_scores):
        V01[j] = np.mean(pos_scores > ns) + 0.5 * np.mean(pos_scores == ns)
    
    # AUC
    auc = np.mean(V10)  # S10
    
    # Variance components
    S10_var = np.var(V10, ddof=1) if P > 1 else 0
    S01_var = np.var(V01, ddof=1) if N > 1 else 0
    
    # DeLong variance of AUC
    var_auc = S10_var / P + S01_var / N
    
    return auc, var_auc
 
def auc_confidence_interval(scores, labels, alpha=0.05):
    """
    Compute AUC with confidence interval.
    
    Returns: (auc, lower_bound, upper_bound)
    """
    auc, var_auc = delong_variance(scores, labels)
    se = np.sqrt(var_auc)
    
    z = stats.norm.ppf(1 - alpha / 2)
    
    lower = max(0, auc - z * se)
    upper = min(1, auc + z * se)
    
    return auc, lower, upper
 
# Example usage
np.random.seed(42)
n = 200
scores = np.concatenate([
    np.random.normal(0.6, 0.2, 50),   # Positives
    np.random.normal(0.4, 0.2, 150)   # Negatives
])
labels = np.array([1]*50 + [0]*150)
 
auc, lower, upper = auc_confidence_interval(scores, labels)
print(f"AUC = {auc:.3f}")
print(f"95% CI: [{lower:.3f}, {upper:.3f}]")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

import numpy as np
from scipy import stats
 
def delong_test(scores1, scores2, labels):
    """
    Compare two classifiers' AUCs using DeLong's test.
    
    H0: AUC1 = AUC2 (no difference)
    H1: AUC1 ≠ AUC2 (two-sided test)
    
    Parameters:
    -----------
    scores1 : array-like
        Scores from first classifier
    scores2 : array-like
        Scores from second classifier (same examples)
    labels : array-like
        True labels
    
    Returns:
    --------
    z_stat : float
        Z-statistic for the difference
    p_value : float
        Two-sided p-value
    """
    scores1 = np.array(scores1)
    scores2 = np.array(scores2)
    labels = np.array(labels)
    
    pos_mask = labels == 1
    neg_mask = labels == 0
    
    pos_scores1 = scores1[pos_mask]
    pos_scores2 = scores2[pos_mask]
    neg_scores1 = scores1[neg_mask]
    neg_scores2 = scores2[neg_mask]
    
    P = len(pos_scores1)
    N = len(neg_scores1)
    
    # Placement values for each classifier
    def placements(pos_s, neg_s):
        V10 = np.array([np.mean(ps > neg_s) + 0.5 * np.mean(ps == neg_s) 
                        for ps in pos_s])
        V01 = np.array([np.mean(pos_s > ns) + 0.5 * np.mean(pos_s == ns) 
                        for ns in neg_s])
        return V10, V01
    
    V10_1, V01_1 = placements(pos_scores1, neg_scores1)
    V10_2, V01_2 = placements(pos_scores2, neg_scores2)
    
    auc1 = np.mean(V10_1)
    auc2 = np.mean(V10_2)
    
    # Covariance structure
    S10 = np.cov(V10_1, V10_2)
    S01 = np.cov(V01_1, V01_2)
    
    # Variance of (AUC1 - AUC2)
    S = S10 / P + S01 / N
    var_diff = S[0, 0] + S[1, 1] - 2 * S[0, 1]
    
    if var_diff <= 0:
        return 0.0, 1.0  # Identical, no difference
    
    z = (auc1 - auc2) / np.sqrt(var_diff)
    p_value = 2 * (1 - stats.norm.cdf(abs(z)))
    
    return z, p_value, auc1, auc2
 
# Example usage
np.random.seed(42)
n = 300
true_signal = np.random.randn(n)
labels = (true_signal > 0).astype(int)
 
# Two models with different noise levels
scores1 = true_signal + np.random.randn(n) * 0.5  # Better model
scores2 = true_signal + np.random.randn(n) * 0.8  # Worse model
 
z, p, auc1, auc2 = delong_test(scores1, scores2, labels)
print(f"Model 1 AUC: {auc1:.3f}")
print(f"Model 2 AUC: {auc2:.3f}")
print(f"Difference: {auc1 - auc2:.3f}")
print(f"Z-statistic: {z:.3f}")
print(f"P-value: {p:.4f}")
print(f"Significant at α=0.05? {p < 0.05}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

from sklearn.metrics import roc_auc_score
from sklearn.utils import resample
 
def bootstrap_auc_comparison(scores1, scores2, labels, n_bootstrap=1000, 
                             confidence_level=0.95):
    """
    Compare AUCs using stratified bootstrap resampling.
    
    Returns confidence interval for (AUC1 - AUC2).
    """
    differences = []
    
    for _ in range(n_bootstrap):
        # Stratified resample (maintain class balance)
        idx = resample(range(len(labels)), stratify=labels)
        
        s1_boot = scores1[idx]
        s2_boot = scores2[idx]
        y_boot = labels[idx]
        
        auc1_boot = roc_auc_score(y_boot, s1_boot)
        auc2_boot = roc_auc_score(y_boot, s2_boot)
        
        differences.append(auc1_boot - auc2_boot)
    
    differences = np.array(differences)
    
    alpha = 1 - confidence_level
    lower = np.percentile(differences, 100 * alpha / 2)
    upper = np.percentile(differences, 100 * (1 - alpha / 2))
    
    # If CI doesn't contain 0, difference is significant
    significant = (lower > 0) or (upper < 0)
    
    return {
        'mean_diff': np.mean(differences),
        'std_diff': np.std(differences),
        'ci_lower': lower,
        'ci_upper': upper,
        'significant': significant
    }