Training, Validation, and Test Sets: The Foundation of Model Evaluation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
 
# Generate synthetic data: y = sin(x) + noise
np.random.seed(42)
n_samples = 20
X = np.linspace(0, 3, n_samples).reshape(-1, 1)
y_true = np.sin(2 * X).ravel()
y = y_true + 0.1 * np.random.randn(n_samples)  # Add noise
 
# Fit models of increasing complexity
degrees = [1, 3, 15]
X_plot = np.linspace(0, 3, 100).reshape(-1, 1)
 
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
for ax, degree in zip(axes, degrees):
    # Create polynomial features
    poly = PolynomialFeatures(degree)
    X_poly = poly.fit_transform(X)
    X_plot_poly = poly.transform(X_plot)
    
    # Fit model
    model = LinearRegression()
    model.fit(X_poly, y)
    
    # Evaluate on training data (THIS IS THE MISTAKE)
    y_pred_train = model.predict(X_poly)
    train_mse = mean_squared_error(y, y_pred_train)
    
    # Plot
    ax.scatter(X, y, color='blue', s=50, label='Training data')
    ax.plot(X_plot, model.predict(X_plot_poly), 'r-', linewidth=2, label='Model')
    ax.set_title(f'Degree {degree}
Train MSE: {train_mse:.4f}')
    ax.legend()
 
plt.tight_layout()
plt.suptitle('Lower Training Error ≠ Better Model!', y=1.02, fontsize=14)
plt.show()
 
# The degree-15 polynomial has near-zero training error
# But it will perform terribly on new data!
print("Lesson: Training error is a BIASED estimate of true performance")
print("The more complex the model, the more optimistic (and misleading) training error becomes")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

from sklearn.model_selection import train_test_split
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
 
# Load data
X, y = load_iris(return_X_y=True)
print(f"Total samples: {len(X)}")
 
# Split data: 80% train, 20% test
# random_state ensures reproducibility
# stratify=y ensures class balance is preserved in both sets
X_train, X_test, y_train, y_test = train_test_split(
    X, y, 
    test_size=0.2,      # 20% for testing
    random_state=42,    # Reproducibility
    stratify=y          # Maintain class proportions
)
 
print(f"Training samples: {len(X_train)} ({len(X_train)/len(X)*100:.0f}%)")
print(f"Test samples: {len(X_test)} ({len(X_test)/len(X)*100:.0f}%)")
 
# Verify stratification
import numpy as np
print(f"
Class distribution in full data: {np.bincount(y)}")
print(f"Class distribution in training: {np.bincount(y_train)}")
print(f"Class distribution in test: {np.bincount(y_test)}")
 
# Train model ONLY on training data
model = LogisticRegression(max_iter=200)
model.fit(X_train, y_train)
 
# Evaluate on BOTH sets to see the difference
train_acc = accuracy_score(y_train, model.predict(X_train))
test_acc = accuracy_score(y_test, model.predict(X_test))
 
print(f"
Training accuracy: {train_acc:.4f}")
print(f"Test accuracy: {test_acc:.4f}")
print(f"Gap: {train_acc - test_acc:.4f} (smaller is better)")
 
# If gap is large → model is overfitting
# If both are low → model is underfitting

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
 
# Generate synthetic data
X, y = make_classification(n_samples=1000, n_features=20, 
                           n_informative=10, random_state=42)
 
# First split: separate test set (final evaluation only)
X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
 
# Second split: separate validation set from training
X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=42, stratify=y_temp
)
# 0.25 of 0.8 = 0.2, so we get 60/20/20 split
 
print(f"Training set: {len(X_train)} samples (60%)")
print(f"Validation set: {len(X_val)} samples (20%)")
print(f"Test set: {len(X_test)} samples (20%)")
 
# Step 1: Train multiple models on TRAINING data only
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'SVM': SVC(kernel='rbf', random_state=42)
}
 
# Step 2: Evaluate on VALIDATION set to choose best model
print("
--- Model Selection (using validation set) ---")
val_scores = {}
for name, model in models.items():
    model.fit(X_train, y_train)
    val_pred = model.predict(X_val)
    val_scores[name] = accuracy_score(y_val, val_pred)
    print(f"{name}: Validation Accuracy = {val_scores[name]:.4f}")
 
# Step 3: Select best model based on validation performance
best_model_name = max(val_scores, key=val_scores.get)
print(f"
Best model: {best_model_name}")
 
# Step 4: Retrain on train + validation combined
X_train_full = np.vstack([X_train, X_val])
y_train_full = np.concatenate([y_train, y_val])
 
final_model = models[best_model_name].__class__(**models[best_model_name].get_params())
final_model.fit(X_train_full, y_train_full)
 
# Step 5: Final evaluation on TEST set (only once!)
test_pred = final_model.predict(X_test)
test_accuracy = accuracy_score(y_test, test_pred)
 
print(f"
--- Final Evaluation (TEST SET - use only once!) ---")
print(f"Test Accuracy: {test_accuracy:.4f}")
print("This is our best estimate of real-world performance.")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

from sklearn.model_selection import (
    cross_val_score, KFold, StratifiedKFold, 
    LeaveOneOut, TimeSeriesSplit
)
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
import numpy as np
 
X, y = load_iris(return_X_y=True)
model = LogisticRegression(max_iter=200)
 
# Standard K-Fold (k=5)
kfold = KFold(n_splits=5, shuffle=True, random_state=42)
scores_kfold = cross_val_score(model, X, y, cv=kfold)
print(f"5-Fold CV: {scores_kfold.mean():.4f} ± {scores_kfold.std():.4f}")
print(f"  Individual fold scores: {scores_kfold.round(4)}")
 
# Stratified K-Fold (maintains class proportions in each fold)
skfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores_strat = cross_val_score(model, X, y, cv=skfold)
print(f"
Stratified 5-Fold CV: {scores_strat.mean():.4f} ± {scores_strat.std():.4f}")
 
# 10-Fold (more folds = more training data per fold, but more variance)
scores_10fold = cross_val_score(model, X, y, cv=10)
print(f"
10-Fold CV: {scores_10fold.mean():.4f} ± {scores_10fold.std():.4f}")
 
# Leave-One-Out (extreme case: k = n)
# Only practical for small datasets!
loo = LeaveOneOut()
scores_loo = cross_val_score(model, X, y, cv=loo)
print(f"
Leave-One-Out CV: {scores_loo.mean():.4f} (n={len(scores_loo)} folds)")
 
# Confidence interval (approximate 95% CI)
mean = scores_kfold.mean()
std = scores_kfold.std()
ci_lower = mean - 1.96 * std / np.sqrt(len(scores_kfold))
ci_upper = mean + 1.96 * std / np.sqrt(len(scores_kfold))
print(f"
95% CI for 5-Fold CV: [{ci_lower:.4f}, {ci_upper:.4f}]")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

import numpy as np
from sklearn.model_selection import train_test_split
 
# Create imbalanced dataset: 95% class 0, 5% class 1
np.random.seed(42)
n = 200
y = np.array([0] * 190 + [1] * 10)  # 95% vs 5%
X = np.random.randn(n, 5)
 
print("Original class distribution:", np.bincount(y))
print(f"Class 0: {np.mean(y==0)*100:.1f}%, Class 1: {np.mean(y==1)*100:.1f}%")
 
# Non-stratified split (risky!)
print("
--- Random Split (non-stratified) ---")
for seed in [0, 1, 2, 3, 4]:
    _, _, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=seed  # No stratify!
    )
    print(f"Seed {seed}: Test class 1 count = {sum(y_test==1)}/{len(y_test)}")
# Notice: Some splits might have 0 or very few class 1 samples in test!
 
# Stratified split (safe!)
print("
--- Stratified Split ---")
for seed in [0, 1, 2, 3, 4]:
    _, _, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=seed, stratify=y  # Stratified!
    )
    print(f"Seed {seed}: Test class 1 count = {sum(y_test==1)}/{len(y_test)}")
# Stratified: Consistent class proportions in every split

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

from sklearn.model_selection import cross_val_score, KFold, GridSearchCV
from sklearn.svm import SVC
from sklearn.datasets import load_breast_cancer
import numpy as np
 
X, y = load_breast_cancer(return_X_y=True)
 
# Hyperparameter grid
param_grid = {
    'C': [0.1, 1, 10, 100],
    'gamma': ['scale', 'auto', 0.01, 0.1]
}
 
# WRONG: Non-nested CV (biased estimate)
# This finds best params and reports the same CV score
print("=== Non-Nested CV (BIASED) ===")
inner_cv = KFold(n_splits=5, shuffle=True, random_state=42)
grid_search = GridSearchCV(SVC(), param_grid, cv=inner_cv, scoring='accuracy')
grid_search.fit(X, y)
print(f"Best params: {grid_search.best_params_}")
print(f"Best CV score: {grid_search.best_score_:.4f}")
print("⚠️ This score is optimistically biased!")
 
# CORRECT: Nested CV (unbiased estimate)
print("
=== Nested CV (UNBIASED) ===")
outer_cv = KFold(n_splits=5, shuffle=True, random_state=42)
inner_cv = KFold(n_splits=3, shuffle=True, random_state=42)  # 3 inner for speed
 
# Inner loop: hyperparameter tuning
clf = GridSearchCV(SVC(), param_grid, cv=inner_cv, scoring='accuracy')
 
# Outer loop: performance estimation
nested_scores = cross_val_score(clf, X, y, cv=outer_cv, scoring='accuracy')
 
print(f"Nested CV scores: {nested_scores.round(4)}")
print(f"Mean: {nested_scores.mean():.4f} ± {nested_scores.std():.4f}")
print("✓ This is an unbiased estimate of tuned model performance")
 
# The nested score is typically lower (less optimistic) than non-nested
print(f"
Difference: {grid_search.best_score_ - nested_scores.mean():.4f}")
print("(Positive difference = non-nested was overly optimistic)")