CatBoost

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

import numpy as np
import pandas as pd
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import GradientBoostingClassifier
 
def demonstrate_target_leakage():
    """
    Shows how naive target encoding causes dramatic overfitting.
    
    We create a dataset where the categorical feature has NO predictive
    power, yet target encoding makes it appear highly predictive.
    """
    np.random.seed(42)
    n_samples = 1000
    n_categories = 200
    
    # Random categorical feature with NO relationship to target
    categories = np.random.randint(0, n_categories, n_samples)
    # Target is pure noise - category provides zero information
    y = np.random.randint(0, 2, n_samples)
    
    # === Method 1: Naive Target Encoding ===
    # Compute mean target per category across ALL data (LEAKAGE!)
    category_means = pd.Series(y).groupby(categories).transform('mean')
    X_leaky = category_means.values.reshape(-1, 1)
    
    # This will appear to have signal, but it's pure leakage
    leaky_cv_scores = cross_val_score(
        GradientBoostingClassifier(n_estimators=50, random_state=42),
        X_leaky, y, cv=5, scoring='roc_auc'
    )
    
    # === Method 2: Proper Leave-One-Out within CV ===
    # More proper approach: encode each fold using only training fold data
    X_proper = np.zeros((n_samples, 1))
    
    from sklearn.model_selection import KFold
    kf = KFold(n_splits=5, shuffle=True, random_state=42)
    
    for train_idx, val_idx in kf.split(X_leaky):
        # Compute means only from training fold
        train_categories = categories[train_idx]
        train_y = y[train_idx]
        
        means = {}
        for cat in np.unique(train_categories):
            mask = train_categories == cat
            means[cat] = train_y[mask].mean() if mask.sum() > 0 else 0.5
        
        # Apply to validation fold
        global_mean = train_y.mean()
        for idx in val_idx:
            X_proper[idx, 0] = means.get(categories[idx], global_mean)
    
    proper_cv_scores = cross_val_score(
        GradientBoostingClassifier(n_estimators=50, random_state=42),
        X_proper, y, cv=5, scoring='roc_auc'
    )
    
    print("Target Leakage Demonstration")
    print("=" * 50)
    print(f"Ground truth: Categories have ZERO predictive power")
    print(f"Expected AUC: 0.50 (random chance)")
    print()
    print(f"Naive target encoding AUC: {leaky_cv_scores.mean():.3f}")
    print(f"  ^ Appears predictive due to leakage!")
    print()
    print(f"Proper fold-isolated encoding AUC: {proper_cv_scores.mean():.3f}")
    print(f"  ^ Correctly shows no predictive power")
 
demonstrate_target_leakage()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

import numpy as np
from typing import Dict, List
 
def compute_ordered_target_statistics(
    categories: np.ndarray,
    targets: np.ndarray,
    prior: float = 0.5,
    prior_weight: float = 1.0,
    random_seed: int = 42
) -> np.ndarray:
    """
    Compute ordered target statistics as used by CatBoost.
    
    For each sample, the encoding uses only targets from samples appearing
    earlier in a random permutation that share the same category.
    
    Parameters:
    -----------
    categories : array of shape (n_samples,)
        Categorical feature values
    targets : array of shape (n_samples,)
        Target values (0/1 for classification, continuous for regression)
    prior : float
        Prior probability/value for regularization
    prior_weight : float
        Weight of prior (higher = more shrinkage toward prior)
    random_seed : int
        Random seed for permutation reproducibility
        
    Returns:
    --------
    encodings : array of shape (n_samples,)
        Ordered target statistics for each sample
    """
    np.random.seed(random_seed)
    n_samples = len(categories)
    
    # Generate random permutation
    permutation = np.random.permutation(n_samples)
    
    # Track running statistics per category
    # sum_targets[cat] = sum of targets seen so far for category cat
    # count[cat] = number of samples seen so far for category cat
    sum_targets: Dict[int, float] = {}
    count: Dict[int, int] = {}
    
    # Encodings for each sample (in original order)
    encodings = np.zeros(n_samples)
    
    # Process samples in permutation order
    for position in range(n_samples):
        sample_idx = permutation[position]
        cat = categories[sample_idx]
        target = targets[sample_idx]
        
        # Compute encoding using statistics BEFORE this sample
        cat_sum = sum_targets.get(cat, 0.0)
        cat_count = count.get(cat, 0)
        
        # Ordered target statistic with prior smoothing
        encodings[sample_idx] = (
            (cat_sum + prior_weight * prior) / 
            (cat_count + prior_weight)
        )
        
        # Update running statistics to include this sample
        # (for subsequent samples in the permutation)
        sum_targets[cat] = cat_sum + target
        count[cat] = cat_count + 1
    
    return encodings
 
 
def verify_no_leakage(categories: np.ndarray, targets: np.ndarray):
    """
    Verify that ordered target statistics don't cause leakage.
    
    For a random target with no relationship to categories,
    ordered encoding should show no predictive power.
    """
    from sklearn.model_selection import cross_val_score
    from sklearn.ensemble import GradientBoostingClassifier
    
    # Compute ordered target statistics
    encodings = compute_ordered_target_statistics(
        categories, targets, prior=0.5, prior_weight=1.0
    )
    
    X = encodings.reshape(-1, 1)
    
    # Cross-validate
    scores = cross_val_score(
        GradientBoostingClassifier(n_estimators=50, random_state=42),
        X, targets, cv=5, scoring='roc_auc'
    )
    
    print(f"Ordered Target Statistics AUC: {scores.mean():.3f} (+/- {scores.std():.3f})")
    print("Expected: ~0.50 for random targets (no leakage)")
    
    return scores.mean()
 
 
# Demonstrate with random data
if __name__ == "__main__":
    np.random.seed(42)
    n_samples = 1000
    categories = np.random.randint(0, 100, n_samples)
    targets = np.random.randint(0, 2, n_samples)  # Random targets
    
    print("Verification: Ordered encoding eliminates leakage")
    print("=" * 55)
    verify_no_leakage(categories, targets)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

from catboost import CatBoostClassifier, Pool
import pandas as pd
import numpy as np
 
# Create sample data with high-cardinality categoricals
def create_sample_data():
    np.random.seed(42)
    n_samples = 10000
    
    data = {
        'user_id': [f'user_{i % 5000}' for i in range(n_samples)],
        'product_category': np.random.choice(
            ['electronics', 'clothing', 'food', 'books', 'sports'],
            n_samples
        ),
        'city': np.random.choice(
            [f'city_{i}' for i in range(200)],  # 200 cities
            n_samples
        ),
        'browser': np.random.choice(
            ['chrome', 'firefox', 'safari', 'edge', 'other'],
            n_samples
        ),
        'numeric_feature': np.random.randn(n_samples),
        'target': (np.random.rand(n_samples) > 0.7).astype(int)
    }
    return pd.DataFrame(data)
 
df = create_sample_data()
train_df = df.iloc[:8000]
test_df = df.iloc[8000:]
 
# Identify categorical columns
cat_features = ['user_id', 'product_category', 'city', 'browser']
 
# CatBoost handles categoricals natively - no preprocessing needed
model = CatBoostClassifier(
    iterations=500,
    learning_rate=0.05,
    depth=6,
    
    # ===========================
    # Categorical Feature Config
    # ===========================
    
    # Identify categorical feature indices or names
    cat_features=cat_features,
    
    # One-hot encoding threshold: categories with <= this many levels
    # are one-hot encoded; others use target statistics
    one_hot_max_size=10,  # One-hot for features with ≤10 categories
    
    # Maximum number of categorical feature combinations to consider
    max_ctr_complexity=2,  # Up to pairs of categoricals
    
    # Prior for target statistics regularization
    # Higher = more shrinkage toward prior
    simple_ctr=[
        'Borders:Prior=0.5:PriorWeight=1.0',
    ],
    
    random_seed=42,
    verbose=100,
)
 
# Create Pool objects (CatBoost's data container)
train_pool = Pool(
    train_df.drop('target', axis=1),
    train_df['target'],
    cat_features=cat_features
)
 
test_pool = Pool(
    test_df.drop('target', axis=1),
    test_df['target'],
    cat_features=cat_features
)
 
# Train - CatBoost automatically:
# 1. Computes ordered target statistics per permutation
# 2. Generates categorical feature combinations
# 3. Uses symmetric trees for efficient processing
model.fit(train_pool, eval_set=test_pool, early_stopping_rounds=50)
 
# Examine feature importance including categorical combinations
feature_importance = model.get_feature_importance(
    train_pool,
    type='PredictionValuesChange'
)
 
feature_names = model.feature_names_
print("
Top Features by Importance:")
for name, importance in sorted(
    zip(feature_names, feature_importance),
    key=lambda x: -x[1]
)[:10]:
    print(f"  {name}: {importance:.3f}")
 
# Handle new categories at inference
# Create a sample with previously unseen categories
new_sample = pd.DataFrame({
    'user_id': ['completely_new_user'],  # Never seen in training
    'product_category': ['new_category'],  # Never seen
    'city': ['city_999'],  # Never seen
    'browser': ['chrome'],  # Seen
    'numeric_feature': [0.5]
})
 
# CatBoost handles this gracefully - no crash, uses priors
prediction = model.predict_proba(new_sample)
print(f"
Prediction for unseen categories: {prediction}")
print("(Graceful fallback to prior - no errors)")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

from catboost import CatBoostClassifier, Pool
import numpy as np
import pandas as pd
 
# Example: Configuring CTR for e-commerce prediction
def configure_advanced_ctr():
    """
    Demonstrates advanced CTR configuration for a realistic
    e-commerce purchase prediction problem.
    """
    
    model = CatBoostClassifier(
        iterations=1000,
        learning_rate=0.03,
        depth=6,
        
        # ===========================================
        # General CTR Configuration
        # ===========================================
        
        # Max combinations of categorical features
        # Higher = more expressive but slower and risk of overfitting
        max_ctr_complexity=3,  # Up to triples
        
        # One-hot threshold: use one-hot for low-cardinality
        one_hot_max_size=5,    # One-hot if ≤5 categories
        
        # ===========================================
        # Simple CTR: Applied to individual features
        # ===========================================
        simple_ctr=[
            # Primary: Target mean with moderate shrinkage
            'Borders:TargetBorderCount=1:PriorWeight=1.0',
            
            # Secondary: Frequency-based (useful for popularity signals)
            'Counter',
        ],
        
        # ===========================================
        # Combination CTR: Applied to feature combinations
        # ===========================================
        combinations_ctr=[
            # For combinations, use stronger shrinkage
            # (combinations have less data per category)
            'Borders:TargetBorderCount=1:PriorWeight=2.0',
        ],
        
        # ===========================================
        # Per-Feature CTR Customization
        # ===========================================
        # Assuming: 0=user_id, 1=category, 2=brand, 3=device
        per_feature_ctr={
            # User ID: Very high cardinality, aggressive shrinkage
            0: ['Borders:PriorWeight=5.0'],
            
            # Category: Moderate cardinality, standard config
            1: ['Borders:PriorWeight=1.0', 'Counter'],
            
            # Brand: May have popularity effects
            2: ['Borders:PriorWeight=1.0', 'Counter'],
            
            # Device: Low cardinality, will be one-hot encoded anyway
            3: [],  # Use one-hot only
        },
        
        # ===========================================
        # Regularization for overfitting prevention
        # ===========================================
        l2_leaf_reg=3.0,
        random_strength=0.5,
        bagging_temperature=0.3,
        
        random_seed=42,
        verbose=100,
    )
    
    return model
 
 
def analyze_ctr_features(model, pool):
    """
    Analyze how CatBoost created CTR-based features.
    """
    # Get all feature importances including CTR combinations
    importance = model.get_feature_importance(pool)
    feature_names = model.feature_names_
    
    # Categorize features
    original_features = []
    ctr_features = []
    combination_features = []
    
    for name, imp in zip(feature_names, importance):
        if '{' in name:  # CTR combination features contain braces
            combination_features.append((name, imp))
        elif ':' in name:  # CTR transforms contain colons
            ctr_features.append((name, imp))
        else:
            original_features.append((name, imp))
    
    print("Feature Importance Analysis")
    print("=" * 60)
    
    print("
Original Features:")
    for name, imp in sorted(original_features, key=lambda x: -x[1])[:5]:
        print(f"  {name}: {imp:.3f}")
    
    print("
CTR-transformed Features:")
    for name, imp in sorted(ctr_features, key=lambda x: -x[1])[:5]:
        print(f"  {name}: {imp:.3f}")
    
    print("
Combination Features:")
    for name, imp in sorted(combination_features, key=lambda x: -x[1])[:5]:
        print(f"  {name}: {imp:.3f}")
 
# Usage example (with appropriate data)
model = configure_advanced_ctr()
print("Model configured with advanced CTR settings")
print(f"Simple CTR: {model.get_params().get('simple_ctr')}")
print(f"Combinations CTR: {model.get_params().get('combinations_ctr')}")