1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

from catboost import CatBoostClassifier, CatBoostRegressor, Pool
import numpy as np
 
# =============================================================
# Basic GPU Training
# =============================================================
 
model = CatBoostClassifier(
    # =====================
    # GPU Configuration
    # =====================
    task_type='GPU',        # Enable GPU training
    
    # Device selection (comma-separated for multi-GPU)
    devices='0',            # Single GPU
    # devices='0:1:2:3',    # Multiple GPUs
    
    # =====================
    # GPU-Specific Parameters
    # =====================
    
    # Number of bins for feature quantization (GPU only)
    # Higher = more accuracy but more memory
    border_count=128,       # Default for GPU (vs 254 for CPU)
    
    # GPU RAM limit in bytes (optional)
    # gpu_ram_part=0.9,     # Use 90% of GPU RAM
    
    # =====================
    # Standard Training Parameters
    # =====================
    iterations=1000,
    learning_rate=0.1,
    depth=6,
    
    # Early stopping still works
    early_stopping_rounds=50,
    
    random_seed=42,
    verbose=100,
)
 
 
# =============================================================
# Training with Validation
# =============================================================
 
def train_with_gpu(X_train, y_train, X_val, y_val, cat_features=None):
    """
    Complete GPU training example with best practices.
    """
    
    # Create Pool objects (data containers)
    # Pools can specify categorical features once
    train_pool = Pool(X_train, y_train, cat_features=cat_features)
    val_pool = Pool(X_val, y_val, cat_features=cat_features)
    
    model = CatBoostClassifier(
        task_type='GPU',
        devices='0',
        
        # GPU-optimized parameters
        iterations=1000,
        learning_rate=0.1,
        depth=8,              # GPU handles deeper trees efficiently
        border_count=128,
        
        # Regularization
        l2_leaf_reg=3,
        random_strength=1,
        
        # Early stopping
        early_stopping_rounds=50,
        
        verbose=100,
    )
    
    # Fit with validation set
    model.fit(
        train_pool,
        eval_set=val_pool,
        use_best_model=True,
    )
    
    print(f"Best iteration: {model.best_iteration_}")
    print(f"Best validation score: {model.best_score_}")
    
    return model
 
 
# =============================================================
# Checking GPU Availability
# =============================================================
 
def check_gpu_status():
    """Check if GPU training is available and configured."""
    from catboost import CatBoost
    
    try:
        n_gpus = CatBoost.get_gpu_device_count()
        print(f"Available GPUs: {n_gpus}")
        
        if n_gpus > 0:
            # Quick test
            X = np.random.randn(1000, 10)
            y = (X[:, 0] > 0).astype(int)
            
            model = CatBoostClassifier(
                task_type='GPU',
                devices='0',
                iterations=10,
                verbose=0
            )
            model.fit(X, y)
            print("GPU training: ✓ Working")
            return True
        else:
            print("No GPU devices found")
            return False
            
    except Exception as e:
        print(f"GPU check failed: {e}")
        return False
 
if __name__ == "__main__":
    check_gpu_status()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

from catboost import CatBoostClassifier, Pool
import numpy as np
import time
 
def benchmark_multi_gpu(X, y, gpu_configs):
    """
    Benchmark training time across different GPU configurations.
    """
    results = []
    
    for n_gpus, devices in gpu_configs:
        # Prepare model
        model = CatBoostClassifier(
            task_type='GPU',
            devices=devices,
            iterations=100,
            depth=8,
            learning_rate=0.1,
            random_seed=42,
            verbose=0,
        )
        
        # Warm-up run
        small_X = X[:min(1000, len(X))]
        small_y = y[:min(1000, len(y))]
        model.fit(small_X, small_y)
        
        # Timed run
        model = CatBoostClassifier(
            task_type='GPU',
            devices=devices,
            iterations=100,
            depth=8,
            learning_rate=0.1,
            random_seed=42,
            verbose=0,
        )
        
        start = time.perf_counter()
        model.fit(X, y)
        elapsed = time.perf_counter() - start
        
        results.append({
            'n_gpus': n_gpus,
            'devices': devices,
            'time_seconds': elapsed,
        })
        
        print(f"GPUs: {n_gpus} ({devices}): {elapsed:.2f}s")
    
    # Calculate speedups
    baseline = results[0]['time_seconds']
    for r in results:
        r['speedup'] = baseline / r['time_seconds']
        print(f"  {r['n_gpus']} GPU(s): {r['speedup']:.2f}x speedup")
    
    return results
 
 
# Example configurations (adjust based on available GPUs)
# gpu_configs = [
#     (1, '0'),
#     (2, '0:1'),
#     (4, '0:1:2:3'),
# ]
 
 
# =============================================================
# GPU Memory Management for Large Datasets
# =============================================================
 
def train_large_dataset_gpu(X, y, available_gpu_ram_gb=8):
    """
    Configure GPU training for datasets approaching GPU memory limits.
    """
    
    # Estimate memory requirements
    n_samples, n_features = X.shape
    bytes_per_sample = n_features * 4  # float32
    estimated_data_gb = (n_samples * bytes_per_sample) / 1e9
    
    # CatBoost needs ~3-5x data size for working memory
    estimated_working_gb = estimated_data_gb * 4
    
    print(f"Dataset: {n_samples:,} samples × {n_features} features")
    print(f"Estimated data size: {estimated_data_gb:.2f} GB")
    print(f"Estimated working memory: {estimated_working_gb:.2f} GB")
    
    if estimated_working_gb > available_gpu_ram_gb:
        print(f"WARNING: May exceed GPU RAM ({available_gpu_ram_gb} GB)")
        print("Consider: (1) reducing border_count, (2) using multi-GPU, "
              "(3) using CPU for this dataset")
    
    # Configure for memory efficiency
    model = CatBoostClassifier(
        task_type='GPU',
        devices='0',
        
        # Memory-efficient settings
        border_count=64,        # Fewer bins = less memory
        depth=6,                # Shallower = less memory
        
        # Limit GPU RAM usage
        gpu_ram_part=0.8,       # Use 80% of GPU RAM max
        
        iterations=500,
        learning_rate=0.1,
        early_stopping_rounds=50,
        verbose=100,
    )
    
    return model
 
 
# =============================================================
# Handling Out-of-Memory on GPU
# =============================================================
 
def train_with_gpu_fallback(X, y, cat_features=None):
    """
    Attempt GPU training with automatic fallback to CPU.
    """
    from catboost import CatBoostError
    
    try:
        model = CatBoostClassifier(
            task_type='GPU',
            devices='0',
            iterations=500,
            depth=6,
            border_count=128,
            verbose=100,
        )
        
        model.fit(X, y, cat_features=cat_features)
        print("Training completed on GPU")
        return model, 'GPU'
        
    except CatBoostError as e:
        if 'out of memory' in str(e).lower() or 'CUDA' in str(e):
            print(f"GPU training failed: {e}")
            print("Falling back to CPU...")
            
            model = CatBoostClassifier(
                task_type='CPU',
                iterations=500,
                depth=6,
                border_count=254,
                verbose=100,
            )
            
            model.fit(X, y, cat_features=cat_features)
            print("Training completed on CPU")
            return model, 'CPU'
        else:
            raise

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

import subprocess
import threading
import time
from catboost import CatBoostClassifier
import numpy as np
 
def monitor_gpu_utilization(duration_seconds=60, interval=1):
    """
    Monitor GPU utilization during training using nvidia-smi.
    """
    samples = []
    stop_flag = threading.Event()
    
    def sample_gpu():
        while not stop_flag.is_set():
            try:
                result = subprocess.run(
                    ['nvidia-smi', '--query-gpu=utilization.gpu,memory.used,memory.total',
                     '--format=csv,noheader,nounits'],
                    capture_output=True, text=True
                )
                if result.returncode == 0:
                    parts = result.stdout.strip().split(', ')
                    samples.append({
                        'gpu_util': float(parts[0]),
                        'memory_used_mb': float(parts[1]),
                        'memory_total_mb': float(parts[2]),
                        'timestamp': time.time()
                    })
            except Exception as e:
                pass
            time.sleep(interval)
    
    monitor_thread = threading.Thread(target=sample_gpu)
    monitor_thread.start()
    
    return stop_flag, samples
 
 
def benchmark_gpu_efficiency(X, y):
    """
    Benchmark GPU efficiency across different configurations.
    """
    configs = [
        {'depth': 4, 'border_count': 64},
        {'depth': 6, 'border_count': 128},
        {'depth': 8, 'border_count': 128},
        {'depth': 8, 'border_count': 254},
    ]
    
    results = []
    
    for config in configs:
        # Start GPU monitoring
        stop_flag, samples = monitor_gpu_utilization()
        samples.clear()
        
        model = CatBoostClassifier(
            task_type='GPU',
            devices='0',
            iterations=100,
            learning_rate=0.1,
            verbose=0,
            **config
        )
        
        start = time.perf_counter()
        model.fit(X, y)
        elapsed = time.perf_counter() - start
        
        stop_flag.set()
        time.sleep(1)  # Allow final samples
        
        # Analyze utilization
        if samples:
            avg_util = np.mean([s['gpu_util'] for s in samples])
            avg_mem = np.mean([s['memory_used_mb'] for s in samples])
        else:
            avg_util = avg_mem = 0
        
        results.append({
            **config,
            'time_seconds': elapsed,
            'avg_gpu_util': avg_util,
            'avg_memory_mb': avg_mem,
        })
        
        print(f"Depth={config['depth']}, Borders={config['border_count']}: "
              f"{elapsed:.2f}s, GPU={avg_util:.0f}%, Mem={avg_mem:.0f}MB")
    
    return results
 
 
# =============================================================
# Optimizing for Production Training
# =============================================================
 
def optimized_gpu_training_config(n_samples, n_features, gpu_ram_gb=16):
    """
    Generate optimized GPU configuration based on data characteristics.
    """
    
    # Base configuration
    config = {
        'task_type': 'GPU',
        'devices': '0',
        'random_seed': 42,
    }
    
    # Scale parameters based on data size
    if n_samples < 50_000:
        # Small data: less aggressive GPU usage
        config.update({
            'iterations': 1000,
            'learning_rate': 0.05,
            'depth': 6,
            'border_count': 128,
        })
        print("Small dataset: Consider CPU training")
        
    elif n_samples < 500_000:
        # Medium data: standard GPU config
        config.update({
            'iterations': 500,
            'learning_rate': 0.1,
            'depth': 8,
            'border_count': 128,
        })
        
    else:
        # Large data: maximize GPU utilization
        config.update({
            'iterations': 300,
            'learning_rate': 0.15,
            'depth': 10,
            'border_count': 254 if gpu_ram_gb >= 16 else 128,
        })
    
    # Adjust for feature count
    if n_features > 100:
        config['depth'] = min(config['depth'], 8)  # Prevent memory explosion
    
    print(f"Optimized config for {n_samples:,} samples, {n_features} features:")
    for k, v in config.items():
        print(f"  {k}: {v}")
    
    return config

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

from catboost import CatBoostClassifier, Pool
import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.metrics import roc_auc_score
 
def validate_gpu_cpu_equivalence(X, y, n_runs=5, tolerance=0.01):
    """
    Validate that GPU training produces equivalent results to CPU.
    
    Parameters:
    -----------
    tolerance : float
        Maximum acceptable difference in AUC between GPU and CPU
    """
    
    # Common parameters (except task_type)
    common_params = {
        'iterations': 200,
        'learning_rate': 0.1,
        'depth': 6,
        'random_seed': 42,
        'verbose': 0,
    }
    
    cpu_scores = []
    gpu_scores = []
    
    for run in range(n_runs):
        # Shuffle data differently each run
        np.random.seed(run)
        indices = np.random.permutation(len(X))
        X_shuffled = X[indices]
        y_shuffled = y[indices]
        
        split_idx = int(0.8 * len(X))
        X_train, X_test = X_shuffled[:split_idx], X_shuffled[split_idx:]
        y_train, y_test = y_shuffled[:split_idx], y_shuffled[split_idx:]
        
        # CPU training
        cpu_model = CatBoostClassifier(task_type='CPU', **common_params)
        cpu_model.fit(X_train, y_train)
        cpu_probs = cpu_model.predict_proba(X_test)[:, 1]
        cpu_auc = roc_auc_score(y_test, cpu_probs)
        cpu_scores.append(cpu_auc)
        
        # GPU training
        gpu_model = CatBoostClassifier(task_type='GPU', **common_params)
        gpu_model.fit(X_train, y_train)
        gpu_probs = gpu_model.predict_proba(X_test)[:, 1]
        gpu_auc = roc_auc_score(y_test, gpu_probs)
        gpu_scores.append(gpu_auc)
    
    # Compare results
    cpu_mean = np.mean(cpu_scores)
    gpu_mean = np.mean(gpu_scores)
    difference = abs(cpu_mean - gpu_mean)
    
    print("GPU vs CPU Validation Results")
    print("=" * 50)
    print(f"CPU Mean AUC: {cpu_mean:.4f} (+/- {np.std(cpu_scores):.4f})")
    print(f"GPU Mean AUC: {gpu_mean:.4f} (+/- {np.std(gpu_scores):.4f})")
    print(f"Difference:   {difference:.4f}")
    
    if difference <= tolerance:
        print(f"✓ PASS: Difference ({difference:.4f}) within tolerance ({tolerance})")
        return True
    else:
        print(f"✗ FAIL: Difference ({difference:.4f}) exceeds tolerance ({tolerance})")
        return False
 
 
def compare_prediction_consistency(cpu_model, gpu_model, X_test):
    """
    Compare individual predictions between CPU and GPU models.
    """
    cpu_probs = cpu_model.predict_proba(X_test)[:, 1]
    gpu_probs = gpu_model.predict_proba(X_test)[:, 1]
    
    # Statistics
    abs_diff = np.abs(cpu_probs - gpu_probs)
    
    print("Prediction Consistency Analysis")
    print("=" * 50)
    print(f"Mean absolute difference:   {np.mean(abs_diff):.6f}")
    print(f"Max absolute difference:    {np.max(abs_diff):.6f}")
    print(f"Std of differences:         {np.std(abs_diff):.6f}")
    print(f"Predictions within 0.01:    {(abs_diff < 0.01).mean()*100:.1f}%")
    print(f"Predictions within 0.001:   {(abs_diff < 0.001).mean()*100:.1f}%")
    
    # Same classification decision
    cpu_preds = (cpu_probs >= 0.5).astype(int)
    gpu_preds = (gpu_probs >= 0.5).astype(int)
    agreement = (cpu_preds == gpu_preds).mean()
    print(f"Classification agreement:   {agreement*100:.2f}%")
    
    return abs_diff

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

from catboost import CatBoostClassifier, Pool
import numpy as np
import os
 
def train_with_checkpointing(X_train, y_train, X_val, y_val, 
                             checkpoint_dir='./catboost_checkpoints',
                             total_iterations=2000,
                             checkpoint_interval=100):
    """
    Train with periodic checkpointing for spot instance resilience.
    
    If training is interrupted, resume from the last checkpoint.
    """
    os.makedirs(checkpoint_dir, exist_ok=True)
    checkpoint_path = os.path.join(checkpoint_dir, 'model_checkpoint')
    
    # Check for existing checkpoint
    start_iteration = 0
    if os.path.exists(checkpoint_path):
        print(f"Found checkpoint at {checkpoint_path}, resuming...")
        # Load checkpoint and continue
        model = CatBoostClassifier()
        model.load_model(checkpoint_path)
        start_iteration = model.tree_count_
        print(f"Resuming from iteration {start_iteration}")
    else:
        model = CatBoostClassifier(
            task_type='GPU',
            devices='0',
            iterations=checkpoint_interval,
            learning_rate=0.05,
            depth=8,
            early_stopping_rounds=100,
            verbose=50,
        )
    
    # Train in chunks with checkpointing
    train_pool = Pool(X_train, y_train)
    val_pool = Pool(X_val, y_val)
    
    current_iteration = start_iteration
    while current_iteration < total_iterations:
        remaining = total_iterations - current_iteration
        chunk_size = min(checkpoint_interval, remaining)
        
        if current_iteration == 0:
            # Initial training
            model.set_params(iterations=chunk_size)
            model.fit(train_pool, eval_set=val_pool)
        else:
            # Continue training
            model.set_params(iterations=chunk_size)
            model.fit(
                train_pool, 
                eval_set=val_pool,
                init_model=model  # Continue from current model
            )
        
        current_iteration += chunk_size
        
        # Save checkpoint
        model.save_model(checkpoint_path)
        print(f"Checkpoint saved at iteration {current_iteration}")
        
        # Check early stopping
        if hasattr(model, 'best_iteration_') and            current_iteration - model.best_iteration_ > 100:
            print(f"Early stopping at iteration {current_iteration}")
            break
    
    print("Training complete")
    return model
 
 
def estimate_training_cost(n_samples, n_features, n_iterations, 
                           instance_type='g4dn.xlarge', provider='aws'):
    """
    Estimate cloud GPU training cost.
    """
    # Approximate instance costs per hour (varies by region)
    hourly_costs = {
        'aws': {
            'g4dn.xlarge': 0.526,
            'p3.2xlarge': 3.06,
            'p3.8xlarge': 12.24,
        },
        'gcp': {
            't4': 0.35,
            'v100': 2.48,
            'a100': 3.67,
        }
    }
    
    # Rough training time estimates (iterations per minute)
    # Based on empirical benchmarks
    iters_per_minute = {
        'g4dn.xlarge': 300 / (1 + n_samples / 500_000),  # Scales with data
        'p3.2xlarge': 500 / (1 + n_samples / 500_000),
        'p3.8xlarge': 800 / (1 + n_samples / 500_000),
    }
    
    if instance_type not in hourly_costs[provider]:
        print(f"Unknown instance type: {instance_type}")
        return None
    
    hourly_cost = hourly_costs[provider][instance_type]
    iter_rate = iters_per_minute.get(instance_type, 200)
    
    estimated_minutes = n_iterations / iter_rate
    estimated_hours = estimated_minutes / 60
    estimated_cost = estimated_hours * hourly_cost
    
    print(f"Training Cost Estimate")
    print(f"=" * 40)
    print(f"Instance:    {instance_type}")
    print(f"Dataset:     {n_samples:,} samples × {n_features} features")
    print(f"Iterations:  {n_iterations:,}")
    print(f"Est. time:   {estimated_minutes:.0f} min ({estimated_hours:.1f} hr)")
    print(f"Est. cost:   ${estimated_cost: .2f
                            }")
    
    # Spot instance savings
    spot_cost = estimated_cost * 0.3  # ~70 % savings
    print(f"Est. spot:   ${spot_cost:.2f} (with spot instances)")
    
    return {
                                'time_minutes': estimated_minutes,
                                'cost_ondemand': estimated_cost,
                                'cost_spot': spot_cost,
                            }
 
 
# Example usage
if __name__ == "__main__":
                            estimate_training_cost(
                                n_samples = 1_000_000,
                                n_features = 100,
                                n_iterations = 1000,
                                instance_type = 'p3.2xlarge'
                            )