1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243

"""
Common Search Space Bounds: A Practical Reference
 
These bounds are derived from extensive empirical studies and represent
reasonable starting points for most tasks. Adjust based on domain knowledge.
"""
from dataclasses import dataclass
from typing import List, Tuple, Optional, Union
import numpy as np
 
 
@dataclass
class HyperparameterSpec:
    """Specification for a single hyperparameter."""
    name: str
    type: str  # 'continuous', 'integer', 'categorical'
    bounds: Union[Tuple[float, float], List[str]]
    log_scale: bool = False
    default: Optional[float] = None
    description: str = ""
 
 
# ============================================================
# Neural Network Hyperparameters
# ============================================================
 
NEURAL_NETWORK_SEARCH_SPACE = [
    HyperparameterSpec(
        name="learning_rate",
        type="continuous",
        bounds=(1e-5, 1.0),
        log_scale=True,  # CRITICAL: Always log-scale for learning rate
        default=1e-3,
        description="Step size for gradient updates. Log-scale essential."
    ),
    HyperparameterSpec(
        name="weight_decay",
        type="continuous",
        bounds=(1e-6, 1e-1),
        log_scale=True,
        default=1e-4,
        description="L2 regularization. Log-scale for wide range."
    ),
    HyperparameterSpec(
        name="dropout",
        type="continuous",
        bounds=(0.0, 0.5),
        log_scale=False,  # Linear scale works for probability
        default=0.1,
        description="Dropout probability. Linear scale, bounded by 0.5."
    ),
    HyperparameterSpec(
        name="batch_size",
        type="integer",
        bounds=(8, 256),
        log_scale=True,  # Consider log-scale for powers of 2
        default=32,
        description="Samples per gradient update. Powers of 2 preferred."
    ),
    HyperparameterSpec(
        name="hidden_units",
        type="integer",
        bounds=(32, 1024),
        log_scale=True,
        default=256,
        description="Units in hidden layers. Log-scale for wide range."
    ),
    HyperparameterSpec(
        name="num_layers",
        type="integer",
        bounds=(1, 10),
        log_scale=False,  # Linear for small integer range
        default=3,
        description="Number of hidden layers."
    ),
    HyperparameterSpec(
        name="optimizer",
        type="categorical",
        bounds=["sgd", "adam", "adamw", "rmsprop"],
        default="adam",
        description="Optimization algorithm choice."
    ),
    HyperparameterSpec(
        name="activation",
        type="categorical",
        bounds=["relu", "gelu", "swish", "tanh"],
        default="relu",
        description="Activation function."
    ),
]
 
 
# ============================================================
# Gradient Boosting Hyperparameters (XGBoost/LightGBM)
# ============================================================
 
GRADIENT_BOOSTING_SEARCH_SPACE = [
    HyperparameterSpec(
        name="learning_rate",
        type="continuous",
        bounds=(0.001, 0.3),
        log_scale=True,
        default=0.1,
        description="Boosting learning rate (shrinkage)."
    ),
    HyperparameterSpec(
        name="n_estimators",
        type="integer",
        bounds=(50, 2000),
        log_scale=True,
        default=500,
        description="Number of boosting rounds."
    ),
    HyperparameterSpec(
        name="max_depth",
        type="integer",
        bounds=(3, 12),
        log_scale=False,
        default=6,
        description="Maximum tree depth."
    ),
    HyperparameterSpec(
        name="min_child_weight",
        type="continuous",
        bounds=(1, 100),
        log_scale=True,
        default=1,
        description="Minimum sum of instance weight in child."
    ),
    HyperparameterSpec(
        name="subsample",
        type="continuous",
        bounds=(0.5, 1.0),
        log_scale=False,
        default=0.8,
        description="Row subsampling ratio."
    ),
    HyperparameterSpec(
        name="colsample_bytree",
        type="continuous",
        bounds=(0.5, 1.0),
        log_scale=False,
        default=0.8,
        description="Column subsampling ratio per tree."
    ),
    HyperparameterSpec(
        name="reg_alpha",
        type="continuous",
        bounds=(1e-8, 10.0),
        log_scale=True,
        default=1e-5,
        description="L1 regularization."
    ),
    HyperparameterSpec(
        name="reg_lambda",
        type="continuous",
        bounds=(1e-8, 10.0),
        log_scale=True,
        default=1.0,
        description="L2 regularization."
    ),
]
 
 
# ============================================================
# SVM Hyperparameters
# ============================================================
 
SVM_SEARCH_SPACE = [
    HyperparameterSpec(
        name="C",
        type="continuous",
        bounds=(1e-4, 1e4),
        log_scale=True,  # 8 orders of magnitude!
        default=1.0,
        description="Regularization parameter. Log-scale essential."
    ),
    HyperparameterSpec(
        name="gamma",
        type="continuous",
        bounds=(1e-5, 1e2),
        log_scale=True,
        default=None,  # Often 1/n_features
        description="RBF kernel bandwidth. Log-scale essential."
    ),
    HyperparameterSpec(
        name="kernel",
        type="categorical",
        bounds=["linear", "poly", "rbf", "sigmoid"],
        default="rbf",
        description="Kernel function."
    ),
]
 
 
def estimate_search_space_size(specs: List[HyperparameterSpec], 
                                resolution: int = 10) -> float:
    """
    Estimate the effective size of a search space.
    
    For continuous parameters: assume 'resolution' distinct values
    For categorical: number of categories
    For integer: (upper - lower + 1), capped at resolution
    
    Args:
        specs: List of hyperparameter specifications
        resolution: Discretization level for continuous params
        
    Returns:
        Total number of grid points (can be astronomical)
    """
    total_configs = 1
    
    for spec in specs:
        if spec.type == "continuous":
            points = resolution
        elif spec.type == "integer":
            low, high = spec.bounds
            points = min(high - low + 1, resolution)
        elif spec.type == "categorical":
            points = len(spec.bounds)
        else:
            points = resolution
            
        total_configs *= points
    
    return total_configs
 
 
# Example: Calculate search space size
if __name__ == "__main__":
    print("Neural Network Search Space Analysis")
    print("=" * 50)
    
    nn_size = estimate_search_space_size(NEURAL_NETWORK_SEARCH_SPACE)
    print(f"Grid points (10 per continuous dim): {nn_size:,}")
    
    nn_size_fine = estimate_search_space_size(NEURAL_NETWORK_SEARCH_SPACE, 
                                               resolution=100)
    print(f"Grid points (100 per continuous dim): {nn_size_fine:,}")
    
    print(f"\nAt 5 min/evaluation: {nn_size * 5 / 60:.1f} hours for coarse grid")
    print(f"This illustrates why grid search is infeasible!")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

"""
Scale Transformations for Hyperparameter Search
 
This demonstrates why scale choice dramatically affects search efficiency.
"""
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats
 
 
def sample_linear(low: float, high: float, n: int = 1000) -> np.ndarray:
    """Sample uniformly on linear scale."""
    return np.random.uniform(low, high, n)
 
 
def sample_log(low: float, high: float, n: int = 1000) -> np.ndarray:
    """Sample uniformly on log scale (log-uniform distribution)."""
    log_low, log_high = np.log10(low), np.log10(high)
    return 10 ** np.random.uniform(log_low, log_high, n)
 
 
def sample_logit(low: float, high: float, n: int = 1000) -> np.ndarray:
    """Sample on logit scale (useful for probabilities near boundaries)."""
    # Map [low, high] to [logit(low), logit(high)], sample, invert
    def logit(p):
        return np.log(p / (1 - p))
    def sigmoid(x):
        return 1 / (1 + np.exp(-x))
    
    logit_low, logit_high = logit(low), logit(high)
    samples = np.random.uniform(logit_low, logit_high, n)
    return sigmoid(samples)
 
 
def compare_sampling_strategies():
    """
    Visualize how different scales distribute samples.
    
    This demonstrates why log-scale is essential for wide ranges.
    """
    learning_rate_range = (1e-5, 1.0)  # 5 orders of magnitude
    
    linear_samples = sample_linear(*learning_rate_range, n=10000)
    log_samples = sample_log(*learning_rate_range, n=10000)
    
    print("Learning Rate Sampling: [1e-5, 1.0]")
    print("=" * 60)
    
    # Count samples in different ranges
    ranges = [
        (1e-5, 1e-4, "1e-5 to 1e-4"),
        (1e-4, 1e-3, "1e-4 to 1e-3"),
        (1e-3, 1e-2, "1e-3 to 1e-2"),
        (1e-2, 1e-1, "1e-2 to 1e-1"),
        (1e-1, 1.0,  "1e-1 to 1.0"),
    ]
    
    print(f"{'Range':<20} {'Linear %':>12} {'Log %':>12} {'Optimal?':<15}")
    print("-" * 60)
    
    for low, high, name in ranges:
        linear_pct = 100 * np.sum((linear_samples >= low) & 
                                   (linear_samples < high)) / len(linear_samples)
        log_pct = 100 * np.sum((log_samples >= low) & 
                                (log_samples < high)) / len(log_samples)
        
        # Most optimal learning rates for Adam are in 1e-4 to 1e-2 range
        is_optimal = "← Sweet spot" if name in ["1e-4 to 1e-3", "1e-3 to 1e-2"] else ""
        
        print(f"{name:<20} {linear_pct:>11.2f}% {log_pct:>11.2f}% {is_optimal}")
    
    print("\n⚠️  Linear sampling wastes 90% of budget on suboptimal region [0.1, 1.0]!")
    print("✓  Log sampling gives equal probability to each order of magnitude.")
 
 
def demonstrate_log_transform_benefit():
    """
    Show quantitatively how log-transform improves optimization.
    
    Simulate a response surface where optimal is at lr=0.001,
    and show how many random samples find the good region.
    """
    np.random.seed(42)
    
    # Simulated response surface: optimum near lr=0.001
    def response_surface(lr):
        """Validation error as function of learning rate."""
        log_lr = np.log10(lr)
        optimal_log_lr = np.log10(0.001)  # -3
        # Parabola in log-space, minimum at log(0.001) = -3
        return (log_lr - optimal_log_lr) ** 2 + 0.1 * np.random.randn()
    
    n_samples = 100
    
    # Random search with linear sampling
    linear_samples = sample_linear(1e-5, 1.0, n_samples)
    linear_errors = [response_surface(lr) for lr in linear_samples]
    
    # Random search with log sampling
    log_samples = sample_log(1e-5, 1.0, n_samples)
    log_errors = [response_surface(lr) for lr in log_samples]
    
    print("\nRandom Search Comparison (100 samples)")
    print("=" * 50)
    print(f"Optimal learning rate: 0.001")
    print(f"\nLinear sampling:")
    print(f"  Best lr found: {linear_samples[np.argmin(linear_errors)]:.6f}")
    print(f"  Best error: {min(linear_errors):.4f}")
    print(f"  Samples in [1e-4, 1e-2]: {np.sum((linear_samples >= 1e-4) & (linear_samples <= 1e-2))}")
    
    print(f"\nLog sampling:")
    print(f"  Best lr found: {log_samples[np.argmin(log_errors)]:.6f}")
    print(f"  Best error: {min(log_errors):.4f}")
    print(f"  Samples in [1e-4, 1e-2]: {np.sum((log_samples >= 1e-4) & (log_samples <= 1e-2))}")
 
 
if __name__ == "__main__":
    compare_sampling_strategies()
    demonstrate_log_transform_benefit()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197

"""
Adaptive Search Space Definition
 
Define search spaces that adapt to problem characteristics.
"""
from dataclasses import dataclass
from typing import Dict, Tuple, List, Optional
import numpy as np
 
 
@dataclass 
class ProblemCharacteristics:
    """Characteristics that inform search space design."""
    n_samples: int
    n_features: int
    n_classes: int  # 1 for regression
    is_imbalanced: bool
    task_type: str  # 'classification', 'regression', 'multilabel'
    compute_budget: str  # 'low', 'medium', 'high'
    memory_gb: float  # Available GPU memory
 
 
def create_neural_network_search_space(
    problem: ProblemCharacteristics
) -> Dict:
    """
    Create an adaptive search space for neural networks.
    
    The bounds adapt based on:
    - Dataset size (more data → can use more capacity)
    - Feature dimensionality (affects first layer)
    - Compute budget (limits what's feasible)
    - Memory constraints (limits batch size)
    """
    space = {}
    
    # ============================================================
    # Learning Rate: Adapt based on expected training dynamics
    # ============================================================
    
    # Smaller datasets often need smaller learning rates
    if problem.n_samples < 1000:
        lr_range = (1e-5, 1e-2)  # More conservative
    elif problem.n_samples < 100000:
        lr_range = (1e-5, 1e-1)  # Standard range
    else:
        lr_range = (1e-4, 0.5)   # Large datasets can use higher lr
    
    space['learning_rate'] = {
        'type': 'continuous',
        'bounds': lr_range,
        'log_scale': True,
        'rationale': f'Adapted for dataset size n={problem.n_samples}'
    }
    
    # ============================================================
    # Model Capacity: Scale with data availability
    # ============================================================
    
    # Rough heuristic: capacity should scale with log(n_samples)
    max_complexity = np.log10(problem.n_samples)  # e.g., 3 for 1K, 6 for 1M
    
    # Number of layers
    max_layers = int(np.clip(max_complexity, 2, 10))
    space['num_layers'] = {
        'type': 'integer',
        'bounds': (1, max_layers),
        'log_scale': False,
        'rationale': f'Max {max_layers} layers for n={problem.n_samples}'
    }
    
    # Hidden units
    max_units = int(np.clip(2 ** (4 + max_complexity), 64, 2048))
    space['hidden_units'] = {
        'type': 'integer',
        'bounds': (32, max_units),
        'log_scale': True,
        'rationale': f'Max {max_units} units for n={problem.n_samples}'
    }
    
    # ============================================================
    # Regularization: More for small data, less for large
    # ============================================================
    
    # Dropout: smaller datasets need more regularization
    if problem.n_samples < 1000:
        dropout_range = (0.3, 0.6)
    elif problem.n_samples < 100000:
        dropout_range = (0.1, 0.5)
    else:
        dropout_range = (0.0, 0.3)
    
    space['dropout'] = {
        'type': 'continuous',
        'bounds': dropout_range,
        'log_scale': False,
        'rationale': f'Dropout adapted for dataset size'
    }
    
    # Weight decay
    if problem.n_samples < 10000:
        wd_range = (1e-4, 1e-1)  # Strong regularization
    else:
        wd_range = (1e-6, 1e-2)  # Can use weaker
    
    space['weight_decay'] = {
        'type': 'continuous',
        'bounds': wd_range,
        'log_scale': True,
        'rationale': 'Weight decay for overfitting control'
    }
    
    # ============================================================
    # Batch Size: Based on memory and dataset size
    # ============================================================
    
    # Estimate memory per sample (rough heuristic)
    bytes_per_sample = problem.n_features * 4  # float32
    max_batch_from_memory = int(problem.memory_gb * 1e9 / bytes_per_sample / 100)
    max_batch_from_data = problem.n_samples // 10  # At least 10 batches
    max_batch = max(16, min(512, max_batch_from_memory, max_batch_from_data))
    
    # Common batch sizes (powers of 2)
    batch_sizes = [bs for bs in [16, 32, 64, 128, 256, 512] if bs <= max_batch]
    if not batch_sizes:
        batch_sizes = [16]
    
    space['batch_size'] = {
        'type': 'categorical',
        'choices': batch_sizes,
        'rationale': f'Batch sizes feasible for {problem.memory_gb}GB memory'
    }
    
    # ============================================================
    # Optimizer: Include gradient clipping for imbalanced data
    # ============================================================
    
    space['optimizer'] = {
        'type': 'categorical',
        'choices': ['adam', 'adamw', 'sgd'],
        'rationale': 'Standard optimizers'
    }
    
    if problem.is_imbalanced:
        space['gradient_clip_norm'] = {
            'type': 'continuous',
            'bounds': (0.5, 5.0),
            'log_scale': False,
            'rationale': 'Gradient clipping for unstable gradients with imbalance'
        }
    
    return space
 
 
def report_search_space(space: Dict) -> None:
    """Pretty print search space with rationale."""
    print("\nAdaptive Search Space")
    print("=" * 70)
    
    for name, config in space.items():
        print(f"\n{name}:")
        for key, value in config.items():
            print(f"  {key}: {value}")
 
 
# Example usage
if __name__ == "__main__":
    # Small tabular dataset
    small_problem = ProblemCharacteristics(
        n_samples=2000,
        n_features=50,
        n_classes=3,
        is_imbalanced=True,
        task_type='classification',
        compute_budget='medium',
        memory_gb=8.0
    )
    
    small_space = create_neural_network_search_space(small_problem)
    print("Search Space for Small Dataset (n=2000)")
    report_search_space(small_space)
    
    # Large dataset
    large_problem = ProblemCharacteristics(
        n_samples=1_000_000,
        n_features=100,
        n_classes=10,
        is_imbalanced=False,
        task_type='classification',
        compute_budget='high',
        memory_gb=32.0
    )
    
    large_space = create_neural_network_search_space(large_problem)
    print("\n" + "=" * 70)
    print("Search Space for Large Dataset (n=1,000,000)")
    report_search_space(large_space)