Grid Search

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

from typing import Dict, List, Tuple
from enum import Enum
 
class HPOMethod(Enum):
    GRID = "Grid Search"
    RANDOM = "Random Search"
    BAYESIAN = "Bayesian Optimization"
    MULTI_FIDELITY = "Multi-fidelity (Hyperband/BOHB)"
    MANUAL = "Manual Tuning"
 
def recommend_hpo_method(
    n_hyperparameters: int,
    training_time_seconds: float,
    budget_hours: float,
    cv_folds: int = 5,
    requires_reproducibility: bool = False,
    has_hpo_expertise: bool = True
) -> Dict:
    """
    Recommend HPO method based on problem characteristics.
    
    Parameters:
    -----------
    n_hyperparameters: Number of hyperparameters to tune
    training_time_seconds: Time to train one model
    budget_hours: Total compute budget in hours
    cv_folds: Cross-validation folds
    requires_reproducibility: Must be exactly reproducible
    has_hpo_expertise: Team has experience with advanced HPO
    """
    budget_seconds = budget_hours * 3600
    max_evaluations = budget_seconds / (training_time_seconds * cv_folds)
    
    # Calculate feasible grid resolution
    if n_hyperparameters > 0:
        grid_resolution = int(max_evaluations ** (1 / n_hyperparameters))
        grid_resolution = max(2, min(10, grid_resolution))
        grid_size = grid_resolution ** n_hyperparameters
    else:
        grid_resolution = 0
        grid_size = 0
    
    # Scoring for each method
    scores = {method: 0.0 for method in HPOMethod}
    reasoning = {method: [] for method in HPOMethod}
    
    # === Dimension-based scoring ===
    if n_hyperparameters <= 2:
        scores[HPOMethod.GRID] += 3
        reasoning[HPOMethod.GRID].append("Low dimensions - grid is optimal")
        scores[HPOMethod.RANDOM] -= 1
        reasoning[HPOMethod.RANDOM].append("Wasteful in low dimensions")
        
    elif n_hyperparameters <= 3:
        scores[HPOMethod.GRID] += 2
        scores[HPOMethod.RANDOM] += 1
        scores[HPOMethod.BAYESIAN] += 1
        reasoning[HPOMethod.GRID].append("3D - grid still effective")
        
    elif n_hyperparameters <= 5:
        scores[HPOMethod.GRID] += 0
        scores[HPOMethod.RANDOM] += 2
        scores[HPOMethod.BAYESIAN] += 2
        reasoning[HPOMethod.RANDOM].append("4-5D - random becomes competitive")
        reasoning[HPOMethod.BAYESIAN].append("4-5D - can model correlations")
        
    elif n_hyperparameters <= 10:
        scores[HPOMethod.GRID] -= 3
        scores[HPOMethod.RANDOM] += 1
        scores[HPOMethod.BAYESIAN] += 3
        scores[HPOMethod.MULTI_FIDELITY] += 2
        reasoning[HPOMethod.GRID].append("6-10D - grid is impractical")
        reasoning[HPOMethod.BAYESIAN].append("6-10D - Bayesian excels")
        
    else:
        scores[HPOMethod.GRID] -= 5
        scores[HPOMethod.RANDOM] += 1
        scores[HPOMethod.BAYESIAN] += 2
        scores[HPOMethod.MULTI_FIDELITY] += 3
        reasoning[HPOMethod.MULTI_FIDELITY].append(">10D - multi-fidelity essential")
    
    # === Budget-based scoring ===
    if grid_size <= max_evaluations:
        scores[HPOMethod.GRID] += 1
        reasoning[HPOMethod.GRID].append(f"Budget allows {grid_resolution}-resolution grid")
    else:
        scores[HPOMethod.GRID] -= 2
        reasoning[HPOMethod.GRID].append("Budget insufficient for meaningful grid")
    
    if training_time_seconds > 300:  # > 5 minutes per train
        scores[HPOMethod.MULTI_FIDELITY] += 2
        scores[HPOMethod.BAYESIAN] += 1
        reasoning[HPOMethod.MULTI_FIDELITY].append("Long training - multi-fidelity saves time")
    
    # === Requirements-based scoring ===
    if requires_reproducibility:
        scores[HPOMethod.GRID] += 3
        scores[HPOMethod.RANDOM] -= 1
        reasoning[HPOMethod.GRID].append("Reproducibility required - grid is deterministic")
        reasoning[HPOMethod.RANDOM].append("Randomness complicates reproducibility")
    
    if not has_hpo_expertise:
        scores[HPOMethod.GRID] += 2
        scores[HPOMethod.RANDOM] += 1
        scores[HPOMethod.BAYESIAN] -= 2
        scores[HPOMethod.MULTI_FIDELITY] -= 3
        reasoning[HPOMethod.GRID].append("No expertise - grid is simplest")
        reasoning[HPOMethod.BAYESIAN].append("Requires expertise to tune properly")
    
    # === Determine recommendation ===
    best_method = max(scores.keys(), key=lambda m: scores[m])
    
    # Build result
    result = {
        'recommended_method': best_method,
        'scores': scores,
        'reasoning': {m: r for m, r in reasoning.items() if r},
        'analysis': {
            'n_hyperparameters': n_hyperparameters,
            'max_evaluations': int(max_evaluations),
            'feasible_grid_resolution': grid_resolution,
            'feasible_grid_size': grid_size,
        }
    }
    
    return result
 
 
def print_recommendation(result: Dict):
    """Pretty-print the recommendation."""
    print("="*60)
    print("HPO METHOD RECOMMENDATION")
    print("="*60)
    
    print(f"\nRecommended Method: {result['recommended_method'].value}")
    
    print(f"\nAnalysis:")
    for key, value in result['analysis'].items():
        print(f"  {key}: {value}")
    
    print(f"\nMethod Scores:")
    for method, score in sorted(result['scores'].items(), key=lambda x: -x[1]):
        print(f"  {method.value}: {score:+.1f}")
    
    print(f"\nReasoning:")
    for method, reasons in result['reasoning'].items():
        if reasons:
            print(f"  {method.value}:")
            for reason in reasons:
                print(f"    - {reason}")
 
 
# Example scenarios
scenarios = [
    {
        'name': "Quick SVM tuning",
        'n_hyperparameters': 2,
        'training_time_seconds': 5,
        'budget_hours': 1,
        'requires_reproducibility': True,
        'has_hpo_expertise': False,
    },
    {
        'name': "XGBoost with many params",
        'n_hyperparameters': 7,
        'training_time_seconds': 30,
        'budget_hours': 8,
        'requires_reproducibility': False,
        'has_hpo_expertise': True,
    },
    {
        'name': "Deep learning NAS",
        'n_hyperparameters': 15,
        'training_time_seconds': 600,
        'budget_hours': 48,
        'requires_reproducibility': False,
        'has_hpo_expertise': True,
    },
]
 
for scenario in scenarios:
    print(f"\n{'='*60}")
    print(f"SCENARIO: {scenario['name']}")
    del scenario['name']
    result = recommend_hpo_method(**scenario)
    print_recommendation(result)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225

import numpy as np
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.gaussian_process import GaussianProcessRegressor
from scipy.stats import uniform, randint
from typing import Dict, List, Any, Callable
from itertools import product
 
def grid_for_primary_random_for_secondary(
    estimator, X, y,
    primary_grid: Dict[str, List],
    secondary_distributions: Dict[str, Any],
    n_random_iter: int = 50,
    cv: int = 5
):
    """
    Pattern 1: Grid search on primary hyperparameters,
    random search on secondary hyperparameters.
    
    Combines thorough coverage of important dimensions
    with efficient exploration of less critical ones.
    """
    print("=== Hybrid: Grid (Primary) + Random (Secondary) ===")
    
    # Phase 1: Grid on primary
    print(f"\nPhase 1: Grid Search on {list(primary_grid.keys())}")
    primary_size = np.prod([len(v) for v in primary_grid.values()])
    print(f"  Grid size: {primary_size}")
    
    grid_search = GridSearchCV(estimator, primary_grid, cv=cv, n_jobs=-1)
    grid_search.fit(X, y)
    best_primary = grid_search.best_params_
    print(f"  Best: {best_primary}, score: {grid_search.best_score_:.4f}")
    
    # Phase 2: Random on secondary with primary fixed
    print(f"\nPhase 2: Random Search on {list(secondary_distributions.keys())}")
    
    full_distributions = {
        **{k: [v] for k, v in best_primary.items()},
        **secondary_distributions,
    }
    
    random_search = RandomizedSearchCV(
        estimator, full_distributions,
        n_iter=n_random_iter, cv=cv, n_jobs=-1, random_state=42
    )
    random_search.fit(X, y)
    
    print(f"  Best: {random_search.best_params_}, score: {random_search.best_score_:.4f}")
    print(f"\nTotal evaluations: {primary_size + n_random_iter}")
    
    return random_search.best_params_, random_search.best_score_
 
 
def grid_as_bayesian_initialization(
    objective_fn: Callable,
    param_bounds: Dict[str, tuple],
    init_grid_resolution: int = 3,
    bo_iterations: int = 50
):
    """
    Pattern 2: Grid search provides initial samples for Bayesian optimization.
    
    The grid ensures diverse initial samples, preventing the surrogate
    model from focusing too early on a single region.
    """
    from scipy.optimize import minimize
    
    print("=== Hybrid: Grid Initialization + Bayesian Optimization ===")
    
    param_names = list(param_bounds.keys())
    n_dims = len(param_names)
    
    # Phase 1: Coarse grid for initialization
    print(f"\nPhase 1: Grid Initialization ({init_grid_resolution}^{n_dims} points)")
    
    grid_points = []
    grid_results = []
    
    for combo in product(*[np.linspace(lo, hi, init_grid_resolution) 
                           for lo, hi in param_bounds.values()]):
        config = dict(zip(param_names, combo))
        score = objective_fn(config)
        grid_points.append(list(combo))
        grid_results.append(score)
    
    best_grid_idx = np.argmin(grid_results)
    print(f"  Evaluated {len(grid_results)} configurations")
    print(f"  Best grid score: {grid_results[best_grid_idx]:.4f}")
    
    # Phase 2: Bayesian optimization starting from grid samples
    print(f"\nPhase 2: Bayesian Optimization ({bo_iterations} iterations)")
    
    X_init = np.array(grid_points)
    y_init = np.array(grid_results)
    
    # Fit initial GP surrogate
    gp = GaussianProcessRegressor(normalize_y=True, random_state=42)
    gp.fit(X_init, y_init)
    
    # Simple expected improvement acquisition
    def expected_improvement(x, gp, y_best, xi=0.01):
        from scipy.stats import norm
        mu, sigma = gp.predict(x.reshape(1, -1), return_std=True)
        sigma = sigma + 1e-9
        z = (y_best - mu - xi) / sigma
        ei = sigma * (z * norm.cdf(z) + norm.pdf(z))
        return -ei.item()  # Minimize negative EI
    
    X_all = list(X_init)
    y_all = list(y_init)
    
    for i in range(bo_iterations):
        y_best = min(y_all)
        
        # Optimize acquisition function
        bounds = list(param_bounds.values())
        best_ei = float('inf')
        best_x = None
        
        # Multi-start optimization
        for _ in range(20):
            x0 = [np.random.uniform(lo, hi) for lo, hi in bounds]
            res = minimize(
                lambda x: expected_improvement(x, gp, y_best),
                x0, bounds=bounds, method='L-BFGS-B'
            )
            if res.fun < best_ei:
                best_ei = res.fun
                best_x = res.x
        
        # Evaluate at best point
        config = dict(zip(param_names, best_x))
        score = objective_fn(config)
        
        X_all.append(best_x)
        y_all.append(score)
        
        # Update GP
        gp.fit(np.array(X_all), np.array(y_all))
    
    best_idx = np.argmin(y_all)
    best_config = dict(zip(param_names, X_all[best_idx]))
    
    print(f"  Best BO score: {y_all[best_idx]:.4f}")
    print(f"  Best config: {best_config}")
    print(f"\nTotal evaluations: {len(grid_results) + bo_iterations}")
    
    return best_config, y_all[best_idx]
 
 
def grid_for_discrete_continuous_hybrid(
    estimator_class,
    discrete_grid: Dict[str, List],
    continuous_distributions: Dict[str, Any],
    X, y,
    n_random_per_discrete: int = 20,
    cv: int = 5
):
    """
    Pattern 4: Grid search for discrete hyperparameters,
    random search for continuous hyperparameters.
    
    For each discrete combination, run random search on continuous.
    This ensures all discrete options are explored while
    efficiently searching continuous spaces.
    """
    print("=== Hybrid: Grid (Discrete) + Random (Continuous) ===")
    
    param_names = list(discrete_grid.keys())
    all_results = []
    
    # Enumerate all discrete combinations
    discrete_combos = list(product(*discrete_grid.values()))
    print(f"\nDiscrete combinations: {len(discrete_combos)}")
    print(f"Random samples per discrete: {n_random_per_discrete}")
    print(f"Total evaluations: {len(discrete_combos) * n_random_per_discrete}")
    
    for combo in discrete_combos:
        discrete_config = dict(zip(param_names, combo))
        print(f"\n  Discrete: {discrete_config}")
        
        # Create estimator with fixed discrete params
        class FixedDiscreteEstimator:
            def __init__(self, **kwargs):
                self.model = estimator_class(**discrete_config, **kwargs)
            def fit(self, X, y): 
                self.model.fit(X, y)
                return self
            def predict(self, X): 
                return self.model.predict(X)
            def score(self, X, y): 
                return self.model.score(X, y)
            def get_params(self, deep=True): 
                return self.model.get_params(deep)
            def set_params(self, **params):
                self.model.set_params(**params)
                return self
        
        # Random search on continuous for this discrete combo
        random_search = RandomizedSearchCV(
            FixedDiscreteEstimator(),
            continuous_distributions,
            n_iter=n_random_per_discrete,
            cv=cv, n_jobs=-1, random_state=42
        )
        random_search.fit(X, y)
        
        result = {
            'discrete': discrete_config,
            'continuous': random_search.best_params_,
            'score': random_search.best_score_,
        }
        all_results.append(result)
        print(f"    Best continuous: {random_search.best_params_}")
        print(f"    Score: {random_search.best_score_:.4f}")
    
    # Find overall best
    best = max(all_results, key=lambda x: x['score'])
    
    print(f"\n=== Best Overall ===")
    print(f"Discrete: {best['discrete']}")
    print(f"Continuous: {best['continuous']}")
    print(f"Score: {best['score']:.4f}")
    
    return best

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178

from enum import Enum
from dataclasses import dataclass
from typing import Optional
 
class HPOMethod(Enum):
    GRID = "Grid Search"
    RANDOM = "Random Search"
    BAYESIAN = "Bayesian Optimization"
    HYPERBAND = "Hyperband / Multi-fidelity"
    BOHB = "BOHB (Bayesian + Hyperband)"
    MANUAL = "Manual / Default"
 
@dataclass
class HPORecommendation:
    primary: HPOMethod
    alternative: Optional[HPOMethod]
    confidence: str  # "high", "medium", "low"
    reasoning: str
 
def hpo_decision_tree(
    n_hyperparameters: int,
    n_discrete: int = 0,
    training_time_minutes: float = 1.0,
    budget_evaluations: int = 100,
    requires_reproducibility: bool = False,
    has_hpo_expertise: bool = True,
    expected_low_effective_dim: bool = False,
) -> HPORecommendation:
    """
    Decision tree for HPO method selection.
    
    Parameters:
    -----------
    n_hyperparameters: Total hyperparameters to tune
    n_discrete: Number that are discrete/categorical
    training_time_minutes: Time to train one model
    budget_evaluations: Maximum model evaluations available
    requires_reproducibility: Must be exactly reproducible
    has_hpo_expertise: Team comfortable with advanced methods
    expected_low_effective_dim: Expect few hyperparameters to matter
    """
    
    n_continuous = n_hyperparameters - n_discrete
    
    # Step 1: Dimensionality check
    if n_hyperparameters <= 2:
        # Low dimensions: Grid is almost always best
        if requires_reproducibility or not has_hpo_expertise:
            return HPORecommendation(
                primary=HPOMethod.GRID,
                alternative=None,
                confidence="high",
                reasoning="1-2 dimensions: Grid search is optimal, "
                         "providing complete coverage with tractable cost."
            )
        else:
            return HPORecommendation(
                primary=HPOMethod.GRID,
                alternative=HPOMethod.RANDOM,
                confidence="high",
                reasoning="1-2 dimensions: Grid search recommended. "
                         "Random search is acceptable but offers no benefit."
            )
    
    elif n_hyperparameters == 3:
        # Transition zone
        grid_size = 5 ** 3  # 125
        if budget_evaluations >= grid_size:
            return HPORecommendation(
                primary=HPOMethod.GRID,
                alternative=HPOMethod.RANDOM,
                confidence="medium",
                reasoning=f"3 dimensions with sufficient budget ({budget_evaluations} >= {grid_size}): "
                         "Grid search provides complete coverage."
            )
        else:
            return HPORecommendation(
                primary=HPOMethod.RANDOM,
                alternative=HPOMethod.BAYESIAN,
                confidence="medium",
                reasoning=f"3 dimensions with limited budget: "
                         "Random search is more efficient than sparse grid."
            )
    
    elif n_hyperparameters <= 5:
        # Medium dimensions
        if expected_low_effective_dim:
            return HPORecommendation(
                primary=HPOMethod.RANDOM,
                alternative=HPOMethod.BAYESIAN,
                confidence="high",
                reasoning="4-5 dimensions with low effective dimensionality: "
                         "Random search efficiently explores important dimensions."
            )
        elif has_hpo_expertise and budget_evaluations >= 50:
            return HPORecommendation(
                primary=HPOMethod.BAYESIAN,
                alternative=HPOMethod.RANDOM,
                confidence="medium",
                reasoning="4-5 dimensions with sufficient budget and expertise: "
                         "Bayesian optimization can model response surface."
            )
        else:
            return HPORecommendation(
                primary=HPOMethod.RANDOM,
                alternative=HPOMethod.GRID,
                confidence="medium",
                reasoning="4-5 dimensions: Random search is reliable and simple. "
                         "Consider staged grid search as alternative."
            )
    
    elif n_hyperparameters <= 10:
        # High dimensions
        if training_time_minutes > 10:
            return HPORecommendation(
                primary=HPOMethod.HYPERBAND if not has_hpo_expertise else HPOMethod.BOHB,
                alternative=HPOMethod.BAYESIAN,
                confidence="high",
                reasoning="6-10 dimensions with long training: "
                         "Multi-fidelity methods essential for efficiency."
            )
        elif has_hpo_expertise and budget_evaluations >= 100:
            return HPORecommendation(
                primary=HPOMethod.BAYESIAN,
                alternative=HPOMethod.RANDOM,
                confidence="medium",
                reasoning="6-10 dimensions: Bayesian optimization with "
                         "sufficient budget outperforms random search."
            )
        else:
            return HPORecommendation(
                primary=HPOMethod.RANDOM,
                alternative=HPOMethod.BAYESIAN,
                confidence="medium",
                reasoning="6-10 dimensions with constraints: "
                         "Random search is robust and simple."
            )
    
    else:
        # Very high dimensions
        if training_time_minutes > 5:
            return HPORecommendation(
                primary=HPOMethod.BOHB,
                alternative=HPOMethod.HYPERBAND,
                confidence="high",
                reasoning=">10 dimensions with expensive training: "
                         "BOHB combines Bayesian optimization with multi-fidelity."
            )
        else:
            return HPORecommendation(
                primary=HPOMethod.BAYESIAN,
                alternative=HPOMethod.RANDOM,
                confidence="medium",
                reasoning=">10 dimensions: Bayesian optimization recommended, "
                         "but consider dimensionality reduction."
            )
 
 
# Example usage
print("HPO METHOD DECISION TREE")
print("="*60)
 
scenarios = [
    {"n_hyperparameters": 2, "budget_evaluations": 50, "training_time_minutes": 0.5},
    {"n_hyperparameters": 3, "budget_evaluations": 200, "training_time_minutes": 1},
    {"n_hyperparameters": 5, "budget_evaluations": 100, "expected_low_effective_dim": True},
    {"n_hyperparameters": 8, "training_time_minutes": 15, "has_hpo_expertise": True},
    {"n_hyperparameters": 12, "training_time_minutes": 30, "budget_evaluations": 200},
]
 
for scenario in scenarios:
    print(f"\nScenario: {scenario}")
    rec = hpo_decision_tree(**scenario)
    print(f"  Primary: {rec.primary.value}")
    if rec.alternative:
        print(f"  Alternative: {rec.alternative.value}")
    print(f"  Confidence: {rec.confidence}")
    print(f"  Reasoning: {rec.reasoning}")