Feature Importance

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

import numpy as np
import pandas as pd
from typing import List, Dict, Optional
 
def generate_importance_narrative(
    importance_df: pd.DataFrame,
    expected_important: List[str] = None,
    expected_unimportant: List[str] = None,
    domain_context: str = "",
    top_n: int = 5
) -> str:
    """
    Generate a natural language narrative from feature importance results.
    
    Args:
        importance_df: DataFrame with 'feature', 'importance', 'ci_lower', 'ci_upper' columns
        expected_important: Features expected to be important based on domain knowledge
        expected_unimportant: Features expected to be unimportant
        domain_context: Description of the prediction problem
        top_n: Number of top features to highlight
    
    Returns:
        Narrative string
    """
    narrative_parts = []
    
    # Sort by importance
    df = importance_df.sort_values('importance', ascending=False).reset_index(drop=True)
    total_importance = df['importance'].sum()
    
    # === HEADLINE ===
    top_features = df.head(top_n)['feature'].tolist()
    top_pct = df.head(top_n)['importance'].sum() / total_importance * 100
    
    headline = f"## Key Finding\n\n"
    headline += f"The top {top_n} features ({', '.join(top_features[:3])}"
    if len(top_features) > 3:
        headline += f", and {len(top_features) - 3} others"
    headline += f") account for **{top_pct:.0f}%** of total feature importance.\n"
    narrative_parts.append(headline)
    
    # === TOP DRIVERS ===
    drivers = "## Top Drivers\n\n"
    for i, row in df.head(top_n).iterrows():
        pct = row['importance'] / total_importance * 100
        ci_info = ""
        if 'ci_lower' in df.columns and 'ci_upper' in df.columns:
            ci_info = f" (95% CI: [{row['ci_lower']:.3f}, {row['ci_upper']:.3f}])"
        drivers += f"{i+1}. **{row['feature']}**: {pct:.1f}% of importance{ci_info}\n"
    narrative_parts.append(drivers)
    
    # === SURPRISES ===
    surprises = "## Notable Findings\n\n"
    surprises_found = False
    
    if expected_important:
        missing_expected = [f for f in expected_important 
                          if f in df['feature'].values and 
                          df[df['feature'] == f].index[0] >= top_n]
        if missing_expected:
            surprises += f"**Unexpectedly low importance:** {', '.join(missing_expected)} "
            surprises += "ranked lower than domain knowledge suggested.\n\n"
            surprises_found = True
    
    if expected_unimportant:
        unexpected_important = [f for f in expected_unimportant 
                               if f in df['feature'].values and 
                               df[df['feature'] == f].index[0] < top_n]
        if unexpected_important:
            surprises += f"**Unexpectedly high importance:** {', '.join(unexpected_important)} "
            surprises += "ranked higher than expected—worth investigating for potential leakage.\n\n"
            surprises_found = True
    
    # Features with negative importance
    negative_features = df[df['importance'] < 0]['feature'].tolist()
    if negative_features:
        surprises += f"**Negative importance (harmful features):** {', '.join(negative_features)} "
        surprises += "appear to hurt predictions and should be investigated.\n\n"
        surprises_found = True
    
    if surprises_found:
        narrative_parts.append(surprises)
    
    # === BOTTOM TIER ===
    bottom = "## Low-Importance Features\n\n"
    bottom_n = min(5, len(df) - top_n)
    bottom_features = df.tail(bottom_n)['feature'].tolist()
    bottom_pct = df.tail(bottom_n)['importance'].sum() / total_importance * 100
    bottom += f"The bottom {bottom_n} features ({', '.join(bottom_features)}) "
    bottom += f"together contribute only **{bottom_pct:.1f}%** of importance. "
    bottom += "Consider whether these are worth maintaining in the feature pipeline.\n"
    narrative_parts.append(bottom)
    
    # === PRESCRIPTION ===
    prescription = "## Recommendations\n\n"
    prescription += "Based on this analysis:\n\n"
    prescription += f"1. **Prioritize data quality** for: {', '.join(top_features[:3])}\n"
    prescription += f"2. **Monitor for drift** in top features during production\n"
    if negative_features:
        prescription += f"3. **Investigate and likely remove**: {', '.join(negative_features)}\n"
    if len(bottom_features) > 2:
        prescription += f"4. **Consider removing** low-importance features to reduce complexity\n"
    narrative_parts.append(prescription)
    
    return "\n".join(narrative_parts)
 
# Example usage
if __name__ == "__main__":
    # Simulated importance results
    importance_data = {
        'feature': ['purchase_frequency', 'session_duration', 'support_tickets',
                   'subscription_tier', 'tenure_months', 'geographic_region',
                   'device_type', 'email_opens', 'random_noise_feature', 'customer_id'],
        'importance': [0.25, 0.20, 0.15, 0.12, 0.08, 0.05, 0.07, 0.06, 0.01, 0.01],
        'ci_lower': [0.22, 0.17, 0.12, 0.09, 0.05, 0.02, 0.04, 0.03, -0.01, -0.02],
        'ci_upper': [0.28, 0.23, 0.18, 0.15, 0.11, 0.08, 0.10, 0.09, 0.03, 0.04],
    }
    df = pd.DataFrame(importance_data)
    
    narrative = generate_importance_narrative(
        df,
        expected_important=['tenure_months', 'geographic_region'],
        expected_unimportant=['random_noise_feature', 'customer_id'],
        domain_context="Customer churn prediction for SaaS product",
        top_n=5
    )
    
    print("=" * 70)
    print("FEATURE IMPORTANCE NARRATIVE")
    print("=" * 70)
    print(narrative)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
 
def plot_importance_with_uncertainty(
    importance_df: pd.DataFrame,
    top_n: int = 15,
    figsize: tuple = (10, 8),
    title: str = "Feature Importance",
    show_significance: bool = True,
    significance_threshold: float = 0.0
):
    """
    Create publication-quality feature importance visualization.
    
    Args:
        importance_df: DataFrame with 'feature', 'importance', 'std' (and optionally 'significant')
        top_n: Number of features to show
        figsize: Figure size
        title: Plot title
        show_significance: Whether to highlight significant features
        significance_threshold: Horizontal line for significance cutoff
    """
    # Prepare data
    df = importance_df.nlargest(top_n, 'importance').copy()
    df = df.sort_values('importance', ascending=True)  # For horizontal bar chart
    
    fig, ax = plt.subplots(figsize=figsize)
    
    # Colors based on significance
    if show_significance and 'significant' in df.columns:
        colors = ['#2E86AB' if sig else '#A6A6A6' 
                 for sig in df['significant']]
    else:
        # Gradient based on importance
        norm_imp = (df['importance'] - df['importance'].min()) /                    (df['importance'].max() - df['importance'].min())
        colors = plt.cm.Blues(0.3 + 0.6 * norm_imp)
    
    # Create horizontal bar chart
    y_pos = np.arange(len(df))
    bars = ax.barh(y_pos, df['importance'], 
                   xerr=df.get('std', 0), 
                   capsize=4,
                   color=colors,
                   edgecolor='white',
                   linewidth=0.5,
                   alpha=0.9)
    
    # Add significance threshold line
    if significance_threshold > 0:
        ax.axvline(x=significance_threshold, color='red', 
                  linestyle='--', linewidth=1.5, alpha=0.7,
                  label=f'Significance threshold ({significance_threshold})')
    
    # Styling
    ax.set_yticks(y_pos)
    ax.set_yticklabels(df['feature'], fontsize=11)
    ax.set_xlabel('Feature Importance', fontsize=12)
    ax.set_title(title, fontsize=14, fontweight='bold', pad=15)
    
    # Add value labels
    for i, (imp, std) in enumerate(zip(df['importance'], df.get('std', [0]*len(df)))):
        label = f'{imp:.3f}'
        if std > 0:
            label += f' ± {std:.3f}'
        ax.text(imp + 0.005, i, label, va='center', fontsize=9, alpha=0.8)
    
    # Grid for readability
    ax.set_axisbelow(True)
    ax.grid(axis='x', linestyle='--', alpha=0.3)
    
    # Legend
    if show_significance and 'significant' in df.columns:
        from matplotlib.patches import Patch
        legend_elements = [
            Patch(facecolor='#2E86AB', label='Statistically significant'),
            Patch(facecolor='#A6A6A6', label='Not significant'),
        ]
        ax.legend(handles=legend_elements, loc='lower right', fontsize=10)
    
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    
    plt.tight_layout()
    return fig, ax
 
def plot_importance_comparison(
    importance_methods: dict,
    feature_names: list,
    top_n: int = 10,
    figsize: tuple = (12, 8)
):
    """
    Compare feature rankings across multiple importance methods.
    
    Args:
        importance_methods: Dict mapping method name to importance array
        feature_names: List of feature names
        top_n: Number of features to show
        figsize: Figure size
    """
    # Get union of top features across all methods
    all_top = set()
    for method, importances in importance_methods.items():
        top_idx = np.argsort(importances)[-top_n:]
        all_top.update(top_idx)
    
    selected_idx = sorted(all_top)
    selected_names = [feature_names[i] for i in selected_idx]
    
    # Prepare data for grouped bar chart
    n_methods = len(importance_methods)
    x = np.arange(len(selected_idx))
    width = 0.8 / n_methods
    
    fig, ax = plt.subplots(figsize=figsize)
    
    colors = plt.cm.Set2(np.linspace(0, 1, n_methods))
    
    for i, (method, importances) in enumerate(importance_methods.items()):
        offset = (i - n_methods/2 + 0.5) * width
        vals = [importances[j] for j in selected_idx]
        ax.bar(x + offset, vals, width, label=method, color=colors[i], alpha=0.85)
    
    ax.set_xticks(x)
    ax.set_xticklabels(selected_names, rotation=45, ha='right', fontsize=10)
    ax.set_ylabel('Feature Importance', fontsize=12)
    ax.set_title('Feature Importance Comparison Across Methods', 
                fontsize=14, fontweight='bold')
    ax.legend(loc='upper right', fontsize=10)
    
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.grid(axis='y', linestyle='--', alpha=0.3)
    
    plt.tight_layout()
    return fig, ax
 
# Example usage
if __name__ == "__main__":
    # Simulated data
    np.random.seed(42)
    n_features = 20
    
    feature_names = [f"feature_{i}" for i in range(n_features)]
    
    # Generate random importance with some clearly important features
    importance = np.random.exponential(0.03, n_features)
    importance[0:5] *= 5  # Make first 5 more important
    importance = importance / importance.sum()  # Normalize
    
    std = importance * 0.2  # 20% relative uncertainty
    significant = importance > 0.03
    
    df = pd.DataFrame({
        'feature': feature_names,
        'importance': importance,
        'std': std,
        'significant': significant
    })
    
    # Create single-method visualization
    fig1, ax1 = plot_importance_with_uncertainty(
        df, 
        top_n=12,
        title="Random Forest Feature Importance (Permutation Method)",
        show_significance=True
    )
    
    # Create multi-method comparison
    methods = {
        'Impurity-based': np.random.exponential(0.05, n_features),
        'Permutation (val)': importance,
        'Drop-column': np.maximum(0, importance + np.random.randn(n_features) * 0.03)
    }
    
    fig2, ax2 = plot_importance_comparison(
        methods, feature_names, top_n=10
    )
    
    plt.show()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

def feature_selection_by_importance(
    importance_df: pd.DataFrame,
    cumulative_threshold: float = 0.90,
    individual_threshold: float = 0.01,
    must_include: list = None,
    must_exclude: list = None
) -> dict:
    """
    Framework for importance-based feature selection.
    
    Selection criteria:
    1. Include features contributing to cumulative_threshold of importance
    2. Include features with individual importance > individual_threshold
    3. Always include must_include features
    4. Never include must_exclude features
    
    Args:
        importance_df: DataFrame with 'feature' and 'importance' columns
        cumulative_threshold: Include enough features to reach this cumulative importance
        individual_threshold: Include any feature above this importance
        must_include: Features to always include (domain requirements)
        must_exclude: Features to never include (known issues)
    
    Returns:
        Dict with selected features and rationale
    """
    must_include = must_include or []
    must_exclude = must_exclude or []
    
    df = importance_df.sort_values('importance', ascending=False).copy()
    df['cumulative'] = df['importance'].cumsum() / df['importance'].sum()
    
    selected = set(must_include)
    rationale = {f: "domain requirement" for f in must_include}
    
    # Add features meeting thresholds
    for _, row in df.iterrows():
        feature = row['feature']
        if feature in must_exclude:
            continue
        if feature in selected:
            continue
            
        if row['importance'] > individual_threshold:
            selected.add(feature)
            rationale[feature] = f"importance = {row['importance']:.3f} > threshold"
        elif row['cumulative'] <= cumulative_threshold:
            selected.add(feature)
            rationale[feature] = f"contributes to {cumulative_threshold*100:.0f}% cumulative"
    
    # Features excluded
    excluded = set(df['feature']) - selected
    excluded_important = [f for f in excluded 
                         if df[df['feature']==f]['importance'].values[0] > 0.005]
    
    return {
        'selected_features': sorted(selected),
        'n_selected': len(selected),
        'n_total': len(df),
        'reduction_pct': (1 - len(selected)/len(df)) * 100,
        'cumulative_importance': df[df['feature'].isin(selected)]['importance'].sum() / 
                                  df['importance'].sum(),
        'rationale': rationale,
        'excluded_notable': excluded_important,
        'excluded_low': list(excluded - set(excluded_important))
    }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

from datetime import datetime
from typing import Dict, List, Optional
import json
 
def generate_importance_report(
    analysis_name: str,
    methodology: Dict,
    results: Dict,
    interpretation: Dict,
    decisions: Dict,
    output_path: Optional[str] = None
) -> str:
    """
    Generate a standardized importance analysis report.
    
    Args:
        analysis_name: Name/ID for this analysis
        methodology: Dict with method, validation, repeats, software details
        results: Dict with feature rankings, scores, stability metrics
        interpretation: Dict with narrative, domain validation, limitations
        decisions: Dict with actions, recommendations, follow-ups
        output_path: Optional path to save report
    
    Returns:
        Formatted report string
    """
    report = []
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    
    # Header
    report.append("=" * 70)
    report.append(f"FEATURE IMPORTANCE ANALYSIS REPORT")
    report.append(f"Analysis: {analysis_name}")
    report.append(f"Generated: {timestamp}")
    report.append("=" * 70)
    report.append("")
    
    # Methodology
    report.append("## METHODOLOGY")
    report.append("-" * 40)
    report.append(f"Importance Method: {methodology.get('method', 'Not specified')}")
    report.append(f"Validation Strategy: {methodology.get('validation', 'Not specified')}")
    report.append(f"Permutation Repeats: {methodology.get('repeats', 'N/A')}")
    report.append(f"Random Seed: {methodology.get('random_seed', 'Not specified')}")
    report.append(f"Software: {methodology.get('software', 'Not specified')}")
    report.append(f"Dataset Size: {methodology.get('dataset_size', 'Not specified')}")
    report.append("")
    
    # Results
    report.append("## RESULTS")
    report.append("-" * 40)
    report.append("Top Features (by importance):")
    for i, feat in enumerate(results.get('top_features', [])[:10], 1):
        imp = feat.get('importance', 0)
        ci = feat.get('ci', [0, 0])
        report.append(f"  {i}. {feat['name']}: {imp:.4f} (95% CI: [{ci[0]:.4f}, {ci[1]:.4f}])")
    
    report.append(f"\nStability Metrics:")
    report.append(f"  Rank Correlation (Kendall τ): {results.get('rank_correlation', 'N/A')}")
    report.append(f"  Features with Stable Rankings: {results.get('stable_count', 'N/A')}")
    report.append("")
    
    # Interpretation
    report.append("## INTERPRETATION")
    report.append("-" * 40)
    report.append(f"Key Finding: {interpretation.get('headline', 'Not specified')}")
    report.append(f"\nDomain Validation: {interpretation.get('domain_validation', 'Not performed')}")
    report.append(f"\nAnomalies Noted: {interpretation.get('anomalies', 'None')}")
    report.append(f"\nLimitations: {interpretation.get('limitations', 'Not specified')}")
    report.append("")
    
    # Decisions
    report.append("## DECISIONS & ACTIONS")
    report.append("-" * 40)
    report.append("Actions Taken:")
    for action in decisions.get('actions', ['None specified']):
        report.append(f"  • {action}")
    
    report.append("\nRecommendations:")
    for rec in decisions.get('recommendations', ['None specified']):
        report.append(f"  • {rec}")
    
    report.append("\nFollow-up Items:")
    for item in decisions.get('follow_ups', ['None specified']):
        report.append(f"  • {item}")
    
    report.append("")
    report.append("=" * 70)
    report.append("END OF REPORT")
    report.append("=" * 70)
    
    report_str = "\n".join(report)
    
    if output_path:
        with open(output_path, 'w') as f:
            f.write(report_str)
        
        # Also save structured data as JSON
        json_path = output_path.replace('.txt', '.json')
        data = {
            'analysis_name': analysis_name,
            'timestamp': timestamp,
            'methodology': methodology,
            'results': results,
            'interpretation': interpretation,
            'decisions': decisions
        }
        with open(json_path, 'w') as f:
            json.dump(data, f, indent=2, default=str)
    
    return report_str
 
# Example usage
if __name__ == "__main__":
    report = generate_importance_report(
        analysis_name="Churn Prediction Feature Analysis Q1 2024",
        methodology={
            'method': 'Permutation Importance (validation set)',
            'validation': '5-fold stratified cross-validation',
            'repeats': 30,
            'random_seed': 42,
            'software': 'scikit-learn 1.2.0, Python 3.10',
            'dataset_size': '50,000 customers (35,000 train, 15,000 validation)'
        },
        results={
            'top_features': [
                {'name': 'days_since_purchase', 'importance': 0.145, 'ci': [0.132, 0.158]},
                {'name': 'support_tickets_30d', 'importance': 0.098, 'ci': [0.085, 0.111]},
                {'name': 'login_frequency', 'importance': 0.076, 'ci': [0.065, 0.087]},
            ],
            'rank_correlation': 0.94,
            'stable_count': '12 of 15 features'
        },
        interpretation={
            'headline': 'Customer engagement metrics dominate churn prediction',
            'domain_validation': 'Findings confirmed by Customer Success team',
            'anomalies': 'Geographic features unexpectedly low—investigating',
            'limitations': 'Analysis reflects current customer base; may not apply to new segments'
        },
        decisions={
            'actions': [
                'Implemented real-time monitoring for top 3 features',
                'Removed 5 low-importance features from pipeline'
            ],
            'recommendations': [
                'Prioritize engagement campaigns for low login_frequency customers',
                'Investigate geographic feature performance gap'
            ],
            'follow_ups': [
                'Re-run analysis after new feature deployment (Q2)',
                'Conduct causal analysis on top 3 features'
            ]
        }
    )
    
    print(report)