Feature Engineering Mastery

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

import pandas as pd
import numpy as np
 
def create_domain_features(df: pd.DataFrame) -> pd.DataFrame:
    """
    Translate domain knowledge into computable features.
    
    Domain context: E-commerce customer churn prediction
    Expert insights encoded as features.
    """
    features = df.copy()
    
    # INSIGHT 1: "Customers who haven't purchased recently are at risk"
    # Translation: Recency score (days since last purchase)
    features['days_since_last_purchase'] = (
        pd.Timestamp.now() - pd.to_datetime(df['last_purchase_date'])
    ).dt.days
    
    # INSIGHT 2: "Declining purchase frequency signals disengagement"
    # Translation: Trend in purchase rate over time
    features['purchase_frequency_trend'] = (
        df['purchases_last_30d'] / df['purchases_days_31_60'].replace(0, 0.1)
    ) - 1  # Negative = declining
    
    # INSIGHT 3: "Customers who complain but don't churn are actually loyal"
    # Translation: Complaint-to-engagement ratio
    features['complaint_engagement_ratio'] = (
        df['complaints_last_90d'] / (df['support_contacts_last_90d'] + 1)
    )
    
    # INSIGHT 4: "Product diversity indicates stickiness"
    # Translation: Category spread (how many product categories purchased)
    features['category_diversity'] = df['unique_categories_purchased']
    
    # INSIGHT 5: "Price-sensitive customers churn when we raise prices"
    # Translation: Discount dependency score
    features['discount_dependency'] = (
        df['discounted_purchases'] / (df['total_purchases'] + 1)
    )
    
    # INSIGHT 6: "Long-tenure customers with sudden behavior change are concerning"
    # Translation: Behavioral anomaly score
    avg_monthly = df['lifetime_purchases'] / (df['tenure_months'] + 1)
    features['behavioral_anomaly'] = (
        df['purchases_last_30d'] - avg_monthly
    ) / (avg_monthly + 0.1)  # Negative = below average
    
    # INSIGHT 7: "Mobile-first customers have different patterns"
    # Translation: Channel concentration
    features['mobile_channel_pct'] = (
        df['mobile_purchases'] / (df['total_purchases'] + 1)
    )
    
    return features
 
# Example of documenting domain features
DOMAIN_FEATURE_CATALOG = {
    'days_since_last_purchase': {
        'source_insight': 'Customers who have not purchased recently are at risk',
        'expert': 'Customer Success Team',
        'expected_direction': 'Higher values increase churn probability',
        'computation': '(current_date - last_purchase_date).days',
        'validation_check': 'Should be 0+ integers; check for future dates',
    },
    'behavioral_anomaly': {
        'source_insight': 'Sudden behavior changes signal potential churn',
        'expert': 'Data Science Lead + Product Team',
        'expected_direction': 'Negative values (decline) increase churn risk',
        'computation': '(recent_monthly - avg_monthly) / avg_monthly',
        'validation_check': 'Should cluster around 0; outliers need review',
    },
    # ... continue for all domain features
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

import pandas as pd
import numpy as np
from scipy import stats
from sklearn.feature_selection import mutual_info_classif
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import GradientBoostingClassifier
import matplotlib.pyplot as plt
 
def validate_domain_feature(
    df: pd.DataFrame,
    feature_name: str,
    target_name: str,
    expected_direction: str = 'positive',  # or 'negative'
    feature_type: str = 'numerical'  # or 'categorical'
) -> dict:
    """
    Comprehensive validation of a domain-hypothesized feature.
    
    Returns validation metrics and diagnostic information.
    """
    feature = df[feature_name].dropna()
    target = df.loc[feature.index, target_name]
    
    validation = {
        'feature_name': feature_name,
        'n_samples': len(feature),
        'missing_pct': (df[feature_name].isna().sum() / len(df)) * 100,
    }
    
    if feature_type == 'numerical':
        # 1. Correlation analysis
        correlation = feature.corr(target)
        validation['correlation'] = correlation
        
        # Check if direction matches hypothesis
        if expected_direction == 'positive':
            validation['direction_match'] = correlation > 0
        else:
            validation['direction_match'] = correlation < 0
        
        # 2. Statistical significance
        _, p_value = stats.pearsonr(feature, target)
        validation['p_value'] = p_value
        validation['significant'] = p_value < 0.05
        
        # 3. Mutual information (non-linear relationship strength)
        mi = mutual_info_classif(
            feature.values.reshape(-1, 1),
            target,
            random_state=42
        )[0]
        validation['mutual_information'] = mi
        
        # 4. Binned analysis (check monotonicity)
        feature_binned = pd.qcut(feature, q=5, duplicates='drop')
        bin_means = target.groupby(feature_binned).mean()
        validation['monotonic'] = bin_means.is_monotonic_increasing or bin_means.is_monotonic_decreasing
        validation['bin_target_rates'] = bin_means.to_dict()
        
    elif feature_type == 'categorical':
        # Chi-square test for independence
        contingency = pd.crosstab(feature, target)
        chi2, p_value, dof, expected = stats.chi2_contingency(contingency)
        validation['chi2_statistic'] = chi2
        validation['p_value'] = p_value
        validation['significant'] = p_value < 0.05
        
        # Category-wise target rates
        cat_rates = target.groupby(feature).mean()
        validation['category_target_rates'] = cat_rates.to_dict()
        
        # Mutual information
        mi = mutual_info_classif(
            pd.factorize(feature)[0].reshape(-1, 1),
            target,
            discrete_features=True,
            random_state=42
        )[0]
        validation['mutual_information'] = mi
    
    # 5. Incremental predictive value
    # Compare model with and without this feature
    X_base = df.drop(columns=[feature_name, target_name]).select_dtypes(include=[np.number]).fillna(0)
    X_with_feature = X_base.copy()
    X_with_feature[feature_name] = df[feature_name].fillna(0)
    
    model = GradientBoostingClassifier(n_estimators=50, max_depth=3, random_state=42)
    
    score_base = cross_val_score(model, X_base, target, cv=3, scoring='roc_auc').mean()
    score_with = cross_val_score(model, X_with_feature, target, cv=3, scoring='roc_auc').mean()
    
    validation['baseline_auc'] = score_base
    validation['with_feature_auc'] = score_with
    validation['auc_lift'] = score_with - score_base
    validation['provides_lift'] = (score_with - score_base) > 0.001
    
    # Final verdict
    validation['validated'] = (
        validation['significant'] and
        validation.get('direction_match', True) and
        validation['provides_lift']
    )
    
    return validation
 
# Example usage
validation_result = validate_domain_feature(
    df=customer_data,
    feature_name='days_since_last_purchase',
    target_name='churned',
    expected_direction='positive',  # Longer recency = higher churn
    feature_type='numerical'
)
 
print(f"Feature: {validation_result['feature_name']}")
print(f"Validated: {validation_result['validated']}")
print(f"Correlation: {validation_result['correlation']:.3f}")
print(f"Direction matches hypothesis: {validation_result['direction_match']}")
print(f"AUC lift: {validation_result['auc_lift']:.4f}")