Activation Functions

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

import numpy as np
 
# Example: Classifying task type to output activation
 
def get_output_activation_and_loss(
    task_type: str,
    num_classes: int = None,
    output_range: tuple = None
):
    """
    Returns the correct output activation and loss function
    based on task specification.
    """
    
    if task_type == "binary_classification":
        return {
            "activation": "sigmoid",
            "loss": "binary_crossentropy",
            "output_neurons": 1,
            "notes": "Output is P(positive class)"
        }
    
    elif task_type == "multiclass_classification":
        assert num_classes is not None, "Specify num_classes"
        return {
            "activation": "softmax",
            "loss": "categorical_crossentropy",
            "output_neurons": num_classes,
            "notes": f"Output is probability distribution over {num_classes} classes"
        }
    
    elif task_type == "multilabel_classification":
        assert num_classes is not None, "Specify num_labels as num_classes"
        return {
            "activation": "sigmoid",  # Applied independently to each output
            "loss": "binary_crossentropy",  # Sum over all labels
            "output_neurons": num_classes,
            "notes": f"Each of {num_classes} outputs is independent P(label=1)"
        }
    
    elif task_type == "regression":
        if output_range is None:
            return {
                "activation": "linear",  # No activation
                "loss": "mse",
                "output_neurons": 1,
                "notes": "Unbounded regression"
            }
        elif output_range == (0, 1):
            return {
                "activation": "sigmoid",
                "loss": "mse",
                "output_neurons": 1,
                "notes": "Bounded to [0, 1]"
            }
        elif output_range == (-1, 1):
            return {
                "activation": "tanh",
                "loss": "mse",
                "output_neurons": 1,
                "notes": "Bounded to [-1, 1]"
            }
        elif output_range[0] >= 0:
            return {
                "activation": "softplus or relu",
                "loss": "mse",
                "output_neurons": 1,
                "notes": "Non-negative output (e.g., counts, prices)"
            }
    
    raise ValueError(f"Unknown task type: {task_type}")
 
# Demo
print("Output Layer Selection Guide:")
print("=" * 60)
 
tasks = [
    ("binary_classification", None, None),
    ("multiclass_classification", 10, None),
    ("multilabel_classification", 5, None),
    ("regression", None, None),
    ("regression", None, (0, 1)),
    ("regression", None, (-1, 1)),
]
 
for task, nc, rng in tasks:
    result = get_output_activation_and_loss(task, nc, rng)
    print(f"\n{task}" + (f" (num_classes={nc})" if nc else "") + 
          (f" (range={rng})" if rng else ""))
    print(f"  Activation: {result['activation']}")
    print(f"  Loss: {result['loss']}")
    print(f"  Outputs: {result['output_neurons']}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

def select_hidden_activation(
    architecture: str,
    layer_type: str = "standard",
    constraints: list = None,
    depth: int = None
) -> dict:
    """
    Systematic selection of hidden layer activation.
    
    Returns recommended activation and initialization.
    """
    constraints = constraints or []
    
    # Special architectural components (non-negotiable)
    special_components = {
        "lstm_gate": {"activation": "sigmoid", "reason": "Bounded [0,1] gating required"},
        "lstm_cell": {"activation": "tanh", "reason": "Bounded [-1,1] values"},
        "gru_gate": {"activation": "sigmoid", "reason": "Bounded gating"},
        "gru_candidate": {"activation": "tanh", "reason": "Bounded values"},
        "attention_weights": {"activation": "softmax", "reason": "Probability distribution"},
        "gating_mechanism": {"activation": "sigmoid", "reason": "Multiplicative gating"},
    }
    
    if layer_type in special_components:
        return special_components[layer_type]
    
    # Architecture-based defaults
    architecture_defaults = {
        "cnn_standard": {"activation": "relu", "init": "he_normal", "use_batchnorm": True},
        "cnn_mobile": {"activation": "hard_swish", "init": "he_normal", "use_batchnorm": True},
        "transformer_nlp": {"activation": "gelu", "init": "xavier", "use_layernorm": True},
        "transformer_vision": {"activation": "gelu", "init": "xavier", "use_layernorm": True},
        "llm_ffn": {"activation": "swiglu", "init": "xavier", "use_layernorm": True},
        "mlp_standard": {"activation": "relu", "init": "he_normal", "use_batchnorm": True},
        "mlp_deep_no_norm": {"activation": "selu", "init": "lecun_normal", "use_alpha_dropout": True},
        "resnet": {"activation": "relu", "init": "he_normal", "use_batchnorm": True},
        "gan_generator": {"activation": "relu", "init": "he_normal", "note": "tanh at output"},
        "gan_discriminator": {"activation": "leaky_relu", "init": "he_normal"},
    }
    
    if architecture in architecture_defaults:
        result = architecture_defaults[architecture].copy()
    else:
        # Safe default
        result = {"activation": "relu", "init": "he_normal", "use_batchnorm": True}
    
    # Apply constraint modifications
    if "efficiency_critical" in constraints:
        if result["activation"] in ["gelu", "swish"]:
            result["activation"] = "relu"
            result["note"] = "Switched to ReLU for efficiency"
    
    if "dead_neurons_observed" in constraints:
        result["activation"] = "leaky_relu"
        result["leaky_alpha"] = 0.01
        result["note"] = "Using Leaky ReLU to prevent dead neurons"
    
    if "gradient_flow_issues" in constraints:
        if result["activation"] == "relu":
            result["activation"] = "swish"
            result["note"] = "Switched to Swish for smoother gradients"
    
    return result
 
# Demonstration
print("Hidden Layer Activation Selection Examples:")
print("=" * 60)
 
examples = [
    ("cnn_standard", "standard", []),
    ("transformer_nlp", "standard", []),
    ("llm_ffn", "standard", []),
    ("transformer_vision", "standard", ["efficiency_critical"]),
    ("mlp_standard", "standard", ["dead_neurons_observed"]),
    ("cnn_standard", "lstm_gate", []),  # Special component
]
 
for arch, layer, constr in examples:
    result = select_hidden_activation(arch, layer, constr)
    print(f"\nArchitecture: {arch}, Layer: {layer}")
    if constr:
        print(f"Constraints: {constr}")
    for k, v in result.items():
        print(f"  {k}: {v}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

import numpy as np
 
def xavier_init(shape, gain=1.0):
    """
    Xavier/Glorot initialization.
    For sigmoid, tanh, linear activations.
    """
    n_in, n_out = shape
    std = gain * np.sqrt(2.0 / (n_in + n_out))
    return np.random.normal(0, std, shape)
 
def he_init(shape, mode='fan_in'):
    """
    He/Kaiming initialization.
    For ReLU and variants.
    """
    n_in, n_out = shape
    fan = n_in if mode == 'fan_in' else n_out
    std = np.sqrt(2.0 / fan)
    return np.random.normal(0, std, shape)
 
def lecun_init(shape):
    """
    LeCun initialization.
    Required for SELU self-normalization.
    """
    n_in, n_out = shape
    std = np.sqrt(1.0 / n_in)
    return np.random.normal(0, std, shape)
 
def get_initialization_for_activation(activation: str, shape: tuple) -> np.ndarray:
    """
    Return properly initialized weights for given activation.
    """
    init_map = {
        'sigmoid': lambda s: xavier_init(s),
        'tanh': lambda s: xavier_init(s, gain=5/3),  # Slightly higher gain for tanh
        'relu': lambda s: he_init(s),
        'leaky_relu': lambda s: he_init(s),  # Small α adjustment often ignored
        'prelu': lambda s: he_init(s),
        'elu': lambda s: he_init(s),
        'selu': lambda s: lecun_init(s),
        'gelu': lambda s: he_init(s),  # ReLU-like for positive
        'swish': lambda s: he_init(s),  # ReLU-like for positive
        'linear': lambda s: xavier_init(s),
    }
    
    if activation not in init_map:
        print(f"Warning: Unknown activation '{activation}', using Xavier")
        return xavier_init(shape)
    
    return init_map[activation](shape)
 
# Demonstrate variance preservation
def test_variance_propagation(activation_fn, init_fn, num_layers=10):
    """
    Test if variance is preserved through layers.
    """
    np.random.seed(42)
    
    layer_size = 512
    batch_size = 1000
    
    # Input with unit variance
    x = np.random.randn(batch_size, layer_size)
    print(f"Input variance: {x.var():.4f}")
    
    for layer_idx in range(num_layers):
        W = init_fn((layer_size, layer_size))
        z = x @ W
        x = activation_fn(z)
        
        if layer_idx % 3 == 0 or layer_idx == num_layers - 1:
            print(f"Layer {layer_idx+1}: variance = {x.var():.4f}")
    
    return x.var()
 
# Test ReLU with He init
print("ReLU + He Initialization:")
print("-" * 40)
test_variance_propagation(
    activation_fn=lambda x: np.maximum(0, x),
    init_fn=lambda s: he_init(s)
)
 
print("\nReLU + Xavier Initialization (WRONG):")
print("-" * 40)
test_variance_propagation(
    activation_fn=lambda x: np.maximum(0, x),
    init_fn=lambda s: xavier_init(s)
)
 
print("\nTanh + Xavier Initialization:")
print("-" * 40)
test_variance_propagation(
    activation_fn=lambda x: np.tanh(x),
    init_fn=lambda s: xavier_init(s)
)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

import numpy as np
 
def diagnose_relu_health(model, data_sample, threshold=0.01):
    """
    Check for dead ReLU neurons.
    
    A neuron is 'dead' if it outputs 0 for all samples.
    A neuron is 'dying' if it outputs 0 for >99% of samples.
    """
    dead_count = 0
    dying_count = 0
    total_neurons = 0
    
    # Track which layers have issues
    problematic_layers = []
    
    for name, layer in model.named_modules():
        if hasattr(layer, 'weight'):
            # Get activations before ReLU
            # (Would hook into forward pass in real implementation)
            pass
    
    # Pseudo-analysis
    print("ReLU Health Check:")
    print("-" * 50)
    print(f"Total neurons: {total_neurons}")
    print(f"Dead neurons (0% active): {dead_count}")
    print(f"Dying neurons (<1% active): {dying_count}")
    print(f"Dead percentage: {100*dead_count/max(total_neurons,1):.1f}%")
    
    if dead_count > total_neurons * 0.1:
        print("\n⚠️  HIGH DEAD NEURON RATE")
        print("Recommendations:")
        print("  1. Reduce learning rate")
        print("  2. Switch to Leaky ReLU")
        print("  3. Check initialization (use He)")
        print("  4. Add BatchNorm before ReLU")
 
def diagnose_gradient_flow(model, loss):
    """
    Check for vanishing/exploding gradients.
    """
    gradient_norms = {}
    
    for name, param in model.named_parameters():
        if param.grad is not None:
            grad_norm = param.grad.norm().item()
            gradient_norms[name] = grad_norm
    
    # Analyze
    norms = list(gradient_norms.values())
    
    print("Gradient Flow Analysis:")
    print("-" * 50)
    print(f"Mean gradient norm: {np.mean(norms):.2e}")
    print(f"Max gradient norm:  {np.max(norms):.2e}")
    print(f"Min gradient norm:  {np.min(norms):.2e}")
    print(f"Std gradient norm:  {np.std(norms):.2e}")
    
    # Detect issues
    if np.mean(norms) < 1e-7:
        print("\n⚠️  VANISHING GRADIENTS DETECTED")
        print("Recommendations:")
        print("  1. Replace sigmoid/tanh with ReLU in hidden layers")
        print("  2. Add skip connections (ResNet-style)")
        print("  3. Use BatchNorm/LayerNorm")
        print("  4. Reduce network depth")
    
    if np.max(norms) > 1e3:
        print("\n⚠️  EXPLODING GRADIENTS DETECTED")
        print("Recommendations:")
        print("  1. Apply gradient clipping")
        print("  2. Reduce learning rate")
        print("  3. Check weight initialization")
        print("  4. Add BatchNorm")
 
def check_activation_statistics(activations_dict):
    """
    Analyze activation statistics per layer.
    """
    print("Activation Statistics by Layer:")
    print("-" * 60)
    
    for layer_name, acts in activations_dict.items():
        mean = np.mean(acts)
        std = np.std(acts)
        zero_frac = np.mean(acts == 0)
        saturated_frac = np.mean((acts < 0.01) | (acts > 0.99))
        
        print(f"{layer_name}:")
        print(f"  Mean: {mean:8.4f}, Std: {std:8.4f}")
        print(f"  Zero fraction: {100*zero_frac:.1f}%")
        
        # Warnings
        if zero_frac > 0.5:
            print(f"  ⚠️  High zero fraction - possible dead neurons")
        if std < 0.01:
            print(f"  ⚠️  Very low std - activations collapsing")
        if std > 10:
            print(f"  ⚠️  Very high std - activations exploding")
 
# Example usage simulation
def simulate_diagnosis():
    """Simulate activation diagnosis without actual model."""
    
    print("=" * 60)
    print("ACTIVATION FUNCTION DIAGNOSTIC REPORT")
    print("=" * 60)
    
    # Simulated healthy network
    print("\n[Scenario 1: Healthy ReLU Network]")
    print("-" * 50)
    layer_stats = {
        'layer1_relu': np.random.exponential(1, 1000),
        'layer2_relu': np.random.exponential(1, 1000) * 0.8,
        'layer3_relu': np.random.exponential(1, 1000) * 0.6,
    }
    for name, acts in layer_stats.items():
        acts = np.maximum(0, acts)  # ReLU
        zero_frac = np.mean(acts == 0)
        print(f"{name}: mean={acts.mean():.3f}, std={acts.std():.3f}, "
              f"zeros={100*zero_frac:.1f}%")
    print("✓ Network appears healthy")
    
    # Simulated dying network
    print("\n[Scenario 2: Dying ReLU Network]")
    print("-" * 50)
    layer_stats = {
        'layer1_relu': np.random.randn(1000) - 0.5,  # Shifted negative
        'layer2_relu': np.random.randn(1000) - 1.0,
        'layer3_relu': np.random.randn(1000) - 2.0,
    }
    for name, acts in layer_stats.items():
        acts = np.maximum(0, acts)
        zero_frac = np.mean(acts == 0)
        print(f"{name}: mean={acts.mean():.3f}, std={acts.std():.3f}, "
              f"zeros={100*zero_frac:.1f}%")
    print("⚠️  SIGNIFICANT DEAD NEURONS DETECTED")
    print("   Recommendation: Use Leaky ReLU or add BatchNorm")
 
simulate_diagnosis()