Dropout

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179

import numpy as np
from typing import Optional, Union, Tuple
 
class InvertedDropout:
    """
    Production-grade inverted dropout implementation.
    
    Key features:
    - Inverted scaling during training for seamless inference
    - Proper gradient handling for backpropagation
    - Support for evaluation mode (no dropout applied)
    - Configurable random seed for reproducibility
    """
    
    def __init__(
        self, 
        p: float = 0.5,
        seed: Optional[int] = None
    ):
        """
        Initialize inverted dropout layer.
        
        Args:
            p: Probability of dropping each neuron [0, 1)
            seed: Optional random seed for reproducibility
        """
        if not 0 <= p < 1:
            raise ValueError(f"Dropout probability must be in [0, 1), got {p}")
        
        self.p = p
        self.seed = seed
        self.rng = np.random.default_rng(seed)
        
        # Cache for backward pass
        self._mask: Optional[np.ndarray] = None
        self._scale: float = 1.0 / (1.0 - p) if p < 1 else 1.0
        
        # Training mode flag
        self._training = True
    
    def train(self, mode: bool = True):
        """Set training mode."""
        self._training = mode
        return self
    
    def eval(self):
        """Set evaluation mode (no dropout)."""
        return self.train(False)
    
    @property
    def training(self) -> bool:
        return self._training
    
    def forward(self, x: np.ndarray) -> np.ndarray:
        """
        Forward pass through inverted dropout.
        
        During training:
            1. Generate Bernoulli mask with P(keep) = 1-p
            2. Apply mask to zero out dropped neurons
            3. Scale by 1/(1-p) to maintain expected value
        
        During evaluation:
            Pass input through unchanged (already at correct scale)
        
        Args:
            x: Input tensor of shape (batch, features) or any shape
        
        Returns:
            Output tensor of same shape, with dropout applied if training
        """
        if not self._training or self.p == 0:
            # No dropout in eval mode or if p=0
            self._mask = None
            return x
        
        # Generate mask: 1 with probability (1-p), 0 with probability p
        # Using Bernoulli: each element independently kept with prob (1-p)
        self._mask = self.rng.binomial(1, 1 - self.p, size=x.shape)
        
        # Apply mask and scale in one operation for efficiency
        # The division by (1-p) ensures E[output] = E[input]
        return x * self._mask * self._scale
    
    def backward(self, grad_output: np.ndarray) -> np.ndarray:
        """
        Backward pass through inverted dropout.
        
        Key insight: Gradient flows through the same paths as activations.
        - Dropped neurons (mask=0): no gradient
        - Kept neurons (mask=1): gradient scaled by 1/(1-p)
        
        This is mathematically equivalent to:
            d_input = d_output * mask * scale
        
        Args:
            grad_output: Gradient from subsequent layer
        
        Returns:
            Gradient with respect to input
        """
        if not self._training or self._mask is None:
            return grad_output
        
        # Same mask and scale as forward pass
        return grad_output * self._mask * self._scale
    
    def __call__(self, x: np.ndarray) -> np.ndarray:
        return self.forward(x)
    
    def reset_rng(self, seed: Optional[int] = None):
        """Reset random number generator (useful for reproducibility)."""
        self.rng = np.random.default_rng(seed if seed is not None else self.seed)
 
 
def compare_standard_vs_inverted():
    """
    Demonstrate the equivalence between standard and inverted dropout.
    """
    np.random.seed(42)
    
    # Parameters
    p = 0.5  # Dropout probability
    num_features = 100
    batch_size = 32
    num_trials = 10000
    
    # Input
    x = np.random.randn(batch_size, num_features)
    
    # Simulated weights (for demonstration)
    W = np.random.randn(num_features, 64) * 0.1
    
    print("Comparing Standard vs Inverted Dropout")
    print("=" * 50)
    
    # Standard dropout: train without scaling, scale at inference
    standard_train_outputs = []
    for _ in range(num_trials):
        mask = np.random.binomial(1, 1 - p, size=x.shape)
        output = (x * mask) @ W  # No scaling during training
        standard_train_outputs.append(output)
    
    standard_train_mean = np.mean(standard_train_outputs, axis=0)
    
    # Standard dropout inference: scale weights
    standard_inference = (x @ (W * (1 - p)))
    
    print(f"\nStandard Dropout:")
    print(f"  Train mean:      {np.mean(standard_train_mean):.6f}")
    print(f"  Inference mean:  {np.mean(standard_inference):.6f}")
    print(f"  Difference:      {abs(np.mean(standard_train_mean) - np.mean(standard_inference)):.6f}")
    
    # Inverted dropout: scale during training, no change at inference
    inverted_train_outputs = []
    scale = 1.0 / (1 - p)
    for _ in range(num_trials):
        mask = np.random.binomial(1, 1 - p, size=x.shape)
        output = (x * mask * scale) @ W  # Scale during training
        inverted_train_outputs.append(output)
    
    inverted_train_mean = np.mean(inverted_train_outputs, axis=0)
    
    # Inverted dropout inference: use weights as-is
    inverted_inference = x @ W
    
    print(f"\nInverted Dropout:")
    print(f"  Train mean:      {np.mean(inverted_train_mean):.6f}")
    print(f"  Inference mean:  {np.mean(inverted_inference):.6f}")
    print(f"  Difference:      {abs(np.mean(inverted_train_mean) - np.mean(inverted_inference)):.6f}")
    
    # Cross-comparison
    print(f"\nCross-comparison:")
    print(f"  Standard inference vs inverted inference: "
          f"{abs(np.mean(standard_inference) - np.mean(inverted_inference)):.6f}")
    print(f"  ✓ Both approaches produce equivalent inference results")
 
 
compare_standard_vs_inverted()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

import torch
import torch.nn as nn
import torch.nn.functional as F
 
# PyTorch's dropout is inverted by default
class PyTorchDropoutExample(nn.Module):
    """
    PyTorch uses inverted dropout in all its dropout functions.
    
    Key behaviors:
    - nn.Dropout(p) drops with probability p
    - Automatically respects model.train() and model.eval()
    - Scales by 1/(1-p) during training
    - No-op during evaluation
    """
    
    def __init__(self, hidden_dim: int = 256, dropout_rate: float = 0.5):
        super().__init__()
        
        # Standard dropout for fully-connected layers
        self.fc1 = nn.Linear(784, hidden_dim)
        self.dropout1 = nn.Dropout(p=dropout_rate)  # Inverted by default
        
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.dropout2 = nn.Dropout(p=dropout_rate)
        
        self.fc3 = nn.Linear(hidden_dim, 10)
    
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        # Dropout applied after activation
        x = F.relu(self.fc1(x))
        x = self.dropout1(x)  # Only active during training
        
        x = F.relu(self.fc2(x))
        x = self.dropout2(x)
        
        return self.fc3(x)
 
 
# Demonstrating train/eval behavior
def demonstrate_pytorch_dropout():
    """Show PyTorch dropout in train vs eval mode."""
    torch.manual_seed(42)
    
    model = PyTorchDropoutExample(dropout_rate=0.5)
    x = torch.randn(32, 784)
    
    # Training mode
    model.train()
    train_outputs = [model(x) for _ in range(5)]
    train_means = [out.mean().item() for out in train_outputs]
    print(f"Training mode outputs vary: {[f'{m:.4f}' for m in train_means]}")
    
    # Evaluation mode
    model.eval()
    eval_outputs = [model(x) for _ in range(5)]
    eval_means = [out.mean().item() for out in eval_outputs]
    print(f"Eval mode outputs identical: {[f'{m:.4f}' for m in eval_means]}")
    
    # Using functional dropout with explicit training flag
    dropout_p = 0.5
    train_out = F.dropout(x, p=dropout_p, training=True)
    eval_out = F.dropout(x, p=dropout_p, training=False)
    
    print(f"\nFunctional dropout:")
    print(f"  Train mean: {train_out.mean():.4f}")
    print(f"  Eval mean:  {eval_out.mean():.4f}")
    print(f"  Input mean: {x.mean():.4f}")
 
 
demonstrate_pytorch_dropout()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

import numpy as np
 
class InvertedDropoutWithGradient:
    """
    Inverted dropout with complete gradient computation demonstration.
    """
    
    def __init__(self, p: float = 0.5):
        self.p = p
        self.scale = 1.0 / (1.0 - p)
        self.mask = None
        self.training = True
    
    def forward(self, x: np.ndarray) -> np.ndarray:
        """Forward pass: apply mask and scale."""
        if not self.training:
            return x
        
        self.mask = np.random.binomial(1, 1 - self.p, size=x.shape)
        return x * self.mask * self.scale
    
    def backward(self, grad_output: np.ndarray) -> np.ndarray:
        """
        Backward pass: gradient flows through same paths as forward.
        
        Critical insight: The mask determines which gradients flow.
        - Dropped neurons (mask=0): no gradient update
        - Kept neurons (mask=1): gradient scaled by 1/(1-p)
        
        The scaling ensures that the expected gradient magnitude is preserved.
        Without scaling, kept neurons would receive smaller expected gradients.
        """
        if not self.training:
            return grad_output
        
        # Same mask and scale as forward pass
        return grad_output * self.mask * self.scale
 
 
def demonstrate_gradient_flow():
    """Visualize how gradients flow through inverted dropout."""
    np.random.seed(42)
    
    # Setup
    p = 0.5
    x = np.array([[1.0, 2.0, 3.0, 4.0, 5.0]])  # Single sample, 5 features
    grad_output = np.array([[0.1, 0.2, 0.3, 0.4, 0.5]])  # Gradient from loss
    
    dropout = InvertedDropoutWithGradient(p=p)
    
    print("Gradient Flow Through Inverted Dropout")
    print("=" * 55)
    
    # Forward pass
    dropout.training = True
    y = dropout.forward(x)
    
    print(f"\nInput:        {x}")
    print(f"Mask:         {dropout.mask}")
    print(f"Scale:        {dropout.scale}")
    print(f"Output:       {y}")
    
    # Backward pass
    grad_input = dropout.backward(grad_output)
    
    print(f"\nGrad output:  {grad_output}")
    print(f"Grad input:   {grad_input}")
    
    # Analysis
    print(f"\nAnalysis:")
    for i in range(5):
        if dropout.mask[0, i] == 0:
            print(f"  Feature {i}: DROPPED - gradient = 0")
        else:
            expected = grad_output[0, i] * dropout.scale
            print(f"  Feature {i}: KEPT - gradient = {grad_output[0, i]:.2f} × {dropout.scale:.2f} = {expected:.3f}")
    
    # Expected gradient magnitude analysis
    print(f"\nExpected Gradient Analysis:")
    print(f"  Original grad magnitude: {np.linalg.norm(grad_output):.4f}")
    print(f"  Actual grad magnitude:   {np.linalg.norm(grad_input):.4f}")
    
    # Over many trials, expected gradient magnitude should match
    num_trials = 10000
    grad_magnitudes = []
    for _ in range(num_trials):
        y = dropout.forward(x)
        grad_in = dropout.backward(grad_output)
        grad_magnitudes.append(np.linalg.norm(grad_in))
    
    print(f"  Average grad magnitude ({num_trials} trials): {np.mean(grad_magnitudes):.4f}")
    print(f"  ✓ Inverted scaling preserves expected gradient magnitude")
 
 
def analyze_gradient_variance():
    """Analyze how dropout increases gradient variance."""
    np.random.seed(42)
    
    # Compare gradient variance with and without dropout
    x = np.random.randn(32, 256)
    grad_output = np.random.randn(32, 256)
    
    dropout_rates = [0.0, 0.2, 0.5, 0.8]
    
    print("\nGradient Variance Analysis")
    print("=" * 45)
    print(f"{'Dropout Rate':<15} {'Mean Grad':<15} {'Std Grad':<15}")
    print("-" * 45)
    
    for p in dropout_rates:
        if p == 0:
            grads = [grad_output for _ in range(100)]
        else:
            dropout = InvertedDropoutWithGradient(p=p)
            grads = []
            for _ in range(100):
                dropout.forward(x)
                grads.append(dropout.backward(grad_output))
        
        mean_grad = np.mean([g.mean() for g in grads])
        std_grad = np.std([g.mean() for g in grads])
        
        print(f"{p:<15.1f} {mean_grad:<15.6f} {std_grad:<15.6f}")
    
    print("\nInsight: Higher dropout → higher gradient variance")
    print("This variance acts as regularization (noisy gradients)")
 
 
demonstrate_gradient_flow()
print()
analyze_gradient_variance()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

import numpy as np
 
class SimpleBatchNorm:
    """Simplified BatchNorm for demonstration."""
    
    def __init__(self, num_features: int, momentum: float = 0.1, eps: float = 1e-5):
        self.num_features = num_features
        self.momentum = momentum
        self.eps = eps
        
        # Learnable parameters
        self.gamma = np.ones(num_features)
        self.beta = np.zeros(num_features)
        
        # Running statistics (for inference)
        self.running_mean = np.zeros(num_features)
        self.running_var = np.ones(num_features)
        
        self.training = True
    
    def forward(self, x):
        if self.training:
            mean = x.mean(axis=0)
            var = x.var(axis=0)
            
            # Update running statistics
            self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean
            self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var
        else:
            mean = self.running_mean
            var = self.running_var
        
        x_norm = (x - mean) / np.sqrt(var + self.eps)
        return self.gamma * x_norm + self.beta
 
 
class InvertedDropout:
    """Simple inverted dropout."""
    
    def __init__(self, p: float = 0.5):
        self.p = p
        self.scale = 1.0 / (1 - p)
        self.training = True
    
    def forward(self, x):
        if not self.training:
            return x
        mask = np.random.binomial(1, 1 - self.p, size=x.shape)
        return x * mask * self.scale
 
 
def analyze_variance_shift():
    """
    Demonstrate the variance shift caused by dropout before/after BatchNorm.
    """
    np.random.seed(42)
    
    batch_size = 128
    num_features = 64
    dropout_rate = 0.5
    num_iterations = 1000
    
    # Simulated activations
    x = np.random.randn(batch_size, num_features)
    
    print("Variance Shift Analysis: Dropout and BatchNorm")
    print("=" * 60)
    
    # Setup 1: Dropout BEFORE BatchNorm (problematic)
    print("\nSetup 1: Dropout → BatchNorm")
    dropout1 = InvertedDropout(dropout_rate)
    bn1 = SimpleBatchNorm(num_features)
    
    bn1.training = True
    for _ in range(num_iterations):
        x_dropped = dropout1.forward(x)
        _ = bn1.forward(x_dropped)
    
    # Now evaluate
    bn1.training = False
    dropout1.training = False
    
    train_stats = []
    eval_stats = []
    
    for _ in range(100):
        dropout1.training = True
        bn1.training = True
        x_dropped = dropout1.forward(x)
        out_train = bn1.forward(x_dropped)
        train_stats.append((out_train.mean(), out_train.var()))
        
        dropout1.training = False
        bn1.training = False
        out_eval = bn1.forward(x)
        eval_stats.append((out_eval.mean(), out_eval.var()))
    
    avg_train_var = np.mean([s[1] for s in train_stats])
    avg_eval_var = np.mean([s[1] for s in eval_stats])
    
    print(f"  Training variance:  {avg_train_var:.4f}")
    print(f"  Inference variance: {avg_eval_var:.4f}")
    print(f"  Variance ratio:     {avg_eval_var / avg_train_var:.2f}x")
    
    # Setup 2: BatchNorm BEFORE Dropout (better)
    print("\nSetup 2: BatchNorm → Dropout")
    bn2 = SimpleBatchNorm(num_features)
    dropout2 = InvertedDropout(dropout_rate)
    
    bn2.training = True
    for _ in range(num_iterations):
        x_normed = bn2.forward(x)
        _ = dropout2.forward(x_normed)
    
    train_stats2 = []
    eval_stats2 = []
    
    for _ in range(100):
        dropout2.training = True
        bn2.training = True
        x_normed = bn2.forward(x)
        out_train = dropout2.forward(x_normed)
        train_stats2.append((out_train.mean(), out_train.var()))
        
        dropout2.training = False
        bn2.training = False
        out_eval = bn2.forward(x)  # No dropout at inference
        eval_stats2.append((out_eval.mean(), out_eval.var()))
    
    avg_train_var2 = np.mean([s[1] for s in train_stats2])
    avg_eval_var2 = np.mean([s[1] for s in eval_stats2])
    
    print(f"  Training variance:  {avg_train_var2:.4f}")
    print(f"  Inference variance: {avg_eval_var2:.4f}")
    print(f"  Variance ratio:     {avg_eval_var2 / avg_train_var2:.2f}x")
    
    # Summary
    print("\n" + "-" * 60)
    print("Summary:")
    print(f"  Dropout→BN variance shift: {abs(1 - avg_eval_var / avg_train_var):.1%}")
    print(f"  BN→Dropout variance shift: {abs(1 - avg_eval_var2 / avg_train_var2):.1%}")
    print("\n  ✓ BN→Dropout has less train/inference mismatch")
 
 
analyze_variance_shift()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

import numpy as np
import hashlib
 
class ReproducibleDropout:
    """
    Dropout implementation with full reproducibility control.
    
    Supports:
    - Deterministic inference (no randomness)
    - Reproducible training with explicit seeds
    - Per-iteration reproducibility for debugging
    """
    
    def __init__(self, p: float = 0.5, seed: int = None):
        self.p = p
        self.base_seed = seed
        self.rng = np.random.default_rng(seed)
        self.training = True
        self.call_count = 0
    
    def reset(self, seed: int = None):
        """Reset RNG state for reproducibility."""
        if seed is not None:
            self.base_seed = seed
        self.rng = np.random.default_rng(self.base_seed)
        self.call_count = 0
    
    def forward(self, x: np.ndarray, iteration: int = None) -> np.ndarray:
        """
        Forward pass with optional iteration-specific seeding.
        
        Args:
            x: Input tensor
            iteration: If provided, derive deterministic seed from iteration
        """
        if not self.training:
            return x  # Deterministic: no dropout
        
        # For reproducible training, optionally seed based on iteration
        if iteration is not None:
            # Derive seed from base seed and iteration
            seed_input = f"{self.base_seed}_{iteration}"
            derived_seed = int(hashlib.md5(seed_input.encode()).hexdigest()[:8], 16)
            local_rng = np.random.default_rng(derived_seed)
            mask = local_rng.binomial(1, 1 - self.p, size=x.shape)
        else:
            mask = self.rng.binomial(1, 1 - self.p, size=x.shape)
        
        self.call_count += 1
        scale = 1.0 / (1 - self.p)
        return x * mask * scale
 
 
def demonstrate_reproducibility():
    """Show reproducibility scenarios with inverted dropout."""
    print("Inverted Dropout Reproducibility")
    print("=" * 50)
    
    x = np.random.randn(4, 8)
    
    # Scenario 1: Deterministic inference
    print("\n1. Deterministic Inference:")
    dropout = ReproducibleDropout(p=0.5, seed=42)
    dropout.training = False
    
    out1 = dropout.forward(x)
    out2 = dropout.forward(x)
    out3 = dropout.forward(x)
    
    print(f"   Output 1 mean: {out1.mean():.6f}")
    print(f"   Output 2 mean: {out2.mean():.6f}")  
    print(f"   Output 3 mean: {out3.mean():.6f}")
    print(f"   All identical: {np.allclose(out1, out2) and np.allclose(out2, out3)}")
    
    # Scenario 2: Reproducible training with reset
    print("\n2. Reproducible Training (with reset):")
    dropout = ReproducibleDropout(p=0.5, seed=42)
    dropout.training = True
    
    # First run
    dropout.reset(42)
    run1_outputs = [dropout.forward(x).mean() for _ in range(3)]
    
    # Second run with reset
    dropout.reset(42)
    run2_outputs = [dropout.forward(x).mean() for _ in range(3)]
    
    print(f"   Run 1: {[f'{o:.4f}' for o in run1_outputs]}")
    print(f"   Run 2: {[f'{o:.4f}' for o in run2_outputs]}")
    print(f"   Runs match: {run1_outputs == run2_outputs}")
    
    # Scenario 3: Iteration-specific reproducibility
    print("\n3. Iteration-Specific Seeds:")
    dropout = ReproducibleDropout(p=0.5, seed=42)
    dropout.training = True
    
    # Can reproduce specific iteration without replaying all previous
    iter_100_out = dropout.forward(x, iteration=100)
    iter_200_out = dropout.forward(x, iteration=200)
    
    # Verify iteration 100 is reproducible
    iter_100_out_again = dropout.forward(x, iteration=100)
    
    print(f"   Iteration 100:       {iter_100_out.mean():.6f}")
    print(f"   Iteration 200:       {iter_200_out.mean():.6f}")
    print(f"   Iteration 100 again: {iter_100_out_again.mean():.6f}")
    print(f"   Iter 100 reproducible: {np.allclose(iter_100_out, iter_100_out_again)}")
 
 
def demonstrate_production_deployment():
    """Show production deployment patterns."""
    print("\n" + "=" * 50)
    print("Production Deployment Checklist")
    print("=" * 50)
    
    checklist = [
        ("✓", "Model set to eval mode", "model.eval() called before inference"),
        ("✓", "Dropout disabled", "Happens automatically in eval mode"),
        ("✓", "Same weights as training", "Inverted dropout uses identical weights"),
        ("✓", "No randomness", "Inference is fully deterministic"),
        ("✓", "Reproducible outputs", "Same input → same output, always"),
    ]
    
    for check, item, detail in checklist:
        print(f"  {check} {item}")
        print(f"      {detail}")
    
    print("\n  Key insight: Inverted dropout requires NO modifications")
    print("  for production deployment. Just switch to eval mode.")
 
 
demonstrate_reproducibility()
demonstrate_production_deployment()