1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

import torch
import torch.nn as nn
import torch.nn.functional as F
 
class GAPClassificationNetwork(nn.Module):
    """
    Demonstrates the structure of a modern classification network
    using Global Average Pooling.
    """
    def __init__(self, num_classes=1000):
        super().__init__()
        
        # Example: final convolutional block outputs 2048 channels
        self.features = nn.Sequential(
            # ... backbone layers (ResNet, EfficientNet, etc.)
            nn.Conv2d(512, 2048, 1),  # Example final 1×1 conv
            nn.BatchNorm2d(2048),
            nn.ReLU(inplace=True)
        )
        
        # Global Average Pooling - the key operation
        self.gap = nn.AdaptiveAvgPool2d(1)  # Works with any spatial size
        
        # Single linear layer for classification
        self.classifier = nn.Linear(2048, num_classes)
        
    def forward(self, x):
        # Feature extraction (any spatial size accepted)
        features = self.features(x)  # (B, 2048, H', W')
        
        # Global pooling collapses spatial dimensions
        pooled = self.gap(features)  # (B, 2048, 1, 1)
        
        # Flatten to remove spatial dimensions
        flattened = pooled.view(pooled.size(0), -1)  # (B, 2048)
        
        # Final classification
        logits = self.classifier(flattened)  # (B, num_classes)
        
        return logits
    
    def extract_features(self, x):
        """
        Extract pre-GAP feature maps for visualization/analysis.
        """
        with torch.no_grad():
            features = self.features(x)
        return features
 
 
def analyze_spatial_statistics():
    """
    Analyze what GAP computes: the mean activation per channel.
    """
    # Simulated feature maps from a trained network
    B, C, H, W = 4, 2048, 7, 7
    features = torch.randn(B, C, H, W)
    
    # GAP computation
    gap_output = F.adaptive_avg_pool2d(features, 1)  # (4, 2048, 1, 1)
    
    # Equivalent manual computation
    manual_gap = features.mean(dim=[2, 3], keepdim=True)
    
    # Verify equivalence
    assert torch.allclose(gap_output, manual_gap), "GAP computes channel-wise mean"
    
    print(f"Feature map shape: {features.shape}")
    print(f"GAP output shape: {gap_output.shape}")
    print(f"Each channel reduced from {H*W} values to 1 (mean)")
    
    # Statistics preserved
    print(f"
Sample channel analysis (channel 0, batch 0):")
    print(f"  Feature map values: mean={features[0,0].mean():.4f}, std={features[0,0].std():.4f}")
    print(f"  GAP output: {gap_output[0,0,0,0]:.4f}")
 
analyze_spatial_statistics()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

import torch
import torch.nn as nn
import torch.nn.functional as F
 
class ConcatenatedGlobalPooling(nn.Module):
    """
    Combines GAP and GMP for richer global representations.
    Used in fastai, many Kaggle winning solutions.
    """
    def __init__(self):
        super().__init__()
        self.gap = nn.AdaptiveAvgPool2d(1)
        self.gmp = nn.AdaptiveMaxPool2d(1)
        
    def forward(self, x):
        avg = self.gap(x)  # (B, C, 1, 1)
        max_ = self.gmp(x)  # (B, C, 1, 1)
        return torch.cat([avg, max_], dim=1)  # (B, 2C, 1, 1)
 
 
class GeMPooling(nn.Module):
    """
    Generalized Mean Pooling (GeM): Interpolates between GAP and GMP.
    
    When p=1: equivalent to GAP
    When p→∞: approaches GMP
    
    Often p≈3 works well empirically (learnable parameter).
    Used in image retrieval and fine-grained recognition.
    """
    def __init__(self, p=3.0, eps=1e-6, learnable=True):
        super().__init__()
        if learnable:
            self.p = nn.Parameter(torch.ones(1) * p)
        else:
            self.p = p
        self.eps = eps
        
    def forward(self, x):
        # Clamp to avoid numerical issues with negative values
        x_clamped = x.clamp(min=self.eps)
        
        # Generalized mean: (mean(x^p))^(1/p)
        return x_clamped.pow(self.p).mean(dim=[2, 3], keepdim=True).pow(1./self.p)
 
 
class AttentiveGlobalPooling(nn.Module):
    """
    Attention-weighted global pooling.
    Learns which spatial positions to emphasize.
    """
    def __init__(self, channels):
        super().__init__()
        self.attention = nn.Sequential(
            nn.Conv2d(channels, channels // 16, 1),
            nn.ReLU(inplace=True),
            nn.Conv2d(channels // 16, 1, 1),
        )
        
    def forward(self, x):
        # Compute attention weights
        attn = self.attention(x)  # (B, 1, H, W)
        attn = F.softmax(attn.view(attn.size(0), -1), dim=1)
        attn = attn.view(attn.size(0), 1, x.size(2), x.size(3))
        
        # Attention-weighted sum
        weighted = (x * attn).sum(dim=[2, 3], keepdim=True)  # (B, C, 1, 1)
        return weighted
 
 
# Comparison on different feature distributions
def compare_pooling_behaviors():
    """
    Show how different global pooling methods respond to
    different feature activation patterns.
    """
    # Scenario 1: Dense, uniform activations
    dense_features = torch.ones(1, 1, 7, 7) * 0.5
    
    # Scenario 2: Sparse, localized activation
    sparse_features = torch.zeros(1, 1, 7, 7)
    sparse_features[0, 0, 3, 3] = 5.0  # Single strong activation
    
    # Scenario 3: Varying intensity
    varying_features = torch.randn(1, 1, 7, 7).abs()
    varying_features[0, 0, 0, 0] = 10.0  # One outlier
    
    gap = nn.AdaptiveAvgPool2d(1)
    gmp = nn.AdaptiveMaxPool2d(1)
    gem = GeMPooling(p=3.0, learnable=False)
    
    for name, features in [("Dense", dense_features), 
                           ("Sparse", sparse_features), 
                           ("Varying", varying_features)]:
        print(f"
{name} features:")
        print(f"  GAP: {gap(features).item():.4f}")
        print(f"  GMP: {gmp(features).item():.4f}")
        print(f"  GeM (p=3): {gem(features).item():.4f}")
 
compare_pooling_behaviors()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

import torch
import torch.nn as nn
import torch.nn.functional as F
 
class SpatialPyramidPooling(nn.Module):
    """
    Spatial Pyramid Pooling layer.
    
    Produces fixed-size output regardless of input spatial dimensions,
    capturing multi-scale spatial information.
    """
    def __init__(self, levels=[1, 2, 4], pool_type='avg'):
        """
        Args:
            levels: List of grid sizes for each pyramid level
                   [1, 2, 4] means: 1×1 (GAP), 2×2, and 4×4 grids
            pool_type: 'avg' for average pooling, 'max' for max pooling
        """
        super().__init__()
        self.levels = levels
        self.pool_type = pool_type
        
    def forward(self, x):
        B, C, H, W = x.shape
        outputs = []
        
        for level in self.levels:
            # Adaptive pooling to level × level grid
            if self.pool_type == 'avg':
                pooled = F.adaptive_avg_pool2d(x, output_size=level)
            else:  # max
                pooled = F.adaptive_max_pool2d(x, output_size=level)
            
            # Flatten spatial dimensions
            pooled_flat = pooled.view(B, -1)  # (B, C * level * level)
            outputs.append(pooled_flat)
        
        # Concatenate all levels
        return torch.cat(outputs, dim=1)
    
    def output_dim(self, channels):
        """Calculate output feature dimension."""
        return channels * sum(l * l for l in self.levels)
 
 
class ImprovedSPP(nn.Module):
    """
    Improved SPP with separate weight projections per level.
    Allows the network to weight different spatial scales differently.
    """
    def __init__(self, in_channels, out_features, levels=[1, 2, 4]):
        super().__init__()
        self.levels = levels
        
        self.level_projections = nn.ModuleList([
            nn.Sequential(
                nn.Linear(in_channels * l * l, out_features // len(levels)),
                nn.ReLU(inplace=True)
            )
            for l in levels
        ])
        
    def forward(self, x):
        B, C, H, W = x.shape
        level_outputs = []
        
        for level, projection in zip(self.levels, self.level_projections):
            pooled = F.adaptive_avg_pool2d(x, output_size=level)
            flat = pooled.view(B, -1)
            projected = projection(flat)
            level_outputs.append(projected)
        
        return torch.cat(level_outputs, dim=1)
 
 
# Example usage and output size calculation
def spp_example():
    batch_size = 8
    channels = 512
    
    # SPP works with any input size
    for H, W in [(7, 7), (14, 14), (28, 28), (13, 17)]:
        x = torch.randn(batch_size, channels, H, W)
        spp = SpatialPyramidPooling(levels=[1, 2, 4])
        
        output = spp(x)
        expected_dim = channels * (1 + 4 + 16)  # 1×1 + 2×2 + 4×4
        
        print(f"Input: {x.shape} → SPP output: {output.shape} (expected {expected_dim})")
        
        assert output.shape[1] == expected_dim
 
spp_example()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
 
class CAMExtractor:
    """
    Extract Class Activation Maps from a model with GAP.
    
    Requirements:
    - Model must have GAP before the final classifier
    - Need access to the final feature maps and classifier weights
    """
    def __init__(self, model, feature_layer_name, classifier_name):
        """
        Args:
            model: The trained model
            feature_layer_name: Name of the layer producing feature maps
            classifier_name: Name of the final linear classifier
        """
        self.model = model
        self.feature_maps = None
        
        # Register hook to capture feature maps
        feature_layer = dict(model.named_modules())[feature_layer_name]
        feature_layer.register_forward_hook(self._save_features)
        
        # Get classifier weights
        classifier = dict(model.named_modules())[classifier_name]
        self.weights = classifier.weight.data  # (num_classes, num_channels)
        
    def _save_features(self, module, input, output):
        self.feature_maps = output
        
    def get_cam(self, image, class_idx=None):
        """
        Generate Class Activation Map for an image.
        
        Args:
            image: Input image tensor (B, C, H, W)
            class_idx: Target class index. If None, uses predicted class.
            
        Returns:
            cam: Class activation map (H, W)
            pred_class: Predicted class index
        """
        self.model.eval()
        with torch.no_grad():
            logits = self.model(image)
            
        if class_idx is None:
            class_idx = logits.argmax(dim=1).item()
            
        # Get weights for target class
        class_weights = self.weights[class_idx]  # (num_channels,)
        
        # Compute CAM: weighted sum of feature maps
        features = self.feature_maps.squeeze(0)  # (C, H, W)
        
        cam = torch.zeros(features.shape[1:])  # (H, W)
        for i, w in enumerate(class_weights):
            cam += w * features[i]
            
        # Normalize and apply ReLU (only positive contributions)
        cam = F.relu(cam)
        cam = cam - cam.min()
        cam = cam / (cam.max() + 1e-8)
        
        return cam.numpy(), class_idx
 
 
class GradCAM:
    """
    Gradient-weighted Class Activation Mapping (Grad-CAM).
    
    Works with any CNN architecture, not just GAP-based networks.
    Uses gradients to weight feature map importance.
    """
    def __init__(self, model, target_layer):
        self.model = model
        self.feature_maps = None
        self.gradients = None
        
        # Register hooks
        target_layer.register_forward_hook(self._save_features)
        target_layer.register_full_backward_hook(self._save_gradients)
        
    def _save_features(self, module, input, output):
        self.feature_maps = output
        
    def _save_gradients(self, module, grad_input, grad_output):
        self.gradients = grad_output[0]
        
    def get_cam(self, image, class_idx=None):
        """Generate Grad-CAM visualization."""
        self.model.eval()
        
        # Forward pass
        image.requires_grad = True
        logits = self.model(image)
        
        if class_idx is None:
            class_idx = logits.argmax(dim=1).item()
            
        # Backward pass for target class
        self.model.zero_grad()
        one_hot = torch.zeros_like(logits)
        one_hot[0, class_idx] = 1
        logits.backward(gradient=one_hot, retain_graph=True)
        
        # Compute importance weights: global average of gradients
        weights = self.gradients.mean(dim=[2, 3], keepdim=True)  # (1, C, 1, 1)
        
        # Weighted sum of feature maps
        cam = (weights * self.feature_maps).sum(dim=1, keepdim=True)  # (1, 1, H, W)
        cam = F.relu(cam).squeeze()
        
        # Normalize
        cam = cam - cam.min()
        cam = cam / (cam.max() + 1e-8)
        
        return cam.detach().numpy(), class_idx
 
 
# Usage example
def visualize_cam_example():
    """
    Example of creating and interpreting CAM visualizations.
    """
    # Simulated scenario (in practice, use a real trained model)
    class SimpleNetwork(nn.Module):
        def __init__(self):
            super().__init__()
            self.features = nn.Sequential(
                nn.Conv2d(3, 64, 3, padding=1),
                nn.ReLU(),
                nn.Conv2d(64, 64, 3, padding=1),
                nn.ReLU()
            )
            self.gap = nn.AdaptiveAvgPool2d(1)
            self.classifier = nn.Linear(64, 10)
            
        def forward(self, x):
            x = self.features(x)
            x = self.gap(x)
            x = x.view(x.size(0), -1)
            return self.classifier(x)
    
    model = SimpleNetwork()
    image = torch.randn(1, 3, 32, 32)
    
    # Get prediction
    with torch.no_grad():
        logits = model(image)
        pred_class = logits.argmax(dim=1).item()
        
    print(f"Predicted class: {pred_class}")
    print("CAM would highlight regions that activated feature maps")
    print("positively weighted for this class")
 
visualize_cam_example()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168

import torch
import torch.nn as nn
import torch.nn.functional as F
 
class SqueezeExcitation(nn.Module):
    """
    Squeeze-and-Excitation block.
    
    Learns to emphasize informative channels and suppress less useful ones
    based on global context captured by GAP.
    """
    def __init__(self, channels, reduction=16):
        super().__init__()
        reduced = channels // reduction
        
        self.squeeze = nn.AdaptiveAvgPool2d(1)  # GAP for global context
        self.excitation = nn.Sequential(
            nn.Linear(channels, reduced, bias=False),
            nn.ReLU(inplace=True),
            nn.Linear(reduced, channels, bias=False),
            nn.Sigmoid()
        )
        
    def forward(self, x):
        B, C, H, W = x.shape
        
        # Squeeze: global average pooling
        squeezed = self.squeeze(x).view(B, C)  # (B, C)
        
        # Excitation: learn channel weights
        weights = self.excitation(squeezed).view(B, C, 1, 1)  # (B, C, 1, 1)
        
        # Scale: apply channel-wise attention
        return x * weights
 
 
class CBAM(nn.Module):
    """
    Convolutional Block Attention Module (CBAM).
    
    Combines channel attention (based on global pooling) with
    spatial attention for comprehensive feature refinement.
    """
    def __init__(self, channels, reduction=16, kernel_size=7):
        super().__init__()
        
        # Channel attention uses both GAP and GMP
        self.channel_attention = ChannelAttention(channels, reduction)
        
        # Spatial attention based on max and avg across channels
        self.spatial_attention = SpatialAttention(kernel_size)
        
    def forward(self, x):
        x = self.channel_attention(x)
        x = self.spatial_attention(x)
        return x
 
 
class ChannelAttention(nn.Module):
    """Channel attention sub-module of CBAM."""
    def __init__(self, channels, reduction=16):
        super().__init__()
        reduced = channels // reduction
        
        self.gap = nn.AdaptiveAvgPool2d(1)
        self.gmp = nn.AdaptiveMaxPool2d(1)
        
        self.fc = nn.Sequential(
            nn.Linear(channels, reduced, bias=False),
            nn.ReLU(inplace=True),
            nn.Linear(reduced, channels, bias=False)
        )
        
    def forward(self, x):
        B, C, H, W = x.shape
        
        # Both pooling types for richer statistics
        avg_pool = self.gap(x).view(B, C)
        max_pool = self.gmp(x).view(B, C)
        
        # Shared MLP processing
        avg_out = self.fc(avg_pool)
        max_out = self.fc(max_pool)
        
        # Combine and apply
        weights = torch.sigmoid(avg_out + max_out).view(B, C, 1, 1)
        return x * weights
 
 
class SpatialAttention(nn.Module):
    """Spatial attention sub-module of CBAM."""
    def __init__(self, kernel_size=7):
        super().__init__()
        padding = kernel_size // 2
        self.conv = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)
        
    def forward(self, x):
        # Channel-wise statistics
        avg_out = x.mean(dim=1, keepdim=True)  # (B, 1, H, W)
        max_out = x.max(dim=1, keepdim=True)[0]  # (B, 1, H, W)
        
        # Concatenate and convolve
        concat = torch.cat([avg_out, max_out], dim=1)  # (B, 2, H, W)
        weights = torch.sigmoid(self.conv(concat))  # (B, 1, H, W)
        
        return x * weights
 
 
class NetVLAD(nn.Module):
    """
    NetVLAD: Learnable pooling for image retrieval.
    
    Learns cluster centers and aggregates residuals from feature
    vectors to those centers, producing a rich global descriptor.
    """
    def __init__(self, feature_dim, num_clusters=64):
        super().__init__()
        self.num_clusters = num_clusters
        
        # Learnable cluster centers
        self.clusters = nn.Parameter(torch.randn(num_clusters, feature_dim))
        
        # Soft assignment convolution
        self.assignment = nn.Conv2d(feature_dim, num_clusters, 1, bias=False)
        
    def forward(self, x):
        B, C, H, W = x.shape
        N = H * W  # Number of local features
        
        # Soft assignment to clusters
        soft_assign = self.assignment(x)  # (B, K, H, W)
        soft_assign = F.softmax(soft_assign.view(B, self.num_clusters, -1), dim=1)
        
        # Reshape features
        x_flat = x.view(B, C, -1)  # (B, C, N)
        
        # Compute VLAD: sum of residuals to cluster centers
        vlad = torch.zeros(B, C, self.num_clusters, device=x.device)
        for k in range(self.num_clusters):
            residual = x_flat - self.clusters[k].unsqueeze(0).unsqueeze(-1)
            vlad[:, :, k] = (soft_assign[:, k:k+1, :] * residual).sum(-1)
        
        # L2 normalize
        vlad = F.normalize(vlad.view(B, -1), p=2, dim=1)
        return vlad
 
 
# Compare pooling methods
def compare_learned_pooling():
    x = torch.randn(4, 512, 14, 14)
    
    # Standard pooling
    gap = nn.AdaptiveAvgPool2d(1)
    gap_out = gap(x).view(4, -1)
    
    # SE-enhanced
    se = SqueezeExcitation(512)
    se_out = gap(se(x)).view(4, -1)
    
    # NetVLAD
    vlad = NetVLAD(512, num_clusters=16)
    vlad_out = vlad(x)
    
    print(f"GAP output: {gap_out.shape}")
    print(f"SE + GAP output: {se_out.shape}")
    print(f"NetVLAD output: {vlad_out.shape}")
 
compare_learned_pooling()