Orphans and Zombies: Handling Parent-Child Process Anomalies

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

/**
 * WARNING: This code deliberately creates zombies
 * Run only in a test environment with resource limits
 * 
 * This demonstrates how zombie accumulation happens
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
 
int main(void) {
    int zombie_count = 0;
    
    printf("=== Zombie Factory (for demonstration only) ===
");
    printf("Parent PID: %d
", getpid());
    printf("Creating zombies... (Ctrl+C to stop)
 
");
    
    while (1) {
        pid_t pid = fork();
        
        if (pid < 0) {
            /* fork() failed - likely out of resources */
            perror("fork failed");
            printf("
Failed after creating %d zombies
", zombie_count);
            printf("This typically means PID exhaustion or memory limits
");
            break;
        }
        
        if (pid == 0) {
            /* Child: immediately exit, become zombie */
            _exit(0);
        }
        
        /* Parent: does NOT call wait() */
        /* Each child becomes a zombie */
        zombie_count++;
        
        if (zombie_count % 100 == 0) {
            printf("Created %d zombies...
", zombie_count);
        }
        
        /* Slow down slightly to observe */
        usleep(1000);  /* 1ms delay */
    }
    
    printf("
=== System State ===
");
    printf("Use 'ps aux | grep -c Z' to count zombies
");
    printf("Use 'cat /proc/sys/kernel/pid_max' to see PID limit
");
    printf("
Parent will sleep to keep zombies visible...
");
    sleep(300);
    
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

#!/bin/bash
# Demonstrates the effect of PID exhaustion (simulation only)
# DO NOT RUN THIS - for educational understanding only
 
echo "=== PID Exhaustion Demonstration (Theory) ==="
echo ""
 
# Get current PID max
pid_max=$(cat /proc/sys/kernel/pid_max)
echo "System pid_max: $pid_max"
 
# Count current processes
current_procs=$(ls /proc | grep -E '^[0-9]+$' | wc -l)
echo "Current processes: $current_procs"
 
# Count zombies
zombie_count=$(ps aux | awk '$8 ~ /^Z/ {count++} END {print count+0}')
echo "Current zombies: $zombie_count"
 
# Calculate headroom
headroom=$((pid_max - current_procs))
echo "Available PIDs: $headroom"
 
echo ""
echo "If zombies consume all $headroom remaining PIDs:"
echo "  - fork() will return EAGAIN (resource temporarily unavailable)"
echo "  - Every new process creation will fail"
echo "  - SSH logins: FAIL"
echo "  - Cron jobs: FAIL"
echo "  - Service restarts: FAIL"
echo "  - The system becomes effectively frozen"
echo ""
echo "Symptoms of active PID exhaustion:"
echo "  - 'bash: fork: Resource temporarily unavailable'"
echo "  - 'Cannot fork: Resource temporarily unavailable'"
echo "  - Services fail to start or restart"
echo "  - Container orchestrators report failures"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

#!/bin/bash
# Comprehensive zombie monitoring script
 
set -e
 
# Thresholds for alerting
WARN_THRESHOLD=10
CRITICAL_THRESHOLD=100
 
echo "=== Zombie Process Monitor ==="
echo "Time: $(date)"
echo ""
 
# Count zombies
zombie_count=$(ps aux | awk '$8 ~ /^Z/ {count++} END {print count+0}')
 
# Get PID info
pid_max=$(cat /proc/sys/kernel/pid_max)
current_pids=$(ls /proc | grep -E '^[0-9]+$' | wc -l)
available=$((pid_max - current_pids))
 
echo "Zombie Count: $zombie_count"
echo "Total Processes: $current_pids"
echo "PID Max: $pid_max"
echo "Available PIDs: $available"
echo "Zombie Percentage of Used: $(echo "scale=2; $zombie_count * 100 / $current_pids" | bc)%"
 
# Alert logic
if [ "$zombie_count" -ge "$CRITICAL_THRESHOLD" ]; then
    echo ""
    echo "🚨 CRITICAL: $zombie_count zombies detected!"
    echo "Immediate investigation required."
    exit 2
elif [ "$zombie_count" -ge "$WARN_THRESHOLD" ]; then
    echo ""
    echo "⚠️  WARNING: $zombie_count zombies detected."
    echo "Investigation recommended."
    exit 1
else
    echo ""
    echo "✅ OK: Zombie count within normal limits."
fi
 
# If zombies exist, show details
if [ "$zombie_count" -gt 0 ]; then
    echo ""
    echo "=== Zombie Details ==="
    ps -eo pid,ppid,stat,user,cmd | awk 'NR==1 || $3 ~ /^Z/'
    
    echo ""
    echo "=== Parent Processes of Zombies ==="
    ps aux | awk '$8 ~ /^Z/ {print $2}' | while read zpid; do
        ppid=$(cat /proc/$zpid/stat 2>/dev/null | awk '{print $4}')
        if [ -n "$ppid" ] && [ -d "/proc/$ppid" ]; then
            pname=$(cat /proc/$ppid/comm 2>/dev/null)
            echo "Zombie $zpid <- Parent $ppid ($pname)"
        fi
    done | sort | uniq -c | sort -rn | head -10
    
    echo ""
    echo "=== Top Zombie-Producing Parents ==="
    ps -eo ppid | sort | uniq -c | sort -rn | while read count ppid; do
        # Check if this parent has zombie children
        zombie_children=$(ps -o stat,ppid | awk -v p="$ppid" '$2 == p && $1 ~ /Z/' | wc -l)
        if [ "$zombie_children" -gt 0 ]; then
            pname=$(cat /proc/$ppid/comm 2>/dev/null || echo "unknown")
            echo "PPID $ppid ($pname): $zombie_children zombies"
        fi
    done | head -5
fi

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

#!/bin/bash
# Deep investigation of zombie accumulation
 
echo "=== Zombie Investigation ==="
echo ""
 
# Step 1: Identify all zombie parent PIDs
echo "Step 1: Finding zombie-producing parents..."
zombie_parents=$(ps -eo ppid,stat | awk '$2 ~ /^Z/ {print $1}' | sort -u)
 
for ppid in $zombie_parents; do
    zombie_count=$(ps -eo ppid,stat | awk -v p="$ppid" '$1==p && $2~/^Z/ {count++} END {print count+0}')
    
    if [ -d "/proc/$ppid" ]; then
        echo ""
        echo "=== Parent PID: $ppid ($zombie_count zombies) ==="
        
        # Basic info
        echo "Command: $(cat /proc/$ppid/cmdline | tr '\0' ' ')"
        echo "Executable: $(readlink /proc/$ppid/exe 2>/dev/null || echo 'N/A')"
        echo "Working Dir: $(readlink /proc/$ppid/cwd 2>/dev/null)"
        echo "Owner: $(stat -c '%U' /proc/$ppid 2>/dev/null)"
        echo "Start Time: $(stat -c '%y' /proc/$ppid 2>/dev/null | cut -d. -f1)"
        
        # Memory and threads
        echo "Threads: $(cat /proc/$ppid/status | grep Threads | awk '{print $2}')"
        echo "RSS: $(cat /proc/$ppid/status | grep VmRSS | awk '{print $2, $3}')"
        
        # Check if SIGCHLD is blocked or ignored
        echo ""
        echo "Signal Handling:"
        sigchld=$(cat /proc/$ppid/status | grep -E '^Sig.*:')
        echo "$sigchld"
        
        # Decode SIGCHLD (signal 17 on most Linux)
        sig_ign=$(cat /proc/$ppid/status | grep SigIgn | awk '{print $2}')
        if [ -n "$sig_ign" ]; then
            # Check bit 17 (SIGCHLD)
            sig_int=$(printf "%d" 0x$sig_ign)
            if [ $((sig_int & (1 << 16))) -ne 0 ]; then
                echo "⚠️  SIGCHLD is IGNORED! This prevents zombie reaping."
            fi
        fi
        
        # Check open files (may indicate what this process does)
        echo ""
        echo "Open Files (first 10):"
        ls -la /proc/$ppid/fd 2>/dev/null | head -10
        
        # Network connections
        echo ""
        echo "Network Connections:"
        ss -tnp | grep "pid=$ppid" | head -5
        
    else
        echo ""
        echo "=== Parent PID: $ppid (DEAD - now under init) ==="
        echo "Parent has died; zombies should be reaped by init soon."
    fi
done
 
echo ""
echo "=== Summary ==="
echo "Total unique zombie parents: $(echo "$zombie_parents" | wc -w)"
echo "Total zombies: $(ps aux | awk '$8 ~ /^Z/ {count++} END {print count+0}')"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

#!/bin/bash
# Check for zombies in Docker containers
 
echo "=== Docker Container Zombie Check ==="
 
for container in $(docker ps -q); do
    name=$(docker inspect -f '{{.Name}}' $container | sed 's/^\///')
    
    # Get zombie count inside container
    zombie_count=$(docker exec $container sh -c         'ps aux 2>/dev/null | grep -c "^[^ ]* *[^ ]* *[^ ]* *[^ ]* *[^ ]* *[^ ]* *[^ ]* *Z"'         2>/dev/null || echo "N/A")
    
    # Get PID 1 process
    pid1=$(docker exec $container sh -c 'cat /proc/1/comm' 2>/dev/null || echo "N/A")
    
    if [ "$zombie_count" = "N/A" ]; then
        echo "Container $name: Unable to check (no shell?)"
    elif [ "$zombie_count" -gt 0 ]; then
        echo "⚠️  Container $name: $zombie_count zombies (PID 1: $pid1)"
        
        # Show the zombies
        docker exec $container ps aux 2>/dev/null | awk '$8 ~ /Z/' | head -5
    else
        echo "✅ Container $name: No zombies (PID 1: $pid1)"
    fi
done
 
echo ""
echo "=== Recommendations ==="
echo "Containers with zombies likely need:"
echo "  1. docker run --init (uses tini)"
echo "  2. Or add tini/dumb-init as ENTRYPOINT"
echo "  3. Or have application properly reap children"