Orphans and Zombies: Handling Parent-Child Process Anomalies

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

/**
 * Complete SIGCHLD handler implementation
 * This is the gold-standard pattern for zombie prevention
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <sys/wait.h>
#include <errno.h>
 
/* Track child statistics (optional) */
volatile sig_atomic_t children_reaped = 0;
volatile sig_atomic_t children_signaled = 0;
 
/**
 * SIGCHLD handler - reaps ALL available children
 * 
 * Key points:
 * 1. Loop with WNOHANG - don't block, reap all ready
 * 2. Preserve errno - signal handlers may interrupt syscalls
 * 3. Handle all termination types (exit, signal)
 */
void sigchld_handler(int sig) {
    int saved_errno = errno;  /* Preserve errno for interrupted syscall */
    pid_t pid;
    int status;
    
    /* 
     * Loop to reap ALL terminated children
     * WNOHANG: return immediately if no child has exited
     * Multiple children may have died before this handler runs
     */
    while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
        if (WIFEXITED(status)) {
            /* Child exited normally */
            children_reaped++;
        } else if (WIFSIGNALED(status)) {
            /* Child killed by signal */
            children_signaled++;
        }
        /* Note: WIFSTOPPED and WIFCONTINUED are filtered by SA_NOCLDSTOP */
    }
    
    /* 
     * waitpid returns -1 with ECHILD when no more children
     * This is expected and not an error
     */
    
    errno = saved_errno;  /* Restore errno */
}
 
/**
 * Install the SIGCHLD handler properly
 * Returns 0 on success, -1 on error
 */
int setup_sigchld_handler(void) {
    struct sigaction sa;
    
    /* Clear the structure */
    sigemptyset(&sa.sa_mask);
    
    /* Set the handler */
    sa.sa_handler = sigchld_handler;
    
    /* 
     * Flags:
     * SA_RESTART: Restart interrupted system calls
     * SA_NOCLDSTOP: Don't notify for stopped children (only terminated)
     */
    sa.sa_flags = SA_RESTART | SA_NOCLDSTOP;
    
    if (sigaction(SIGCHLD, &sa, NULL) == -1) {
        perror("sigaction SIGCHLD");
        return -1;
    }
    
    return 0;
}
 
/**
 * Alternative: Ignore SIGCHLD entirely
 * When SIG_IGN is set for SIGCHLD, children are automatically reaped
 * No zombies are created - exit status is discarded
 * Use this when you don't care about child exit status
 */
int setup_sigchld_ignore(void) {
    struct sigaction sa;
    sa.sa_handler = SIG_IGN;
    sigemptyset(&sa.sa_mask);
    sa.sa_flags = 0;
    
    if (sigaction(SIGCHLD, &sa, NULL) == -1) {
        perror("sigaction SIGCHLD SIG_IGN");
        return -1;
    }
    
    return 0;
}
 
/* Example usage */
int main(void) {
    /* Setup handler */
    if (setup_sigchld_handler() != 0) {
        exit(EXIT_FAILURE);
    }
    
    printf("Parent PID: %d\n", getpid());
    printf("Spawning children...\n\n");
    
    /* Spawn some children */
    for (int i = 0; i < 5; i++) {
        pid_t pid = fork();
        if (pid == 0) {
            /* Child */
            int sleep_time = i + 1;
            printf("Child %d (PID %d): sleeping %d seconds\n", 
                   i, getpid(), sleep_time);
            sleep(sleep_time);
            exit(i);  /* Exit with different codes */
        }
    }
    
    /* Parent: do other work while children run */
    printf("\nParent: Working while children run...\n");
    
    for (int i = 0; i < 10; i++) {
        sleep(1);
        printf("  Parent: Reaped so far: %d, Signaled: %d\n", 
               children_reaped, children_signaled);
    }
    
    printf("\nFinal: Reaped %d children, %d killed by signal\n",
           children_reaped, children_signaled);
    
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

/**
 * The double-fork technique for daemon creation
 * Creates a properly detached background process
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
 
/**
 * Daemonize the current process
 * Returns: 0 in daemon (grandchild), -1 on error, never returns in parent
 */
int daemonize(void) {
    pid_t pid;
    
    /* 
     * First fork
     * Allows the original shell to wait() and return
     * Child runs in background
     */
    pid = fork();
    if (pid < 0) {
        perror("First fork failed");
        return -1;
    }
    if (pid > 0) {
        /* Parent: exit so shell gets prompt back */
        _exit(0);
    }
    
    /* 
     * Child: Become session leader
     * This detaches from controlling terminal
     */
    if (setsid() < 0) {
        perror("setsid failed");
        return -1;
    }
    
    /*
     * Second fork
     * The session leader could acquire a terminal if it opened one
     * By forking again, the grandchild can NEVER acquire a terminal
     */
    pid = fork();
    if (pid < 0) {
        perror("Second fork failed");
        return -1;
    }
    if (pid > 0) {
        /* First child: exit immediately */
        /* Parent (original) already exited after first fork */
        /* Grandchild becomes orphan, adopted by init */
        _exit(0);
    }
    
    /*
     * Grandchild: Now a proper daemon
     * - Not a session leader (cannot get terminal)
     * - Orphaned, adopted by init (PPID will be 1)
     * - Completely detached from original process tree
     */
    
    /* Standard daemon housekeeping */
    
    /* Change working directory to root (not on mounted filesystem) */
    if (chdir("/") < 0) {
        perror("chdir failed");
        return -1;
    }
    
    /* Set file creation mask */
    umask(0);
    
    /* Close standard file descriptors */
    close(STDIN_FILENO);
    close(STDOUT_FILENO);
    close(STDERR_FILENO);
    
    /* Redirect to /dev/null */
    int fd = open("/dev/null", O_RDWR);
    if (fd != STDIN_FILENO) dup2(fd, STDIN_FILENO);
    if (fd != STDOUT_FILENO) dup2(fd, STDOUT_FILENO);
    if (fd != STDERR_FILENO) dup2(fd, STDERR_FILENO);
    if (fd > STDERR_FILENO) close(fd);
    
    return 0;  /* Success - we are the daemon */
}
 
/* Example usage */
int main(void) {
    printf("Starting daemon process...\n");
    printf("Original PID: %d, PPID: %d\n", getpid(), getppid());
    
    if (daemonize() != 0) {
        fprintf(stderr, "Failed to daemonize\n");
        exit(EXIT_FAILURE);
    }
    
    /* We are now the daemon */
    /* Log to syslog since stdout is closed */
    #include <syslog.h>
    openlog("mydaemon", LOG_PID, LOG_DAEMON);
    syslog(LOG_INFO, "Daemon started, PID: %d, PPID: %d", 
           getpid(), getppid());
    
    /* Daemon main loop */
    while (1) {
        syslog(LOG_INFO, "Daemon heartbeat");
        sleep(60);
    }
    
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

/**
 * Process group management for clean cleanup
 * Useful for managing worker pools or child pipelines
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <sys/wait.h>
 
/**
 * Spawn workers in a new process group
 * Makes cleanup easy: just kill the group
 */
pid_t spawn_worker_group(int count) {
    pid_t group_leader = 0;
    
    for (int i = 0; i < count; i++) {
        pid_t pid = fork();
        
        if (pid < 0) {
            perror("fork");
            /* Kill already-spawned workers */
            if (group_leader > 0) {
                kill(-group_leader, SIGTERM);
            }
            return -1;
        }
        
        if (pid == 0) {
            /* Child: join the group */
            if (group_leader > 0) {
                setpgid(0, group_leader);
            } else {
                /* First child becomes group leader */
                setpgid(0, 0);
            }
            
            /* Do worker work */
            printf("Worker %d (PID %d, PGID %d) starting\n",
                   i, getpid(), getpgid(0));
            sleep(10 + i);  /* Simulate work */
            exit(i);
        }
        
        /* Parent: track the group */
        if (i == 0) {
            group_leader = pid;
        }
        setpgid(pid, group_leader);  /* Put child in group */
    }
    
    return group_leader;
}
 
/**
 * Wait for entire process group
 */
int wait_for_group(pid_t pgid) {
    int status;
    pid_t pid;
    int count = 0;
    
    /* waitpid with negative pgid waits for group members */
    while ((pid = waitpid(-pgid, &status, 0)) > 0) {
        count++;
        printf("Group member PID %d exited\n", pid);
    }
    
    return count;
}
 
/**
 * Kill entire process group
 */
void kill_group(pid_t pgid, int sig) {
    printf("Killing process group %d with signal %d\n", pgid, sig);
    kill(-pgid, sig);  /* Negative PID = process group */
}
 
/* Example: clean timeout-based worker management */
int main(void) {
    printf("Spawning worker group...\n");
    
    pid_t group = spawn_worker_group(3);
    if (group < 0) {
        fprintf(stderr, "Failed to spawn workers\n");
        exit(EXIT_FAILURE);
    }
    
    printf("Worker group leader: %d\n", group);
    
    /* Give workers 5 seconds */
    sleep(5);
    
    /* Kill the entire group */
    printf("Timeout! Killing worker group...\n");
    kill_group(group, SIGTERM);
    
    /* Wait for all group members */
    printf("Waiting for workers to terminate...\n");
    int reaped = wait_for_group(group);
    printf("Reaped %d workers\n", reaped);
    
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

/**
 * RAII wrapper for child process management (C++)
 * Ensures child is always waited for
 */
#include <iostream>
#include <functional>
#include <unistd.h>
#include <sys/wait.h>
 
class ChildProcess {
public:
    ChildProcess(std::function<int()> child_func) : pid_(-1) {
        pid_ = fork();
        
        if (pid_ == 0) {
            // Child process
            int result = child_func();
            _exit(result);
        } else if (pid_ < 0) {
            throw std::runtime_error("fork failed");
        }
        // Parent continues with valid pid_
    }
    
    ~ChildProcess() {
        if (pid_ > 0) {
            // Destructor ALWAYS waits for child
            // Even if exception was thrown
            int status;
            waitpid(pid_, &status, 0);
            std::cerr << "Child " << pid_ << " reaped in destructor\n";
        }
    }
    
    // Explicit wait for result
    int wait() {
        if (pid_ <= 0) return -1;
        
        int status;
        pid_t result = waitpid(pid_, &status, 0);
        pid_ = -1;  // Mark as already waited
        
        if (result > 0 && WIFEXITED(status)) {
            return WEXITSTATUS(status);
        }
        return -1;
    }
    
    pid_t pid() const { return pid_; }
    
    // Non-copyable
    ChildProcess(const ChildProcess&) = delete;
    ChildProcess& operator=(const ChildProcess&) = delete;
    
private:
    pid_t pid_;
};
 
// Usage - child is ALWAYS reaped
void example() {
    ChildProcess child([]() {
        sleep(1);
        return 42;
    });
    
    // Even if exception thrown here, child is reaped in destructor
    do_something_that_might_throw();
    
    int result = child.wait();
    std::cout << "Child returned: " << result << "\n";
}  // child destructor called, waits if not already waited

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

"""
Python patterns for robust process management
"""
import os
import signal
import subprocess
from contextlib import contextmanager
 
# Pattern 1: Use subprocess instead of os.fork
# subprocess handles wait() automatically
def safe_spawn(cmd):
    """subprocess module manages child lifecycle"""
    result = subprocess.run(cmd, capture_output=True)
    # Child is automatically waited for
    return result
 
# Pattern 2: Context manager for manual fork
@contextmanager
def child_process():
    """Ensure child is waited for"""
    pid = os.fork()
    if pid == 0:
        try:
            yield True  # In child
        finally:
            os._exit(0)
    else:
        yield False  # In parent
        os.waitpid(pid, 0)  # Always wait
 
# Usage:
# with child_process() as is_child:
#     if is_child:
#         do_child_work()
#     else:
#         do_parent_work()
 
# Pattern 3: Signal handler for async children
def setup_sigchld():
    """Install zombie reaper for async children"""
    def reap_children(signum, frame):
        while True:
            try:
                pid, status = os.waitpid(-1, os.WNOHANG)
                if pid == 0:
                    break
            except ChildProcessError:
                break
    
    signal.signal(signal.SIGCHLD, reap_children)
 
# Pattern 4: Pool-based management
from multiprocessing import Pool
 
def managed_workers():
    """Pool handles all lifecycle management"""
    with Pool(processes=4) as pool:
        results = pool.map(do_work, items)
    # All workers cleaned up automatically
    return results

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

# Prometheus alerting rules for process health
groups:
  - name: process_health
    interval: 30s
    rules:
      # Alert on zombie accumulation
      - alert: ZombieProcessesWarning
        expr: node_procs_zombie > 20
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Zombie processes on {{ $labels.instance }}"
          description: "{{ $value }} zombie processes detected"
          runbook: "Check parent processes with SIGCHLD issues"
      
      - alert: ZombieProcessesCritical
        expr: node_procs_zombie > 100
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Critical zombie accumulation on {{ $labels.instance }}"
          description: "{{ $value }} zombies - PID exhaustion risk"
          runbook: "Identify and restart zombie-producing parent"
      
      # Alert on rapid zombie growth (more sensitive)
      - alert: ZombieGrowthRate
        expr: rate(node_procs_zombie[5m]) > 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Zombie count increasing on {{ $labels.instance }}"
          description: "Zombies growing at {{ $value }}/sec"
      
      # Alert on PID exhaustion risk
      - alert: PIDExhaustionRisk
        expr: (1 - node_procs_running / node_kernel_pid_max) < 0.1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "PID space nearly exhausted on {{ $labels.instance }}"
          description: "Only {{ $value | percentage }} PIDs remaining"
 
# Grafana dashboard JSON (simplified)
# Panels:
# 1. Zombie count over time (time series)
# 2. Total processes vs pid_max (gauge)
# 3. Top zombie-producing parents (table, from custom exporter)
# 4. Recent zombie reaping events (log panel)