1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

// Why PID 1 must reap zombies (conceptual)
 
// When a process exits, it becomes a zombie until its parent
// calls wait() to collect the exit status
 
// Without proper reaping in init:
void bad_init() {
    // Fork child processes to do work
    for (int i = 0; i < 100; i++) {
        if (fork() == 0) {
            // Child does work then exits
            do_service_work();
            exit(0);  // Child terminates
        }
    }
    // Parent (init) never calls wait()
    // Result: 100 zombie processes consuming PID table space
    
    while(1) pause();  // Just wait forever
}
 
// Proper init implementation:
void good_init() {
    // Set up SIGCHLD handler
    signal(SIGCHLD, sigchld_handler);
    
    // Start services
    start_services();
    
    // Main loop
    while(1) {
        pause();  // Wait for signals
    }
}
 
void sigchld_handler(int sig) {
    int status;
    pid_t pid;
    
    // Reap ALL available children (multiple may have died)
    while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
        // Log child termination
        log_child_exit(pid, status);
        
        // Restart service if configured for auto-restart
        if (should_restart(pid)) {
            restart_service(pid);
        }
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# Traditional SysVinit inittab example
 
# Default runlevel (3 = multi-user, 5 = graphical)
id:3:initdefault:
 
# System initialization script (runs before any runlevel)
si::sysinit:/etc/rc.d/rc.sysinit
 
# Runlevel scripts
l0:0:wait:/etc/rc.d/rc 0    # Halt
l1:1:wait:/etc/rc.d/rc 1    # Single-user
l2:2:wait:/etc/rc.d/rc 2    # Multi-user, no network
l3:3:wait:/etc/rc.d/rc 3    # Multi-user with network
l4:4:wait:/etc/rc.d/rc 4    # Unused (custom)
l5:5:wait:/etc/rc.d/rc 5    # Multi-user with graphical
l6:6:wait:/etc/rc.d/rc 6    # Reboot
 
# Getty processes for virtual consoles
1:2345:respawn:/sbin/mingetty tty1
2:2345:respawn:/sbin/mingetty tty2
3:2345:respawn:/sbin/mingetty tty3
4:2345:respawn:/sbin/mingetty tty4
5:2345:respawn:/sbin/mingetty tty5
6:2345:respawn:/sbin/mingetty tty6
 
# Ctrl+Alt+Delete handling
ca::ctrlaltdel:/sbin/shutdown -t3 -r now
 
# Power failure script
pf::powerfail:/sbin/shutdown -f -h +2 "Power Failure"
 
# inittab format: id:runlevels:action:process
# id       - unique 1-4 character identifier
# runlevels - runlevels where this entry applies
# action   - wait, respawn, once, boot, sysinit, etc.
# process  - command to execute

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

# Example systemd service unit file
# /etc/systemd/system/myapp.service
 
[Unit]
# Metadata and dependencies
Description=My Application Server
Documentation=https://example.com/docs
After=network.target postgresql.service
Wants=postgresql.service
# After=  : we start after these (ordering only)
# Wants=  : try to start these with us (weak dependency)
# Requires= : must have these (strong dependency, fail if they fail)
 
[Service]
# How to run the service
Type=simple          # simple, forking, oneshot, notify, dbus, idle
User=myapp           # Run as this user
Group=myapp          # Run as this group
Environment=NODE_ENV=production
EnvironmentFile=-/etc/myapp/env    # "-" means optional
WorkingDirectory=/opt/myapp
 
ExecStartPre=/opt/myapp/pre-start.sh   # Run before main process
ExecStart=/opt/myapp/bin/server --port 8080
ExecReload=/bin/kill -HUP $MAINPID     # How to reload config
ExecStop=/opt/myapp/graceful-shutdown.sh
 
# Restart behavior
Restart=on-failure   # no, always, on-failure, on-abnormal, on-abort, on-watchdog
RestartSec=5         # Wait 5 seconds before restart
StartLimitBurst=3    # Max 3 restarts...
StartLimitIntervalSec=60  # ...per 60 seconds
 
# Resource limits
LimitNOFILE=65536    # Max open files
LimitNPROC=4096      # Max processes
MemoryMax=2G         # cgroup memory limit
CPUQuota=200%        # Max 2 CPU cores worth
 
# Security hardening
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
ReadWritePaths=/var/lib/myapp
 
[Install]
# How "systemctl enable" integrates this unit
WantedBy=multi-user.target
# This creates symlink:
# /etc/systemd/system/multi-user.target.wants/myapp.service

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# Essential systemctl commands
 
# Service management
systemctl start nginx.service      # Start now
systemctl stop nginx.service       # Stop now
systemctl restart nginx.service    # Stop then start
systemctl reload nginx.service     # Reload configuration (SIGHUP)
systemctl status nginx.service     # Show status and recent logs
systemctl is-active nginx          # Exit 0 if running
 
# Enable/disable (auto-start at boot)
systemctl enable nginx.service     # Create symlinks for boot
systemctl disable nginx.service    # Remove symlinks
systemctl is-enabled nginx         # Exit 0 if enabled
 
# Masking (completely prevent starting)
systemctl mask nginx.service       # Link to /dev/null
systemctl unmask nginx.service     # Remove mask
 
# System state
systemctl list-units               # Show all loaded units
systemctl list-units --failed      # Show failed units
systemctl list-unit-files          # Show all installed units
 
# Dependencies
systemctl list-dependencies nginx  # Show dependency tree
systemctl list-dependencies --reverse nginx  # What depends on this
 
# Reload configuration after editing unit files
systemctl daemon-reload            # Required after modifying .service files
 
# Debugging
systemctl cat nginx.service        # Show unit file contents
systemctl show nginx.service       # Show all properties
systemctl edit nginx.service       # Create override file in /etc
journalctl -u nginx.service        # View service logs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# Viewing and debugging dependencies
 
# Show what a unit needs
$ systemctl list-dependencies nginx.service
nginx.service
● ├─-.slice
● ├─system.slice
● └─sysinit.target
●   ├─dev-hugepages.mount
●   ├─dev-mqueue.mount
●   ...
 
# Show what needs this unit (reverse)
$ systemctl list-dependencies --reverse nginx.service
nginx.service
● └─multi-user.target
●   └─graphical.target
 
# Show ordering
$ systemctl show nginx.service -p After
After=basic.target system.slice -.slice sysinit.target network.target
 
# Dependency debugging: why did unit start?
$ systemctl status nginx
   Triggered By: ● multi-user.target
 
# Check if ordering is satisfied
$ systemctl list-jobs  # Shows pending jobs and their state
 
# Common dependency patterns:
 
# Web app needing database
[Unit]
After=network.target postgresql.service
Requires=postgresql.service
 
# Service needing specific mount
[Unit]
After=mnt-data.mount
BindsTo=mnt-data.mount   # Stop service if unmounted
 
# Mutually exclusive configs
[Unit]
Conflicts=nginx.service  # Can't run with nginx

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# Socket-activated service example
 
# /etc/systemd/system/myapp.socket
[Unit]
Description=My App Socket
 
[Socket]
ListenStream=8080              # TCP port
# Or: ListenStream=/run/myapp.sock  # Unix socket
# Or: ListenDatagram=5353          # UDP
Accept=no                       # no: pass socket to one service instance
                               # yes: fork new instance per connection (like inetd)
 
[Install]
WantedBy=sockets.target
 
# /etc/systemd/system/myapp.service
[Unit]
Description=My App Service
# Note: no After/Wants for socket - systemd handles it
 
[Service]
Type=simple
ExecStart=/opt/myapp/server    # Receives socket as FD 3
# Server must accept socket via sd_listen_fds() or inherit FD 3
 
# For Accept=no (single instance):
# StandardInput=socket          # Socket becomes stdin (for simple protocols)
 
# Restart handling with socket activation
Restart=on-failure
# New connections queue on socket while service restarts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# Working with socket-activated services
 
# List all sockets
$ systemctl list-sockets
LISTEN                        UNIT                  ACTIVATES
/run/dbus/system_bus_socket   dbus.socket          dbus.service
/run/docker.sock              docker.socket        docker.service
[::]:22                       sshd.socket          sshd.service
 
# Enable socket activation for SSH
$ systemctl stop sshd.service    # Stop direct service
$ systemctl disable sshd.service
$ systemctl enable sshd.socket   # Enable socket instead
$ systemctl start sshd.socket
 
# Now sshd only runs when someone connects
$ ss -tlnp | grep 22
LISTEN  0  128  *:22  *:*  users:(("systemd",pid=1,fd=36))
# Note: systemd holds socket, not sshd
 
$ ssh localhost
# NOW sshd starts
$ systemctl status sshd.service
Active: active (running)  # Started on demand
 
# Converting existing service to socket activation:
# 1. Create .socket unit matching .service name
# 2. Ensure service can inherit socket (sd_listen_fds or fd=3)
# 3. Disable .service, enable .socket

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# Essential journalctl commands
 
# View all logs (newest first)
$ journalctl
$ journalctl -r          # Reverse (newest first)
$ journalctl -f          # Follow (like tail -f)
$ journalctl -n 100      # Last 100 lines
 
# Filter by unit
$ journalctl -u nginx.service
$ journalctl -u nginx -u php-fpm    # Multiple units
 
# Filter by priority (0=emerg to 7=debug)
$ journalctl -p err                  # Errors and above
$ journalctl -p warning..err         # Warning through error
 
# Filter by time
$ journalctl --since "2024-01-15"
$ journalctl --since "10 minutes ago"
$ journalctl --since "09:00" --until "10:00"
 
# Filter by boot
$ journalctl -b           # Current boot only
$ journalctl -b -1        # Previous boot
$ journalctl --list-boots # List all stored boots
 
# Kernel messages (like dmesg)
$ journalctl -k
$ journalctl -k -b -1     # Kernel msgs from previous boot
 
# Output formats
$ journalctl -o short     # Default: syslog-like
$ journalctl -o verbose   # All metadata fields
$ journalctl -o json      # JSON format
$ journalctl -o cat       # Message only (no timestamp)
 
# Disk usage and maintenance
$ journalctl --disk-usage
$ journalctl --vacuum-size=500M   # Trim to 500MB
$ journalctl --vacuum-time=7d     # Keep only 7 days
 
# Advanced filtering
$ journalctl _SYSTEMD_UNIT=nginx.service _PID=1234
$ journalctl CONTAINER_NAME=mycontainer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

# systemd Troubleshooting Techniques
 
# 1. Service won't start - check status and logs
$ systemctl status myapp.service
● myapp.service - My Application
     Loaded: loaded (/etc/systemd/system/myapp.service; enabled)
     Active: failed (Result: exit-code) since ...
    Process: 1234 ExecStart=/opt/myapp/bin/server (code=exited, status=1/FAILURE)
$ journalctl -u myapp.service -n 50
 
# 2. Service starts but immediately dies
$ systemctl start myapp
$ systemctl status myapp
# Check: Is Type= correct? Forking service with Type=simple fails
# Check: Does ExecStart run in foreground?
# Check: Does the binary exist and have execute permission?
 
# 3. Boot hangs - identify the blocking unit
# At stuck boot, press Escape to see status
# Or add 'systemd.log_level=debug systemd.log_target=console' to kernel cmdline
$ systemctl list-jobs  # Shows blocked/waiting jobs
 
# 4. View startup time analysis
$ systemd-analyze
Startup finished in 3.456s (kernel) + 7.890s (userspace) = 11.346s
$ systemd-analyze blame    # Slowest units
$ systemd-analyze critical-chain  # Blocking path to target
$ systemd-analyze plot > boot.svg  # Visual timeline
 
# 5. Service fails to stop (timeout)
$ systemctl stop myapp  # Hangs...
# Check: Is ExecStop doing something wrong?
# Check: Is the process ignoring SIGTERM?
$ systemctl kill myapp  # Send SIGKILL
 
# 6. Dependency issues
$ systemctl list-dependencies myapp --all
$ systemctl list-dependencies myapp --reverse
 
# 7. Unit file syntax errors
$ systemd-analyze verify /etc/systemd/system/myapp.service
# Shows parsing errors
 
# 8. Emergency recovery
# Boot with: systemd.unit=rescue.target (single-user)
# Or: systemd.unit=emergency.target (pre-mount root only)
# Or: init=/bin/bash (bypass init entirely)