Health Check Endpoints

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

// TypeScript/Express Example: Minimal Liveness Endpoint
 
import express from 'express';
import { threadPoolHealth } from './monitoring';
 
const app = express();
 
/**
 * Liveness endpoint - answers "should we restart this process?"
 * 
 * Rules:
 * - Must respond extremely fast (< 10ms target)
 * - Must NOT check external dependencies
 * - Should only fail for unrecoverable local state
 */
app.get('/health/live', (req, res) => {
    // Check 1: Can we even respond? (Implicit - if we got here, yes)
    
    // Check 2: Are worker threads responsive?
    const threadHealth = threadPoolHealth();
    if (threadHealth.deadlockedThreads > 0) {
        return res.status(503).json({
            status: 'unhealthy',
            reason: 'thread_deadlock',
            deadlockedThreads: threadHealth.deadlockedThreads
        });
    }
    
    // Check 3: Is our event loop responsive?
    // (If this handler runs, the event loop is alive)
    
    // All checks pass - we're alive
    res.status(200).json({
        status: 'healthy',
        timestamp: new Date().toISOString()
    });
});
 
// Anti-pattern: DO NOT include database checks in liveness
// ❌ BAD - This can cause cascading restarts
app.get('/health/live-bad', async (req, res) => {
    try {
        await database.query('SELECT 1');  // DON'T DO THIS IN LIVENESS
        res.json({ status: 'healthy' });
    } catch (error) {
        res.status(503).json({ status: 'unhealthy' });  // Will restart!
    }
});

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

// Go/Gin Example: Minimal Liveness Endpoint
 
package main
 
import (
    "net/http"
    "runtime"
    "time"
    
    "github.com/gin-gonic/gin"
)
 
// LivenessHandler checks if the process is fundamentally alive
// and should continue running (vs being restarted)
func LivenessHandler(c *gin.Context) {
    // Check 1: Verify the runtime is healthy
    // If we couldn't call NumGoroutine, we'd have a serious problem
    numGoroutines := runtime.NumGoroutine()
    
    // Check 2: Watch for goroutine leaks that might indicate deadlock
    // This threshold should be tuned per-application
    const maxGoroutines = 10000
    if numGoroutines > maxGoroutines {
        c.JSON(http.StatusServiceUnavailable, gin.H{
            "status":      "unhealthy",
            "reason":      "goroutine_leak_suspected",
            "goroutines":  numGoroutines,
            "threshold":   maxGoroutines,
        })
        return
    }
    
    // Check 3: Verify memory isn't critically exhausted
    var memStats runtime.MemStats
    runtime.ReadMemStats(&memStats)
    
    // If heap is over 90% of system memory, we may need restart
    const heapThreshold = 0.9
    heapUsageRatio := float64(memStats.HeapAlloc) / float64(memStats.HeapSys)
    if heapUsageRatio > heapThreshold {
        c.JSON(http.StatusServiceUnavailable, gin.H{
            "status":        "unhealthy",
            "reason":        "heap_exhaustion",
            "heapAlloc":     memStats.HeapAlloc,
            "heapSys":       memStats.HeapSys,
            "usageRatio":    heapUsageRatio,
        })
        return
    }
    
    // All checks pass
    c.JSON(http.StatusOK, gin.H{
        "status":     "healthy",
        "goroutines": numGoroutines,
        "timestamp":  time.Now().UTC().Format(time.RFC3339),
    })
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

// TypeScript/Express: Comprehensive Readiness Endpoint
 
import express, { Request, Response } from 'express';
import { Pool } from 'pg';
import Redis from 'ioredis';
 
interface HealthCheckResult {
    service: string;
    status: 'healthy' | 'unhealthy' | 'degraded';
    latencyMs?: number;
    message?: string;
}
 
interface ReadinessResponse {
    status: 'ready' | 'not_ready';
    timestamp: string;
    checks: HealthCheckResult[];
    version?: string;
}
 
// Dependency references
let pgPool: Pool;
let redisClient: Redis;
let isInitialized = false;
 
// Configurable timeouts
const CHECK_TIMEOUT_MS = 2000;
const MAX_CONNECTION_POOL_USAGE = 0.8;  // 80% threshold
 
/**
 * Check PostgreSQL connectivity and pool health
 */
async function checkDatabase(): Promise<HealthCheckResult> {
    const start = Date.now();
    try {
        // Verify we can execute a query
        await Promise.race([
            pgPool.query('SELECT 1'),
            new Promise((_, reject) => 
                setTimeout(() => reject(new Error('timeout')), CHECK_TIMEOUT_MS)
            )
        ]);
        
        // Check connection pool saturation
        const poolStats = pgPool.totalCount;
        const idleCount = pgPool.idleCount;
        const waitingCount = pgPool.waitingCount;
        
        if (waitingCount > 0) {
            return {
                service: 'postgresql',
                status: 'degraded',
                latencyMs: Date.now() - start,
                message: `${waitingCount} requests waiting for connections`
            };
        }
        
        return {
            service: 'postgresql',
            status: 'healthy',
            latencyMs: Date.now() - start
        };
    } catch (error) {
        return {
            service: 'postgresql',
            status: 'unhealthy',
            latencyMs: Date.now() - start,
            message: error instanceof Error ? error.message : 'Unknown error'
        };
    }
}
 
/**
 * Check Redis connectivity
 */
async function checkRedis(): Promise<HealthCheckResult> {
    const start = Date.now();
    try {
        await Promise.race([
            redisClient.ping(),
            new Promise((_, reject) => 
                setTimeout(() => reject(new Error('timeout')), CHECK_TIMEOUT_MS)
            )
        ]);
        
        return {
            service: 'redis',
            status: 'healthy',
            latencyMs: Date.now() - start
        };
    } catch (error) {
        return {
            service: 'redis',
            status: 'unhealthy',
            latencyMs: Date.now() - start,
            message: error instanceof Error ? error.message : 'Unknown error'
        };
    }
}
 
/**
 * Check initialization status
 */
function checkInitialization(): HealthCheckResult {
    if (!isInitialized) {
        return {
            service: 'initialization',
            status: 'unhealthy',
            message: 'Application initialization not complete'
        };
    }
    return {
        service: 'initialization',
        status: 'healthy'
    };
}
 
/**
 * Readiness endpoint handler
 */
async function readinessHandler(req: Request, res: Response) {
    // Run all checks in parallel
    const checkResults = await Promise.all([
        checkInitialization(),
        checkDatabase(),
        checkRedis()
    ]);
    
    // Determine overall readiness
    const hasUnhealthy = checkResults.some(c => c.status === 'unhealthy');
    const hasDegraded = checkResults.some(c => c.status === 'degraded');
    
    const response: ReadinessResponse = {
        status: hasUnhealthy ? 'not_ready' : 'ready',
        timestamp: new Date().toISOString(),
        checks: checkResults,
        version: process.env.APP_VERSION
    };
    
    const statusCode = hasUnhealthy ? 503 : 200;
    res.status(statusCode).json(response);
}
 
const app = express();
app.get('/health/ready', readinessHandler);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

// TypeScript: Deep Health Endpoint for Diagnostics
 
interface DeepHealthResponse {
    status: 'healthy' | 'unhealthy' | 'degraded';
    timestamp: string;
    uptime: number;
    version: {
        app: string;
        git_sha: string;
        deployed_at: string;
    };
    system: {
        hostname: string;
        node_version: string;
        memory: {
            heap_used_mb: number;
            heap_total_mb: number;
            external_mb: number;
            rss_mb: number;
        };
        cpu: {
            user_percent: number;
            system_percent: number;
        };
    };
    dependencies: DependencyHealth[];
    configuration: {
        environment: string;
        region: string;
        log_level: string;
        feature_flags: Record<string, boolean>;
    };
}
 
interface DependencyHealth {
    name: string;
    type: 'database' | 'cache' | 'queue' | 'service' | 'storage';
    status: 'healthy' | 'unhealthy' | 'degraded' | 'unknown';
    latency_ms?: number;
    details?: Record<string, unknown>;
    error?: string;
}
 
async function deepHealthHandler(req: Request, res: Response) {
    const startTime = process.hrtime.bigint();
    
    // Gather system metrics
    const memUsage = process.memoryUsage();
    const cpuUsage = process.cpuUsage();
    
    // Check all dependencies with detailed information
    const dependencies: DependencyHealth[] = await Promise.all([
        checkDatabaseDeep(),
        checkRedisDeep(),
        checkKafkaDeep(),
        checkS3Deep(),
        checkDownstreamServicesDeep()
    ]).then(results => results.flat());
    
    // Determine overall status
    const unhealthyCount = dependencies.filter(d => d.status === 'unhealthy').length;
    const degradedCount = dependencies.filter(d => d.status === 'degraded').length;
    
    let status: 'healthy' | 'unhealthy' | 'degraded' = 'healthy';
    if (unhealthyCount > 0) status = 'unhealthy';
    else if (degradedCount > 0) status = 'degraded';
    
    const response: DeepHealthResponse = {
        status,
        timestamp: new Date().toISOString(),
        uptime: process.uptime(),
        version: {
            app: process.env.APP_VERSION || 'unknown',
            git_sha: process.env.GIT_SHA || 'unknown',
            deployed_at: process.env.DEPLOYED_AT || 'unknown'
        },
        system: {
            hostname: os.hostname(),
            node_version: process.version,
            memory: {
                heap_used_mb: Math.round(memUsage.heapUsed / 1024 / 1024),
                heap_total_mb: Math.round(memUsage.heapTotal / 1024 / 1024),
                external_mb: Math.round(memUsage.external / 1024 / 1024),
                rss_mb: Math.round(memUsage.rss / 1024 / 1024)
            },
            cpu: {
                user_percent: cpuUsage.user / 1000000,  // Convert to seconds
                system_percent: cpuUsage.system / 1000000
            }
        },
        dependencies,
        configuration: {
            environment: process.env.NODE_ENV || 'development',
            region: process.env.AWS_REGION || 'unknown',
            log_level: process.env.LOG_LEVEL || 'info',
            feature_flags: await getFeatureFlags()
        }
    };
    
    // Always return 200 for deep health - it's diagnostic, not routing
    // Use the response body for status interpretation
    res.status(200).json(response);
}
 
async function checkDatabaseDeep(): Promise<DependencyHealth[]> {
    try {
        const start = Date.now();
        
        // Get detailed database stats
        const poolStats = pgPool;
        const [versionResult] = await pgPool.query('SELECT version()');
        const [statsResult] = await pgPool.query(`
            SELECT 
                numbackends as connections,
                xact_commit as transactions_committed,
                xact_rollback as transactions_rolled_back,
                blks_hit as cache_hits,
                blks_read as disk_reads
            FROM pg_stat_database 
            WHERE datname = current_database()
        `);
        
        return [{
            name: 'postgresql',
            type: 'database',
            status: 'healthy',
            latency_ms: Date.now() - start,
            details: {
                version: versionResult.rows[0].version,
                pool_total: poolStats.totalCount,
                pool_idle: poolStats.idleCount,
                pool_waiting: poolStats.waitingCount,
                stats: statsResult.rows[0]
            }
        }];
    } catch (error) {
        return [{
            name: 'postgresql',
            type: 'database',
            status: 'unhealthy',
            error: error instanceof Error ? error.message : 'Unknown error'
        }];
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

// TypeScript: Performance-Optimized Health Checking
 
import { performance } from 'perf_hooks';
 
// Cache for expensive health check results
interface CachedHealth {
    result: HealthCheckResult;
    cachedAt: number;
}
 
const healthCache = new Map<string, CachedHealth>();
const CACHE_TTL_MS = 2000;  // 2 second cache
 
/**
 * Get cached health check or execute if stale
 */
async function getCachedHealthCheck(
    name: string,
    checker: () => Promise<HealthCheckResult>
): Promise<HealthCheckResult> {
    const now = Date.now();
    const cached = healthCache.get(name);
    
    if (cached && (now - cached.cachedAt) < CACHE_TTL_MS) {
        return { ...cached.result, cached: true };
    }
    
    const result = await checker();
    healthCache.set(name, { result, cachedAt: now });
    return result;
}
 
/**
 * Run multiple health checks concurrently with global timeout
 */
async function runHealthChecks(
    checks: Array<{ name: string; check: () => Promise<HealthCheckResult> }>,
    globalTimeoutMs: number
): Promise<HealthCheckResult[]> {
    const start = performance.now();
    
    // Create promise for global timeout
    const timeout = new Promise<never>((_, reject) => {
        setTimeout(() => {
            reject(new Error(`Health checks exceeded global timeout of ${globalTimeoutMs}ms`));
        }, globalTimeoutMs);
    });
    
    // Run all checks with individual handling
    const checkPromises = checks.map(async ({ name, check }) => {
        try {
            return await getCachedHealthCheck(name, check);
        } catch (error) {
            // Individual check failure doesn't fail the batch
            return {
                service: name,
                status: 'unhealthy' as const,
                message: error instanceof Error ? error.message : 'Unknown error'
            };
        }
    });
    
    try {
        // Race against global timeout
        const results = await Promise.race([
            Promise.all(checkPromises),
            timeout
        ]);
        
        const duration = performance.now() - start;
        console.log(`Health checks completed in ${duration.toFixed(2)}ms`);
        
        return results;
    } catch (error) {
        // Global timeout exceeded - return partial results with timeout markers
        const duration = performance.now() - start;
        console.error(`Health checks timed out after ${duration.toFixed(2)}ms`);
        
        return checks.map(({ name }) => ({
            service: name,
            status: 'unhealthy' as const,
            message: 'Health check timed out'
        }));
    }
}
 
// Express middleware to track health endpoint performance
function healthMetricsMiddleware(req: Request, res: Response, next: NextFunction) {
    const start = performance.now();
    
    res.on('finish', () => {
        const duration = performance.now() - start;
        
        // Record metrics
        healthCheckDuration.observe({
            path: req.path,
            status: res.statusCode.toString()
        }, duration);
        
        // Warn if health check is slow
        if (duration > 100 && req.path.includes('/ready')) {
            console.warn(`Slow readiness check: ${duration.toFixed(2)}ms`);
        } else if (duration > 10 && req.path.includes('/live')) {
            console.warn(`Slow liveness check: ${duration.toFixed(2)}ms`);
        }
    });
    
    next();
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

{
    "status": "pass",
    "version": "1.2.3",
    "releaseId": "abc123",
    "serviceId": "api-gateway",
    "description": "Primary API Gateway Service",
    "output": "",
    "notes": ["All systems operational"],
    "checks": {
        "postgresql:connections": [
            {
                "componentId": "database-001",
                "componentType": "datastore",
                "observedValue": 25,
                "observedUnit": "connections",
                "status": "pass",
                "time": "2024-01-15T10:30:00Z"
            }
        ],
        "redis:responseTime": [
            {
                "componentId": "cache-001",
                "componentType": "datastore",
                "observedValue": 2.5,
                "observedUnit": "ms",
                "status": "pass",
                "time": "2024-01-15T10:30:00Z"
            }
        ],
        "memory:utilization": [
            {
                "componentId": "self",
                "componentType": "system",
                "observedValue": 68.5,
                "observedUnit": "percent",
                "status": "warn",
                "time": "2024-01-15T10:30:00Z",
                "output": "Memory usage approaching threshold"
            }
        ]
    },
    "links": {
        "about": "https://docs.company.com/api-gateway",
        "docs": "https://docs.company.com/api-gateway/health"
    }
}