1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

# Consul Service Definition with Multiple Health Checks
 
service {
  name = "order-service"
  id   = "order-service-1"
  port = 8080
  
  tags = ["v2.3.1", "production"]
  
  meta = {
    version  = "2.3.1"
    protocol = "http"
  }
 
  # HTTP health check - verifies application responds correctly
  check {
    id       = "order-http-check"
    name     = "HTTP Health Check"
    http     = "http://localhost:8080/health"
    method   = "GET"
    interval = "10s"
    timeout  = "5s"
    
    # Tuning parameters
    success_before_passing   = 2  # Must pass 2x before healthy
    failures_before_critical = 3  # Must fail 3x before unhealthy
    
    header {
      Accept = ["application/json"]
    }
  }
 
  # TCP check - verifies port is listening
  check {
    id       = "order-tcp-check"
    name     = "TCP Port Check"
    tcp      = "localhost:8080"
    interval = "5s"
    timeout  = "2s"
  }
 
  # TTL check - service must actively report health
  check {
    id   = "order-ttl-check"
    name = "Service Heartbeat"
    ttl  = "30s"
    
    # Service must call: consul.agent.check.pass("order-ttl-check")
    # within every 30 seconds or check fails
  }
 
  # Deregister service if critical for too long
  # Prevents stale entries from accumulating
  check {
    id                             = "order-deregister"
    name                           = "Deregister Check"
    http                           = "http://localhost:8080/health"
    interval                       = "10s"
    deregister_critical_service_after = "5m"
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

import Consul from 'consul';
 
const consul = new Consul({ host: 'consul.internal' });
 
// STRONGLY CONSISTENT READ
// Queries the leader directly, always returns current state
// Higher latency, may fail if leader unavailable
async function getInstancesStrong(serviceName: string) {
  return consul.health.service({
    service: serviceName,
    passing: true,  // Only healthy instances
    consistent: true  // Force strongly consistent read
  });
}
 
// EVENTUALLY CONSISTENT READ (STALE)
// May return from local cache, potentially stale
// Lower latency, always available
async function getInstancesStale(serviceName: string) {
  return consul.health.service({
    service: serviceName,
    passing: true,
    stale: true  // Allow stale reads from any server
  });
}
 
// CACHED READ
// Uses locally cached data, refreshed periodically
// Lowest latency, but may be out of date
async function getInstancesCached(serviceName: string) {
  return consul.health.service({
    service: serviceName,
    passing: true,
    cached: true,  // Use agent's local cache
    'max-age': 30  // Accept cached data up to 30 seconds old
  });
}
 
// PRACTICAL STRATEGY:
// 1. Normal operations: Use cached/stale reads for low latency
// 2. Critical decisions: Use consistent reads
// 3. Have fallbacks: Cache last-known-good endpoints locally
 
class ServiceDiscovery {
  private cache = new Map<string, Endpoint[]>();
  
  async getEndpoints(serviceName: string): Promise<Endpoint[]> {
    try {
      // Try stale read first (fast, usually accurate)
      const endpoints = await getInstancesStale(serviceName);
      this.cache.set(serviceName, endpoints);
      return endpoints;
    } catch (error) {
      // Fall back to cache if registry unavailable
      const cached = this.cache.get(serviceName);
      if (cached && cached.length > 0) {
        console.warn(`Using cached endpoints for ${serviceName}`);
        return cached;
      }
      throw new Error(`No endpoints available for ${serviceName}`);
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

import Consul from 'consul';
 
class DirectDiscoveryClient {
  private consul: Consul.Consul;
  private cache = new Map<string, CachedEndpoints>();
  
  constructor() {
    this.consul = new Consul({ host: 'consul.internal' });
  }
 
  async callService(serviceName: string, path: string): Promise<Response> {
    // Get endpoints from registry (with caching)
    const endpoints = await this.getEndpoints(serviceName);
    
    // Client-side load balancing
    const endpoint = this.selectEndpoint(endpoints);
    
    // Make the actual call
    return fetch(`http://${endpoint.address}:${endpoint.port}${path}`);
  }
 
  private async getEndpoints(serviceName: string): Promise<Endpoint[]> {
    const cached = this.cache.get(serviceName);
    if (cached && Date.now() - cached.timestamp < 30000) {
      return cached.endpoints;
    }
 
    const services = await this.consul.health.service({
      service: serviceName,
      passing: true,
    });
 
    const endpoints = services.map(s => ({
      address: s.Service.Address,
      port: s.Service.Port,
      meta: s.Service.Meta,
    }));
 
    this.cache.set(serviceName, { endpoints, timestamp: Date.now() });
    return endpoints;
  }
 
  private selectEndpoint(endpoints: Endpoint[]): Endpoint {
    // Simple round-robin (real implementations use more sophisticated algorithms)
    return endpoints[Math.floor(Math.random() * endpoints.length)];
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

// Application uses standard DNS resolution
// Consul provides DNS interface at port 8600
 
import dns from 'dns';
import { Resolver } from 'dns';
 
// Configure resolver to use Consul DNS
const resolver = new Resolver();
resolver.setServers(['consul.internal:8600']);
 
async function discoverViaConsulDNS(serviceName: string): Promise<string[]> {
  // Query SRV records for full service info
  return new Promise((resolve, reject) => {
    resolver.resolveSrv(`${serviceName}.service.consul`, (err, addresses) => {
      if (err) reject(err);
      else resolve(addresses.map(a => `${a.name}:${a.port}`));
    });
  });
}
 
// Or simply use the service name as hostname
// Consul DNS returns round-robin A record
async function callServiceViaDNS() {
  // Application doesn't know about Consul - just uses hostname
  // Consul DNS at 8600 resolves 'order-service.service.consul' to instance IP
  return fetch('http://order-service.service.consul:8080/api/orders');
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# Prometheus alerting rules for Consul service registry
groups:
  - name: consul-alerts
    rules:
      # Alert if cluster loses quorum (critical)
      - alert: ConsulClusterNoLeader
        expr: consul_raft_leader == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Consul cluster has no leader"
          description: "Consul cluster has been without a leader for more than 1 minute"
 
      # Alert if peer count drops below threshold
      - alert: ConsulInsufficientPeers
        expr: consul_raft_peers < 3
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Consul cluster has fewer than 3 peers"
 
      # Alert on high health check failure rate
      - alert: ConsulHighHealthCheckFailures
        expr: rate(consul_health_checks_critical[5m]) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High rate of health check failures"
 
      # Alert on high query latency
      - alert: ConsulHighQueryLatency
        expr: histogram_quantile(0.99, consul_http_request_duration_seconds) > 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Consul query latency is high (p99 > 500ms)"
 
      # Alert on replication lag in multi-DC setup
      - alert: ConsulReplicationLag
        expr: consul_rpc_raft_verify_leader_duration_seconds > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Consul replication lag detected"