1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number: 0a:bc:de:f1:23:45:67:89
        Signature Algorithm: sha256WithRSAEncryption
        Issuer: CN = Internal CA, O = Example Corp
        Validity:
            Not Before: Jan  1 00:00:00 2024 GMT
            Not After : Jan  1 00:00:00 2025 GMT
        Subject: CN = payment-service.internal
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                RSA Public-Key: (2048 bit)
        X509v3 extensions:
            X509v3 Key Usage: critical
                Digital Signature, Key Encipherment
            X509v3 Extended Key Usage: 
                TLS Web Client Authentication, TLS Web Server Authentication
            X509v3 Subject Alternative Name: 
                DNS:payment-service.internal
                DNS:payment-service.prod.svc.cluster.local
                URI:spiffe://example.com/ns/prod/sa/payment-service
    Signature Algorithm: sha256WithRSAEncryption
         a1:b2:c3:d4:e5...

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

# Internal CA configuration for service certificates
 
ca:
  root:
    algorithm: RSA
    keySize: 4096
    validity: "87600h"  # 10 years
    storage: HSM  # Hardware Security Module
    pathLenConstraint: 2  # Allow 2 levels of intermediate CAs
    
  intermediate:
    production:
      algorithm: RSA
      keySize: 4096
      validity: "26280h"  # 3 years
      pathLenConstraint: 0  # Cannot sign other CAs
      nameConstraints:
        permitted:
          - ".prod.internal"
          - ".prod.svc.cluster.local"
        excluded:
          - ".stage.internal"
          - ".dev.internal"
    
    staging:
      algorithm: ECDSA
      curve: P-256
      validity: "8760h"  # 1 year
      pathLenConstraint: 0
      nameConstraints:
        permitted:
          - ".stage.internal"
 
# End-entity (service) certificate policy
service_certificates:
  algorithm: ECDSA
  curve: P-256
  validity: "24h"  # Short-lived certificates
  max_validity: "168h"  # 1 week maximum
  
  required_extensions:
    - keyUsage: ["digitalSignature", "keyEncipherment"]
    - extKeyUsage: ["clientAuth", "serverAuth"]
    
  san_policies:
    # Must include SPIFFE URI
    require_spiffe: true
    spiffe_trust_domain: "example.com"
    
    # Allowed DNS patterns
    allowed_dns_patterns:
      - "*.prod.internal"
      - "*.prod.svc.cluster.local"
    
    # Allowed IP SANs (for pod IPs if needed)
    allow_ip_san: true

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

# Envoy Proxy mTLS Configuration
 
static_resources:
  listeners:
    - name: mtls_listener
      address:
        socket_address:
          address: 0.0.0.0
          port_value: 8443
      filter_chains:
        - filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: ingress_mtls
                route_config:
                  name: local_route
                  virtual_hosts:
                    - name: backend
                      domains: ["*"]
                      routes:
                        - match:
                            prefix: "/"
                          route:
                            cluster: backend_cluster
          transport_socket:
            name: envoy.transport_sockets.tls
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
              require_client_certificate: true
              common_tls_context:
                # Server certificate and key
                tls_certificates:
                  - certificate_chain:
                      filename: "/certs/gateway.crt"
                    private_key:
                      filename: "/certs/gateway.key"
                
                # Trusted CAs for client certificates
                validation_context:
                  trusted_ca:
                    filename: "/certs/ca-bundle.crt"
                  # Optional: Certificate Revocation List
                  crl:
                    filename: "/certs/crl.pem"
                  
                  # Verify client certificate against specific SANs
                  match_typed_subject_alt_names:
                    - san_type: DNS
                      matcher:
                        suffix: ".internal"
                    - san_type: URI
                      matcher:
                        prefix: "spiffe://example.com/"
                
                # TLS parameters
                tls_params:
                  tls_minimum_protocol_version: TLSv1_2
                  tls_maximum_protocol_version: TLSv1_3
                  cipher_suites:
                    - ECDHE-ECDSA-AES128-GCM-SHA256
                    - ECDHE-RSA-AES128-GCM-SHA256
                    - ECDHE-ECDSA-AES256-GCM-SHA384
                    - ECDHE-RSA-AES256-GCM-SHA384

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

# Envoy Cluster with mTLS to backend
 
clusters:
  - name: payment_service
    type: STRICT_DNS
    lb_policy: ROUND_ROBIN
    load_assignment:
      cluster_name: payment_service
      endpoints:
        - lb_endpoints:
            - endpoint:
                address:
                  socket_address:
                    address: payment-service.internal
                    port_value: 8443
    transport_socket:
      name: envoy.transport_sockets.tls
      typed_config:
        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
        common_tls_context:
          # Client certificate for gateway-to-service auth
          tls_certificates:
            - certificate_chain:
                filename: "/certs/gateway-client.crt"
              private_key:
                filename: "/certs/gateway-client.key"
          
          # Validate server certificate
          validation_context:
            trusted_ca:
              filename: "/certs/internal-ca.crt"
            match_typed_subject_alt_names:
              - san_type: DNS
                matcher:
                  exact: "payment-service.internal"
        
        # SNI for proper routing
        sni: payment-service.internal

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

interface CertificateInfo {
    certificate: string;      // PEM-encoded certificate
    privateKey: string;       // PEM-encoded private key
    chain: string[];          // Intermediate CA chain
    notBefore: Date;
    notAfter: Date;
    serialNumber: string;
    subjectAltNames: string[];
}
 
class CertificateManager {
    private currentCert: CertificateInfo | null = null;
    private rotationTimer: NodeJS.Timer | null = null;
    
    constructor(
        private caClient: CAClient,
        private config: CertificateConfig
    ) {}
    
    async start(): Promise<void> {
        // Initial certificate fetch
        await this.rotateCertificate();
        
        // Schedule rotation
        this.scheduleNextRotation();
    }
    
    async rotateCertificate(): Promise<void> {
        console.log("Requesting new certificate from CA...");
        
        // Generate new key pair
        const { publicKey, privateKey } = await crypto.subtle.generateKey(
            {
                name: "ECDSA",
                namedCurve: "P-256",
            },
            true,
            ["sign", "verify"]
        );
        
        // Create Certificate Signing Request (CSR)
        const csr = await this.generateCSR(publicKey, privateKey);
        
        // Request signed certificate from CA
        const signedCert = await this.caClient.signCSR({
            csr: csr,
            validity: this.config.validity,
            sans: this.config.subjectAltNames,
            spiffeId: this.config.spiffeId,
        });
        
        // Prepare new certificate bundle
        const newCert: CertificateInfo = {
            certificate: signedCert.certificate,
            privateKey: await this.exportPrivateKey(privateKey),
            chain: signedCert.chain,
            notBefore: new Date(signedCert.notBefore),
            notAfter: new Date(signedCert.notAfter),
            serialNumber: signedCert.serialNumber,
            subjectAltNames: signedCert.sans,
        };
        
        // Atomic swap of current certificate
        const previousCert = this.currentCert;
        this.currentCert = newCert;
        
        // Notify listeners (e.g., reload TLS context)
        await this.notifyCertificateChange(newCert, previousCert);
        
        console.log(`Certificate rotated. New expiry: ${newCert.notAfter.toISOString()}`);
    }
    
    private scheduleNextRotation(): void {
        if (!this.currentCert) return;
        
        // Rotate at 75% of certificate lifetime
        const lifetime = this.currentCert.notAfter.getTime() - this.currentCert.notBefore.getTime();
        const rotationPoint = this.currentCert.notBefore.getTime() + (lifetime * 0.75);
        const delay = rotationPoint - Date.now();
        
        console.log(`Next rotation scheduled in ${Math.round(delay / 1000)} seconds`);
        
        this.rotationTimer = setTimeout(async () => {
            try {
                await this.rotateCertificate();
                this.scheduleNextRotation();
            } catch (error) {
                console.error("Certificate rotation failed:", error);
                // Retry with exponential backoff
                await this.retryRotation();
            }
        }, Math.max(delay, 60000));  // Minimum 1 minute
    }
    
    private async retryRotation(): Promise<void> {
        const maxRetries = 5;
        for (let attempt = 1; attempt <= maxRetries; attempt++) {
            const delay = Math.pow(2, attempt) * 1000;  // Exponential backoff
            console.log(`Retry attempt ${attempt}/${maxRetries} in ${delay}ms`);
            
            await new Promise(resolve => setTimeout(resolve, delay));
            
            try {
                await this.rotateCertificate();
                this.scheduleNextRotation();
                return;
            } catch (error) {
                console.error(`Rotation retry ${attempt} failed:`, error);
            }
        }
        
        // All retries failed - alert and continue with existing cert
        this.alertCritical("Certificate rotation failed after all retries");
    }
    
    getCertificate(): CertificateInfo | null {
        return this.currentCert;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

# cert-manager Certificate for service
 
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: payment-service
  namespace: production
spec:
  # Secret to store certificate and key
  secretName: payment-service-tls
  
  # Certificate validity
  duration: 24h
  renewBefore: 8h  # Rotate when 16 hours of validity remain
  
  # Subject
  commonName: payment-service.production.svc.cluster.local
  
  # Subject Alternative Names
  dnsNames:
    - payment-service
    - payment-service.production
    - payment-service.production.svc
    - payment-service.production.svc.cluster.local
  uris:
    - spiffe://cluster.local/ns/production/sa/payment-service
  
  # Private key settings
  privateKey:
    algorithm: ECDSA
    size: 256
    rotationPolicy: Always  # Generate new key on each renewal
  
  # Issuer reference (Intermediate CA)
  issuerRef:
    name: internal-ca-issuer
    kind: ClusterIssuer
    
  # Key usages for mTLS
  usages:
    - digital signature
    - key encipherment
    - client auth
    - server auth
 
---
# Issuer configuration (CA setup)
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: internal-ca-issuer
spec:
  # Using HashiCorp Vault as CA
  vault:
    server: https://vault.internal:8200
    path: pki/sign/service-role
    auth:
      kubernetes:
        role: cert-manager
        mountPath: auth/kubernetes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

import { X509Certificate, X509Crl } from "node:crypto";
 
class CrlChecker {
    private crlCache: Map<string, { crl: X509Crl; fetchedAt: Date }> = new Map();
    private refreshInterval = 30 * 60 * 1000;  // 30 minutes
    
    async isRevoked(certificate: X509Certificate): Promise<boolean> {
        // Extract CRL Distribution Point from certificate
        const crlUrls = this.extractCrlDistributionPoints(certificate);
        
        for (const url of crlUrls) {
            const crl = await this.getCrl(url);
            if (crl && this.checkCrl(crl, certificate)) {
                return true;  // Certificate is revoked
            }
        }
        
        return false;  // Not found in any CRL
    }
    
    private async getCrl(url: string): Promise<X509Crl | null> {
        // Check cache
        const cached = this.crlCache.get(url);
        if (cached && this.isFresh(cached.fetchedAt)) {
            return cached.crl;
        }
        
        // Fetch fresh CRL
        try {
            const response = await fetch(url, {
                signal: AbortSignal.timeout(5000),
            });
            
            if (!response.ok) {
                console.warn(`Failed to fetch CRL from ${url}: HTTP ${response.status}`);
                return cached?.crl || null;  // Use stale cache if available
            }
            
            const derData = await response.arrayBuffer();
            const crl = new X509Crl(Buffer.from(derData));
            
            // Cache the CRL
            this.crlCache.set(url, { crl, fetchedAt: new Date() });
            
            return crl;
        } catch (error) {
            console.error(`Error fetching CRL from ${url}:`, error);
            return cached?.crl || null;
        }
    }
    
    private checkCrl(crl: X509Crl, certificate: X509Certificate): boolean {
        const serialNumber = certificate.serialNumber;
        // Check if serial number is in the revoked list
        return crl.isRevoked(certificate);
    }
    
    private isFresh(fetchedAt: Date): boolean {
        return Date.now() - fetchedAt.getTime() < this.refreshInterval;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

import { X509Certificate } from "node:crypto";
 
interface OcspResponse {
    status: "good" | "revoked" | "unknown";
    producedAt: Date;
    thisUpdate: Date;
    nextUpdate: Date;
    revocationReason?: string;
    revocationTime?: Date;
}
 
class OcspChecker {
    private responseCache: Map<string, OcspResponse> = new Map();
    
    async checkStatus(
        certificate: X509Certificate,
        issuerCertificate: X509Certificate
    ): Promise<OcspResponse> {
        const cacheKey = certificate.serialNumber;
        
        // Check cache
        const cached = this.responseCache.get(cacheKey);
        if (cached && this.isValid(cached)) {
            return cached;
        }
        
        // Build OCSP request
        const ocspRequest = this.buildOcspRequest(certificate, issuerCertificate);
        
        // Get OCSP responder URL from certificate
        const ocspUrl = this.extractOcspResponder(certificate);
        if (!ocspUrl) {
            throw new Error("No OCSP responder URL in certificate");
        }
        
        // Send OCSP request
        const response = await fetch(ocspUrl, {
            method: "POST",
            headers: {
                "Content-Type": "application/ocsp-request",
            },
            body: ocspRequest,
            signal: AbortSignal.timeout(5000),
        });
        
        if (!response.ok) {
            throw new Error(`OCSP request failed: HTTP ${response.status}`);
        }
        
        const ocspResponseData = await response.arrayBuffer();
        const ocspResponse = this.parseOcspResponse(Buffer.from(ocspResponseData));
        
        // Verify OCSP response signature
        if (!this.verifyOcspSignature(ocspResponse, issuerCertificate)) {
            throw new Error("OCSP response signature verification failed");
        }
        
        // Cache successful response
        this.responseCache.set(cacheKey, ocspResponse);
        
        return ocspResponse;
    }
    
    private isValid(response: OcspResponse): boolean {
        const now = new Date();
        return now >= response.thisUpdate && now <= response.nextUpdate;
    }
    
    // ... implementation details for building/parsing OCSP messages
}
 
// OCSP Stapling: Server-side optimization
// Gateway fetches OCSP response and "staples" it to TLS handshake
// Eliminates client's need to contact OCSP responder
 
// Envoy OCSP stapling configuration:
// tls_certificates:
//   - certificate_chain: { filename: "/certs/server.crt" }
//     private_key: { filename: "/certs/server.key" }
//     ocsp_staple: { filename: "/certs/server.ocsp" }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

# Namespace-wide mTLS enforcement
 
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default
  namespace: production
spec:
  mtls:
    mode: STRICT  # Require mTLS for all traffic
 
---
# Authorization based on mTLS identity
 
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: payment-service-policy
  namespace: production
spec:
  selector:
    matchLabels:
      app: payment-service
  rules:
    # Allow only specific services to access payment-service
    - from:
        - source:
            principals:
              - "cluster.local/ns/production/sa/order-service"
              - "cluster.local/ns/production/sa/refund-service"
      to:
        - operation:
            methods: ["GET", "POST"]
            paths: ["/api/v1/payments/*"]
    
    # Deny all other traffic
    - from:
        - source:
            notPrincipals:
              - "cluster.local/ns/production/sa/*"
 
---
# Certificate configuration
 
apiVersion: security.istio.io/v1
kind: CertificateConfig
metadata:
  name: default
  namespace: istio-system
spec:
  # Certificate validity
  certificateValidity: 24h
  
  # Refresh before expiration
  certificateRefreshGracePeriod: 1h
  
  # Root CA configuration
  rootCertificates:
    - secretName: cacerts
      namespace: istio-system

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

# Istio Ingress Gateway with mTLS termination
 
apiVersion: networking.istio.io/v1beta1
kind: Gateway
metadata:
  name: api-gateway
  namespace: istio-system
spec:
  selector:
    istio: ingressgateway
  servers:
    # External HTTPS with client certificate (mTLS)
    - port:
        number: 443
        name: https
        protocol: HTTPS
      tls:
        mode: MUTUAL  # Require client certificate
        credentialName: api-gateway-tls  # Server cert Secret
        caCertificates: /etc/istio/client-certs/ca.crt  # Client CA
      hosts:
        - "api.example.com"
 
---
# Virtual Service routing
 
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: api-routes
  namespace: production
spec:
  hosts:
    - "api.example.com"
  gateways:
    - istio-system/api-gateway
  http:
    - match:
        - uri:
            prefix: /payments
      route:
        - destination:
            host: payment-service
            port:
              number: 8080
      # Gateway-to-service: automatic mTLS via mesh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# 1. Test mTLS connection with OpenSSL
openssl s_client   -connect payment-service:8443   -cert /certs/client.crt   -key /certs/client.key   -CAfile /certs/ca.crt   -verify 2   -verify_return_error
 
# Expected output includes:
# - Verify return code: 0 (ok)
# - Server certificate chain
# - "Acceptable client certificate CA names" (what CAs server trusts)
 
# 2. Check certificate details
openssl x509 -in /certs/client.crt -text -noout
 
# Key things to verify:
# - Validity dates (Not Before, Not After)
# - Subject and Issuer
# - Subject Alternative Names (SANs)
# - Key Usage and Extended Key Usage (must include clientAuth)
 
# 3. Verify certificate chain
openssl verify   -CAfile /certs/root-ca.crt   -untrusted /certs/intermediate-ca.crt   /certs/client.crt
 
# 4. Test with curl (verbose)
curl -v   --cert /certs/client.crt   --key /certs/client.key   --cacert /certs/ca.crt   https://payment-service:8443/health
 
# 5. Check for common issues
# - Clock skew between client and server (affects validity checks)
# - Wrong CA in trust store
# - Certificate missing required Extended Key Usage
# - Expired certificate or CRL
# - Hostname/SAN mismatch

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

# Prometheus alerting rules for certificate monitoring
 
groups:
  - name: certificate-alerts
    rules:
      # Alert when certificate expires within 24 hours
      - alert: CertificateExpiringSoon
        expr: |
          (x509_cert_not_after - time()) / 3600 < 24
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Certificate expiring within 24 hours"
          description: "Certificate {{ $labels.subject }} expires at {{ $labels.not_after }}"
      
      # Alert when certificate expires within 7 days
      - alert: CertificateExpiringInWeek
        expr: |
          (x509_cert_not_after - time()) / 3600 / 24 < 7
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Certificate expiring within 7 days"
      
      # Alert on mTLS handshake failures
      - alert: MTLSHandshakeFailures
        expr: |
          rate(envoy_ssl_handshake_error_total[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Elevated mTLS handshake errors"
          description: "{{ $value | humanize }} errors per second on {{ $labels.pod }}"
      
      # Alert on certificate rotation failure
      - alert: CertificateRotationFailed
        expr: |
          certmanager_certificate_ready_status{condition="False"} == 1
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Certificate rotation failed"
          description: "cert-manager failed to renew {{ $labels.name }}"
 
---
# Metrics to collect
 
# From Envoy sidecar:
# - envoy_ssl_handshake_total
# - envoy_ssl_handshake_error_total
# - envoy_ssl_connection_duration_ms
 
# From cert-manager:
# - certmanager_certificate_ready_status
# - certmanager_certificate_renewal_timestamp_seconds
# - certmanager_certificate_expiration_timestamp_seconds