Physical Clocks and Clock Drift

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

# Clock Drift Calculations for Distributed Systems
 
def calculate_max_drift(ppm: float, duration_seconds: float) -> float:
    """
    Calculate maximum clock drift over a time period.
    
    Args:
        ppm: Clock accuracy in parts per million (e.g., 50.0 for 50 ppm)
        duration_seconds: Time in seconds since last synchronization
    
    Returns:
        Maximum drift in seconds
    """
    return (ppm / 1_000_000) * duration_seconds
 
 
def calculate_max_skew(ppm_a: float, ppm_b: float, duration_seconds: float) -> float:
    """
    Calculate maximum skew between two clocks.
    Both clocks could drift in opposite directions (worst case).
    
    Args:
        ppm_a: Clock A accuracy in ppm
        ppm_b: Clock B accuracy in ppm  
        duration_seconds: Time since both were synchronized
    
    Returns:
        Maximum skew in seconds (difference between the two clocks)
    """
    # Worst case: one clock runs fast, other runs slow
    max_drift_a = calculate_max_drift(ppm_a, duration_seconds)
    max_drift_b = calculate_max_drift(ppm_b, duration_seconds)
    return max_drift_a + max_drift_b
 
 
# Example: Server farm with 50 ppm clocks, NTP sync every 10 minutes
ppm_typical_server = 50.0
sync_interval_seconds = 10 * 60  # 10 minutes = 600 seconds
 
max_drift = calculate_max_drift(ppm_typical_server, sync_interval_seconds)
print(f"Max drift per clock: {max_drift * 1000:.1f} ms")  # 30.0 ms
 
max_skew = calculate_max_skew(ppm_typical_server, ppm_typical_server, sync_interval_seconds)
print(f"Max skew between any two servers: {max_skew * 1000:.1f} ms")  # 60.0 ms
 
 
# Extended example: Worst-case scenarios
scenarios = [
    ("Same rack, NTP every 1 min", 50, 60),
    ("Same datacenter, NTP every 10 min", 50, 600),
    ("Cross-datacenter, NTP every 1 hour", 50, 3600),
    ("Poor NTP, sync every 24 hours", 50, 86400),
    ("Server with TCXO (5 ppm), 10 min sync", 5, 600),
    ("Google TrueTime scenario", 0.001, 600),  # GPS-disciplined
]
 
print("
Maximum clock skew scenarios:")
print("-" * 55)
for scenario, ppm, interval in scenarios:
    skew = calculate_max_skew(ppm, ppm, interval)
    print(f"{scenario}:")
    print(f"  Max skew: {skew * 1000:.2f} ms ({skew:.4f} s)")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

import time
 
# DANGEROUS: Using wall-clock time for timeout/duration
# This code WILL break when NTP adjusts the clock
 
def dangerous_timeout():
    """This function has a subtle bug that causes production failures."""
    
    start = time.time()  # Wall-clock time
    timeout_seconds = 30
    
    while True:
        # Process some work...
        process_next_item()
        
        elapsed = time.time() - start
        if elapsed > timeout_seconds:
            print("Timeout reached")
            break
    
    # BUG: If NTP jumps time backward by 1 minute during this loop,
    # 'elapsed' becomes negative, and this loop runs for 1 minute
    # longer than intended.
    #
    # If NTP jumps time forward by 1 hour, the loop exits immediately
    # even if it just started, potentially before processing any items.
 
 
def safe_timeout():
    """Correct implementation using monotonic clock."""
    
    start = time.monotonic()  # Monotonic time - never moves backward
    timeout_seconds = 30
    
    while True:
        # Process some work...
        process_next_item()
        
        elapsed = time.monotonic() - start
        if elapsed > timeout_seconds:
            print("Timeout reached")
            break
    
    # SAFE: time.monotonic() is guaranteed to never move backward.
    # NTP adjustments don't affect it. The timeout works correctly
    # regardless of wall-clock adjustments.
 
 
# CRITICAL: Different languages have different APIs
# 
# Python:
#   time.time()       -> Wall-clock (AVOID for durations)
#   time.monotonic()  -> Monotonic (USE for durations)
#
# Java:
#   System.currentTimeMillis() -> Wall-clock (AVOID for durations)
#   System.nanoTime()          -> Monotonic (USE for durations)
#
# C (Linux):
#   clock_gettime(CLOCK_REALTIME, ...)  -> Wall-clock
#   clock_gettime(CLOCK_MONOTONIC, ...) -> Monotonic
#
# Go:
#   time.Now()             -> Wall-clock
#   time.Now().UnixNano()  after runtime changes tracks monotonic
#   // Go 1.9+ time.Time includes monotonic component
#
# Rust:
#   std::time::SystemTime::now() -> Wall-clock
#   std::time::Instant::now()    -> Monotonic

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

import time
from dataclasses import dataclass
from typing import Optional
 
@dataclass
class ClockHealth:
    """Tracks the health and stability of the system clock."""
    last_wall_time: float
    last_mono_time: float
    detected_jumps: int = 0
    max_jump_detected: float = 0.0
    
 
def detect_clock_adjustment(previous: ClockHealth) -> tuple[ClockHealth, Optional[float]]:
    """
    Detect if a clock adjustment has occurred since last check.
    
    Returns:
        Tuple of (updated ClockHealth, adjustment amount if detected else None)
    """
    current_wall = time.time()
    current_mono = time.monotonic()
    
    # Calculate elapsed time according to each clock
    wall_elapsed = current_wall - previous.last_wall_time
    mono_elapsed = current_mono - previous.last_mono_time
    
    # If these differ significantly, wall-clock was adjusted
    # Small differences are normal due to timer resolution
    discrepancy = wall_elapsed - mono_elapsed
    
    jump_detected = None
    if abs(discrepancy) > 0.1:  # More than 100ms discrepancy
        jump_detected = discrepancy
        previous.detected_jumps += 1
        if abs(discrepancy) > abs(previous.max_jump_detected):
            previous.max_jump_detected = discrepancy
            
        if discrepancy > 0:
            print(f"⚠️ Clock jumped FORWARD by {discrepancy:.3f} seconds")
        else:
            print(f"🚨 Clock jumped BACKWARD by {-discrepancy:.3f} seconds")
    
    # Update tracking
    previous.last_wall_time = current_wall
    previous.last_mono_time = current_mono
    
    return previous, jump_detected
 
 
def monitor_clock(check_interval_seconds: float = 1.0):
    """
    Continuously monitor for clock adjustments.
    In production, this would emit metrics and potentially alerts.
    """
    health = ClockHealth(
        last_wall_time=time.time(),
        last_mono_time=time.monotonic()
    )
    
    print("Starting clock monitoring...")
    while True:
        time.sleep(check_interval_seconds)
        health, jump = detect_clock_adjustment(health)
        
        if jump is not None:
            # In production, emit metrics or alerts here
            # metrics.emit("clock.adjustment", jump)
            # if abs(jump) > THRESHOLD:
            #     alert_oncall("Large clock adjustment detected")
            pass
 
 
# Design pattern: Clock-safe timestamp generation
class SafeTimestamp:
    """
    Generates timestamps that are safe for ordering within a node.
    Handles clock jumps by using monotonic time to ensure monotonicity.
    """
    
    def __init__(self):
        self._last_timestamp = 0
        self._wall_offset = time.time() - time.monotonic()
    
    def now(self) -> int:
        """
        Returns monotonically increasing timestamp in microseconds.
        Uses wall-clock for approximate absolute time, but ensures
        strict monotonicity using monotonic clock as backstop.
        """
        # Use wall clock for absolute time
        wall_time_us = int(time.time() * 1_000_000)
        
        # Ensure monotonicity: never return a timestamp <= previous
        if wall_time_us <= self._last_timestamp:
            # Wall clock went backward or didn't advance
            # Use last timestamp + 1 (microsecond increment)
            self._last_timestamp += 1
        else:
            self._last_timestamp = wall_time_us
            
        return self._last_timestamp