etcd Locks: Modern Coordination for Cloud-Native Systems

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

etcd Cluster (3 or 5 nodes typical):
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
                    ┌─────────────────────────────────────┐
                    │             Clients                  │
                    │   gRPC clients, etcdctl, Kubernetes │
                    └───────────┬─────────────────────────┘
                                │
              ┌─────────────────┼─────────────────┐
              │                 │                 │
         ┌────┴────┐      ┌────┴────┐       ┌────┴────┐
         │  etcd   │      │  etcd   │       │  etcd   │
         │  Node 1 │◄────►│  Node 2 │◄─────►│  Node 3 │
         │(Follower)│      │ (LEADER)│       │(Follower)│
         └────┬────┘      └────┬────┘       └────┬────┘
              │                 │                 │
         ┌────┴────┐      ┌────┴────┐       ┌────┴────┐
         │  Raft   │      │  Raft   │       │  Raft   │
         │   Log   │      │   Log   │       │   Log   │
         └─────────┘      └─────────┘       └─────────┘
 
Key Components:
- All nodes run identical etcd binary
- Leader handles all write operations
- Followers replicate log and serve linearizable reads
- Raft ensures safety even with minority failures
- gRPC API (v3) for all client communication

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

etcd Namespace (Flat with Prefix Convention):
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
Key                                          Value
────────────────────────────────────────     ─────────────────────
/locks/inventory/holder                      client-id-123
/locks/order-processing/holder               client-id-456
/locks/payment/queue/00000000000000000001    client-id-789
/locks/payment/queue/00000000000000000002    client-id-abc
 
/config/database/connection-string           postgresql://...
/config/feature-flags                        {"dark_mode": true}
 
/services/api-gateway/192.168.1.10           {"port": 8080}
/services/api-gateway/192.168.1.11           {"port": 8080}
/services/worker/192.168.1.20                {"port": 9000}
 
Prefix Queries:
- GET /locks/ (prefix)     → Returns all lock-related keys
- GET /services/api-gateway/ (prefix) → Returns all api-gateway instances
 
No Parent-Child Relationship:
- /locks/inventory can exist without /locks existing
- Deleting /locks does NOT delete /locks/inventory
- This differs fundamentally from Zookeeper's tree!

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

MVCC Revision System:
━━━━━━━━━━━━━━━━━━━━━
 
Global Revision: Monotonically increasing across ALL keys
                 Similar to Zookeeper's zxid
 
Operation History:
  Rev 1: PUT /config/db = "postgres://v1"
  Rev 2: PUT /locks/x = "client-A"
  Rev 3: PUT /config/db = "postgres://v2"
  Rev 4: DELETE /locks/x
  Rev 5: PUT /locks/x = "client-B"
 
Key /config/db at:
  - Rev 1: "postgres://v1"
  - Rev 2: "postgres://v1" (no change to this key)
  - Rev 3: "postgres://v2"
  - Current: "postgres://v2"
 
Key /locks/x at:
  - Rev 1: (does not exist)
  - Rev 2: "client-A" (create_revision=2, mod_revision=2)
  - Rev 3: "client-A"
  - Rev 4: (deleted)
  - Rev 5: "client-B" (create_revision=5, mod_revision=5)
 
Use for Locks:
- create_revision: When the key was created (changes on delete/recreate)
- mod_revision: When the key was last modified
- Compare create_revision to determine lock ordering!

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

LEASE LIFECYCLE:
━━━━━━━━━━━━━━━━
 
1. CREATE LEASE
   Client → etcd: LeaseGrant(TTL=30s)
   etcd → Client: Lease ID = 1234567890
 
2. ATTACH KEYS TO LEASE
   Client → etcd: PUT /locks/resource = "client-A" (lease=1234567890)
   Key now bound to lease: expires when lease expires
 
3. KEEP LEASE ALIVE (Heartbeat)
   Client → etcd: LeaseKeepAlive(1234567890)
   etcd → Client: TTL refreshed to 30s
   (Client must send keep-alives at ~TTL/3 interval)
 
4. NORMAL RELEASE
   Client → etcd: LeaseRevoke(1234567890)
   Result: Lease deleted, ALL attached keys deleted immediately
 
5. EXPIRATION (Client Crash)
   Client crashes, stop sending keep-alives
   etcd: Lease 1234567890 TTL countdown...
   After 30s: Lease EXPIRED
   Result: All attached keys (/locks/resource) deleted automatically
 
 
MULTIPLE KEYS PER LEASE:
━━━━━━━━━━━━━━━━━━━━━━━━━
 
Lease 1234567890 (TTL=30s):
  Attached keys:
    - /locks/resource-A = "client-1"
    - /locks/resource-B = "client-1"
    - /session/client-1/heartbeat = timestamp
 
If client-1 crashes:
  → Lease expires
  → All three keys deleted atomically
  → Resources A and B both released

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

package main
 
import (
    "context"
    "log"
    "time"
    
    clientv3 "go.etcd.io/etcd/client/v3"
    "go.etcd.io/etcd/client/v3/concurrency"
)
 
func leaseExample() {
    cli, err := clientv3.New(clientv3.Config{
        Endpoints:   []string{"localhost:2379"},
        DialTimeout: 5 * time.Second,
    })
    if err != nil { log.Fatal(err) }
    defer cli.Close()
 
    ctx := context.Background()
 
    // Create lease with 30-second TTL
    lease, err := cli.Grant(ctx, 30)
    if err != nil { log.Fatal(err) }
    
    log.Printf("Created lease with ID: %x, TTL: %d", lease.ID, lease.TTL)
 
    // Put key with lease attachment
    _, err = cli.Put(ctx, "/locks/my-resource", "holder-id", 
        clientv3.WithLease(lease.ID))
    if err != nil { log.Fatal(err) }
 
    // Start keep-alive goroutine
    keepAliveCh, err := cli.KeepAlive(ctx, lease.ID)
    if err != nil { log.Fatal(err) }
 
    // Monitor keep-alive responses
    go func() {
        for ka := range keepAliveCh {
            if ka == nil {
                log.Println("Keep-alive channel closed, lease expired!")
                return
            }
            log.Printf("Lease renewed, TTL: %d", ka.TTL)
        }
    }()
 
    // Do critical work...
    time.Sleep(60 * time.Second)
 
    // Explicit release (optional - can also just let lease expire)
    cli.Revoke(ctx, lease.ID)
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

ETCD LOCK ALGORITHM:
━━━━━━━━━━━━━━━━━━━━━
 
ACQUIRE LOCK on prefix "/locks/resource/":
1. Create a lease (e.g., 30s TTL)
2. PUT a key with unique suffix under the lock prefix:
   Key: /locks/resource/{lease-id}
   Value: (empty or client info)
   Condition: Only if key doesn't exist (using transaction)
   
3. GET all keys with prefix /locks/resource/
   Sort by create_revision (ascending)
   
4. If our key has the LOWEST create_revision:
   → We hold the lock! Enter critical section.
   
5. Otherwise, find the key with the next-lower create_revision:
   WATCH that key for deletion
   
6. When watch triggers (predecessor deleted):
   → Re-check if we're now lowest (go to step 3)
 
 
EXAMPLE:
━━━━━━━━
 
Initial state: /locks/resource/ prefix is empty
 
Client A (lease 1001):
  1. PUT /locks/resource/1001 (create_revision = 100)
  2. GET prefix: [/locks/resource/1001]
  3. We're lowest → Lock acquired!
 
Client B (lease 1002):
  1. PUT /locks/resource/1002 (create_revision = 101)
  2. GET prefix: [/locks/resource/1001, /locks/resource/1002]
  3. 100 < 101 → We're not lowest
  4. Watch /locks/resource/1001 for deletion
 
Client C (lease 1003):
  1. PUT /locks/resource/1003 (create_revision = 102)
  2. GET prefix: [/locks/resource/1001, /locks/resource/1002, /locks/resource/1003]
  3. Predecessor with lower revision: /locks/resource/1002
  4. Watch /locks/resource/1002 for deletion (NOT 1001!)
 
Client A releases (lease 1001 revoked):
  → /locks/resource/1001 deleted
  → B's watch triggers
  → B re-checks: [/locks/resource/1002, /locks/resource/1003]
  → B is now lowest → Lock acquired!
  → C is NOT notified (watching 1002, not 1001)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

package main
 
import (
    "context"
    "fmt"
    "log"
    "time"
 
    clientv3 "go.etcd.io/etcd/client/v3"
    "go.etcd.io/etcd/client/v3/concurrency"
)
 
func etcdLockExample() {
    cli, err := clientv3.New(clientv3.Config{
        Endpoints:   []string{"localhost:2379"},
        DialTimeout: 5 * time.Second,
    })
    if err != nil { log.Fatal(err) }
    defer cli.Close()
 
    // Create a session (wraps lease management)
    // TTL: 10 seconds, auto-renewed via keep-alives
    session, err := concurrency.NewSession(cli, concurrency.WithTTL(10))
    if err != nil { log.Fatal(err) }
    defer session.Close()
 
    // Create mutex on a given prefix
    mutex := concurrency.NewMutex(session, "/locks/my-critical-resource/")
 
    // Acquire lock (blocks until acquired)
    ctx := context.Background()
    if err := mutex.Lock(ctx); err != nil {
        log.Fatal(err)
    }
    
    fmt.Println("Lock acquired!")
    fmt.Printf("Lock key: %s
", mutex.Key())  // The actual key created
 
    // CRITICAL SECTION
    // Only one process with this lock prefix executes here
    doExclusiveWork()
 
    // Release lock
    if err := mutex.Unlock(ctx); err != nil {
        log.Fatal(err)
    }
    fmt.Println("Lock released!")
}
 
// With timeout
func etcdLockWithTimeout() {
    // ... client setup ...
    
    mutex := concurrency.NewMutex(session, "/locks/resource/")
    
    // Context with timeout
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()
    
    if err := mutex.Lock(ctx); err != nil {
        if err == context.DeadlineExceeded {
            log.Println("Could not acquire lock within timeout")
            return
        }
        log.Fatal(err)
    }
    defer mutex.Unlock(context.Background())
    
    // Critical section...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

ETCD TRANSACTION (Txn):
━━━━━━━━━━━━━━━━━━━━━━━
 
Structure:
  IF (conditions)
  THEN (operations)
  ELSE (operations)
 
All conditions are evaluated atomically.
If ALL conditions pass: THEN operations execute.
If ANY condition fails: ELSE operations execute.
 
Condition Types:
  - Compare(Key).Version() == 0     // Key doesn't exist
  - Compare(Key).CreateRevision()   // When key was created
  - Compare(Key).ModRevision()      // When key was last modified
  - Compare(Key).Value()            // Key's current value
  - Compare(Key).Lease()            // Key's attached lease
 
Operations:
  - Put(key, value, opts...)
  - Delete(key)
  - Get(key)  // Returns value in response

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

// Atomic lock acquisition with transaction
func acquireLockWithTxn(cli *clientv3.Client, lockKey string, leaseID clientv3.LeaseID) (bool, error) {
    ctx := context.Background()
    
    // Transaction: Only create lock key if it doesn't exist
    txn := cli.Txn(ctx).If(
        // Condition: Key doesn't exist (version == 0)
        clientv3.Compare(clientv3.Version(lockKey), "=", 0),
    ).Then(
        // If condition passes: Create the lock key
        clientv3.OpPut(lockKey, "holder-info", clientv3.WithLease(leaseID)),
    ).Else(
        // If condition fails: Get current holder info
        clientv3.OpGet(lockKey),
    )
    
    resp, err := txn.Commit()
    if err != nil {
        return false, err
    }
    
    if resp.Succeeded {
        // We created the lock key, we hold the lock
        return true, nil
    }
    
    // Lock already held by someone else
    // resp.Responses[0].GetResponseRange() contains current holder
    return false, nil
}
 
// Safe lock release: Only delete if we're still the holder
func releaseLockSafely(cli *clientv3.Client, lockKey string, expectedLeaseID clientv3.LeaseID) error {
    ctx := context.Background()
    
    txn := cli.Txn(ctx).If(
        // Condition: Key exists AND has our lease
        clientv3.Compare(clientv3.LeaseValue(lockKey), "=", expectedLeaseID),
    ).Then(
        // Only delete if we still hold it
        clientv3.OpDelete(lockKey),
    )
    
    _, err := txn.Commit()
    return err
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

func waitForLockRelease(cli *clientv3.Client, predecessorKey string, ctx context.Context) error {
    // Watch the predecessor key for deletion
    watchCh := cli.Watch(ctx, predecessorKey)
    
    for watchResp := range watchCh {
        if watchResp.Canceled {
            return fmt.Errorf("watch canceled: %v", watchResp.Err())
        }
        
        for _, event := range watchResp.Events {
            if event.Type == mvccpb.DELETE {
                // Predecessor key deleted, we may now hold the lock
                log.Printf("Predecessor %s deleted, checking lock status", predecessorKey)
                return nil
            }
            // Other events (PUT) on this key - predecessor is still holding
        }
    }
    
    return fmt.Errorf("watch channel closed unexpectedly")
}
 
// Watch with starting revision (ensure no missed events)
func watchFromRevision(cli *clientv3.Client, key string, startRev int64) {
    ctx := context.Background()
    
    watchCh := cli.Watch(ctx, key, 
        clientv3.WithRev(startRev),           // Start from specific revision
        clientv3.WithPrevKV(),                // Include previous value in events
        clientv3.WithProgressNotify(),        // Periodic progress notifications
    )
    
    for watchResp := range watchCh {
        if watchResp.IsProgressNotify() {
            log.Printf("Watch progress: revision %d", watchResp.Header.Revision)
            continue
        }
        
        for _, event := range watchResp.Events {
            log.Printf("Event: Type=%s, Key=%s, Rev=%d",
                event.Type, event.Kv.Key, event.Kv.ModRevision)
        }
    }
}