gRPC: High-Performance Remote Procedure Calls

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

// Syntax declaration - MUST be first non-empty, non-comment line
syntax = "proto3";
 
// Package declaration - prevents naming conflicts between projects
// Maps to package in many languages (Java, Go, C#)
package com.example.users.v1;
 
// Import statements - for using definitions from other .proto files
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";
 
// Options customize code generation behavior
option java_package = "com.example.users.v1";
option java_outer_classname = "UserProtos";
option go_package = "github.com/example/users/v1;usersv1";
 
// Enum definition - strongly typed enumerated values
enum UserStatus {
  USER_STATUS_UNSPECIFIED = 0;  // Proto3 requires 0 as first value
  USER_STATUS_ACTIVE = 1;
  USER_STATUS_SUSPENDED = 2;
  USER_STATUS_DELETED = 3;
}
 
// Message definition - the core data structure
message User {
  // Scalar types with field numbers
  string id = 1;                    // Unique identifier
  string email = 2;                 // User email
  string display_name = 3;          // Display name
  
  // Nested message reference
  UserProfile profile = 4;          // Embedded profile
  
  // Repeated field (list/array)
  repeated string roles = 5;        // User roles
  
  // Map type (associative array)
  map<string, string> metadata = 6; // Arbitrary key-value pairs
  
  // Enum field
  UserStatus status = 7;
  
  // Well-known type (imported)
  google.protobuf.Timestamp created_at = 8;
  google.protobuf.Timestamp updated_at = 9;
  
  // Wrapper types for nullable primitives
  google.protobuf.StringValue nickname = 10;
}
 
// Nested message for profile data
message UserProfile {
  string first_name = 1;
  string last_name = 2;
  string bio = 3;
  string avatar_url = 4;
  Address address = 5;
}
 
message Address {
  string street = 1;
  string city = 2;
  string state = 3;
  string country = 4;
  string postal_code = 5;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14

message OptimizedMessage {
  // Fields 1-15: Most frequently accessed fields (1-byte tag)
  string id = 1;              // Almost always present and used
  int64 timestamp = 2;        // Usually present
  string type = 3;            // Frequently filtered on
  
  // Fields 16+: Less common fields (2-byte tag)
  string description = 16;    // Optional in many cases
  map<string, string> tags = 17;  // Often empty
  
  // Reserved field numbers for removed fields (safety)
  reserved 100, 101, 102;
  reserved "old_field_name", "deprecated_field";
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

// Understanding how Protobuf encodes this message:
// message Example {
//   int32 id = 1;      // field number 1
//   string name = 2;   // field number 2
// }
 
// Encoded message: { id: 150, name: "test" }
// Binary: 08 96 01 12 04 74 65 73 74
 
// Breaking it down:
// 
// Field 1 (id = 150):
//   Tag: 08 = (1 << 3) | 0 = field_num 1, wire_type 0 (varint)
//   Value: 96 01 = 150 in varint encoding
//     - 0x96 = 1001 0110 (MSB set = more bytes follow)
//     - 0x01 = 0000 0001 (MSB clear = last byte)
//     - Decode: (0x16 | (0x01 << 7)) = 22 + 128 = 150
//
// Field 2 (name = "test"):
//   Tag: 12 = (2 << 3) | 2 = field_num 2, wire_type 2 (length-delimited)
//   Length: 04 = 4 bytes follow
//   Value: 74 65 73 74 = "test" in UTF-8
 
function decodeVarint(buffer: Uint8Array, offset: number): [number, number] {
    let result = 0;
    let shift = 0;
    let bytesRead = 0;
    
    while (true) {
        const byte = buffer[offset + bytesRead];
        bytesRead++;
        
        // Extract 7 data bits, add to result at correct position
        result |= (byte & 0x7F) << shift;
        shift += 7;
        
        // If MSB is 0, this is the last byte
        if ((byte & 0x80) === 0) {
            break;
        }
    }
    
    return [result, bytesRead];
}
 
function parseTag(tagVarint: number): { fieldNumber: number; wireType: number } {
    return {
        fieldNumber: tagVarint >>> 3,    // Upper bits = field number
        wireType: tagVarint & 0x07       // Lower 3 bits = wire type
    };
}
 
// Example usage
const encoded = new Uint8Array([0x08, 0x96, 0x01, 0x12, 0x04, 0x74, 0x65, 0x73, 0x74]);
let offset = 0;
 
// Parse first field
const [tag1, tagLen1] = decodeVarint(encoded, offset);
offset += tagLen1;
const { fieldNumber: fn1, wireType: wt1 } = parseTag(tag1);
console.log(`Field ${fn1}, WireType ${wt1}`); // Field 1, WireType 0
 
const [value1, valueLen1] = decodeVarint(encoded, offset);
offset += valueLen1;
console.log(`Value: ${value1}`); // Value: 150

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

// Version 1: Initial schema
message UserV1 {
  string id = 1;
  string name = 2;
  string email = 3;
}
 
// Version 2: Added fields, deprecated one (SAFE)
message UserV2 {
  string id = 1;
  // Renamed conceptually but same field number = SAFE
  string display_name = 2;  // was "name" in V1
  string email = 3;
  
  // New fields are always SAFE to add
  int64 created_at = 4;
  repeated string roles = 5;
  
  // Mark deprecated fields (still compatible)
  // Old readers ignore, new readers skip
}
 
// Version 3: Removed fields properly (SAFE)
message UserV3 {
  string id = 1;
  string display_name = 2;
  // email removed - MUST reserve the field number
  reserved 3;
  reserved "email";  // Reserve name too for documentation
  
  int64 created_at = 4;
  repeated string roles = 5;
  
  // More new fields
  UserProfile profile = 6;
  UserStatus status = 7;
}
 
// Evolution best practices in action:
message RobustMessage {
  // Reserve ranges for future use
  reserved 1000 to 1999;  // Reserved for experimental features
  reserved 9000 to 9999;  // Reserved for internal use
  
  // Explicit defaults via wrapper types when needed
  google.protobuf.Int32Value optional_count = 1;
  
  // Use oneof for mutually exclusive fields
  oneof notification_target {
    string email = 2;
    string phone = 3;
    string push_token = 4;
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

#!/bin/bash
# Comprehensive protobuf compilation script
 
# Directory structure
PROTO_DIR="./proto"
OUT_DIR="./generated"
 
# Compile for multiple languages
protoc \
  --proto_path=${PROTO_DIR} \
  --proto_path=./third_party/googleapis \
  --go_out=${OUT_DIR}/go \
  --go_opt=paths=source_relative \
  --go-grpc_out=${OUT_DIR}/go \
  --go-grpc_opt=paths=source_relative \
  --java_out=${OUT_DIR}/java \
  --python_out=${OUT_DIR}/python \
  --js_out=import_style=commonjs:${OUT_DIR}/js \
  --grpc-web_out=import_style=typescript,mode=grpcwebtext:${OUT_DIR}/js \
  ${PROTO_DIR}/**/*.proto
 
# For TypeScript (using ts-proto plugin)
protoc \
  --plugin=./node_modules/.bin/protoc-gen-ts_proto \
  --ts_proto_out=${OUT_DIR}/typescript \
  --ts_proto_opt=outputEncodeMethods=true \
  --ts_proto_opt=outputJsonMethods=true \
  --ts_proto_opt=outputClientImpl=true \
  --ts_proto_opt=useOptionals=messages \
  ${PROTO_DIR}/**/*.proto

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

// Auto-generated from user.proto
// DO NOT EDIT manually
 
export interface User {
  id: string;
  email: string;
  displayName: string;
  profile: UserProfile | undefined;
  roles: string[];
  metadata: { [key: string]: string };
  status: UserStatus;
  createdAt: Date | undefined;
  updatedAt: Date | undefined;
  nickname: string | undefined;  // wrapper type = optional
}
 
export const User = {
  // Encode message to binary Uint8Array
  encode(message: User): Uint8Array {
    const writer = new BinaryWriter();
    if (message.id !== "") {
      writer.uint32(10);  // (1 << 3) | 2 = tag for string field 1
      writer.string(message.id);
    }
    if (message.email !== "") {
      writer.uint32(18);  // (2 << 3) | 2 = tag for string field 2
      writer.string(message.email);
    }
    // ... encoding for all fields
    return writer.finish();
  },
 
  // Decode binary Uint8Array to message
  decode(input: Uint8Array): User {
    const reader = new BinaryReader(input);
    const message = createBaseUser();
    
    while (reader.pos < reader.len) {
      const tag = reader.uint32();
      switch (tag >>> 3) {  // Extract field number
        case 1:
          message.id = reader.string();
          break;
        case 2:
          message.email = reader.string();
          break;
        // ... decoding for all fields
        default:
          reader.skipType(tag & 7);  // Skip unknown fields
          break;
      }
    }
    return message;
  },
 
  // Convert to JSON-compatible object
  toJSON(message: User): unknown {
    const obj: any = {};
    obj.id = message.id;
    obj.email = message.email;
    // ... conversion for all fields
    return obj;
  },
 
  // Create from JSON-compatible object
  fromJSON(object: any): User {
    return {
      id: isSet(object.id) ? String(object.id) : "",
      email: isSet(object.email) ? String(object.email) : "",
      // ... parsing for all fields
    };
  },
 
  // Create with default values
  create(base?: DeepPartial<User>): User {
    return User.fromPartial(base ?? {});
  },
 
  // Merge partial values into full message
  fromPartial(object: DeepPartial<User>): User {
    const message = createBaseUser();
    message.id = object.id ?? "";
    message.email = object.email ?? "";
    // ... for all fields
    return message;
  },
};
 
function createBaseUser(): User {
  return {
    id: "",
    email: "",
    displayName: "",
    profile: undefined,
    roles: [],
    metadata: {},
    status: UserStatus.UNSPECIFIED,
    createdAt: undefined,
    updatedAt: undefined,
    nickname: undefined,
  };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

// Example message comparison
 
// JSON representation (122 bytes):
const json = {
  "id": "user-12345",           // 17 bytes (key + value + quotes + colon)
  "email": "alice@example.com", // 29 bytes
  "age": 28,                    // 10 bytes
  "isActive": true,             // 17 bytes
  "roles": ["admin", "user"]    // ~32 bytes
  // Plus: braces, commas, whitespace ≈ 17 bytes
};
// Total: ~122 bytes
 
// Protobuf representation (~47 bytes):
// message User {
//   string id = 1;       // tag(1) + len(11) + "user-12345" = 13 bytes
//   string email = 2;    // tag(1) + len(17) + email = 19 bytes
//   int32 age = 3;       // tag(1) + varint(28) = 2 bytes
//   bool is_active = 4;  // tag(1) + varint(1) = 2 bytes
//   repeated string roles = 5; 
//                        // tag(1) + len(5) + "admin" = 7 bytes
//                        // tag(1) + len(4) + "user" = 6 bytes
// }
// Total: ~47 bytes (62% smaller)
 
// Size difference compounds with nesting and arrays:
const complexJson = {
  users: Array(1000).fill({
    id: "user-12345",
    email: "alice@example.com",
    profile: {
      firstName: "Alice",
      lastName: "Smith",
      preferences: { theme: "dark", language: "en" }
    }
  })
};
// JSON: ~180KB, Protobuf: ~65KB (64% reduction)
 
// At 1M messages/second: 115 GB/hour saved in bandwidth

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

// PATTERN 1: Union types with oneof
message Notification {
  string id = 1;
  int64 timestamp = 2;
  
  // Only ONE of these can be set at a time
  oneof content {
    TextNotification text = 10;
    ImageNotification image = 11;
    VideoNotification video = 12;
    ActionNotification action = 13;
  }
}
 
message TextNotification {
  string title = 1;
  string body = 2;
}
 
message ImageNotification {
  string image_url = 1;
  string alt_text = 2;
}
 
// PATTERN 2: Nested messages for composition
message Order {
  string order_id = 1;
  Customer customer = 2;
  repeated LineItem items = 3;
  PaymentInfo payment = 4;
  ShippingAddress shipping = 5;
  
  // Nested message defined within parent (tightly coupled)
  message LineItem {
    string product_id = 1;
    int32 quantity = 2;
    int64 price_cents = 3;
    map<string, string> options = 4;
  }
}
 
// PATTERN 3: Self-referential structures (trees, graphs)
message TreeNode {
  string id = 1;
  string value = 2;
  repeated TreeNode children = 3;  // Recursive reference
}
 
message LinkedListNode {
  string value = 1;
  LinkedListNode next = 2;  // Optional self-reference
}
 
// PATTERN 4: Polymorphism via Any type
import "google/protobuf/any.proto";
 
message Event {
  string event_id = 1;
  string event_type = 2;  // Discriminator
  google.protobuf.Any payload = 3;  // Can be any message type
}
 
// Usage: Pack specific message into Any
// event.payload = Any.pack(UserCreatedEvent{...})
 
// PATTERN 5: Wrapper types for optional primitives
import "google/protobuf/wrappers.proto";
 
message SearchFilters {
  // Can distinguish between "not provided" and "provided as 0/empty"
  google.protobuf.Int32Value min_price = 1;
  google.protobuf.Int32Value max_price = 2;
  google.protobuf.StringValue category = 3;
  google.protobuf.BoolValue in_stock_only = 4;
}
 
// PATTERN 6: API request/response wrappers
message ListUsersRequest {
  int32 page_size = 1;
  string page_token = 2;
  string filter = 3;
  string order_by = 4;
}
 
message ListUsersResponse {
  repeated User users = 1;
  string next_page_token = 2;
  int32 total_size = 3;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

import "google/protobuf/descriptor.proto";
 
// Define custom options
extend google.protobuf.FieldOptions {
  optional bool deprecated_in_v2 = 50000;
  optional string validation_regex = 50001;
  optional bool sensitive = 50002;  // PII, don't log
}
 
extend google.protobuf.MessageOptions {
  optional string api_version = 51000;
}
 
message User {
  option (api_version) = "v2";
  
  string id = 1;
  string email = 2 [(validation_regex) = "^[\w.-]+@[\w.-]+\.\w+$"];
  string ssn = 3 [(sensitive) = true];  // Don't log this field
  string legacy_field = 4 [(deprecated_in_v2) = true];
}