1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

// Segment Descriptor Protection Fields (x86 style)
 
struct SegmentDescriptor {
    // Base and limit fields (addressing)
    uint32_t base;          // Starting physical address
    uint32_t limit;         // Maximum offset (bounds check)
    
    // === PROTECTION FIELDS ===
    
    // Access type (Read, Write, Execute combinations)
    uint8_t readable    : 1;   // Can code be read as data?
    uint8_t writable    : 1;   // Can data be written?
    uint8_t executable  : 1;   // Can this be executed?
    
    // Typical protection combinations:
    // Code segment:  R-X (readable, executable, not writable)
    // Data segment:  RW- (readable, writable, not executable)
    // Stack segment: RW- (readable, writable, not executable)
    // Constants:     R-- (read-only, not executable)
    
    // Privilege level (rings in x86)
    uint8_t dpl         : 2;   // Descriptor Privilege Level (0-3)
    // Ring 0: Kernel (highest privilege)
    // Ring 1: Device drivers (rarely used in practice)
    // Ring 2: Device drivers (rarely used in practice)
    // Ring 3: User applications (lowest privilege)
    
    // CPL (Current Privilege Level) must be <= DPL to access
    // otherwise: GENERAL PROTECTION FAULT (#GP)
    
    // Segment type
    uint8_t type        : 4;   // System segment, code, data, etc.
    uint8_t system      : 1;   // 0=system segment, 1=code/data
    
    // Status flags
    uint8_t present     : 1;   // Segment is in memory
    uint8_t accessed    : 1;   // Segment has been accessed
    
    // Granularity
    uint8_t granularity : 1;   // Limit in bytes (0) or 4KB (1)
    uint8_t default_op  : 1;   // Default operation size (16/32 bit)
};
 
/*
 * Key protection properties of segmentation:
 *
 * 1. SEMANTIC ALIGNMENT
 *    Protection matches program structure:
 *    - Code segment: executable, read-only → prevents code modification
 *    - Data segment: read-write, non-executable → W^X policy natural
 *    - Stack segment: read-write, non-executable → stack execution blocked
 *
 * 2. BOUNDS CHECKING
 *    Every access: if (offset >= segment.limit) → FAULT
 *    This PREVENTS buffer overflows at hardware level!
 *    
 *    Example: char buffer[100]; buffer[500] = 'x';
 *    
 *    With segmentation (if buffer is in its own segment):
 *    - offset 500 > limit 100 → SEGMENTATION FAULT
 *    - Overflow CAUGHT before memory corrupted
 *    
 * 3. ISOLATION BY DEFAULT
 *    Each process has separate segment table
 *    Segment numbers are local to each process
 *    Process A's segment 2 ≠ Process B's segment 2
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

// Page Table Entry Protection Fields (x86-64)
 
struct PageTableEntry {
    // Physical frame number
    uint64_t frame      : 40;   // Physical page frame number
    
    // === PROTECTION FIELDS ===
    
    // Read/Write permission
    uint64_t rw         : 1;    // 0=read-only, 1=read-write
    // Note: Pages are always readable if present
    // There is NO read-disable bit in x86!
    
    // User/Supervisor permission
    uint64_t us         : 1;    // 0=supervisor only, 1=user accessible
    // Supervisor mode (kernel): can access any page
    // User mode: can only access pages with us=1
    
    // Execute Disable (NX bit)
    uint64_t xd         : 1;    // 1=execution disabled (data page)
    // Critical for W^X (Write XOR Execute) policy
    // Prevents code injection attacks
    
    // Present bit (NOT protection, but related)
    uint64_t present    : 1;    // Page is in physical memory
    // Access to non-present page → PAGE FAULT
    
    // Status bits (modified by hardware)
    uint64_t accessed   : 1;    // Page has been read
    uint64_t dirty      : 1;    // Page has been written
    
    // Cache control
    uint64_t pwt        : 1;    // Write-through caching
    uint64_t pcd        : 1;    // Cache disable
    
    // Other bits
    uint64_t pat        : 1;    // Page Attribute Table
    uint64_t global     : 1;    // Global page (not flushed on CR3 change)
    uint64_t available  : 3;    // For OS use
    
    // Reserved bits (must be zero)
    uint64_t reserved   : 7;
};
 
/*
 * Key protection properties of paging:
 *
 * 1. EFFICIENT CHECKING
 *    Protection check is simple bit test:
 *    - Check present bit
 *    - Check rw bit if writing
 *    - Check us bit against current privilege
 *    - Check xd bit if executing
 *    All done in parallel with translation!
 *
 * 2. PAGE GRANULARITY
 *    Protection changes only at page boundaries (4KB)
 *    Can't protect individual bytes or structures
 *    
 *    Example: struct { 
 *        int readable_field;    // offset 0
 *        int secret_field;      // offset 4
 *    };
 *    
 *    Both fields share the same page and protection!
 *    Can't make secret_field more protected than readable_field.
 *    
 * 3. NO BOUNDS CHECKING
 *    Any offset within a page is valid
 *    Buffer overflow within a page → NOT DETECTED
 *    
 *    char buffer[100];  // At offset 0 in page
 *    char password[100]; // At offset 100 in same page
 *    
 *    buffer[150] overwrites password!
 *    Paging allows this - both are in the same page!
 *
 * 4. W^X SUPPORT (via NX bit)
 *    Modern security policy: memory is EITHER writable OR executable
 *    Mark code pages: RX (readable, executable, NOT writable)
 *    Mark data pages: RW (readable, writable, NOT executable)
 *    Attacker can't inject code AND execute it
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

= = = SEGMENTATION: Fine-Grained Protection = = =
 
Process memory layout with segments:
┌─────────────────────────────────────────────────────────────────────┐
│ Segment 0: Code         │ Base: 0x1000  │ Limit: 4000 │ R-X │ Ring3 │
│ Segment 1: Global Data  │ Base: 0x2000  │ Limit: 512  │ RW- │ Ring3 │
│ Segment 2: Stack        │ Base: 0x3000  │ Limit: 8192 │ RW- │ Ring3 │
│ Segment 3: Heap         │ Base: 0x4000  │ Limit: 65536│ RW- │ Ring3 │
│ Segment 4: Config Data  │ Base: 0x5000  │ Limit: 128  │ R-- │ Ring3 │
│ Segment 5: Secret Keys  │ Base: 0x6000  │ Limit: 256  │ R-- │ Ring0 │
└─────────────────────────────────────────────────────────────────────┘
 
Observations:
- Secret Keys require Ring 0 (kernel) access
- Config Data is read-only even for userspace
- Each segment has EXACTLY the right size (no overflow possible)
- Attempt to write Code segment → IMMEDIATE FAULT
- Attempt to read Secret Keys from Ring 3 → IMMEDIATE FAULT
 
= = = PAGING: Page-Granular Protection = = =
 
Process memory layout with pages (4KB each):
┌─────────────────────────────────────────────────────────────────────┐
│ Page 0: Code (part 1)        │ Frame: 0x100 │ R-X │ User │         │
│ Page 1: Code (part 2)        │ Frame: 0x101 │ R-X │ User │ 4KB     │
│ Page 2: Data + Config (MIXED)│ Frame: 0x200 │ RW- │ User │ 4KB     │
│ Page 3: Stack                │ Frame: 0x300 │ RW- │ User │ 4KB     │
│ Page 4: Heap (part 1)        │ Frame: 0x400 │ RW- │ User │ 4KB     │
│ Page 5: Heap (part 2)        │ Frame: 0x401 │ RW- │ User │ 4KB     │
│ Page 6: Secret + Padding     │ Frame: 0x500 │ R-- │ Kern │ 4KB     │
└─────────────────────────────────────────────────────────────────────┘
 
Problems:
- Page 2 mixes 512B data + 128B config + 3456B padding/other
  → Config can be overwritten by data buffer overflow!
  → They share the same RW- permission
  
- Page 6 has 256B secrets + 3840B wasted padding
  → Internal fragmentation wastes kernel memory
  → But at least secrets are protected (supervisor-only)
  
- No hardware detection of buffer overflow within Page 4-5
  → Heap corruption goes undetected if it stays within heap pages

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

                     x86 Protection Rings
                     ═══════════════════════
                     
                           ┌─────────────┐
                           │   Ring 0    │  ← KERNEL (highest privilege)
                           │   Kernel    │     - Full hardware access
                           │             │     - All instructions permitted
                       ┌───┴─────────────┴───┐
                       │      Ring 1         │  ← DEVICE DRIVERS (rarely used)
                       │   Device Drivers    │     - Limited I/O access
                       │                     │     - Historical x86 feature
                   ┌───┴─────────────────────┴───┐
                   │        Ring 2               │  ← SERVICES (rarely used)
                   │      System Services        │     - Even more restricted
                   │                             │     - Also historical
               ┌───┴─────────────────────────────┴───┐
               │            Ring 3                   │  ← USER APPLICATIONS
               │       User Applications             │     (lowest privilege)
               │                                     │     - No direct hardware
               │                                     │     - Only user pages
               └─────────────────────────────────────┘
 
In practice, most OSes use:
  Ring 0: Kernel
  Ring 3: Everything else (drivers run in kernel or use user-mode drivers)
 
Modern alternatives:
  - Hypervisors use Ring -1 (VMX root mode)
  - System Management Mode (SMM) is even more privileged

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

// Classic buffer overflow vulnerability
 
void vulnerable_function(char* user_input) {
    char buffer[64];               // Local buffer on stack
    char* function_pointer = ...;  // Function pointer on same stack
    
    strcpy(buffer, user_input);    // No bounds check!
    
    (*function_pointer)();         // Calls attacker-controlled address?
}
 
/* === PAGING BEHAVIOR === */
// Both 'buffer' and 'function_pointer' are on the stack
// Stack is contiguous pages with RW- permission
// 
// Attack: user_input = "AAAA..." (> 64 bytes) + evil_address
// 
// 1. strcpy copies beyond buffer[63] into function_pointer
// 2. NO PAGE FAULT - all within the same page(s) with RW permission
// 3. Attack SUCCEEDS - function_pointer is overwritten
// 4. Program calls attacker's code
//
// Paging protection: FAILED TO DETECT
 
/* === SEGMENTATION BEHAVIOR (with fine-grained segments) === */
// Hypothetical system where each variable has own segment:
//   Segment 5: buffer[64]        Limit: 64 bytes
//   Segment 6: function_pointer  Limit: 8 bytes
//
// Attack: same as above
//
// 1. strcpy writes to Segment 5 (buffer)
// 2. When offset reaches 64, hardware checks: 64 >= limit(64)?
// 3. YES! SEGMENTATION FAULT raised
// 4. Attack BLOCKED - function_pointer never touched
//
// Segmentation protection: DETECTED AND PREVENTED
 
/*
 * Note: This fine-grained segmentation isn't how x86 segments
 * were actually used in practice (too much overhead).
 * 
 * But the principle is being revived in:
 * - Intel MPX (Memory Protection Extensions) - deprecated
 * - CHERI (Capability Hardware Enhanced RISC Instructions)
 * - AddressSanitizer (software-based, similar effect)
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

Stack with Guard Pages:
 
┌─────────────────────────────────────────────────────────────────────┐
│                     Virtual Address Space                           │
├─────────────────────────────────────────────────────────────────────┤
│    GUARD PAGE     │     STACK AREA      │    GUARD PAGE            │
│   (no access)     │   (read-write)      │   (no access)            │
│    Present=0      │    Present=1        │    Present=0             │
│                   │    RW=1             │                          │
│ Page Fault if     │                     │ Page Fault if            │
│ stack underflows  │    Normal stack     │ stack overflows          │
│                   │    operations       │                          │
└─────────────────────────────────────────────────────────────────────┘
 
If stack overflow crosses into guard page:
1. Access to guard page (present=0)
2. PAGE FAULT raised
3. OS detects stack overflow attempt
4. Process terminated or stack extended (with new guard)
 
Limitations:
- Only catches overflows that HIT the guard page
- A carefully sized overflow can skip over the guard page!
- Large allocations or multiple pages of overflow may miss detection
- Single 4KB guard = ~4KB "detection zone"
 
Heap Guard Pages:
- Similar concept for heap regions
- GlibC/jemalloc use guard pages between large allocations
- Still doesn't catch small intra-page overflows

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

// Stack Canary Protection (GCC -fstack-protector)
 
void protected_function(char* input) {
    // Compiler-inserted canary check
    uintptr_t __stack_chk_guard_copy = __stack_chk_guard;
    
    char buffer[64];
    strcpy(buffer, input);  // Potential overflow
    
    // Compiler-inserted canary verification
    if (__stack_chk_guard_copy != __stack_chk_guard) {
        // CANARY CORRUPTED! Buffer overflow detected
        __stack_chk_fail();  // Terminate process
    }
    
    // Normal return if canary intact
    return;
}
 
/*
 * Stack layout with canary:
 * 
 * Higher addresses
 * ─────────────────
 * │ Return address │  ← Target of attack
 * │ Saved RBP      │  
 * │ CANARY VALUE   │  ← Random value, checked before return
 * │ buffer[63]     │
 * │ ...            │
 * │ buffer[0]      │
 * ─────────────────
 * Lower addresses
 * 
 * Overflow path: buffer[0] → buffer[63] → CANARY → Saved RBP → Return
 * 
 * To reach return address, attacker MUST overwrite canary
 * Check at function return detects corruption
 * 
 * Limitations: 
 * - Doesn't prevent corruption, only detects it
 * - Format string bugs can leak canary value
 * - Heap overflows don't have canaries
 */