Buffer Cache

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
 
/*
 * sync() - Synchronize all filesystems
 *
 * Schedules or performs writeback of all dirty data and metadata
 * across all mounted filesystems.
 *
 * Returns: void (cannot fail in traditional UNIX semantics)
 */
void sync(void);  /* Flushes entire system */
 
/*
 * syncfs(fd) - Synchronize single filesystem
 *
 * Forces writes for all dirty data on the filesystem containing fd.
 * More targeted than sync() for systems with many filesystems.
 *
 * Returns: 0 on success, -1 on error
 */
int backup_filesystem_sync(const char *mountpoint) {
    int fd = open(mountpoint, O_RDONLY);
    if (fd < 0) {
        perror("open");
        return -1;
    }
    
    /* Sync only this filesystem */
    if (syncfs(fd) != 0) {
        perror("syncfs");
        close(fd);
        return -1;
    }
    
    close(fd);
    return 0;
}
 
/*
 * fsync(fd) - Synchronize single file
 *
 * Forces all data and metadata for fd to stable storage.
 * Returns only after data is confirmed durable.
 *
 * This is the PRIMARY durability mechanism for applications.
 */
int durable_write(int fd, const void *data, size_t len) {
    /* Write data to kernel buffer */
    ssize_t written = write(fd, data, len);
    if (written != len) {
        return -1;
    }
    
    /* Force to stable storage */
    if (fsync(fd) != 0) {
        perror("fsync");
        return -1;  /* Data may not be durable! */
    }
    
    return 0;  /* Data is now on stable storage */
}
 
/*
 * fdatasync(fd) - Synchronize file data only
 *
 * Like fsync() but omits metadata that doesn't affect data retrieval.
 * Faster when you don't need timestamp updates to be durable.
 */
int durable_data_write(int fd, const void *data, size_t len) {
    write(fd, data, len);
    
    /* Only sync data and essential metadata (like file size) */
    if (fdatasync(fd) != 0) {
        return -1;
    }
    
    return 0;
    /* File content is durable; mtime may not be */
}
 
/*
 * sync_file_range() - Fine-grained sync control (Linux-specific)
 *
 * Offers surgical control over which portions of a file to sync
 * and whether to wait for completion.
 */
#define SYNC_FILE_RANGE_WAIT_BEFORE  1
#define SYNC_FILE_RANGE_WRITE        2
#define SYNC_FILE_RANGE_WAIT_AFTER   4
 
void streaming_write_example(int fd) {
    char buffer[1024 * 1024];  /* 1MB chunks */
    off_t offset = 0;
    
    while (/* have more data */) {
        /* Fill buffer with data */
        // ...
        
        /* Write to kernel buffer */
        pwrite(fd, buffer, sizeof(buffer), offset);
        
        /* Initiate async writeback for this chunk */
        /* Don't wait - let it happen in background */
        sync_file_range(fd, offset, sizeof(buffer), SYNC_FILE_RANGE_WRITE);
        
        offset += sizeof(buffer);
    }
    
    /* At end, ensure everything is synced */
    sync_file_range(fd, 0, offset,
                    SYNC_FILE_RANGE_WAIT_BEFORE |
                    SYNC_FILE_RANGE_WRITE |
                    SYNC_FILE_RANGE_WAIT_AFTER);
}
 
/*
 * msync() - Synchronize memory-mapped file region
 */
void mmap_sync_example(void *addr, size_t len) {
    /* MS_SYNC: Wait for writeback to complete */
    if (msync(addr, len, MS_SYNC) != 0) {
        perror("msync");
    }
    
    /* MS_ASYNC: Schedule writeback, don't wait */
    msync(addr, len, MS_ASYNC);
    
    /* MS_INVALIDATE: Invalidate cached pages (force re-read) */
    msync(addr, len, MS_INVALIDATE);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

/*
 * Linux fsync() implementation (simplified)
 * Real code in fs/sync.c and specific FS implementations
 */
 
/* System call entry point */
SYSCALL_DEFINE1(fsync, unsigned int, fd) {
    struct fd f = fdget(fd);
    int ret;
    
    if (!f.file)
        return -EBADF;
    
    ret = vfs_fsync(f.file, 0);  /* 0 = sync data AND metadata */
    fdput(f);
    return ret;
}
 
/* VFS layer fsync implementation */
int vfs_fsync(struct file *file, int datasync) {
    return vfs_fsync_range(file, 0, LLONG_MAX, datasync);
}
 
int vfs_fsync_range(struct file *file, loff_t start, loff_t end, 
                    int datasync) {
    struct inode *inode = file_inode(file);
    int ret;
    
    /* Call filesystem-specific fsync if provided */
    if (file->f_op->fsync) {
        return file->f_op->fsync(file, start, end, datasync);
    }
    
    /* Generic implementation */
    ret = sync_inode_metadata(inode, 1);  /* 1 = wait */
    if (!ret)
        ret = filemap_fdatawrite_range(file->f_mapping, start, end);
    if (!ret)
        ret = filemap_fdatawait_range(file->f_mapping, start, end);
    return ret;
}
 
/* Write all dirty pages in the specified range */
int filemap_fdatawrite_range(struct address_space *mapping,
                              loff_t start, loff_t end) {
    struct writeback_control wbc = {
        .sync_mode = WB_SYNC_ALL,       /* Wait for completion */
        .nr_to_write = LONG_MAX,        /* No page limit */
        .range_start = start,
        .range_end = end,
    };
    
    return mapping->a_ops->writepages(mapping, &wbc);
}
 
/* Wait for all I/O on pages in range to complete */
int filemap_fdatawait_range(struct address_space *mapping,
                            loff_t start, loff_t end) {
    pgoff_t start_idx = start >> PAGE_SHIFT;
    pgoff_t end_idx = end >> PAGE_SHIFT;
    struct page *page;
    int ret = 0;
    
    /* Iterate through all pages in range */
    for (pgoff_t idx = start_idx; idx <= end_idx; idx++) {
        page = find_get_page(mapping, idx);
        if (!page)
            continue;
        
        /* Wait if page has writeback in progress */
        if (PageWriteback(page))
            wait_on_page_writeback(page);
        
        /* Check for I/O error */
        if (PageError(page))
            ret = -EIO;
        
        put_page(page);
    }
    
    return ret;
}
 
/*
 * Ext4-specific fsync (more sophisticated)
 */
int ext4_sync_file(struct file *file, loff_t start, loff_t end, 
                   int datasync) {
    struct inode *inode = file_inode(file);
    int ret;
    
    /* Handle journaled data case */
    if (EXT4_JOURNAL(inode)) {
        /* Wait for journal commit covering our data */
        ret = ext4_jbd2_inode_add_wait(inode, start, end);
        if (ret)
            return ret;
    }
    
    /* Write file data */
    ret = file_write_and_wait_range(file, start, end);
    if (ret)
        return ret;
    
    /* Sync metadata if needed */
    if (!datasync || ext4_inode_data_dirty(inode)) {
        ret = ext4_write_inode(inode, WB_SYNC_ALL);
        if (ret)
            return ret;
    }
    
    /* Issue storage device flush */
    if (needs_barrier(inode->i_sb)) {
        ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL, NULL);
    }
    
    return ret;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

/*
 * When to use fsync() vs fdatasync()
 */
 
#include <unistd.h>
#include <fcntl.h>
#include <time.h>
 
/*
 * SCENARIO 1: Appending to a log file
 *
 * Each append extends the file, so size changes.
 * fdatasync() syncs both data and the new size.
 * Timestamps don't matter for log integrity.
 */
void append_log_entry(int fd, const char *entry, size_t len) {
    write(fd, entry, len);
    fdatasync(fd);  /* Syncs data + size */
    /* Faster than fsync() because skips mtime/ctime */
}
 
/*
 * SCENARIO 2: Overwriting existing data (no size change)
 *
 * If file size doesn't change, fdatasync() only syncs data.
 * This is even faster - no metadata I/O at all.
 */
void update_record(int fd, off_t offset, const void *data, size_t len) {
    pwrite(fd, data, len, offset);
    fdatasync(fd);  /* Only data, no metadata (size unchanged) */
    /* This can be 2x faster than fsync() */
}
 
/*
 * SCENARIO 3: Creating a new file (fsync required)
 *
 * New file needs directory entry synced too!
 * fsync() on file PLUS fsync() on directory
 */
void create_durable_file(const char *path, const void *data, size_t len) {
    int fd = open(path, O_CREAT | O_WRONLY | O_TRUNC, 0644);
    
    write(fd, data, len);
    fsync(fd);  /* Use fsync for new files */
    close(fd);
    
    /* CRITICAL: Sync the directory to make filename durable */
    int dir_fd = open(dirname_path(path), O_RDONLY);
    fsync(dir_fd);
    close(dir_fd);
}
 
/*
 * SCENARIO 4: Preserving timestamps matters
 *
 * Some applications rely on mtime for synchronization
 * (e.g., rsync, make). Use fsync() in these cases.
 */
void timestamp_critical_write(int fd, const void *data, size_t len) {
    write(fd, data, len);
    fsync(fd);  /* Ensure mtime is also persisted */
}
 
/*
 * Performance comparison: fsync vs fdatasync
 * 
 * On a test system with ext4:
 * - fsync() on overwrite: ~8ms
 * - fdatasync() on overwrite: ~4ms (2x faster!)
 * 
 * The difference comes from:
 * 1. Writing inode block for timestamp update
 * 2. Possibly journal transaction for metadata
 * 
 * For high-frequency sync (like database commits),
 * fdatasync() can significantly improve throughput.
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

/*
 * How the kernel issues device flush commands
 */
 
/*
 * Issue a flush/sync to a block device
 * Called at end of fsync() path
 */
int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
                       sector_t *error_sector) {
    struct bio *bio;
    int ret = 0;
    
    /* Create a bio (block I/O request) with no data */
    bio = bio_alloc(gfp_mask, 0);
    bio_set_dev(bio, bdev);
    bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
    
    /* Submit and wait for completion */
    ret = submit_bio_wait(bio);
    
    bio_put(bio);
    return ret;
}
 
/*
 * NVMe flush command handling (in driver)
 */
void nvme_execute_flush(struct nvme_ns *ns, struct request *req) {
    struct nvme_command cmd = {
        .common = {
            .opcode = nvme_cmd_flush,
            .nsid = cpu_to_le32(ns->head->ns_id),
        },
    };
    
    /* Send flush command to controller */
    nvme_submit_sync_cmd(ns->ctrl, &cmd, NULL, 0);
    /* Returns when flush is complete */
}
 
/*
 * Force Unit Access (FUA) - Alternative to flush
 *
 * FUA writes bypass the volatile cache entirely.
 * Used for individual critical writes without draining entire cache.
 */
void write_with_fua(struct block_device *bdev, void *data, 
                     sector_t sector, size_t len) {
    struct bio *bio = bio_alloc(GFP_KERNEL, 1);
    
    bio_set_dev(bio, bdev);
    bio->bi_iter.bi_sector = sector;
    bio_add_page(bio, virt_to_page(data), len, offset_in_page(data));
    
    /* REQ_FUA: Force Unit Access - bypass volatile cache */
    bio->bi_opf = REQ_OP_WRITE | REQ_FUA;
    
    submit_bio_wait(bio);
    bio_put(bio);
    /* This write is directly on stable media */
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#!/bin/bash
# Managing storage device write cache
 
# Check current write cache status (SATA/ATA)
hdparm -W /dev/sda
# /dev/sda:
#  write-caching =  1 (on)
 
# Disable write cache (sacrifices performance for safety)
hdparm -W0 /dev/sda
 
# For SCSI devices
sdparm --get=WCE /dev/sdb  # Get Write Cache Enable
sdparm --set=WCE=0 /dev/sdb  # Disable
 
# Check NVMe volatile write cache
nvme id-ctrl /dev/nvme0 | grep vwc
#  vwc   : 1  (volatile write cache present)
 
# Enterprise SSDs with Power Loss Protection (PLP)
# These are SAFE to have write cache enabled because
# capacitors flush data on power loss

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

/*
 * Performance optimization strategies for sync-heavy workloads
 */
 
/*
 * STRATEGY 1: Group Commit
 *
 * Instead of syncing each write, batch multiple writes
 * and sync once for the entire batch.
 */
struct batch {
    int count;
    int max_count;
    int fd;
};
 
void batch_write(struct batch *b, const void *data, size_t len) {
    write(b->fd, data, len);
    b->count++;
    
    if (b->count >= b->max_count) {
        /* Batch is full, sync now */
        fdatasync(b->fd);
        b->count = 0;
    }
}
 
/* Result: 
 * 100 individual writes with sync: 100 × 5ms = 500ms
 * 100 writes batched, 1 sync:      100 × 0.001ms + 5ms ≈ 5ms
 * 100x improvement!
 */
 
 
/*
 * STRATEGY 2: Parallel Sync
 *
 * Multiple devices can be synced in parallel.
 * Distribute data across devices for higher throughput.
 */
void parallel_sync(int *fds, int count) {
    pthread_t threads[count];
    
    for (int i = 0; i < count; i++) {
        pthread_create(&threads[i], NULL, 
                       (void*)fsync_thread, &fds[i]);
    }
    
    for (int i = 0; i < count; i++) {
        pthread_join(threads[i], NULL);
    }
    
    /* With 4 SSDs: 4 × 200 TPS = 800 TPS aggregate */
}
 
 
/*
 * STRATEGY 3: Asynchronous Writeback + Barrier
 *
 * Use sync_file_range() to start writeback early,
 * then sync at commit points.
 */
void pipelined_write(int fd, void *data, size_t chunk_size, int chunks) {
    for (int i = 0; i < chunks; i++) {
        off_t offset = i * chunk_size;
        
        /* Write to page cache */
        pwrite(fd, data + offset, chunk_size, offset);
        
        /* Start async writeback (don't wait) */
        sync_file_range(fd, offset, chunk_size, 
                        SYNC_FILE_RANGE_WRITE);
    }
    
    /* Final sync - wait for all writeback */
    sync_file_range(fd, 0, chunks * chunk_size,
                    SYNC_FILE_RANGE_WAIT_BEFORE |
                    SYNC_FILE_RANGE_WRITE |
                    SYNC_FILE_RANGE_WAIT_AFTER);
    fdatasync(fd);
}
 
 
/*
 * STRATEGY 4: Use O_DSYNC or O_SYNC for implicit sync
 *
 * Opens file with automatic sync on every write.
 * Good for small, critical writes.
 */
void always_sync_writes() {
    /* O_DSYNC: Every write is like write + fdatasync */
    int fd = open("critical.dat", O_WRONLY | O_DSYNC);
    write(fd, data, len);  /* Returns after data durable */
    
    /* O_SYNC: Every write is like write + fsync */
    int fd2 = open("more_critical.dat", O_WRONLY | O_SYNC);
    write(fd2, data, len);  /* Returns after data+metadata durable */
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

/*
 * Database Sync Patterns
 */
 
/*
 * PATTERN 1: Synchronous Commit (Default PostgreSQL)
 *
 * Every commit waits for WAL fsync.
 * Maximum durability, lowest throughput.
 */
void synchronous_commit(struct transaction *txn) {
    /* Write transaction to WAL buffer */
    write_wal_record(txn);
    
    /* Force WAL to disk */
    fdatasync(wal_fd);  /* ~4ms */
    
    /* Acknowledge commit */
    txn->status = COMMITTED;
    /* Client can proceed - data is durable */
}
/* Throughput: ~250 TPS per disk */
 
 
/*
 * PATTERN 2: Group Commit (PostgreSQL, MySQL)
 *
 * Batch multiple commits into one fsync.
 * Slight latency increase, huge throughput gain.
 */
void group_commit() {
    /* Collect commits for a short window */
    collect_pending_commits(COMMIT_DELAY_MS);
    
    /* Write all pending to WAL */
    for_each_pending(txn) {
        write_wal_record(txn);
    }
    
    /* Single fsync for all */
    fdatasync(wal_fd);
    
    /* Acknowledge all commits */
    for_each_pending(txn) {
        txn->status = COMMITTED;
        wake_client(txn);
    }
}
/* Throughput: 10,000+ TPS with concurrent connections */
 
 
/*
 * PATTERN 3: Asynchronous Commit (PostgreSQL)
 *
 * Don't wait for fsync; accept data loss risk.
 * Maximum throughput, potential loss window.
 */
void async_commit(struct transaction *txn) {
    /* Write to WAL buffer (not yet on disk) */
    write_wal_record(txn);
    
    /* Immediately acknowledge - data NOT durable! */
    txn->status = COMMITTED;
    
    /* Background thread eventually syncs */
    /* Risk: last ~10ms of transactions lost on crash */
}
/* Throughput: 50,000+ TPS (memory speed) */
 
 
/*
 * PATTERN 4: Commit=No_wait + Group Sync (MySQL InnoDB)
 *
 * innodb_flush_log_at_trx_commit settings:
 * 0: Log written & synced once/second (may lose 1s data)
 * 1: Log written & synced on each commit (full durability)
 * 2: Log written on commit, synced once/second (lose 1s)
 */
 
/*
 * PATTERN 5: Parallel WAL (PostgreSQL 9.4+)
 *
 * Multiple backends can populate WAL concurrently.
 * Single sync still needed, but write phase parallelized.
 */
 
 
/*
 * PATTERN 6: Double-Write Buffer (MySQL InnoDB)
 *
 * Problem: Partial page writes on crash (torn pages).
 * Solution: Write pages to doublewrite buffer first.
 */
void doublewrite_flush(struct page **pages, int count) {
    /* Write all pages to sequential doublewrite space */
    for (int i = 0; i < count; i++) {
        pwrite(dblwr_fd, pages[i], PAGE_SIZE, dblwr_offset);
        dblwr_offset += PAGE_SIZE;
    }
    
    /* Single fsync for doublewrite */
    fdatasync(dblwr_fd);
    
    /* Now safe to write to actual locations */
    for (int i = 0; i < count; i++) {
        pwrite(datafile_fd, pages[i], PAGE_SIZE, pages[i]->offset);
    }
    
    /* On crash: recover torn pages from doublewrite */
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

#!/bin/bash
# Testing sync behavior and durability
 
# 1. STRACE: Verify sync calls are happening
strace -e fsync,fdatasync,sync,sync_file_range \
    ./my_database_app 2>&1 | head -50
 
# Expected output:
# fsync(5) = 0
# fdatasync(5) = 0
# ...
 
# 2. Track device I/O during sync
# In terminal 1:
iostat -x 1 /dev/sda
 
# In terminal 2:
fsync_test_program
 
# Look for:
# - Writes during sync
# - Device utilization spikes
# - await (average wait time)
 
# 3. BLKTRACE: Detailed block I/O tracing
blktrace -d /dev/sda -o trace &
./my_app --run-test
kill %1
blkparse trace.* | grep -i flush
# Look for FLUSH operations
 
# 4. Use dm-flakey to simulate failures
# Create a device that randomly fails
dmsetup create flakey --table "0 $(blockdev --getsz /dev/loop0) \
    flakey /dev/loop0 0 30 5 1 drop_writes"
# 30 seconds good, 5 seconds dropping writes
 
# Run your application against /dev/mapper/flakey
# Check data integrity after simulated failure
 
# 5. Crash testing with VMs
# - Run workload in VM
# - Abruptly terminate VM process
# - Boot VM and check data integrity
 
# 6. SystemTap probes for sync tracking
stap -e 'probe kernel.function("blkdev_issue_flush") { 
    printf("Flush on %s
", kernel_string($bdev->bd_disk->disk_name)) 
}'