Buffer Cache

# Application writes to a file rapidly:
write(fd, "Hello", 5);     # Modify bytes 0-4
write(fd, "World", 5);     # Modify bytes 5-9  
write(fd, "!", 1);         # Modify byte 10
write(fd, "...", 3);       # Modify bytes 11-13
 
# With WRITE-THROUGH (4 disk operations):
Time 0ms:   Write bytes 0-4 to disk     [5ms disk latency]
Time 5ms:   Write bytes 5-9 to disk     [5ms disk latency]
Time 10ms:  Write bytes 10 to disk      [5ms disk latency]
Time 15ms:  Write bytes 11-13 to disk   [5ms disk latency]
Total: 20ms for 4 write syscalls
 
# With WRITE-BACK (1 disk operation, deferred):
Time 0.001ms:  Update bytes 0-4 in cache   [~1 µs]
Time 0.002ms:  Update bytes 5-9 in cache   [~1 µs]
Time 0.003ms:  Update bytes 10 in cache    [~1 µs]
Time 0.004ms:  Update bytes 11-13 in cache [~1 µs]
Total: ~0.004ms for 4 write syscalls (5000x faster!)
 
# Later, writeback thread combines all into ONE disk write:
Time 5000ms:  Write bytes 0-13 to disk    [5ms disk latency]
 
# Even better: if bytes 0-13 are modified 100 more times
# before writeback, still only ONE disk write occurs!

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

/*
 * Linux page flags related to dirty state
 * Defined in include/linux/page-flags.h
 */
 
/* Page flag bits */
#define PG_dirty        4    /* Page has been modified */
#define PG_writeback    15   /* Page is being written back now */
#define PG_reclaim      18   /* Page marked for reclaim */
 
/*
 * Dirty page lifecycle:
 *
 * 1. Page read from disk -> Clean (PG_dirty clear)
 * 2. Process modifies page -> Mark dirty (PG_dirty set)
 * 3. Writeback initiated -> PG_writeback set, PG_dirty stays
 * 4. Writeback completes -> PG_writeback cleared, PG_dirty cleared
 *
 * If page modified during writeback:
 * 3b. PG_dirty gets re-set while PG_writeback is set
 * 4b. After writeback, PG_writeback cleared but PG_dirty remains
 *     -> Page needs another writeback
 */
 
/* Check and set dirty state */
static inline int PageDirty(struct page *page) {
    return test_bit(PG_dirty, &page->flags);
}
 
static inline void SetPageDirty(struct page *page) {
    set_bit(PG_dirty, &page->flags);
}
 
static inline void ClearPageDirty(struct page *page) {
    clear_bit(PG_dirty, &page->flags);
}
 
/*
 * Mark a page as dirty (called when page is modified)
 * This is a simplified version - real implementation handles:
 * - Inode dirty list management
 * - Memory cgroup accounting
 * - Block device backing
 */
int set_page_dirty(struct page *page) {
    struct address_space *mapping = page_mapping(page);
    
    if (!PageDirty(page)) {
        /* First time dirtying this page */
        SetPageDirty(page);
        
        if (mapping) {
            /* Add to inode's dirty page tracking */
            spin_lock(&mapping->private_lock);
            
            /* Add to radix tree dirty tag */
            radix_tree_tag_set(&mapping->page_tree,
                              page_index(page), PAGECACHE_TAG_DIRTY);
            
            /* If this is first dirty page for inode, add inode to 
             * backing device's dirty list */
            if (mapping->nrpages_dirty++ == 0) {
                inode_mark_dirty(mapping->host);
            }
            
            spin_unlock(&mapping->private_lock);
        }
        
        return 1;  /* Page was newly dirtied */
    }
    
    return 0;  /* Page was already dirty */
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

/*
 * Per-Backing Device Info (BDI) writeback architecture
 * 
 * Each block device has its own writeback thread:
 *   flush-8:0   <- /dev/sda (major 8, minor 0)
 *   flush-8:16  <- /dev/sdb (major 8, minor 16)
 *   flush-253:0 <- /dev/dm-0 (device mapper)
 *
 * This allows parallel writeback to different devices
 */
 
struct bdi_writeback {
    struct backing_dev_info *bdi;    /* Parent BDI */
    
    /* Lists of dirty inodes */
    struct list_head b_dirty;        /* Dirty inodes */
    struct list_head b_io;           /* Inodes being written */
    struct list_head b_more_io;      /* Inodes needing more I/O */
    struct list_head b_dirty_time;   /* Time-expired dirty inodes */
    
    /* Writeback control */
    unsigned long last_old_flush;    /* When last periodic flush ran */
    struct delayed_work dwork;       /* The actual work item */
};
 
/*
 * Main writeback function - called by flusher thread
 */
void wb_workfn(struct work_struct *work) {
    struct bdi_writeback *wb = container_of(to_delayed_work(work),
                                             struct bdi_writeback, dwork);
    long pages_written;
    
    /* Set up writeback control */
    struct writeback_control wbc = {
        .sync_mode = WB_SYNC_NONE,   /* Non-blocking */
        .nr_to_write = LONG_MAX,     /* No page limit */
        .range_start = 0,
        .range_end = LLONG_MAX,
    };
    
    /* Write dirty pages until we've caught up */
    while (wb_has_dirty_io(wb)) {
        pages_written = writeback_inodes_wb(wb, &wbc);
        
        if (pages_written <= 0)
            break;
        
        /* Yield CPU to prevent monopolizing */
        cond_resched();
    }
    
    /* Schedule next run if there's more work or for periodic flush */
    if (wb_has_dirty_io(wb) || time_for_periodic_flush(wb)) {
        queue_delayed_work(bdi_wq, &wb->dwork,
                          msecs_to_jiffies(dirty_writeback_interval));
    }
}
 
/*
 * Write back dirty inodes for this device
 */
long writeback_inodes_wb(struct bdi_writeback *wb,
                         struct writeback_control *wbc) {
    struct inode *inode;
    long pages_written = 0;
    
    spin_lock(&wb->list_lock);
    
    while (!list_empty(&wb->b_io)) {
        inode = list_first_entry(&wb->b_io, struct inode, i_io_list);
        
        spin_unlock(&wb->list_lock);
        
        /* Write this inode's dirty pages */
        pages_written += writeback_single_inode(inode, wbc);
        
        spin_lock(&wb->list_lock);
        
        /* Move inode based on result */
        if (inode_has_dirty_pages(inode)) {
            /* More pages to write - put on b_more_io */
            list_move(&inode->i_io_list, &wb->b_more_io);
        } else {
            /* All clean - remove from dirty lists */
            list_del_init(&inode->i_io_list);
        }
    }
    
    spin_unlock(&wb->list_lock);
    return pages_written;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

#!/bin/bash
# Writeback tuning examples
 
# View current values
cat /proc/sys/vm/dirty_ratio
cat /proc/sys/vm/dirty_background_ratio
cat /proc/sys/vm/dirty_expire_centisecs
cat /proc/sys/vm/dirty_writeback_centisecs
 
# SCENARIO 1: Database Server (prioritize durability)
# Low dirty thresholds = less data at risk
# Frequent writeback = faster flush
echo 5 > /proc/sys/vm/dirty_ratio
echo 2 > /proc/sys/vm/dirty_background_ratio
echo 500 > /proc/sys/vm/dirty_expire_centisecs    # 5 seconds
echo 100 > /proc/sys/vm/dirty_writeback_centisecs # 1 second
 
# SCENARIO 2: File Server (prioritize throughput)
# Higher dirty limits = more write coalescing
# Slower writeback = fewer interruptions
echo 40 > /proc/sys/vm/dirty_ratio
echo 20 > /proc/sys/vm/dirty_background_ratio
echo 6000 > /proc/sys/vm/dirty_expire_centisecs   # 60 seconds
echo 1000 > /proc/sys/vm/dirty_writeback_centisecs # 10 seconds
 
# SCENARIO 3: Machine with limited RAM (128MB)
# Use absolute bytes instead of percentages
# Prevents small RAM from having tiny dirty limits
echo 0 > /proc/sys/vm/dirty_ratio  # Disable ratio
echo 50000000 > /proc/sys/vm/dirty_bytes  # 50MB hard limit
echo 0 > /proc/sys/vm/dirty_background_ratio
echo 25000000 > /proc/sys/vm/dirty_background_bytes  # 25MB background
 
# SCENARIO 4: Laptop (prioritize battery life)
# Less frequent writeback = more disk idle time
# Allow disk to spin down longer
echo 90 > /proc/sys/vm/laptop_mode  # Enable laptop mode (seconds)
echo 60 > /proc/sys/vm/dirty_ratio
echo 40 > /proc/sys/vm/dirty_background_ratio
echo 60000 > /proc/sys/vm/dirty_expire_centisecs   # 10 minutes
echo 60000 > /proc/sys/vm/dirty_writeback_centisecs # 10 minutes
 
# Monitor dirty page status
watch -n1 'grep -E "Dirty|Writeback" /proc/meminfo'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

/*
 * Write barriers ensure ordering across cache layers
 *
 * A write barrier has this semantics:
 *   All writes BEFORE the barrier complete to stable storage
 *   BEFORE any writes AFTER the barrier begin.
 */
 
/*
 * File system journal write with barriers
 * Ensures data integrity after crash
 */
void journal_commit_transaction(journal_t *journal, 
                                 transaction_t *transaction) {
    
    /* Step 1: Write journal descriptor block */
    /* Describes what data is in this transaction */
    write_journal_descriptor(journal, transaction);
    
    /* Step 2: Write all data/metadata blocks to journal */
    /* These are copies that can be replayed on crash */
    list_for_each_entry(block, &transaction->blocks, list) {
        write_to_journal(journal, block);
    }
    
    /* BARRIER: Ensure all journal data is on stable storage */
    /* before we write the commit record */
    blkdev_issue_flush(journal->j_dev, GFP_KERNEL, NULL);
    
    /* Step 3: Write commit record */
    /* This marks the transaction as complete */
    write_journal_commit(journal, transaction);
    
    /* BARRIER: Ensure commit record is on stable storage */
    /* before we write to final locations */
    blkdev_issue_flush(journal->j_dev, GFP_KERNEL, NULL);
    
    /* Step 4: Now safe to write to actual file system locations */
    /* (called "checkpointing") */
    checkpoint_transaction(journal, transaction);
}
 
/*
 * The blkdev_issue_flush function
 * Sends a cache flush command to the storage device
 */
int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
                       sector_t *error_sector) {
    struct bio *bio;
    int ret;
    
    /* Allocate a bio for the flush command */
    bio = bio_alloc(GFP_KERNEL, 0);
    bio_set_dev(bio, bdev);
    bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;  /* Flush operation */
    
    /* Submit and wait */
    ret = submit_bio_wait(bio);
    
    bio_put(bio);
    return ret;
}
 
/*
 * Storage device perspective:
 *
 * When device receives FLUSH command:
 * 1. Complete all pending writes in device cache
 * 2. Force data to stable media (platters for HDD, cells for SSD)
 * 3. Return success only after data is truly persisted
 *
 * Commands: 
 *   SATA: FLUSH CACHE / FLUSH CACHE EXT
 *   SCSI: SYNCHRONIZE CACHE
 *   NVMe: Flush command
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

/*
 * Demonstrating proper durable write patterns
 */
 
#include <unistd.h>
#include <fcntl.h>
 
/*
 * WRONG: Data can be lost on crash
 * The close() doesn't guarantee durability!
 */
void write_file_wrong(const char *path, const char *data, size_t len) {
    int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
    write(fd, data, len);
    close(fd);  /* Data may still be in buffer cache! */
}
 
/*
 * CORRECT: Data is durable after fsync returns
 */
void write_file_correct(const char *path, const char *data, size_t len) {
    int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
    write(fd, data, len);
    
    /* fsync ensures data + metadata reach stable storage */
    if (fsync(fd) != 0) {
        perror("fsync failed");
        /* Handle error - data may not be durable! */
    }
    
    close(fd);
}
 
/*
 * Atomic file update using rename
 * This is the GOLD STANDARD for durable writes
 */
void atomic_write_file(const char *path, const char *data, size_t len) {
    char tmp_path[256];
    snprintf(tmp_path, sizeof(tmp_path), "%s.tmp.XXXXXX", path);
    
    /* Create temporary file */
    int fd = mkstemp(tmp_path);
    if (fd < 0) {
        perror("mkstemp");
        return;
    }
    
    /* Write data */
    ssize_t written = write(fd, data, len);
    if (written != len) {
        close(fd);
        unlink(tmp_path);
        return;
    }
    
    /* CRITICAL: fsync the file data */
    if (fsync(fd) != 0) {
        close(fd);
        unlink(tmp_path);
        return;
    }
    close(fd);
    
    /* Rename is atomic on POSIX systems */
    if (rename(tmp_path, path) != 0) {
        unlink(tmp_path);
        return;
    }
    
    /* CRITICAL: fsync the directory to make rename durable */
    int dir_fd = open(dirname(path), O_RDONLY);
    if (dir_fd >= 0) {
        fsync(dir_fd);
        close(dir_fd);
    }
    
    /* Now the update is fully durable and atomic:
     * - If crash before rename: old file unchanged
     * - If crash after rename: new file complete
     * Never a partial or corrupted state!
     */
}
 
/*
 * fdatasync vs fsync: When to use which
 */
void fdatasync_example(int fd) {
    /*
     * fdatasync() is like fsync() but:
     * - Only syncs data, not all metadata
     * - Skips metadata that doesn't affect data retrieval
     *   (e.g., access time, modification time)
     * - Still syncs size if file was extended
     *
     * Faster when only data integrity matters
     */
    
    /* Use fdatasync for performance-critical paths */
    fdatasync(fd);  /* ~30% faster than fsync on some systems */
    
    /* Use fsync when metadata matters */
    fsync(fd);  /* Guarantees timestamps, permissions, etc. */
}

# System has 16GB RAM
# dirty_background_ratio = 10% (1.6GB)
# dirty_ratio = 20% (3.2GB)
# dirty_writeback_centisecs = 500 (5 seconds)
 
Phase 1: Fast writes (seconds 0-1)
├── Application writes 100MB burst
├── All writes go to page cache (memory speed: ~10GB/s)
├── Takes ~0.01 seconds to complete all writes
├── Dirty pages: ~100MB
├── Below background threshold: NO writeback triggered
└── Application continues immediately
 
Phase 2: Background writeback (seconds 5-6)
├── Timer fires after 5 seconds
├── Writeback thread wakes up
├── Writes 100MB to disk (disk speed: ~100MB/s)
├── Takes ~1 second
└── Dirty pages: 0MB
 
# What if application writes 5GB burst?
 
Phase 1: Fast writes (seconds 0-0.5)
├── Application writes 1.6GB (reaches background threshold)
├── Writeback starts in background
├── Application continues writing...
├── Application writes 3.2GB (reaches dirty_ratio!)
├── APPLICATION BLOCKS - cannot write more
└── Waits until dirty pages drop below 3.2GB
 
Phase 2: Throttled writes (seconds 0.5-15)
├── Writeback thread runs at disk speed (~100MB/s)
├── Application writes blocked intermittently
├── Each write that pushes over limit causes ~10ms sleep
├── Effective write speed: ~100MB/s (disk limited)
└── 5GB takes ~50 seconds instead of 0.5 seconds