Designing Cloud File Storage Systems

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

// Core version history data model
interface File {
    id: string;
    path: string;
    currentVersionId: string;  // Points to latest version
    createdAt: Date;
    createdBy: string;
    isDeleted: boolean;
    deletedAt?: Date;
    deletedBy?: string;
}
 
interface FileVersion {
    id: string;
    fileId: string;
    versionNumber: number;      // 1, 2, 3, ... (monotonic)
    
    // Content reference
    contentHash: string;        // SHA-256 of content
    size: number;               // Size in bytes
    storageRef: string;         // Reference to actual content
    
    // Delta information (for storage efficiency)
    storageType: 'full' | 'delta';
    baseVersionId?: string;     // If delta, the base version
    deltaSize?: number;         // Size of delta data
    
    // Modification metadata
    createdAt: Date;
    modifiedBy: string;
    deviceInfo: string;         // "John's MacBook Pro"
    clientVersion: string;      // "Dropbox 150.4.5423"
    
    // Change description
    changeType: 'create' | 'modify' | 'rename' | 'restore';
    previousPath?: string;      // If renamed, the old path
    restoredFromVersion?: number; // If restore, source version
    
    // Expiration
    expiresAt?: Date;           // When version can be deleted
    pinned: boolean;            // User-pinned, never auto-delete
}
 
// Query: Get all versions of a file
async function getFileVersions(fileId: string): Promise<FileVersion[]> {
    return db.versions
        .where({ fileId })
        .orderBy('versionNumber', 'desc')
        .limit(100);  // Paginate for files with many versions
}
 
// Query: Get a specific version for restoration
async function getVersion(fileId: string, version: number): Promise<FileVersion> {
    return db.versions.findOne({ fileId, versionNumber: version });
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

// Retention policy engine
interface RetentionPolicy {
    fullRetentionDays: number;     // Keep all versions
    dailySnapshotDays: number;     // Keep one per day
    weeklySnapshotDays: number;    // Keep one per week
    monthlySnapshotMonths: number; // Keep one per month
    keepPinnedForever: boolean;
}
 
class RetentionEnforcer {
    async enforcePolicy(
        fileId: string, 
        policy: RetentionPolicy
    ): Promise<void> {
        const now = new Date();
        const versions = await this.getVersions(fileId);
        
        for (const version of versions) {
            if (version.pinned && policy.keepPinnedForever) {
                continue; // Never delete pinned
            }
            
            const age = daysBetween(version.createdAt, now);
            
            // Full retention period: keep all
            if (age <= policy.fullRetentionDays) {
                continue;
            }
            
            // Daily snapshot period: keep best per day
            if (age <= policy.dailySnapshotDays) {
                if (this.isDailySnapshot(version, versions)) {
                    continue;
                }
                await this.markForDeletion(version);
                continue;
            }
            
            // Weekly snapshot period
            if (age <= policy.weeklySnapshotDays) {
                if (this.isWeeklySnapshot(version, versions)) {
                    continue;
                }
                await this.markForDeletion(version);
                continue;
            }
            
            // Monthly snapshot period
            if (age <= policy.monthlySnapshotMonths * 30) {
                if (this.isMonthlySnapshot(version, versions)) {
                    continue;
                }
                await this.markForDeletion(version);
                continue;
            }
            
            // Beyond all retention: delete
            await this.markForDeletion(version);
        }
    }
    
    // Is this the "representative" version for its day?
    isDailySnapshot(version: FileVersion, allVersions: FileVersion[]): boolean {
        const sameDay = allVersions.filter(v => 
            isSameDay(v.createdAt, version.createdAt)
        );
        
        // Keep the last version of each day
        const lastOfDay = sameDay.reduce((latest, v) => 
            v.createdAt > latest.createdAt ? v : latest
        );
        
        return version.id === lastOfDay.id;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

// Version restoration logic
class VersionRestorer {
    // Restore file to a previous version
    async restoreVersion(
        fileId: string, 
        targetVersionNumber: number,
        options: RestoreOptions
    ): Promise<FileVersion> {
        const targetVersion = await this.getVersion(fileId, targetVersionNumber);
        
        if (!targetVersion) {
            throw new VersionNotFoundError(fileId, targetVersionNumber);
        }
        
        switch (options.mode) {
            case 'replace':
                return this.restoreAsReplacement(fileId, targetVersion);
            
            case 'new_file':
                return this.restoreAsNewFile(fileId, targetVersion, options.newName);
            
            case 'download':
                return this.generateDownloadUrl(targetVersion);
        }
    }
    
    private async restoreAsReplacement(
        fileId: string, 
        targetVersion: FileVersion
    ): Promise<FileVersion> {
        // Get current file info
        const file = await this.getFile(fileId);
        const currentVersion = await this.getVersion(fileId, file.currentVersionNumber);
        
        // Create new version that references the old content
        // (No data copy needed if using content-addressed storage!)
        const newVersion = await this.createVersion({
            fileId,
            versionNumber: currentVersion.versionNumber + 1,
            contentHash: targetVersion.contentHash,
            size: targetVersion.size,
            storageRef: targetVersion.storageRef,
            changeType: 'restore',
            restoredFromVersion: targetVersion.versionNumber,
            modifiedBy: this.currentUser.id,
        });
        
        // Update file to point to new version
        await this.updateFile(fileId, { 
            currentVersionId: newVersion.id,
            modifiedAt: new Date(),
        });
        
        // Trigger sync to all devices
        await this.notifySync(fileId, newVersion);
        
        return newVersion;
    }
    
    // Reconstruct content for delta-stored version
    async getVersionContent(version: FileVersion): Promise<Buffer> {
        if (version.storageType === 'full') {
            return this.storage.get(version.storageRef);
        }
        
        // Delta chain: need to reconstruct
        const chain = await this.getDeltaChain(version);
        
        // Start from the nearest full snapshot
        let content = await this.storage.get(chain[0].storageRef);
        
        // Apply deltas in sequence
        for (let i = 1; i < chain.length; i++) {
            const delta = await this.storage.get(chain[i].storageRef);
            content = this.applyDelta(content, delta);
        }
        
        return content;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

// Ransomware detection heuristics
interface SyncBatch {
    fileChanges: FileChange[];
    timestamp: Date;
    deviceId: string;
}
 
class RansomwareDetector {
    // Check sync batch for ransomware indicators
    async checkForRansomware(batch: SyncBatch): Promise<ThreatAssessment> {
        const indicators: ThreatIndicator[] = [];
        
        // Indicator 1: Mass file extension changes
        const extensionChanges = batch.fileChanges.filter(c => 
            c.type === 'rename' && 
            this.hasExtensionChange(c.oldPath, c.newPath)
        );
        if (extensionChanges.length > 50) {
            indicators.push({
                type: 'MASS_EXTENSION_CHANGE',
                severity: 'HIGH',
                details: `${extensionChanges.length} files renamed with extension changes`,
            });
        }
        
        // Indicator 2: Known ransomware extensions
        const suspiciousExtensions = ['.encrypted', '.locked', '.crypted', '.crypt'];
        const knownRansomware = batch.fileChanges.filter(c =>
            suspiciousExtensions.some(ext => c.newPath?.endsWith(ext))
        );
        if (knownRansomware.length > 0) {
            indicators.push({
                type: 'KNOWN_RANSOMWARE_EXTENSION',
                severity: 'CRITICAL',
                details: `Files with known ransomware extensions detected`,
            });
        }
        
        // Indicator 3: Entropy increase (encrypted files have high entropy)
        const entropyIncreases = batch.fileChanges.filter(c =>
            c.type === 'modify' && 
            this.calculateEntropy(c.newContent) > 0.95 &&  // Near-random
            this.calculateEntropy(c.oldContent) < 0.7      // Was structured
        );
        if (entropyIncreases.length > 10) {
            indicators.push({
                type: 'ENTROPY_INCREASE',
                severity: 'HIGH',
                details: `${entropyIncreases.length} files show encryption-like entropy increase`,
            });
        }
        
        // Indicator 4: Rapid bulk modifications
        const recentBatches = await this.getRecentBatches(batch.deviceId, 60); // Last hour
        const totalChanges = recentBatches.reduce((sum, b) => sum + b.fileChanges.length, 0);
        if (totalChanges > 500) {
            indicators.push({
                type: 'RAPID_BULK_MODIFICATION',
                severity: 'MEDIUM',
                details: `${totalChanges} file changes in last hour`,
            });
        }
        
        return this.assessThreat(indicators);
    }
    
    // Respond to detected threat
    async handleThreat(
        assessment: ThreatAssessment, 
        batch: SyncBatch
    ): Promise<void> {
        if (assessment.overallSeverity === 'CRITICAL') {
            // Quarantine device immediately
            await this.quarantineDevice(batch.deviceId);
            
            // Hold sync batch for review
            await this.holdBatch(batch);
            
            // Alert user on other devices
            await this.alertUser(batch.userId, {
                title: 'Suspicious Activity Detected',
                body: 'Sync from ${batch.deviceInfo} has been paused. Possible ransomware detected.',
                actions: ['Review Changes', 'Unlink Device', 'Restore Files'],
            });
            
            // Alert admin for enterprise accounts
            await this.alertAdmin(batch.userId, assessment);
        }
    }
}