Log Retention for Compliance

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

interface RetentionPolicy {
  tierId: 1 | 2 | 3 | 4 | 5;
  retentionDays: number;
  storageClass: 'HOT' | 'WARM' | 'COLD' | 'ARCHIVE';
  immutabilityRequired: boolean;
  legalHoldEligible: boolean;
}
 
interface RetentionRule {
  name: string;
  priority: number; // Higher priority wins on conflict
  condition: (event: AuditEvent) => boolean;
  policy: RetentionPolicy;
}
 
class RetentionClassifier {
  private rules: RetentionRule[] = [
    // Tier 1: Extended Retention
    {
      name: 'Financial Transactions',
      priority: 100,
      condition: (e) => e.category === 'FINANCIAL' || 
                        e.target?.type === 'FINANCIAL_RECORD',
      policy: {
        tierId: 1,
        retentionDays: 2557, // 7 years
        storageClass: 'ARCHIVE',
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    {
      name: 'Privileged Access',
      priority: 95,
      condition: (e) => e.actor?.role === 'ADMIN' || 
                        e.action?.operation === 'PRIVILEGE_ESCALATION',
      policy: {
        tierId: 1,
        retentionDays: 2557,
        storageClass: 'WARM',
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    
    // Tier 2: Standard Retention
    {
      name: 'PHI Access',
      priority: 85,
      condition: (e) => e.target?.classification === 'PHI',
      policy: {
        tierId: 2,
        retentionDays: 2192, // 6 years (HIPAA)
        storageClass: 'WARM',
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    {
      name: 'Authentication Events',
      priority: 80,
      condition: (e) => e.category === 'AUTHENTICATION',
      policy: {
        tierId: 2,
        retentionDays: 1826, // 5 years
        storageClass: 'WARM',
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    {
      name: 'Configuration Changes',
      priority: 75,
      condition: (e) => e.category === 'CONFIGURATION' ||
                        e.action?.operation === 'ADMIN',
      policy: {
        tierId: 2,
        retentionDays: 1095, // 3 years
        storageClass: 'COLD',
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    
    // Tier 3: Operational Retention
    {
      name: 'Cardholder Data Access',
      priority: 70,
      condition: (e) => e.target?.classification === 'PCI',
      policy: {
        tierId: 3,
        retentionDays: 365, // 1 year (PCI DSS)
        storageClass: 'HOT', // 90 days hot requirement
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    {
      name: 'Standard Data Access',
      priority: 50,
      condition: (e) => e.category === 'DATA_ACCESS',
      policy: {
        tierId: 3,
        retentionDays: 730, // 2 years
        storageClass: 'WARM',
        immutabilityRequired: true,
        legalHoldEligible: true,
      },
    },
    
    // Default: Tier 4
    {
      name: 'Default Audit Event',
      priority: 0,
      condition: () => true,
      policy: {
        tierId: 4,
        retentionDays: 365,
        storageClass: 'COLD',
        immutabilityRequired: false,
        legalHoldEligible: false,
      },
    },
  ];
  
  classify(event: AuditEvent): RetentionPolicy {
    // Find highest-priority matching rule
    const matchingRules = this.rules
      .filter(rule => rule.condition(event))
      .sort((a, b) => b.priority - a.priority);
    
    return matchingRules[0].policy;
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

┌─────────────────────────────────────────────────────────────────────────────────┐
│                              HOT TIER (0-90 days)                                │
│                                                                                  │
│  ┌────────────────────────────────────────────────────────────────────────────┐ │
│  │                    ELASTICSEARCH / OPENSEARCH CLUSTER                       │ │
│  │                                                                             │ │
│  │  • Full-text search across all fields        • Sub-second query response  │ │
│  │  • Real-time dashboards and alerting         • Index lifecycle management │ │
│  │  • Aggregations, analytics                   • Hot-warm-cold node types   │ │
│  │                                                                             │ │
│  │  Storage: SSD/NVMe          Replication: 2x         Cost: $$$              │ │
│  └────────────────────────────────────────────────────────────────────────────┘ │
│                                      │                                           │
│                                      │ ILM Policy: After 90 days                │
│                                      ▼                                           │
└─────────────────────────────────────────────────────────────────────────────────┘
 
                                       │
                                       ▼
┌─────────────────────────────────────────────────────────────────────────────────┐
│                            WARM TIER (90 days - 1 year)                          │
│                                                                                  │
│  ┌────────────────────────────────────────────────────────────────────────────┐ │
│  │                       OBJECT STORAGE (S3 Standard-IA)                       │ │
│  │                                                                             │ │
│  │  • Parquet format for efficient querying     • Partitioned by date/type   │ │
│  │  • Queryable via Athena/Presto              • Compressed with ZSTD        │ │
│  │  • Reduced index (key fields only)          • Object Lock: Governance     │ │
│  │                                                                             │ │
│  │  Storage: HDD-backed object    Replication: 3x AZ     Cost: $$             │ │
│  └────────────────────────────────────────────────────────────────────────────┘ │
│                                      │                                           │
│                                      │ Lifecycle: After 1 year                  │
│                                      ▼                                           │
└─────────────────────────────────────────────────────────────────────────────────┘
 
                                       │
                                       ▼
┌─────────────────────────────────────────────────────────────────────────────────┐
│                           COLD TIER (1-3 years)                                  │
│                                                                                  │
│  ┌────────────────────────────────────────────────────────────────────────────┐ │
│  │                          OBJECT STORAGE (Glacier)                           │ │
│  │                                                                             │ │
│  │  • Highly compressed archives               • Expedited retrieval: 5 min   │ │
│  │  • Metadata index for discovery             • Standard retrieval: 5 hours  │ │
│  │  • Object Lock: Compliance mode             • Bulk retrieval: 12 hours     │ │
│  │                                                                             │ │
│  │  Storage: Tape-equivalent      Replication: 3+ AZ     Cost: $              │ │
│  └────────────────────────────────────────────────────────────────────────────┘ │
│                                      │                                           │
│                                      │ Lifecycle: After 3 years                 │
│                                      ▼                                           │
└─────────────────────────────────────────────────────────────────────────────────┘
 
                                       │
                                       ▼
┌─────────────────────────────────────────────────────────────────────────────────┐
│                        ARCHIVE TIER (3-7+ years)                                 │
│                                                                                  │
│  ┌────────────────────────────────────────────────────────────────────────────┐ │
│  │                     OBJECT STORAGE (Glacier Deep Archive)                   │ │
│  │                                                                             │ │
│  │  • Maximum compression                      • Retrieval: 12-48 hours       │ │
│  │  • Minimal metadata (legal discovery)       • Object Lock: Compliance      │ │
│  │  • Cryptographic verification receipts      • Legal hold support           │ │
│  │                                                                             │ │
│  │  Storage: Deep archive         Replication: Multi-region  Cost: ¢          │ │
│  └────────────────────────────────────────────────────────────────────────────┘ │
└─────────────────────────────────────────────────────────────────────────────────┘

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

interface LogLifecycleManager {
  /**
   * Lifecycle state machine for audit logs:
   * 
   * ACTIVE → WARM → COLD → ARCHIVE → DELETED
   *            ↑              ↑          ↑
   *            └── LEGAL_HOLD can freeze at any stage
   */
}
 
class AuditLogLifecycleManager {
  private s3: S3Client;
  private legalHoldService: LegalHoldService;
  private metadataIndex: MetadataIndex;
  
  /**
   * Execute daily lifecycle processing
   * Runs as scheduled job with exclusive lock
   */
  async processLifecycle(): Promise<LifecycleReport> {
    const report = new LifecycleReport();
    
    // Phase 1: Transition eligible logs between tiers
    await this.processTransitions(report);
    
    // Phase 2: Delete logs past retention (respecting legal holds)
    await this.processDeletions(report);
    
    // Phase 3: Verify integrity of archived data
    await this.verifyArchiveIntegrity(report);
    
    return report;
  }
  
  private async processTransitions(report: LifecycleReport): Promise<void> {
    // Hot → Warm (after 90 days)
    const hotToWarm = await this.findLogsForTransition('HOT', 90);
    for (const batch of chunk(hotToWarm, 1000)) {
      await this.transitionBatch(batch, 'HOT', 'WARM');
      report.transitioned.hotToWarm += batch.length;
    }
    
    // Warm → Cold (after 1 year)
    const warmToCold = await this.findLogsForTransition('WARM', 365);
    for (const batch of chunk(warmToCold, 1000)) {
      await this.transitionBatch(batch, 'WARM', 'COLD');
      report.transitioned.warmToCold += batch.length;
    }
    
    // Cold → Archive (after 3 years)
    const coldToArchive = await this.findLogsForTransition('COLD', 1095);
    for (const batch of chunk(coldToArchive, 1000)) {
      await this.transitionBatch(batch, 'COLD', 'ARCHIVE');
      report.transitioned.coldToArchive += batch.length;
    }
  }
  
  private async processDeletions(report: LifecycleReport): Promise<void> {
    // Find logs past retention period
    const deletionCandidates = await this.findLogsForDeletion();
    
    for (const log of deletionCandidates) {
      // CRITICAL: Check for legal hold before any deletion
      const hasLegalHold = await this.legalHoldService.checkHold(log.id);
      
      if (hasLegalHold) {
        // Skip deletion, log is under legal preservation
        report.deletionSkipped.legalHold++;
        continue;
      }
      
      // Verify retention period has truly elapsed
      const retentionEndDate = this.calculateRetentionEnd(log);
      if (new Date() < retentionEndDate) {
        report.deletionSkipped.retentionNotMet++;
        continue;
      }
      
      // Execute deletion with audit trail
      await this.executeSecureDeletion(log, report);
    }
  }
  
  private async executeSecureDeletion(
    log: AuditLogReference,
    report: LifecycleReport
  ): Promise<void> {
    // Create deletion audit record BEFORE deletion
    const deletionRecord = await this.auditService.recordDeletion({
      logId: log.id,
      reason: 'RETENTION_EXPIRED',
      retentionPolicy: log.retentionTier,
      originalCreatedAt: log.createdAt,
      deletedAt: new Date(),
      performedBy: 'LIFECYCLE_MANAGER',
    });
    
    // Verify deletion record is persisted
    await this.verifyDeletionRecordPersisted(deletionRecord);
    
    // Now execute deletion
    await this.s3.deleteObject({
      Bucket: this.getBucketForTier(log.currentTier),
      Key: log.storageKey,
    });
    
    // Remove from metadata index
    await this.metadataIndex.remove(log.id);
    
    report.deleted++;
  }
  
  /**
   * Handle legal hold application
   * Called when litigation or investigation begins
   */
  async applyLegalHold(
    holdId: string,
    criteria: LegalHoldCriteria
  ): Promise<LegalHoldApplication> {
    // Find all logs matching criteria
    const affectedLogs = await this.metadataIndex.search(criteria);
    
    // Apply S3 Object Lock legal hold to each
    for (const log of affectedLogs) {
      await this.s3.putObjectLegalHold({
        Bucket: this.getBucketForTier(log.currentTier),
        Key: log.storageKey,
        LegalHold: { Status: 'ON' },
      });
    }
    
    // Record hold in legal hold service
    return this.legalHoldService.createHold({
      holdId,
      criteria,
      affectedLogIds: affectedLogs.map(l => l.id),
      appliedAt: new Date(),
    });
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

interface ArchivePackage {
  /**
   * Self-contained archive that can be read by future systems
   */
  
  // Metadata that explains how to read this archive
  manifest: {
    version: '1.0';
    created: string; // ISO 8601 timestamp
    creator: string; // System that created this archive
    
    // Content description
    contentType: 'AUDIT_LOGS';
    recordCount: number;
    dateRange: { start: string; end: string };
    
    // Format information
    dataFormat: {
      type: 'PARQUET';
      version: '2.6';
      compression: 'ZSTD';
      schemaVersion: string;
      schemaLocation: './schema.json';
    };
    
    // Encryption (if applicable)
    encryption: {
      algorithm: 'AES-256-GCM';
      keyId: string;
      keyManagementSystem: 'AWS_KMS';
      keyWrappingAlgorithm: 'RSA-OAEP-256';
    };
    
    // Integrity verification
    integrity: {
      algorithm: 'SHA-256';
      dataChecksum: string;
      manifestChecksum: string; // Self-referential after data checksum
    };
    
    // External anchoring references
    anchoring: {
      merkleRoot: string;
      tsaReceipt: string; // Path to RFC 3161 receipt
      blockchainAnchor?: {
        chain: 'bitcoin';
        txid: string;
        blockNumber: number;
      };
    };
  };
  
  // Embedded schema for self-description
  schema: {
    type: 'object';
    properties: {
      eventId: { type: 'string'; description: 'UUID v7 identifier' };
      timestamp: { type: 'string'; format: 'date-time' };
      // ... complete field definitions
    };
  };
  
  // The actual audit log data
  data: Buffer; // Parquet file content
  
  // Integrity proofs
  proofs: {
    merkleProofs: MerkleProof[]; // Per-record proofs
    tsaReceipt: Buffer;          // RFC 3161 timestamp token
  };
  
  // Decoding tools (optional, for maximum preservation)
  tools?: {
    parquetReader: {
      platform: 'linux-x64';
      binary: Buffer;
      checksum: string;
    };
    verificationScript: {
      language: 'python3';
      requirements: string; // pip freeze
      script: string;
    };
  };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

class AuditLogOptimizer {
  /**
   * Transform raw JSON logs to optimized Parquet format
   * Typical compression ratio: 8-12x
   */
  async optimizeForArchival(
    logs: AuditEvent[],
    options: OptimizationOptions
  ): Promise<OptimizedArchive> {
    
    // Step 1: Field pruning - remove fields not needed for compliance
    const pruned = logs.map(log => this.pruneNonEssentialFields(log, options));
    
    // Step 2: Convert to columnar format
    // Parquet stores each column together, enabling better compression
    const parquetBuffer = await this.convertToParquet(pruned, {
      compression: 'ZSTD',          // Best compression ratio
      compressionLevel: 19,          // Maximum compression (slow write, fast read)
      rowGroupSize: 100_000,         // Balance between compression and granularity
      version: '2.6',
    });
    
    // Step 3: Calculate storage metrics
    const originalSize = JSON.stringify(logs).length;
    const optimizedSize = parquetBuffer.length;
    const ratio = originalSize / optimizedSize;
    
    return {
      data: parquetBuffer,
      metadata: {
        recordCount: logs.length,
        originalSizeBytes: originalSize,
        optimizedSizeBytes: optimizedSize,
        compressionRatio: ratio,
        format: 'PARQUET',
        compression: 'ZSTD',
      },
    };
  }
  
  /**
   * Cost projection for retention policy
   */
  calculateRetentionCost(
    dailyLogVolume: DataSize,
    retentionDays: number,
    tierStrategy: TierStrategy
  ): CostProjection {
    
    // Storage rates (USD per GB-month, approximate)
    const rates = {
      HOT: 0.023,      // S3 Standard
      WARM: 0.0125,    // S3 Standard-IA
      COLD: 0.004,     // Glacier Instant
      ARCHIVE: 0.00099, // Glacier Deep Archive
    };
    
    let totalCost = 0;
    let currentVolume = 0;
    
    // Model cumulative storage over retention period
    for (let day = 1; day <= retentionDays; day++) {
      currentVolume += dailyLogVolume.gigabytes;
      
      // Apply compression based on age
      const compressionRatio = this.getCompressionRatio(day);
      const storedVolume = currentVolume / compressionRatio;
      
      // Determine storage tier based on age
      const tier = this.getTierForAge(day, tierStrategy);
      
      // Calculate monthly cost contribution
      totalCost += (storedVolume * rates[tier]) / 30;
    }
    
    return {
      totalRetentionCostUSD: totalCost,
      averageMonthlyCostUSD: totalCost / (retentionDays / 30),
      costPerMillionEvents: this.calculatePerEventCost(totalCost, dailyLogVolume, retentionDays),
      breakdownByTier: this.getBreakdownByTier(retentionDays, tierStrategy),
    };
  }
}
 
// Example: 1TB/day of logs for 7 years
// Without optimization: ~$500K/year ($3.5M total)
// With tiering + compression: ~$50K/year ($350K total)
// Savings: 90%