Data Retention Policies

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154

# Enterprise Data Retention Policy Schema
# Defines retention rules for all data categories
 
version: "2.0"
effective_date: "2024-01-01"
review_schedule: "annual"
policy_owner: "chief_data_officer"
 
data_categories:
  # Customer Personal Data
  - category: "customer_pii"
    description: "Customer personally identifiable information"
    includes:
      - "name, email, phone, address"
      - "date of birth, gender"
      - "account credentials (hashed)"
    retention:
      period: "account_lifetime + 30 days"
      trigger: "account_closure_date"
      rationale: "GDPR data minimization; retained briefly for reactivation window"
    archive:
      after: "notification"
      tier: "cold_storage"
    deletion:
      method: "cryptographic_erasure"
      verification: "sampling_audit"
 
  # Transaction Records
  - category: "financial_transactions"
    description: "Purchase, payment, and billing records"
    includes:
      - "transaction amounts, dates, items"
      - "payment method references (tokenized)"
      - "invoices and receipts"
    retention:
      period: "7 years"
      trigger: "transaction_date"
      rationale: "IRS/SOX requirements for financial records"
    archive:
      after: "1 year"
      tier: "archive_storage"
    deletion:
      method: "secure_delete"
      verification: "certificate_of_destruction"
 
  # System Logs
  - category: "application_logs"
    description: "Application event and error logs"
    includes:
      - "HTTP access logs"
      - "error and exception logs"
      - "performance metrics"
    retention:
      period: "90 days"
      trigger: "log_timestamp"
      rationale: "Operational troubleshooting; minimize PII exposure in logs"
    archive:
      after: "30 days"
      tier: "cold_logs"
    deletion:
      method: "standard_delete"
      verification: "automated_confirmation"
 
  # Security Audit Logs
  - category: "security_audit_logs"
    description: "Authentication, authorization, and security events"
    includes:
      - "login attempts (success/failure)"
      - "permission changes"
      - "data access audit trails"
    retention:
      period: "3 years"
      trigger: "event_timestamp"
      rationale: "SOC 2 / compliance audit requirements"
    archive:
      after: "1 year"
      tier: "immutable_archive"
    deletion:
      method: "verified_destruction"
      verification: "compliance_audit"
 
  # Marketing Data
  - category: "marketing_analytics"
    description: "Marketing campaign data and user preferences"
    includes:
      - "email campaign metrics"
      - "consent records"
      - "preference data"
    retention:
      period: "consent_validity OR 3 years"
      trigger: "last_interaction_date OR consent_withdrawal"
      rationale: "GDPR consent requirements; business analytics needs"
    archive:
      after: "1 year inactive"
      tier: "cold_storage"
    deletion:
      method: "standard_delete"
      verification: "automated_confirmation"
 
  # Analytics (Anonymized)
  - category: "anonymized_analytics"
    description: "Fully anonymized aggregate analytics"
    includes:
      - "aggregated usage statistics"
      - "anonymous behavior patterns"
      - "trend data"
    retention:
      period: "indefinite"
      trigger: "N/A"
      rationale: "Anonymized data not subject to PII retention limits"
    archive:
      after: "2 years"
      tier: "archive_storage"
    deletion:
      method: "N/A"
      verification: "N/A"
 
  # Backups
  - category: "system_backups"
    description: "Database and system backups"
    includes:
      - "database snapshots"
      - "file system backups"
      - "configuration backups"
    retention:
      period: "90 days (rolling)"
      trigger: "backup_creation_date"
      rationale: "Disaster recovery window; minimize stale data in backups"
    archive:
      after: "N/A"
      tier: "backup_vault"
    deletion:
      method: "secure_overwrite"
      verification: "backup_inventory_audit"
 
exception_handling:
  legal_hold:
    description: "Suspend deletion for legal/regulatory investigation"
    authority: "legal_counsel"
    notification: "data_owner"
    duration: "until_hold_released"
    documentation: "hold_order_ticket"
 
  data_subject_request:
    description: "Accelerated deletion for verified data subject request"
    authority: "privacy_team"
    timeline: "30 days"
    exceptions: "legal_retention_requirements"
 
  active_investigation:
    description: "Preserve data for active security/fraud investigation"
    authority: "security_team"
    duration: "investigation_completion + 30 days"
    documentation: "investigation_ticket"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214

// Data Lifecycle Management Service
// Manages data transitions through lifecycle stages
 
interface LifecycleRule {
  dataCategory: string;
  transitions: LifecycleTransition[];
  deletionPolicy: DeletionPolicy;
}
 
interface LifecycleTransition {
  fromStage: LifecycleStage;
  toStage: LifecycleStage;
  trigger: TransitionTrigger;
  action: TransitionAction;
}
 
enum LifecycleStage {
  ACTIVE = 'active',
  WARM = 'warm',
  COLD = 'cold',
  ARCHIVE = 'archive',
  PENDING_DELETION = 'pending_deletion',
  DELETED = 'deleted',
}
 
interface TransitionTrigger {
  type: 'age' | 'last_access' | 'condition';
  threshold?: number;  // Days
  condition?: string;  // For custom conditions
}
 
class DataLifecycleService {
  private policyEngine: LifecyclePolicyEngine;
  private storageManager: StorageTierManager;
  private deletionService: DataDeletionService;
  private auditLogger: LifecycleAuditLogger;
  private holdService: LegalHoldService;
 
  async processLifecycleTransitions(): Promise<LifecycleReport> {
    const report: LifecycleReport = {
      processedRecords: 0,
      transitions: [],
      deletions: 0,
      errors: [],
      executedAt: new Date(),
    };
 
    // Get all data categories with lifecycle rules
    const rules = await this.policyEngine.getAllRules();
 
    for (const rule of rules) {
      try {
        const categoryReport = await this.processCategory(rule);
        report.processedRecords += categoryReport.recordsProcessed;
        report.transitions.push(...categoryReport.transitions);
        report.deletions += categoryReport.deletions;
      } catch (error) {
        report.errors.push({
          category: rule.dataCategory,
          error: error.message,
        });
      }
    }
 
    return report;
  }
 
  private async processCategory(rule: LifecycleRule): Promise<CategoryReport> {
    const report: CategoryReport = {
      category: rule.dataCategory,
      recordsProcessed: 0,
      transitions: [],
      deletions: 0,
    };
 
    // Find records eligible for transition
    for (const transition of rule.transitions) {
      const eligibleRecords = await this.findEligibleRecords(
        rule.dataCategory,
        transition
      );
 
      for (const record of eligibleRecords) {
        // Check for legal holds before any transition
        const hasHold = await this.holdService.hasActiveHold(record.id);
        
        if (hasHold) {
          await this.auditLogger.logHoldPrevention({
            recordId: record.id,
            attemptedTransition: transition.toStage,
            holdReason: 'legal_hold_active',
          });
          continue;
        }
 
        // Execute the transition
        await this.executeTransition(record, transition);
 
        report.transitions.push({
          recordId: record.id,
          from: transition.fromStage,
          to: transition.toStage,
        });
        report.recordsProcessed++;
      }
    }
 
    // Process deletions for records past retention
    const deletionEligible = await this.findDeletionEligible(rule);
    
    for (const record of deletionEligible) {
      const hasHold = await this.holdService.hasActiveHold(record.id);
      
      if (!hasHold) {
        await this.deletionService.scheduleSecureDeletion(
          record,
          rule.deletionPolicy
        );
        report.deletions++;
      }
    }
 
    return report;
  }
 
  private async executeTransition(
    record: DataRecord,
    transition: LifecycleTransition
  ): Promise<void> {
    switch (transition.action.type) {
      case 'move_storage_tier':
        await this.storageManager.moveToTier(
          record,
          transition.action.targetTier
        );
        break;
 
      case 'compress':
        await this.storageManager.compressRecord(record);
        break;
 
      case 'archive':
        await this.storageManager.archiveRecord(record, {
          tier: transition.action.targetTier,
          indexRetention: transition.action.indexRetention,
        });
        break;
 
      case 'anonymize':
        // For analytics data approaching deletion
        await this.anonymizationService.anonymizeForRetention(record);
        break;
    }
 
    // Update record metadata
    await this.updateRecordStage(record.id, transition.toStage);
 
    // Audit log
    await this.auditLogger.logTransition({
      recordId: record.id,
      category: record.category,
      fromStage: transition.fromStage,
      toStage: transition.toStage,
      action: transition.action.type,
      timestamp: new Date(),
    });
  }
 
  private async findEligibleRecords(
    category: string,
    transition: LifecycleTransition
  ): Promise<DataRecord[]> {
    const query: RecordQuery = {
      category,
      currentStage: transition.fromStage,
    };
 
    switch (transition.trigger.type) {
      case 'age':
        query.createdBefore = new Date(
          Date.now() - transition.trigger.threshold * 24 * 60 * 60 * 1000
        );
        break;
 
      case 'last_access':
        query.lastAccessedBefore = new Date(
          Date.now() - transition.trigger.threshold * 24 * 60 * 60 * 1000
        );
        break;
 
      case 'condition':
        query.customCondition = transition.trigger.condition;
        break;
    }
 
    return this.dataRepository.findByQuery(query);
  }
 
  // Schedule recurring lifecycle processing
  async scheduleLifecycleProcessing(): Promise<void> {
    // Daily processing for most categories
    scheduleJob('lifecycle-daily', '0 2 * * *', async () => {
      const report = await this.processLifecycleTransitions();
      await this.sendLifecycleReport(report);
    });
 
    // Hourly processing for high-volume log data
    scheduleJob('lifecycle-logs', '0 * * * *', async () => {
      await this.processCategory(
        await this.policyEngine.getRule('application_logs')
      );
    });
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218

// Legal Hold Management Service
// Manages preservation orders that suspend normal retention
 
interface LegalHold {
  id: string;
  matterName: string;
  matterNumber: string;
  holdType: HoldType;
  status: HoldStatus;
  issuedAt: Date;
  issuedBy: string;      // Legal counsel authorizing hold
  expiresAt?: Date;      // Optional expiration
  releasedAt?: Date;
  scope: HoldScope;
  custodians: string[];  // Users whose data is preserved
  dataCategories: string[];
  searchCriteria?: SearchCriteria;
  legalJustification: string;
}
 
interface HoldScope {
  type: 'custodian' | 'category' | 'query' | 'all';
  custodians?: string[];       // Specific user IDs
  categories?: string[];       // Data categories
  dateRange?: DateRange;       // Temporal scope
  searchQuery?: string;        // Content-based scope
  systems?: string[];          // Specific systems
}
 
enum HoldStatus {
  ACTIVE = 'active',
  RELEASED = 'released',
  EXPIRED = 'expired',
  PENDING = 'pending',
}
 
class LegalHoldService {
  private holdRepository: LegalHoldRepository;
  private notificationService: HoldNotificationService;
  private auditLogger: LegalHoldAuditLogger;
  private dataIndexer: DataIndexingService;
 
  async createHold(request: CreateHoldRequest): Promise<LegalHold> {
    // Validate authorizer has legal authority
    await this.validateHoldAuthority(request.issuedBy);
 
    const hold: LegalHold = {
      id: generateId(),
      matterName: request.matterName,
      matterNumber: request.matterNumber,
      holdType: request.holdType,
      status: HoldStatus.ACTIVE,
      issuedAt: new Date(),
      issuedBy: request.issuedBy,
      scope: request.scope,
      custodians: request.custodians || [],
      dataCategories: request.dataCategories || [],
      legalJustification: request.justification,
    };
 
    // Store hold
    await this.holdRepository.create(hold);
 
    // Mark affected data records
    await this.applyHoldToData(hold);
 
    // Notify affected custodians (employees) of preservation obligations
    if (hold.custodians.length > 0) {
      await this.notifyAffectedCustodians(hold);
    }
 
    // Notify IT/data teams of hold scope
    await this.notificationService.notifyDataTeams({
      holdId: hold.id,
      scope: hold.scope,
      action: 'hold_created',
    });
 
    // Comprehensive audit logging
    await this.auditLogger.logHoldCreation({
      hold,
      authorizedBy: request.issuedBy,
      timestamp: new Date(),
    });
 
    return hold;
  }
 
  async releaseHold(
    holdId: string,
    releaseInfo: HoldReleaseRequest
  ): Promise<void> {
    const hold = await this.holdRepository.getById(holdId);
    
    if (!hold) {
      throw new Error(`Hold ${holdId} not found`);
    }
 
    // Validate release authority
    await this.validateReleaseAuthority(releaseInfo.releasedBy, hold);
 
    // Update hold status
    hold.status = HoldStatus.RELEASED;
    hold.releasedAt = new Date();
    await this.holdRepository.update(hold);
 
    // Remove hold markers from data (unless other holds apply)
    await this.removeHoldFromData(hold);
 
    // Notify relevant parties
    await this.notificationService.notifyHoldRelease({
      holdId,
      releasedBy: releaseInfo.releasedBy,
      reason: releaseInfo.reason,
    });
 
    // Audit log
    await this.auditLogger.logHoldRelease({
      holdId,
      releasedBy: releaseInfo.releasedBy,
      reason: releaseInfo.reason,
      timestamp: new Date(),
    });
  }
 
  async hasActiveHold(recordId: string): Promise<boolean> {
    // Check if any active holds apply to this record
    const holdMarker = await this.holdRepository.getHoldMarker(recordId);
    
    if (!holdMarker) {
      return false;
    }
 
    // Verify referenced holds are still active
    for (const holdId of holdMarker.holdIds) {
      const hold = await this.holdRepository.getById(holdId);
      if (hold && hold.status === HoldStatus.ACTIVE) {
        return true;
      }
    }
 
    return false;
  }
 
  private async applyHoldToData(hold: LegalHold): Promise<void> {
    // Find all records matching hold scope
    const affectedRecords = await this.findAffectedRecords(hold.scope);
 
    // Batch apply hold markers
    const batchSize = 1000;
    for (let i = 0; i < affectedRecords.length; i += batchSize) {
      const batch = affectedRecords.slice(i, i + batchSize);
      await this.holdRepository.applyHoldMarkers(
        batch.map(r => r.id),
        hold.id
      );
    }
 
    // Log scope of preservation
    await this.auditLogger.logHoldApplication({
      holdId: hold.id,
      recordCount: affectedRecords.length,
      timestamp: new Date(),
    });
  }
 
  private async findAffectedRecords(scope: HoldScope): Promise<DataRecord[]> {
    switch (scope.type) {
      case 'custodian':
        return this.dataIndexer.findByOwners(scope.custodians);
 
      case 'category':
        return this.dataIndexer.findByCategories(scope.categories);
 
      case 'query':
        return this.dataIndexer.searchContent(scope.searchQuery);
 
      case 'all':
        // Extremely broad - use with caution
        return this.dataIndexer.findByDateRange(scope.dateRange);
 
      default:
        return [];
    }
  }
 
  private async notifyAffectedCustodians(hold: LegalHold): Promise<void> {
    for (const custodianId of hold.custodians) {
      await this.notificationService.sendPreservationNotice({
        recipientId: custodianId,
        holdId: hold.id,
        matterName: hold.matterName,
        instructions: [
          'Do not delete, modify, or destroy any relevant documents',
          'Preserve all communications related to this matter',
          'Contact legal if unsure about any preservation questions',
        ],
        acknowledgmentRequired: true,
      });
    }
  }
 
  // Reporting for legal team
  async generateHoldReport(holdId: string): Promise<HoldDetailReport> {
    const hold = await this.holdRepository.getById(holdId);
    const affectedRecords = await this.getAffectedRecords(holdId);
    const custodianAcknowledgments = await this.getAcknowledgmentStatus(holdId);
 
    return {
      hold,
      recordsPreserved: affectedRecords.length,
      dataVolumeGB: this.calculateDataVolume(affectedRecords),
      custodianStatus: custodianAcknowledgments,
      systemsCovered: this.getUniqueSystems(affectedRecords),
      auditTrail: await this.auditLogger.getHoldHistory(holdId),
    };
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239

// Retention Policy Enforcement Engine
// Automated enforcement across distributed data systems
 
interface EnforcementConfig {
  batchSize: number;
  parallelism: number;
  dryRunMode: boolean;      // Test without actual deletion
  notifyOwners: boolean;
  alertThresholds: AlertThresholds;
}
 
interface AlertThresholds {
  deletionVolumeWarning: number;   // GB
  deletionRecordWarning: number;   // Record count
  errorRateThreshold: number;      // Percentage
}
 
class RetentionEnforcementEngine {
  private policyRepository: RetentionPolicyRepository;
  private dataRegistry: DataSourceRegistry;
  private holdService: LegalHoldService;
  private deletionService: SecureDeletionService;
  private auditLogger: RetentionAuditLogger;
  private alertService: AlertService;
 
  constructor(private config: EnforcementConfig) {}
 
  async runEnforcementCycle(): Promise<EnforcementReport> {
    const report: EnforcementReport = {
      cycleId: generateId(),
      startedAt: new Date(),
      completedAt: null,
      recordsEvaluated: 0,
      transitionsExecuted: 0,
      deletionsScheduled: 0,
      deletionsExecuted: 0,
      holdBlockedActions: 0,
      errors: [],
    };
 
    try {
      // Load current policies
      const policies = await this.policyRepository.getActivePolicies();
 
      // Process each data source
      for (const dataSource of await this.dataRegistry.getAllSources()) {
        const sourceReport = await this.processDataSource(
          dataSource,
          policies
        );
        this.mergeReports(report, sourceReport);
      }
 
      // Check alert thresholds
      await this.checkAlertThresholds(report);
 
    } catch (error) {
      report.errors.push({
        phase: 'orchestration',
        error: error.message,
        fatal: true,
      });
    }
 
    report.completedAt = new Date();
    
    // Log complete enforcement report
    await this.auditLogger.logEnforcementCycle(report);
 
    return report;
  }
 
  private async processDataSource(
    source: DataSource,
    policies: RetentionPolicy[]
  ): Promise<EnforcementReport> {
    const sourceReport: EnforcementReport = {
      // ... initialize
    };
 
    // Find applicable policies for this source's data categories
    const applicablePolicies = policies.filter(p =>
      source.dataCategories.some(cat => 
        p.appliesToCategory(cat)
      )
    );
 
    if (applicablePolicies.length === 0) {
      return sourceReport; // No policies for this source
    }
 
    // Stream through records in batches
    const recordStream = source.createRecordStream({
      batchSize: this.config.batchSize,
    });
 
    for await (const batch of recordStream) {
      const batchReport = await this.processBatch(
        batch,
        applicablePolicies,
        source
      );
      this.mergeReports(sourceReport, batchReport);
    }
 
    return sourceReport;
  }
 
  private async processBatch(
    records: DataRecord[],
    policies: RetentionPolicy[],
    source: DataSource
  ): Promise<BatchReport> {
    const toTransition: TransitionAction[] = [];
    const toDelete: DeletionAction[] = [];
    const holdBlocked: string[] = [];
 
    // Evaluate each record against policies
    for (const record of records) {
      const policy = this.findMatchingPolicy(record, policies);
      if (!policy) continue;
 
      // Check for active holds
      if (await this.holdService.hasActiveHold(record.id)) {
        holdBlocked.push(record.id);
        continue;
      }
 
      // Determine required action
      const action = this.evaluateRetentionStatus(record, policy);
 
      if (action.type === 'transition') {
        toTransition.push(action);
      } else if (action.type === 'delete') {
        toDelete.push(action);
      }
    }
 
    // Execute transitions (usually safe to parallelize)
    await Promise.all(
      toTransition.map(t => this.executeTransition(t, source))
    );
 
    // Deletions require more care
    if (!this.config.dryRunMode) {
      await this.processDeletions(toDelete, source);
    }
 
    return {
      evaluated: records.length,
      transitioned: toTransition.length,
      deleted: this.config.dryRunMode ? 0 : toDelete.length,
      holdBlocked: holdBlocked.length,
    };
  }
 
  private evaluateRetentionStatus(
    record: DataRecord,
    policy: RetentionPolicy
  ): RetentionAction {
    const now = new Date();
    const triggerDate = this.getTriggerDate(record, policy);
    const ageInDays = this.daysBetween(triggerDate, now);
 
    // Check if past retention period (should delete)
    if (ageInDays >= policy.retentionDays) {
      return {
        type: 'delete',
        recordId: record.id,
        reason: 'retention_expired',
        policy: policy.id,
      };
    }
 
    // Check lifecycle transitions
    for (const transition of policy.lifecycleTransitions) {
      if (
        record.lifecycleStage === transition.fromStage &&
        ageInDays >= transition.afterDays
      ) {
        return {
          type: 'transition',
          recordId: record.id,
          fromStage: transition.fromStage,
          toStage: transition.toStage,
          policy: policy.id,
        };
      }
    }
 
    return { type: 'none' };
  }
 
  private async processDeletions(
    deletions: DeletionAction[],
    source: DataSource
  ): Promise<void> {
    // Pre-deletion validation
    const volumeCheck = await this.validateDeletionVolume(deletions, source);
    if (volumeCheck.requiresApproval) {
      await this.requestDeletionApproval(deletions, volumeCheck);
      return; // Wait for approval before proceeding
    }
 
    // Execute deletions through secure deletion service
    for (const deletion of deletions) {
      await this.deletionService.scheduleDeletion({
        recordId: deletion.recordId,
        source: source.id,
        policy: deletion.policy,
        method: source.deletionMethod,
        verification: true,
      });
    }
  }
 
  private async checkAlertThresholds(
    report: EnforcementReport
  ): Promise<void> {
    if (report.deletionsExecuted > this.config.alertThresholds.deletionRecordWarning) {
      await this.alertService.sendAlert({
        severity: 'warning',
        title: 'High Volume Retention Deletion',
        message: `Deleted ${report.deletionsExecuted} records in single cycle`,
        report,
      });
    }
 
    const errorRate = (report.errors.length / report.recordsEvaluated) * 100;
    if (errorRate > this.config.alertThresholds.errorRateThreshold) {
      await this.alertService.sendAlert({
        severity: 'error',
        title: 'Retention Enforcement Error Rate High',
        message: `${errorRate.toFixed(2)}% error rate in enforcement cycle`,
        report,
      });
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215

// Cross-System Deletion Orchestrator
// Coordinates deletion across all data systems for complete removal
 
interface DeletionRequest {
  recordId: string;
  dataCategory: string;
  reason: DeletionReason;
  requestedBy: string;
  verificationRequired: boolean;
}
 
interface SystemDeletionStatus {
  systemId: string;
  systemType: SystemType;
  status: 'pending' | 'completed' | 'failed' | 'not_applicable';
  deletedAt?: Date;
  error?: string;
  verificationResult?: VerificationResult;
}
 
class CrossSystemDeletionOrchestrator {
  private systemRegistry: DataSystemRegistry;
  private deletionTracker: DeletionTrackingRepository;
  private eventBus: EventBus;
  private verificationService: DeletionVerificationService;
 
  async orchestrateDeletion(
    request: DeletionRequest
  ): Promise<DeletionOrchestrationResult> {
    // Create deletion tracking record
    const orchestration = await this.deletionTracker.create({
      requestId: generateId(),
      recordId: request.recordId,
      category: request.dataCategory,
      status: 'in_progress',
      initiatedAt: new Date(),
      initiatedBy: request.requestedBy,
      reason: request.reason,
      systems: [],
    });
 
    // Identify all systems containing this record
    const affectedSystems = await this.identifyAffectedSystems(
      request.recordId,
      request.dataCategory
    );
 
    // Initialize system status tracking
    for (const system of affectedSystems) {
      orchestration.systems.push({
        systemId: system.id,
        systemType: system.type,
        status: 'pending',
      });
    }
 
    // Execute deletions in dependency order
    const orderedSystems = this.orderByDependency(affectedSystems);
 
    for (const system of orderedSystems) {
      try {
        await this.deleteFromSystem(
          request.recordId,
          system,
          orchestration
        );
      } catch (error) {
        // Log failure but continue with other systems
        this.updateSystemStatus(
          orchestration,
          system.id,
          'failed',
          error.message
        );
      }
    }
 
    // Verify deletion if required
    if (request.verificationRequired) {
      await this.verifyCompleteDeletion(orchestration);
    }
 
    // Emit deletion complete event (for downstream cleanup)
    await this.eventBus.publish('data.deleted', {
      recordId: request.recordId,
      category: request.dataCategory,
      completedAt: new Date(),
      verificationStatus: orchestration.verificationResult,
    });
 
    return this.generateResult(orchestration);
  }
 
  private async identifyAffectedSystems(
    recordId: string,
    category: string
  ): Promise<DataSystem[]> {
    const systems: DataSystem[] = [];
 
    for (const system of await this.systemRegistry.getAll()) {
      // Check if system handles this data category
      if (!system.handlesCategory(category)) {
        continue;
      }
 
      // Check if record exists in this system
      const exists = await system.client.recordExists(recordId);
      if (exists) {
        systems.push(system);
      }
    }
 
    return systems;
  }
 
  private orderByDependency(systems: DataSystem[]): DataSystem[] {
    // Delete in order: derivatives first, source last
    // e.g., search index before DB, cache before DB
    const priority: Record<SystemType, number> = {
      'cache': 1,          // Delete caches first
      'search_index': 2,   // Then search indexes
      'read_replica': 3,   // Then replicas
      'analytics': 4,      // Then analytics copies
      'primary_db': 5,     // Primary DB near last
      'backup': 6,         // Backups last (may be immutable/scheduled)
    };
 
    return systems.sort(
      (a, b) => (priority[a.type] || 99) - (priority[b.type] || 99)
    );
  }
 
  private async deleteFromSystem(
    recordId: string,
    system: DataSystem,
    orchestration: DeletionOrchestration
  ): Promise<void> {
    const startTime = Date.now();
 
    switch (system.type) {
      case 'primary_db':
        await system.client.deleteRecord(recordId);
        break;
 
      case 'cache':
        await system.client.invalidate(recordId);
        break;
 
      case 'search_index':
        await system.client.removeFromIndex(recordId);
        break;
 
      case 'read_replica':
        // May need to wait for replication to sync delete
        await system.client.waitForDeletion(recordId, {
          timeout: 30000, // 30 seconds
        });
        break;
 
      case 'analytics':
        await system.client.purgeRecord(recordId);
        break;
 
      case 'backup':
        // Backups typically use scheduled rotation or crypto erasure
        await system.client.schedulePurge(recordId);
        break;
    }
 
    this.updateSystemStatus(orchestration, system.id, 'completed');
  }
 
  private async verifyCompleteDeletion(
    orchestration: DeletionOrchestration
  ): Promise<void> {
    const failedSystems = orchestration.systems.filter(
      s => s.status === 'failed'
    );
 
    const pendingSystems = orchestration.systems.filter(
      s => s.status === 'pending'
    );
 
    if (failedSystems.length > 0 || pendingSystems.length > 0) {
      orchestration.verificationResult = {
        complete: false,
        incompleteSystems: [
          ...failedSystems.map(s => s.systemId),
          ...pendingSystems.map(s => s.systemId),
        ],
      };
 
      // Alert for manual remediation
      await this.alertIncompleteRemoval(orchestration);
      return;
    }
 
    // Verify by attempting to retrieve record
    for (const system of orchestration.systems) {
      const stillExists = await this.systemRegistry
        .getById(system.systemId)
        .client.recordExists(orchestration.recordId);
 
      if (stillExists) {
        system.status = 'failed';
        system.error = 'Record still exists after deletion';
      }
    }
 
    orchestration.verificationResult = {
      complete: orchestration.systems.every(s => s.status === 'completed'),
      verifiedAt: new Date(),
    };
  }
}