Back-of-Envelope Estimation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

# Detailed size calculation for a single tweet
 
class Tweet:
    """
    Tweet storage size breakdown
    All sizes in bytes
    """
    # Core identifiers
    tweet_id: int = 8           # int64
    user_id: int = 8            # int64
    
    # Content
    text_max: int = 560         # 280 chars × ~2 bytes (UTF-8 average)
    
    # Timestamps
    created_at: int = 8         # datetime
    updated_at: int = 8         # datetime
    
    # Engagement counters (denormalized for read performance)
    like_count: int = 4         # int32
    retweet_count: int = 4      # int32
    reply_count: int = 4        # int32
    quote_count: int = 4        # int32
    
    # Metadata
    language_code: int = 3      # 'en', 'es', 'jp', etc.
    source_app: int = 50        # 'Twitter for iPhone'
    
    # References
    reply_to_id: int = 8        # Nullable - original tweet ID
    quoted_tweet_id: int = 8    # Nullable - quoted tweet ID
    
    # Media references (actual media stored separately)
    media_ids: int = 32         # Up to 4 media items × 8 bytes
    
    # Location (optional)
    geo_lat: int = 8            # double
    geo_lng: int = 8            # double
    place_id: int = 24          # String reference
    
    # Flags
    is_sensitive: int = 1       # boolean
    is_reply: int = 1           # boolean
    has_media: int = 1          # boolean
    
    def total_size(self) -> int:
        return (
            self.tweet_id + self.user_id + self.text_max +
            self.created_at + self.updated_at +
            self.like_count + self.retweet_count + 
            self.reply_count + self.quote_count +
            self.language_code + self.source_app +
            self.reply_to_id + self.quoted_tweet_id +
            self.media_ids + self.geo_lat + self.geo_lng +
            self.place_id + self.is_sensitive + 
            self.is_reply + self.has_media
        )
 
tweet = Tweet()
print(f"Single tweet size: {tweet.total_size()} bytes ≈ {tweet.total_size()/1024:.2f} KB")
 
# Scale calculation
tweets_per_day = 500_000_000  # 500 million tweets/day
raw_daily_storage = tweets_per_day * tweet.total_size()
print(f"
Daily tweet storage: {raw_daily_storage / (1024**4):.2f} TB (raw)")
print(f"With 3x replication: {raw_daily_storage * 3 / (1024**4):.2f} TB")
print(f"Yearly (365 days): {raw_daily_storage * 365 / (1024**5):.2f} PB")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

# Storage calculation for a YouTube-like platform
 
# Video upload assumptions
hours_uploaded_per_minute = 500  # YouTube actual stat: 500 hours/min
minutes_per_day = 60 * 24
 
# Minutes of video uploaded daily
video_minutes_daily = hours_uploaded_per_minute * 60 * minutes_per_day
print(f"Video minutes uploaded daily: {video_minutes_daily:,}")
 
# Storage per minute of video (all quality levels)
storage_per_minute_mb = {
    "144p": 1.5,
    "240p": 3,
    "360p": 4,
    "480p": 8,
    "720p": 22,
    "1080p": 45,
    "1440p": 90,
    "2160p (4K)": 150,
}
 
# Not all videos are encoded at all qualities
# Assume average encoding profile
average_mb_per_minute = 100  # Weighted average
 
# Daily storage (raw)
daily_storage_tb = video_minutes_daily * average_mb_per_minute / (1024 * 1024)
print(f"
Daily raw video storage: {daily_storage_tb:,.0f} TB")
 
# With CDN distribution (multiple copies across edge locations)
cdn_copies = 3  # Minimum copies for global coverage
# Plus origin storage with replication
origin_replication = 3
 
# Actual storage (origin + some CDN)
# CDN typically caches popular 20% of content
cdn_cached_percentage = 0.20
 
total_daily = (daily_storage_tb * origin_replication + 
               daily_storage_tb * cdn_cached_percentage * cdn_copies)
print(f"Daily storage with replication: {total_daily:,.0f} TB")
print(f"
Annual storage growth: {total_daily * 365 / 1000:,.1f} PB/year")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

# 5-Year storage projection model
 
from dataclasses import dataclass
from typing import List
 
@dataclass
class YearlyProjection:
    year: int
    dau: int
    objects_per_user_day: float
    object_size_bytes: int
    retention_days: int
    new_storage_pb: float
    cumulative_storage_pb: float
 
def project_storage(
    initial_dau: int,
    dau_growth_rate: float,
    engagement_growth_rate: float,
    initial_objects_per_user: float,
    object_size_bytes: int,
    retention_days: int,
    years: int
) -> List[YearlyProjection]:
    
    projections = []
    cumulative_storage = 0
    
    for year in range(1, years + 1):
        # Calculate metrics for this year
        dau = int(initial_dau * (1 + dau_growth_rate) ** (year - 1))
        objects_per_user = initial_objects_per_user * (1 + engagement_growth_rate) ** (year - 1)
        
        # Annual data creation
        daily_objects = dau * objects_per_user
        annual_objects = daily_objects * 365
        annual_storage_bytes = annual_objects * object_size_bytes
        annual_storage_pb = annual_storage_bytes / (1024 ** 5)
        
        # Account for data retention
        # If retention is 365 days, we keep 1 year of data
        # If retention is unlimited, cumulative grows indefinitely
        if retention_days >= 365:
            cumulative_storage += annual_storage_pb
        else:
            # Rolling retention - only keep retention_days worth
            daily_bytes = (dau * objects_per_user * object_size_bytes)
            cumulative_storage = (daily_bytes * retention_days) / (1024 ** 5)
        
        projections.append(YearlyProjection(
            year=year,
            dau=dau,
            objects_per_user_day=objects_per_user,
            object_size_bytes=object_size_bytes,
            retention_days=retention_days,
            new_storage_pb=annual_storage_pb,
            cumulative_storage_pb=cumulative_storage
        ))
    
    return projections
 
# Example: Social media platform with photos
projections = project_storage(
    initial_dau=50_000_000,         # 50M DAU year 1
    dau_growth_rate=0.25,           # 25% YoY user growth
    engagement_growth_rate=0.10,    # 10% more photos per user each year
    initial_objects_per_user=3,     # 3 photos/day initially
    object_size_bytes=3 * 1024 * 1024,  # 3MB per photo (all sizes)
    retention_days=36500,           # Keep forever (100 years)
    years=5
)
 
print("5-Year Storage Projection (Photo Platform)")
print("=" * 70)
for p in projections:
    print(f"Year {p.year}: DAU={p.dau/1e6:.0f}M | "
          f"Photos/user={p.objects_per_user_day:.1f} | "
          f"New={p.new_storage_pb:.1f}PB | "
          f"Total={p.cumulative_storage_pb:.1f}PB")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

# Calculate tiered storage savings
 
class StorageTierAnalysis:
    def __init__(self, total_storage_pb: float, monthly_growth_pb: float):
        self.total_storage_pb = total_storage_pb
        self.monthly_growth_pb = monthly_growth_pb
    
    def calculate_flat_cost(self) -> float:
        """All data in S3 Standard"""
        cost_per_gb = 0.023
        gb = self.total_storage_pb * 1024 * 1024  # PB to GB
        return gb * cost_per_gb
    
    def calculate_tiered_cost(self) -> float:
        """
        Distribution:
        - 15% Hot (last 30 days of new data)
        - 25% Warm (31-90 days)
        - 30% Cold (91-365 days)
        - 30% Archive (365+ days)
        """
        gb = self.total_storage_pb * 1024 * 1024
        
        hot_pct, warm_pct, cold_pct, archive_pct = 0.15, 0.25, 0.30, 0.30
        
        hot_cost = gb * hot_pct * 0.023       # S3 Standard
        warm_cost = gb * warm_pct * 0.0125    # S3 Standard-IA
        cold_cost = gb * cold_pct * 0.004     # Glacier Instant
        archive_cost = gb * archive_pct * 0.001  # Glacier Deep Archive
        
        return hot_cost + warm_cost + cold_cost + archive_cost
    
    def savings_analysis(self) -> dict:
        flat = self.calculate_flat_cost()
        tiered = self.calculate_tiered_cost()
        savings = flat - tiered
        savings_pct = (savings / flat) * 100
        
        return {
            "flat_cost_monthly": flat,
            "tiered_cost_monthly": tiered,
            "monthly_savings": savings,
            "savings_percentage": savings_pct,
            "annual_savings": savings * 12
        }
 
# Example: 100PB photo storage platform
analysis = StorageTierAnalysis(total_storage_pb=100, monthly_growth_pb=5)
results = analysis.savings_analysis()
 
print("Storage Cost Analysis: 100PB Photo Platform")
print("=" * 50)
print(f"Flat pricing (all S3 Standard): ${results['flat_cost_monthly']:, .0f
                        } / month")
print(f"Tiered pricing:                 ${results['tiered_cost_monthly']:,.0f}/month")
print(f"Monthly savings:                ${results['monthly_savings']:,.0f}")
print(f"Savings percentage:             {results['savings_percentage']:.1f}%")
print(f"Annual savings:                 ${results['annual_savings']/1e6:.1f}M")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

# URL Shortening Service Storage Estimation
 
# Service assumptions
daily_url_creations = 100_000_000  # 100M URLs created / day
service_lifespan_years = 10        # URLs never expire
 
# Data model per URL
url_record = {
                            "short_code": 7,           # 7 character code
    "original_url": 200,       # Average URL length
    "user_id": 8,              # int64(nullable for anonymous)
    "created_at": 8,           # timestamp
    "click_count": 4,          # int32
    "last_clicked": 8,         # timestamp(nullable)
    "metadata": 50,            # custom tracking params, title
}
bytes_per_url = sum(url_record.values())
print(f"Bytes per URL: {bytes_per_url}")
 
# Analytics data(per click)
click_record = {
    "click_id": 8,
    "short_code": 7,
    "timestamp": 8,
    "ip_hash": 16,             # Anonymized
    "user_agent": 100,
    "referer": 100,
    "country": 2,
    "device_type": 1,
}
bytes_per_click = sum(click_record.values())
print(f"Bytes per click: {bytes_per_click}")
 
# Assume each URL gets clicked 50 times on average
clicks_per_url = 50
 
# Daily storage
daily_url_storage = daily_url_creations * bytes_per_url
daily_click_storage = daily_url_creations * clicks_per_url * bytes_per_click
daily_total = daily_url_storage + daily_click_storage
 
print(f"
Daily URL storage: {daily_url_storage / 1e9:.1f} GB")
print(f"Daily click storage: {daily_click_storage / 1e9:.1f} GB")
print(f"Daily total: {daily_total / 1e9:.1f} GB")
 
# With replication and overhead(3x replication, 1.5x indexes / overhead)
storage_multiplier = 3 * 1.5
daily_actual = daily_total * storage_multiplier
 
# 10 - year projection
ten_year_storage = daily_actual * 365 * service_lifespan_years
print(f"
10-year storage: {ten_year_storage / 1e15:.1f} PB")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

# Chat Application Storage Estimation
 
# User assumptions
monthly_active_users = 20_000_000   # 20M MAU
dau_mau_ratio = 0.65                # High engagement
daily_active_users = monthly_active_users * dau_mau_ratio
 
# Message patterns
messages_per_user_per_day = 40      # Active messengers
direct_message_ratio = 0.4         # 40 % DMs, 60 % channels
 
# Message data model
text_message = {
    "message_id": 16,              # Snowflake ID
    "channel_id": 16,              # Where sent
    "user_id": 16,
    "content": 500,                # Average message(incl.emojis, links)
    "created_at": 8,
    "edited_at": 8,
    "attachments_meta": 50,        # References to files
    "reactions_count": 4,
    "is_pinned": 1,
    "thread_id": 16,               # If reply
}
bytes_per_message = sum(text_message.values())
 
# File attachments(images, documents)
messages_with_attachments_ratio = 0.15  # 15 % have attachments
average_attachment_size_mb = 2
 
# Reactions(separate table for many - to - many)
    reactions_per_message = 0.5        # Average reactions
reaction_record_bytes = 32         # message_id + user_id + emoji
 
# Daily calculations
daily_messages = daily_active_users * messages_per_user_per_day
daily_text_storage = daily_messages * bytes_per_message
daily_attachment_storage = daily_messages * messages_with_attachments_ratio * average_attachment_size_mb * 1e6
daily_reaction_storage = daily_messages * reactions_per_message * reaction_record_bytes
 
print(f"Daily messages: {daily_messages/1e6:.0f}M")
print(f"Daily text storage: {daily_text_storage/1e9:.1f} GB")
print(f"Daily attachment storage: {daily_attachment_storage/1e9:.1f} GB")
print(f"Daily reaction storage: {daily_reaction_storage/1e9:.2f} GB")
 
total_daily = daily_text_storage + daily_attachment_storage + daily_reaction_storage
print(f"
Total daily storage: {total_daily/1e9:.1f} GB")
 
# Retention: Keep messages forever, but tier attachments
# With 3x replication
annual_storage_tb = total_daily * 365 * 3 / 1e12
print(f"Annual storage (replicated): {annual_storage_tb:.0f} TB")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

# E - Commerce Platform Storage Estimation
 
# Business scale
products = 500_000_000           # 500M product listings
daily_orders = 50_000_000        # 50M orders / day
daily_product_views = 5_000_000_000  # 5B product views / day
 
# Product catalog storage
product_record = {
    "product_id": 8,
    "seller_id": 8,
    "title": 200,
    "description": 2000,
    "category_ids": 32,          # Multiple categories
    "price": 8,
    "inventory": 4,
    "ratings_summary": 24,
    "attributes": 500,           # JSON attributes
    "created_at": 8,
    "updated_at": 8,
}
bytes_per_product = sum(product_record.values())
 
# Product images(separate storage)
images_per_product = 7            # Main + gallery
image_size_all_versions_mb = 3    # All resolutions
 
# Product catalog total
catalog_data = products * bytes_per_product
catalog_images = products * images_per_product * image_size_all_versions_mb * 1e6
print(f"Catalog data: {catalog_data/1e12:.1f} TB")
print(f"Catalog images: {catalog_images/1e15:.1f} PB")
 
# Order storage
order_record = {
    "order_id": 16,
    "user_id": 8,
    "total": 8,
    "status": 1,
    "payment_status": 1,
    "shipping_address": 500,
    "created_at": 8,
    "updated_at": 8,
}
order_item_record = {
    "order_item_id": 16,
    "order_id": 16,
    "product_id": 8,
    "quantity": 4,
    "price": 8,
}
bytes_per_order = sum(order_record.values())
bytes_per_order_item = sum(order_item_record.values())
items_per_order = 3  # Average
 
daily_order_storage = daily_orders * (bytes_per_order + items_per_order * bytes_per_order_item)
print(f"
Daily order storage: {daily_order_storage/1e9:.1f} GB")
 
# View / click history for recommendations
view_event = 50  # bytes per event
daily_view_storage = daily_product_views * view_event
print(f"Daily view event storage: {daily_view_storage/1e12:.1f} TB")
 
# Retain views for 90 days(recommendation training)
rolling_view_storage = daily_view_storage * 90
print(f"90-day view history: {rolling_view_storage/1e15:.1f} PB")