Stop Words: When Common Words Matter (and When They Don't)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

// Zipf's Law: Word frequency follows a power law distribution
// The most common word appears ~2x as often as the second most common,
// ~3x as often as the third, and so on.
 
// Example from analyzing 1 million documents:
const wordFrequencies = [
  { word: "the",    frequency: 69_971_528, rank: 1, percentOfCorpus: 7.0 },
  { word: "of",     frequency: 36_411_928, rank: 2, percentOfCorpus: 3.6 },
  { word: "and",    frequency: 28_765_432, rank: 3, percentOfCorpus: 2.9 },
  { word: "to",     frequency: 26_148_764, rank: 4, percentOfCorpus: 2.6 },
  { word: "a",      frequency: 21_341_218, rank: 5, percentOfCorpus: 2.1 },
  { word: "in",     frequency: 20_312_987, rank: 6, percentOfCorpus: 2.0 },
  { word: "is",     frequency: 14_987_234, rank: 7, percentOfCorpus: 1.5 },
  { word: "that",   frequency: 12_384_756, rank: 8, percentOfCorpus: 1.2 },
  { word: "it",     frequency: 10_124_879, rank: 9, percentOfCorpus: 1.0 },
  { word: "was",    frequency: 9_876_543,  rank: 10, percentOfCorpus: 1.0 },
];
 
// The top 10 words alone represent ~25% of all word occurrences!
// The top 100 words represent ~50% of all word occurrences.
 
// Implication for search:
// If we keep these words in the index, "the" appears in effectively
// 100% of documents. A search for "the" would match everything,
// providing zero discriminative value while consuming massive storage.
 
interface IndexImpact {
  withStopwords: {
    indexSize: string;
    averageTokensPerDoc: number;
    searchTimeMs: number;
  };
  withoutStopwords: {
    indexSize: string;
    averageTokensPerDoc: number;
    searchTimeMs: number;
  };
}
 
const realWorldImpact: IndexImpact = {
  withStopwords: {
    indexSize: "42 GB",
    averageTokensPerDoc: 847,
    searchTimeMs: 23
  },
  withoutStopwords: {
    indexSize: "28 GB",               // 33% smaller!
    averageTokensPerDoc: 423,         // 50% fewer tokens
    searchTimeMs: 14                  // 40% faster search
  }
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

// Query execution comparison: with and without stopwords
 
// User query: "what is the best programming language for web development"
 
// WITH stopwords indexed:
// Query becomes: 
//   (what OR is OR the OR best OR programming OR language OR for OR web OR development)
// 
// Posting lists to merge:
//   "what"        → 4,892,341 documents
//   "is"          → 9,123,456 documents  ← almost every document!
//   "the"         → 9,456,789 documents  ← almost every document!
//   "best"        → 1,234,567 documents
//   "programming" → 567,890 documents
//   "language"    → 891,234 documents
//   "for"         → 8,765,432 documents  ← almost every document!
//   "web"         → 2,345,678 documents
//   "development" → 1,987,654 documents
//
// Total posting list entries to process: ~39 million
// Most work is wasted on low-value terms
 
// WITHOUT stopwords:
// Query becomes:
//   (best OR programming OR language OR web OR development)
//
// Posting lists to merge:
//   "best"        → 1,234,567 documents
//   "programming" → 567,890 documents
//   "language"    → 891,234 documents
//   "web"         → 2,345,678 documents
//   "development" → 1,987,654 documents
//
// Total posting list entries to process: ~7 million
// 82% reduction in work for the query engine
 
// Additionally, relevance scoring focuses on meaningful terms:
// Without stopwords, "programming language web development" 
// receives full scoring weight instead of being diluted by
// high-frequency low-value terms.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

// Real-world stopword removal failures
 
const problemQueries = [
  // Named entities broken by stopword removal
  {
    query: "The Office",
    withStopwords: "The Office",    // Correct: finds the TV show
    withoutStopwords: "Office",     // Broken: finds Microsoft Office, generic offices
    failure: "Article is part of proper name"
  },
  
  // Phrases destroyed
  {
    query: "to be or not to be",
    withStopwords: "to be or not to be",
    withoutStopwords: "",            // Complete disaster: empty query!
    failure: "Entire query is stopwords"
  },
  
  // Negation inverted
  {
    query: "not working",
    withStopwords: "not working",    // Finds bug reports, troubleshooting
    withoutStopwords: "working",     // Finds opposite: things that ARE working
    failure: "Meaning inverted by removing 'not'"
  },
  
  // Technical meaning lost
  {
    query: "if else statement",
    withStopwords: "if else statement",
    withoutStopwords: "statement",    // How do you search for if/else now?
    failure: "Keywords are programming constructs"
  },
  
  // Song/media titles
  {
    query: "Let It Be",
    withStopwords: "Let It Be",
    withoutStopwords: "Let",          // Loses the song entirely
    failure: "Song title depends on stopwords"
  },
  
  // Abbreviation expansion gone wrong
  {
    query: "the university of texas",
    withStopwords: "the university of texas",
    withoutStopwords: "university texas",  // Might work, but misses "UT"
    failure: "Prepositions help identify proper institutions"
  }
];
 
// The key insight: stopwords are only "stop" words when they truly
// provide no discriminative value. Context matters enormously.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

// Recommended: Selective stopword removal
 
// Start with a conservative list and expand based on analysis
const conservativeStopwords = {
  // Articles - generally safe to remove
  articles: ["a", "an", "the"],
  
  // Prepositions - mostly safe, but watch for named entities
  prepositions: ["in", "on", "at", "to", "for", "with", "by", "from", "up", "down"],
  
  // Conjunctions - safe except in programming contexts
  conjunctions: ["and", "or", "but", "so", "yet"],
  
  // Auxiliary verbs - safe for general search
  auxiliaries: ["is", "are", "was", "were", "be", "been", "being",
                "have", "has", "had", "do", "does", "did"],
  
  // Common adverbs - usually safe
  adverbs: ["very", "just", "also", "even", "still", "already"],
  
  // Pronouns - safe except in quote/lyric search
  pronouns: ["i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them"],
  
  // EXPLICITLY EXCLUDED from stopwords (these carry meaning):
  keepThese: [
    "not", "no", "never", "without",   // Negation - critical for meaning
    "if", "else", "then",               // Programming keywords
    "null", "true", "false",            // Programming values
    "all", "none", "any", "every",      // Quantifiers often matter
    "this", "that",                     // Often part of names
  ]
};
 
// Elasticsearch configuration
const selectiveStopwordConfig = {
  "settings": {
    "analysis": {
      "filter": {
        "custom_stop": {
          "type": "stop",
          "stopwords": [
            // Combine safe categories
            ...conservativeStopwords.articles,
            ...conservativeStopwords.prepositions,
            ...conservativeStopwords.conjunctions,
            ...conservativeStopwords.auxiliaries,
            ...conservativeStopwords.adverbs,
            ...conservativeStopwords.pronouns
          ],
          "ignore_case": true
        }
      },
      "analyzer": {
        "content_analyzer": {
          "tokenizer": "standard",
          "filter": ["lowercase", "custom_stop", "porter_stem"]
        }
      }
    }
  }
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

// Automated stopword list generation from corpus analysis
 
interface TermStats {
  term: string;
  documentFrequency: number;    // How many docs contain this term
  collectionFrequency: number;  // Total occurrences across all docs
  idfScore: number;             // Inverse document frequency
}
 
async function analyzeCorpusForStopwords(
  client: ElasticsearchClient,
  index: string,
  minDocFrequencyPercent: number = 30,  // Appears in 30%+ of docs
  topN: number = 200
): Promise<string[]> {
  
  // Get total document count
  const countResult = await client.count({ index });
  const totalDocs = countResult.count;
  
  // Get term statistics from the index
  const termVectors = await client.termvectors({
    index,
    id: "_all",  // Aggregate across all docs
    fields: ["content"],
    term_statistics: true
  });
  
  // Calculate document frequency for each term
  const termStats: TermStats[] = [];
  
  for (const [term, stats] of Object.entries(termVectors.term_vectors)) {
    const docFreq = stats.doc_freq || 0;
    const docFreqPercent = (docFreq / totalDocs) * 100;
    
    // If term appears in more than threshold of documents
    if (docFreqPercent >= minDocFrequencyPercent) {
      termStats.push({
        term,
        documentFrequency: docFreq,
        collectionFrequency: stats.ttf || 0,
        idfScore: Math.log(totalDocs / docFreq)
      });
    }
  }
  
  // Sort by document frequency (highest first)
  termStats.sort((a, b) => b.documentFrequency - a.documentFrequency);
  
  // Return top N candidates
  const candidates = termStats.slice(0, topN).map(t => t.term);
  
  console.log("Stopword candidates (manual review recommended):");
  console.log("Term | DocFreq% | IDF Score");
  console.log("-".repeat(40));
  
  termStats.slice(0, 50).forEach(t => {
    const pct = ((t.documentFrequency / totalDocs) * 100).toFixed(1);
    console.log(`${t.term.padEnd(15)} | ${pct.padStart(6)}% | ${t.idfScore.toFixed(3)}`);
  });
  
  return candidates;
}
 
// Example output for an e-commerce product catalog:
// Term           | DocFreq% | IDF Score
// ----------------------------------------
// product        |   89.3%  | 0.113     ← "product" on a product site - true stopword!  
// for            |   78.4%  | 0.243
// with           |   71.2%  | 0.340
// and            |   69.8%  | 0.359
// the            |   65.4%  | 0.424
// new            |   45.2%  | 0.794     ← Careful! "New" might be a feature
// free           |   42.1%  | 0.865     ← Careful! "Free shipping" is important
// ...
 
// Domain-specific stopwords for different contexts:
 
const domainStopwords = {
  ecommerce: [
    "product", "item", "listing", "sale", "shop", "buy",
    "order", "shipping", "available", "stock"
    // These appear on almost every product page but don't help search
  ],
  
  legal: [
    "court", "case", "law", "legal", "attorney", "counsel",
    "plaintiff", "defendant", "pursuant"
    // Wait - these might actually be useful for faceting!
  ],
  
  medical: [
    "patient", "treatment", "condition", "doctor", "medical",
    "hospital", "clinical", "diagnosis"
    // Careful: these help distinguish medical content
  ],
  
  recipe: [
    "recipe", "ingredients", "minutes", "serves", "cook",
    "preparation", "instructions"
    // Every recipe has these - true stopwords for recipe search
  ]
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

// How position-aware indexing preserves phrase matching
 
// Original document: "The quick brown fox"
// 
// Traditional stopword removal:
//   Tokens: ["quick", "brown", "fox"]  
//   Positions: [0, 1, 2]
//   Problem: Token positions are renumbered. Phrase queries fail.
//
// Position-preserving stopword removal:
//   Tokens: ["quick", "brown", "fox"]
//   Positions: [1, 2, 3]  ← Original positions preserved!
//   "The" was at position 0 but is not indexed.
//
// Now a phrase query for "brown fox" (adjacent in original) still works
// because "brown" is at position 2 and "fox" is at position 3.
 
// Elasticsearch position_increment_gap strategy:
const positionAwareConfig = {
  "settings": {
    "analysis": {
      "filter": {
        "english_stop": {
          "type": "stop",
          "stopwords": "_english_"
        }
      },
      "analyzer": {
        "position_aware_english": {
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "english_stop"  // Position increments are preserved by default
          ]
        }
      }
    }
  }
};
 
// How phrase queries work with stopwords removed:
 
// Document indexed: "The Lord of the Rings"
// After stopword removal with positions:
//   "lord" at position 1
//   "rings" at position 4
//   Positions 0, 2, 3 have no tokens (stopwords removed)
 
// Query: phrase "Lord of the Rings" (exact match)
// After stopword removal with positions:
//   "lord" at position 0
//   "rings" at position 3
//
// Position difference: 3 (from 0 to 3 in query, from 1 to 4 in document)
// The relative positions match! Phrase query succeeds.
 
// However, there's a subtlety:
// Query: phrase "lord rings" vs "lord of the rings"
// Both would match because we only check relative positions.
// This can be a feature (flexibility) or a bug (precision loss).
 
// For exact phrase matching including stopwords:
// Use a separate field that doesn't remove stopwords
const hybridMapping = {
  "mappings": {
    "properties": {
      "title": {
        "type": "text",
        "analyzer": "position_aware_english",  // Stopwords removed
        "fields": {
          "exact": {
            "type": "text",
            "analyzer": "standard"  // Keeps stopwords
          }
        }
      }
    }
  }
};
 
// Query strategy:
// 1. For flexible matching: query "title" field
// 2. For exact phrase matching: query "title.exact" field

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

// How BM25 naturally handles stopwords
 
// BM25 scoring formula (simplified):
// score(q, d) = Σ IDF(qi) * (tf(qi, d) * (k1 + 1)) / (tf(qi, d) + k1 * (1 - b + b * |d|/avgdl))
 
// The key is IDF (Inverse Document Frequency):
// IDF(t) = log((N - df(t) + 0.5) / (df(t) + 0.5))
// Where:
//   N = total documents
//   df(t) = documents containing term t
 
// For stopwords:
// - They appear in nearly every document: df(t) ≈ N
// - Therefore: IDF ≈ log((N - N + 0.5) / (N + 0.5)) ≈ log(0.5 / N) ≈ very small negative number
// - Many implementations floor IDF at 0 for terms appearing in >50% of docs
 
// Example with real numbers:
const calculateIDF = (totalDocs: number, docFreq: number): number => {
  // Elasticsearch's BM25 IDF formula
  return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
};
 
const corpus = {
  totalDocs: 1_000_000
};
 
// Regular term: "programming"
const programmingIDF = calculateIDF(corpus.totalDocs, 50_000);  // In 5% of docs
// IDF ≈ 2.97 - High weight, this term matters!
 
// Stopword: "the"
const theIDF = calculateIDF(corpus.totalDocs, 950_000);  // In 95% of docs
// IDF ≈ 0.05 - Near-zero weight, term is almost ignored
 
// Practical implications:
// If query is "the best programming books"
// Weights: "the"(0.05) + "best"(1.2) + "programming"(2.97) + "books"(1.8)
// "The" contributes only 0.8% of the score.
 
// So why remove stopwords if BM25 handles them?
//
// 1. SPEED: Even with low weight, we still process the posting list
//    "the" has 950,000 documents to iterate through
//
// 2. INDEX SIZE: "the" still consumes storage for 950,000 postings
//
// 3. PHRASE QUERIES: Position matching still considers stopwords
//    Without special handling, "the best" won't match "best"
//
// 4. HIGHLIGHTING: We still need to track stopword positions for snippets
 
// Conclusion: BM25 makes stopword removal an optimization, not a necessity
// But for large-scale systems, that optimization matters significantly