Full-Text Search

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

// Zipf's Law: Word frequency follows a power law distribution
// The most common word appears ~2x as often as the second most common,
// ~3x as often as the third, and so on.
 
// Example from analyzing 1 million documents:
const wordFrequencies = [
  { word: "the",    frequency: 69_971_528, rank: 1, percentOfCorpus: 7.0 },
  { word: "of",     frequency: 36_411_928, rank: 2, percentOfCorpus: 3.6 },
  { word: "and",    frequency: 28_765_432, rank: 3, percentOfCorpus: 2.9 },
  { word: "to",     frequency: 26_148_764, rank: 4, percentOfCorpus: 2.6 },
  { word: "a",      frequency: 21_341_218, rank: 5, percentOfCorpus: 2.1 },
  { word: "in",     frequency: 20_312_987, rank: 6, percentOfCorpus: 2.0 },
  { word: "is",     frequency: 14_987_234, rank: 7, percentOfCorpus: 1.5 },
  { word: "that",   frequency: 12_384_756, rank: 8, percentOfCorpus: 1.2 },
  { word: "it",     frequency: 10_124_879, rank: 9, percentOfCorpus: 1.0 },
  { word: "was",    frequency: 9_876_543,  rank: 10, percentOfCorpus: 1.0 },
];
 
// The top 10 words alone represent ~25% of all word occurrences!
// The top 100 words represent ~50% of all word occurrences.
 
// Implication for search:
// If we keep these words in the index, "the" appears in effectively
// 100% of documents. A search for "the" would match everything,
// providing zero discriminative value while consuming massive storage.
 
interface IndexImpact {
  withStopwords: {
    indexSize: string;
    averageTokensPerDoc: number;
    searchTimeMs: number;
  };
  withoutStopwords: {
    indexSize: string;
    averageTokensPerDoc: number;
    searchTimeMs: number;
  };
}
 
const realWorldImpact: IndexImpact = {
  withStopwords: {
    indexSize: "42 GB",
    averageTokensPerDoc: 847,
    searchTimeMs: 23
  },
  withoutStopwords: {
    indexSize: "28 GB",               // 33% smaller!
    averageTokensPerDoc: 423,         // 50% fewer tokens
    searchTimeMs: 14                  // 40% faster search
  }
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

// Query execution comparison: with and without stopwords
 
// User query: "what is the best programming language for web development"
 
// WITH stopwords indexed:
// Query becomes: 
//   (what OR is OR the OR best OR programming OR language OR for OR web OR development)
// 
// Posting lists to merge:
//   "what"        → 4,892,341 documents
//   "is"          → 9,123,456 documents  ← almost every document!
//   "the"         → 9,456,789 documents  ← almost every document!
//   "best"        → 1,234,567 documents
//   "programming" → 567,890 documents
//   "language"    → 891,234 documents
//   "for"         → 8,765,432 documents  ← almost every document!
//   "web"         → 2,345,678 documents
//   "development" → 1,987,654 documents
//
// Total posting list entries to process: ~39 million
// Most work is wasted on low-value terms
 
// WITHOUT stopwords:
// Query becomes:
//   (best OR programming OR language OR web OR development)
//
// Posting lists to merge:
//   "best"        → 1,234,567 documents
//   "programming" → 567,890 documents
//   "language"    → 891,234 documents
//   "web"         → 2,345,678 documents
//   "development" → 1,987,654 documents
//
// Total posting list entries to process: ~7 million
// 82% reduction in work for the query engine
 
// Additionally, relevance scoring focuses on meaningful terms:
// Without stopwords, "programming language web development" 
// receives full scoring weight instead of being diluted by
// high-frequency low-value terms.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

// Real-world stopword removal failures
 
const problemQueries = [
  // Named entities broken by stopword removal
  {
    query: "The Office",
    withStopwords: "The Office",    // Correct: finds the TV show
    withoutStopwords: "Office",     // Broken: finds Microsoft Office, generic offices
    failure: "Article is part of proper name"
  },
  
  // Phrases destroyed
  {
    query: "to be or not to be",
    withStopwords: "to be or not to be",
    withoutStopwords: "",            // Complete disaster: empty query!
    failure: "Entire query is stopwords"
  },
  
  // Negation inverted
  {
    query: "not working",
    withStopwords: "not working",    // Finds bug reports, troubleshooting
    withoutStopwords: "working",     // Finds opposite: things that ARE working
    failure: "Meaning inverted by removing 'not'"
  },
  
  // Technical meaning lost
  {
    query: "if else statement",
    withStopwords: "if else statement",
    withoutStopwords: "statement",    // How do you search for if/else now?
    failure: "Keywords are programming constructs"
  },
  
  // Song/media titles
  {
    query: "Let It Be",
    withStopwords: "Let It Be",
    withoutStopwords: "Let",          // Loses the song entirely
    failure: "Song title depends on stopwords"
  },
  
  // Abbreviation expansion gone wrong
  {
    query: "the university of texas",
    withStopwords: "the university of texas",
    withoutStopwords: "university texas",  // Might work, but misses "UT"
    failure: "Prepositions help identify proper institutions"
  }
];
 
// The key insight: stopwords are only "stop" words when they truly
// provide no discriminative value. Context matters enormously.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

// Recommended: Selective stopword removal
 
// Start with a conservative list and expand based on analysis
const conservativeStopwords = {
  // Articles - generally safe to remove
  articles: ["a", "an", "the"],
  
  // Prepositions - mostly safe, but watch for named entities
  prepositions: ["in", "on", "at", "to", "for", "with", "by", "from", "up", "down"],
  
  // Conjunctions - safe except in programming contexts
  conjunctions: ["and", "or", "but", "so", "yet"],
  
  // Auxiliary verbs - safe for general search
  auxiliaries: ["is", "are", "was", "were", "be", "been", "being",
                "have", "has", "had", "do", "does", "did"],
  
  // Common adverbs - usually safe
  adverbs: ["very", "just", "also", "even", "still", "already"],
  
  // Pronouns - safe except in quote/lyric search
  pronouns: ["i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them"],
  
  // EXPLICITLY EXCLUDED from stopwords (these carry meaning):
  keepThese: [
    "not", "no", "never", "without",   // Negation - critical for meaning
    "if", "else", "then",               // Programming keywords
    "null", "true", "false",            // Programming values
    "all", "none", "any", "every",      // Quantifiers often matter
    "this", "that",                     // Often part of names
  ]
};
 
// Elasticsearch configuration
const selectiveStopwordConfig = {
  "settings": {
    "analysis": {
      "filter": {
        "custom_stop": {
          "type": "stop",
          "stopwords": [
            // Combine safe categories
            ...conservativeStopwords.articles,
            ...conservativeStopwords.prepositions,
            ...conservativeStopwords.conjunctions,
            ...conservativeStopwords.auxiliaries,
            ...conservativeStopwords.adverbs,
            ...conservativeStopwords.pronouns
          ],
          "ignore_case": true
        }
      },
      "analyzer": {
        "content_analyzer": {
          "tokenizer": "standard",
          "filter": ["lowercase", "custom_stop", "porter_stem"]
        }
      }
    }
  }
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

// Automated stopword list generation from corpus analysis
 
interface TermStats {
  term: string;
  documentFrequency: number;    // How many docs contain this term
  collectionFrequency: number;  // Total occurrences across all docs
  idfScore: number;             // Inverse document frequency
}
 
async function analyzeCorpusForStopwords(
  client: ElasticsearchClient,
  index: string,
  minDocFrequencyPercent: number = 30,  // Appears in 30%+ of docs
  topN: number = 200
): Promise<string[]> {
  
  // Get total document count
  const countResult = await client.count({ index });
  const totalDocs = countResult.count;
  
  // Get term statistics from the index
  const termVectors = await client.termvectors({
    index,
    id: "_all",  // Aggregate across all docs
    fields: ["content"],
    term_statistics: true
  });
  
  // Calculate document frequency for each term
  const termStats: TermStats[] = [];
  
  for (const [term, stats] of Object.entries(termVectors.term_vectors)) {
    const docFreq = stats.doc_freq || 0;
    const docFreqPercent = (docFreq / totalDocs) * 100;
    
    // If term appears in more than threshold of documents
    if (docFreqPercent >= minDocFrequencyPercent) {
      termStats.push({
        term,
        documentFrequency: docFreq,
        collectionFrequency: stats.ttf || 0,
        idfScore: Math.log(totalDocs / docFreq)
      });
    }
  }
  
  // Sort by document frequency (highest first)
  termStats.sort((a, b) => b.documentFrequency - a.documentFrequency);
  
  // Return top N candidates
  const candidates = termStats.slice(0, topN).map(t => t.term);
  
  console.log("Stopword candidates (manual review recommended):");
  console.log("Term | DocFreq% | IDF Score");
  console.log("-".repeat(40));
  
  termStats.slice(0, 50).forEach(t => {
    const pct = ((t.documentFrequency / totalDocs) * 100).toFixed(1);
    console.log(`${t.term.padEnd(15)} | ${pct.padStart(6)}% | ${t.idfScore.toFixed(3)}`);
  });
  
  return candidates;
}
 
// Example output for an e-commerce product catalog:
// Term           | DocFreq% | IDF Score
// ----------------------------------------
// product        |   89.3%  | 0.113     ← "product" on a product site - true stopword!  
// for            |   78.4%  | 0.243
// with           |   71.2%  | 0.340
// and            |   69.8%  | 0.359
// the            |   65.4%  | 0.424
// new            |   45.2%  | 0.794     ← Careful! "New" might be a feature
// free           |   42.1%  | 0.865     ← Careful! "Free shipping" is important
// ...
 
// Domain-specific stopwords for different contexts:
 
const domainStopwords = {
  ecommerce: [
    "product", "item", "listing", "sale", "shop", "buy",
    "order", "shipping", "available", "stock"
    // These appear on almost every product page but don't help search
  ],
  
  legal: [
    "court", "case", "law", "legal", "attorney", "counsel",
    "plaintiff", "defendant", "pursuant"
    // Wait - these might actually be useful for faceting!
  ],
  
  medical: [
    "patient", "treatment", "condition", "doctor", "medical",
    "hospital", "clinical", "diagnosis"
    // Careful: these help distinguish medical content
  ],
  
  recipe: [
    "recipe", "ingredients", "minutes", "serves", "cook",
    "preparation", "instructions"
    // Every recipe has these - true stopwords for recipe search
  ]
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

// How position-aware indexing preserves phrase matching
 
// Original document: "The quick brown fox"
// 
// Traditional stopword removal:
//   Tokens: ["quick", "brown", "fox"]  
//   Positions: [0, 1, 2]
//   Problem: Token positions are renumbered. Phrase queries fail.
//
// Position-preserving stopword removal:
//   Tokens: ["quick", "brown", "fox"]
//   Positions: [1, 2, 3]  ← Original positions preserved!
//   "The" was at position 0 but is not indexed.
//
// Now a phrase query for "brown fox" (adjacent in original) still works
// because "brown" is at position 2 and "fox" is at position 3.
 
// Elasticsearch position_increment_gap strategy:
const positionAwareConfig = {
  "settings": {
    "analysis": {
      "filter": {
        "english_stop": {
          "type": "stop",
          "stopwords": "_english_"
        }
      },
      "analyzer": {
        "position_aware_english": {
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "english_stop"  // Position increments are preserved by default
          ]
        }
      }
    }
  }
};
 
// How phrase queries work with stopwords removed:
 
// Document indexed: "The Lord of the Rings"
// After stopword removal with positions:
//   "lord" at position 1
//   "rings" at position 4
//   Positions 0, 2, 3 have no tokens (stopwords removed)
 
// Query: phrase "Lord of the Rings" (exact match)
// After stopword removal with positions:
//   "lord" at position 0
//   "rings" at position 3
//
// Position difference: 3 (from 0 to 3 in query, from 1 to 4 in document)
// The relative positions match! Phrase query succeeds.
 
// However, there's a subtlety:
// Query: phrase "lord rings" vs "lord of the rings"
// Both would match because we only check relative positions.
// This can be a feature (flexibility) or a bug (precision loss).
 
// For exact phrase matching including stopwords:
// Use a separate field that doesn't remove stopwords
const hybridMapping = {
  "mappings": {
    "properties": {
      "title": {
        "type": "text",
        "analyzer": "position_aware_english",  // Stopwords removed
        "fields": {
          "exact": {
            "type": "text",
            "analyzer": "standard"  // Keeps stopwords
          }
        }
      }
    }
  }
};
 
// Query strategy:
// 1. For flexible matching: query "title" field
// 2. For exact phrase matching: query "title.exact" field

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

// How BM25 naturally handles stopwords
 
// BM25 scoring formula (simplified):
// score(q, d) = Σ IDF(qi) * (tf(qi, d) * (k1 + 1)) / (tf(qi, d) + k1 * (1 - b + b * |d|/avgdl))
 
// The key is IDF (Inverse Document Frequency):
// IDF(t) = log((N - df(t) + 0.5) / (df(t) + 0.5))
// Where:
//   N = total documents
//   df(t) = documents containing term t
 
// For stopwords:
// - They appear in nearly every document: df(t) ≈ N
// - Therefore: IDF ≈ log((N - N + 0.5) / (N + 0.5)) ≈ log(0.5 / N) ≈ very small negative number
// - Many implementations floor IDF at 0 for terms appearing in >50% of docs
 
// Example with real numbers:
const calculateIDF = (totalDocs: number, docFreq: number): number => {
  // Elasticsearch's BM25 IDF formula
  return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
};
 
const corpus = {
  totalDocs: 1_000_000
};
 
// Regular term: "programming"
const programmingIDF = calculateIDF(corpus.totalDocs, 50_000);  // In 5% of docs
// IDF ≈ 2.97 - High weight, this term matters!
 
// Stopword: "the"
const theIDF = calculateIDF(corpus.totalDocs, 950_000);  // In 95% of docs
// IDF ≈ 0.05 - Near-zero weight, term is almost ignored
 
// Practical implications:
// If query is "the best programming books"
// Weights: "the"(0.05) + "best"(1.2) + "programming"(2.97) + "books"(1.8)
// "The" contributes only 0.8% of the score.
 
// So why remove stopwords if BM25 handles them?
//
// 1. SPEED: Even with low weight, we still process the posting list
//    "the" has 950,000 documents to iterate through
//
// 2. INDEX SIZE: "the" still consumes storage for 950,000 postings
//
// 3. PHRASE QUERIES: Position matching still considers stopwords
//    Without special handling, "the best" won't match "best"
//
// 4. HIGHLIGHTING: We still need to track stopword positions for snippets
 
// Conclusion: BM25 makes stopword removal an optimization, not a necessity
// But for large-scale systems, that optimization matters significantly

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

// Production-ready stopword configuration
 
const stopwordConfig = {
  "settings": {
    "analysis": {
      "filter": {
        // Option 1: Built-in English stopwords (35 words)
        "english_stop_builtin": {
          "type": "stop",
          "stopwords": "_english_"
        },
        
        // Option 2: Custom conservative list (recommended)
        "conservative_stop": {
          "type": "stop",
          "stopwords": [
            // Articles
            "a", "an", "the",
            // Prepositions
            "in", "on", "at", "to", "for", "with", "by", "from", "up", "down", "into", "onto",
            // Conjunctions
            "and", "or", "but", "so", "yet",
            // Auxiliaries
            "is", "am", "are", "was", "were", "be", "been", "being",
            "have", "has", "had", "do", "does", "did", "will", "would", "could", "should",
            // Common pronouns
            "i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them",
            "my", "your", "his", "its", "our", "their",
            // Common adverbs
            "very", "just", "also", "even", "still", "already", "only"
          ],
          "ignore_case": true
        },
        
        // Option 3: Aggressive list for large corpuses
        "aggressive_stop": {
          "type": "stop",
          "stopwords_path": "analysis/aggressive_stopwords.txt",  // External file
          "ignore_case": true
        },
        
        // Option 4: Custom domain list loaded from file
        "domain_stop": {
          "type": "stop",
          "stopwords_path": "analysis/domain_stopwords.txt",
          "ignore_case": true
        }
      },
      
      "analyzer": {
        // Default analyzer with conservative stopwords
        "default_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "conservative_stop",
            "porter_stem"
          ]
        },
        
        // Analyzer that keeps stopwords for exact phrase matching
        "phrase_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "lowercase"
            // No stopword filter
          ]
        },
        
        // Domain-optimized analyzer
        "domain_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "conservative_stop",
            "domain_stop",  // Additional domain-specific removal
            "porter_stem"
          ]
        }
      }
    }
  },
  
  "mappings": {
    "properties": {
      "content": {
        "type": "text",
        "analyzer": "default_analyzer",
        "fields": {
          // For exact phrase queries that need stopwords
          "phrase": {
            "type": "text",
            "analyzer": "phrase_analyzer"
          }
        }
      },
      
      // Title: more conservative (stopwords might be in names)
      "title": {
        "type": "text",
        "analyzer": "phrase_analyzer",  // Keep stopwords
        "fields": {
          "search": {
            "type": "text",
            "analyzer": "default_analyzer"  // Removes stopwords
          }
        }
      }
    }
  }
};
 
// Query-time strategy:
async function searchWithFallback(query: string, client: ElasticsearchClient) {
  // First: try phrase match (stopwords preserved)
  const phraseResult = await client.search({
    index: "content",
    body: {
      query: {
        match_phrase: {
          "content.phrase": {
            query,
            slop: 1  // Allow small variations
          }
        }
      }
    }
  });
  
  if (phraseResult.hits.total.value > 0) {
    return phraseResult;
  }
  
  // Fallback: regular match (stopwords removed)
  return client.search({
    index: "content",
    body: {
      query: {
        match: {
          content: query  // Uses default_analyzer
        }
      }
    }
  });
}