Relevance Scoring: TF-IDF, BM25, and Ranking Search Results

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

// Intuition behind relevance scoring
 
interface Document {
  id: string;
  content: string;
  wordCount: number;
}
 
// Query: "python asyncio tutorial"
 
const candidates: Document[] = [
  {
    id: "doc1",
    content: "Python asyncio tutorial: Learn Python asyncio from scratch...",
    wordCount: 500,
    // "python" appears 15 times, "asyncio" 12 times, "tutorial" 5 times
  },
  {
    id: "doc2", 
    content: "Programming tutorials for beginners. Python is popular...",
    wordCount: 2000,
    // "python" appears 3 times, "tutorial" 8 times, no "asyncio"
  },
  {
    id: "doc3",
    content: "The complete guide to asyncio patterns in Python applications...",
    wordCount: 800,
    // "python" appears 6 times, "asyncio" 20 times, no "tutorial"
  }
];
 
// Intuitive ranking:
// 1. doc1 - All query terms, high frequency, focused document
// 2. doc3 - Most asyncio mentions (rare term), good Python coverage
// 3. doc2 - Common terms only, no asyncio, less focused
 
// TF-IDF and BM25 formalize this intuition mathematically

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

// TF-IDF Implementation and Examples
 
// =============== TERM FREQUENCY (TF) ===============
// Several variants exist:
 
// Raw TF: Simply count occurrences
const rawTF = (term: string, document: string[]): number => {
  return document.filter(word => word === term).length;
};
 
// Logarithmic TF: Dampens the effect of high frequencies
// Prevents a document with "python" 100 times from scoring 100x 
// higher than one with it 1 time
const logTF = (term: string, document: string[]): number => {
  const raw = rawTF(term, document);
  return raw > 0 ? 1 + Math.log(raw) : 0;
};
 
// Augmented TF: Normalizes by maximum term frequency in document
// Prevents bias toward longer documents
const augmentedTF = (term: string, document: string[]): number => {
  const raw = rawTF(term, document);
  const maxTF = Math.max(...getTermFrequencies(document));
  return 0.5 + (0.5 * raw / maxTF);
};
 
// =============== INVERSE DOCUMENT FREQUENCY (IDF) ===============
// Measures how rare/informative a term is across the corpus
 
// Standard IDF
const idf = (term: string, corpus: string[][]): number => {
  const N = corpus.length;  // Total documents
  const df = corpus.filter(doc => doc.includes(term)).length;  // Docs containing term
  
  // Add 1 to prevent division by zero for unseen terms
  return Math.log(N / (df + 1));
};
 
// Smooth IDF (used by Elasticsearch)
const smoothIDF = (totalDocs: number, docFreq: number): number => {
  // log(1 + (N - df + 0.5) / (df + 0.5))
  return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
};
 
// =============== COMPLETE TF-IDF ===============
const tfIdf = (term: string, document: string[], corpus: string[][]): number => {
  const tf = logTF(term, document);
  const idfScore = idf(term, corpus);
  return tf * idfScore;
};
 
// =============== EXAMPLE CALCULATION ===============
// Corpus of 10,000 documents
// Query: "python asyncio"
// Document: 500 words, "python" appears 10 times, "asyncio" appears 5 times
 
const exampleScoring = {
  // Term: "python"
  python: {
    tf: Math.log(1 + 10),           // = 3.4 (log TF)
    docsContaining: 5000,            // 50% of corpus
    idf: Math.log(10000 / 5000),     // = 0.69 (common term, low IDF)
    tfIdf: 3.4 * 0.69                // = 2.35
  },
  
  // Term: "asyncio"
  asyncio: {
    tf: Math.log(1 + 5),             // = 2.79 (log TF)
    docsContaining: 200,             // 2% of corpus
    idf: Math.log(10000 / 200),      // = 3.91 (rare term, high IDF)
    tfIdf: 2.79 * 3.91               // = 10.91
  },
  
  // Total document score for query
  totalScore: 2.35 + 10.91           // = 13.26
};
 
// Note: "asyncio" contributes more despite lower TF
// because its rarity (high IDF) makes it more informative

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

// BM25 Algorithm Implementation
 
/**
 * BM25 Scoring Formula:
 * 
 * For a query Q containing terms q1, q2, ... qn:
 * 
 * score(D, Q) = Σ IDF(qi) × (f(qi, D) × (k1 + 1)) / (f(qi, D) + k1 × (1 - b + b × |D|/avgdl))
 * 
 * Where:
 *   f(qi, D) = frequency of term qi in document D
 *   |D| = document length (in words)
 *   avgdl = average document length in the corpus
 *   k1 = term frequency saturation parameter (typically 1.2-2.0)
 *   b = document length normalization parameter (typically 0.75)
 */
 
interface BM25Config {
  k1: number;    // Term frequency saturation (1.2-2.0)
  b: number;     // Length normalization (0-1, typically 0.75)
}
 
interface CorpusStats {
  totalDocs: number;
  avgDocLength: number;
  docFrequencies: Map<string, number>;  // term -> docs containing term
}
 
// IDF component (same concept as TF-IDF, slightly different formula)
function bm25IDF(term: string, stats: CorpusStats): number {
  const N = stats.totalDocs;
  const df = stats.docFrequencies.get(term) || 0;
  
  // Elasticsearch formula: log(1 + (N - df + 0.5) / (df + 0.5))
  return Math.log(1 + (N - df + 0.5) / (df + 0.5));
}
 
// Score for a single term
function bm25TermScore(
  termFreq: number,
  docLength: number,
  idf: number,
  config: BM25Config,
  avgDocLength: number
): number {
  const { k1, b } = config;
  
  // Length normalization factor
  const lengthNorm = 1 - b + b * (docLength / avgDocLength);
  
  // TF saturation formula
  const tfComponent = (termFreq * (k1 + 1)) / (termFreq + k1 * lengthNorm);
  
  return idf * tfComponent;
}
 
// Complete BM25 score for a document
function bm25Score(
  queryTerms: string[],
  document: { terms: Map<string, number>; length: number },
  stats: CorpusStats,
  config: BM25Config = { k1: 1.2, b: 0.75 }
): number {
  let totalScore = 0;
  
  for (const term of queryTerms) {
    const termFreq = document.terms.get(term) || 0;
    
    if (termFreq > 0) {
      const idf = bm25IDF(term, stats);
      const termScore = bm25TermScore(
        termFreq,
        document.length,
        idf,
        config,
        stats.avgDocLength
      );
      totalScore += termScore;
    }
  }
  
  return totalScore;
}
 
// =============== WORKED EXAMPLE ===============
 
const exampleStats: CorpusStats = {
  totalDocs: 10000,
  avgDocLength: 500,
  docFrequencies: new Map([
    ["python", 5000],    // 50% of docs
    ["asyncio", 200],    // 2% of docs
    ["tutorial", 3000],  // 30% of docs
  ])
};
 
const exampleDoc = {
  terms: new Map([
    ["python", 10],
    ["asyncio", 5],
    ["tutorial", 2],
  ]),
  length: 400  // Shorter than average
};
 
// Score breakdown:
// "python":  IDF = 0.69, TF=10, score ≈ 2.1
// "asyncio": IDF = 3.91, TF=5, score ≈ 11.2
// "tutorial": IDF = 1.20, TF=2, score ≈ 2.4
// 
// Total ≈ 15.7
 
// Compare to a longer document (1000 words) with same term frequencies:
// Scores would be lower due to length normalization

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

// BM25 parameter tuning strategies
 
// =============== Elasticsearch BM25 Configuration ===============
 
const customBM25Settings = {
  "settings": {
    "index": {
      "similarity": {
        "custom_bm25": {
          "type": "BM25",
          "k1": 1.2,   // Default
          "b": 0.75    // Default
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "content": {
        "type": "text",
        "similarity": "custom_bm25"
      }
    }
  }
};
 
// =============== Use Case-Specific Tuning ===============
 
const tuningGuide = {
  // Tweet/short text search
  shortText: {
    k1: 0.5,  // Saturate quickly - in 280 chars, term appearing once is enough signal
    b: 0.3,   // Minimal length normalization - all docs are short anyway
    rationale: "Short documents have limited term frequency range; length differences are meaningful"
  },
  
  // Legal/academic documents
  longDocuments: {
    k1: 2.0,  // Allow TF to keep mattering in 50-page documents
    b: 0.75,  // Standard length normalization
    rationale: "Long documents benefit from high TF distinguishing comprehensive coverage"
  },
  
  // Product titles
  productTitles: {
    k1: 1.5,  // Moderate saturation
    b: 0.0,   // NO length normalization - "iPhone 15 Pro" shouldn't beat "iPhone" just for being longer
    rationale: "Title length is intentional and meaningful"
  },
  
  // Q&A / Stack Overflow style
  questionsAndAnswers: {
    k1: 1.2,
    b: 0.9,   // Strong length normalization - concise answers often better
    rationale: "Longer answers aren't necessarily better; reward focused content"
  },
  
  // Log/event search
  logSearch: {
    k1: 0.7,  // Logs are often sparse; term appearing once is significant
    b: 0.1,   // Log length varies but doesn't indicate quality
    rationale: "Structure matters more than term frequency in logs"
  }
};
 
// =============== A/B Testing for Parameters ===============
 
interface SearchExperiment {
  name: string;
  k1: number;
  b: number;
  ndcg: number;  // Normalized Discounted Cumulative Gain
  mrr: number;   // Mean Reciprocal Rank
}
 
const experimentResults: SearchExperiment[] = [
  { name: "baseline", k1: 1.2, b: 0.75, ndcg: 0.72, mrr: 0.68 },
  { name: "low_k1", k1: 0.8, b: 0.75, ndcg: 0.69, mrr: 0.71 },
  { name: "high_k1", k1: 1.8, b: 0.75, ndcg: 0.74, mrr: 0.67 },
  { name: "low_b", k1: 1.2, b: 0.4, ndcg: 0.73, mrr: 0.69 },
  { name: "high_b", k1: 1.2, b: 0.9, ndcg: 0.70, mrr: 0.72 },
];
 
// Best configuration depends on your evaluation metric priorities
// NDCG values overall result quality; MRR values top-1 precision

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

// Field boosting strategies in Elasticsearch
 
// =============== QUERY-TIME BOOSTING ===============
 
// Multi-match with field boosts
const multiMatchQuery = {
  "query": {
    "multi_match": {
      "query": "python asyncio tutorial",
      "fields": [
        "title^5",       // Title matches worth 5x
        "headings^3",    // Section headings worth 3x
        "content^1",     // Body content baseline
        "tags^4",        // Tags are highly relevant
        "metadata^0.5"   // Metadata less important
      ],
      "type": "best_fields",  // Score based on best matching field
      "tie_breaker": 0.3      // Also consider other field matches
    }
  }
};
 
// Understanding tie_breaker:
// - 0: Only best field counts (default for best_fields)
// - 0.3: Best field + 30% of other matching fields
// - 1.0: Sum of all fields (like most_fields)
 
// =============== MULTI-MATCH TYPES ===============
 
const multiMatchTypes = {
  // best_fields: Use score from best matching field
  // Good for: Finding documents where query appears in ONE key field
  bestFields: {
    "type": "best_fields",
    "fields": ["title^2", "content"]
  },
  
  // most_fields: Sum scores from all matching fields
  // Good for: Documents where query might appear across multiple fields
  mostFields: {
    "type": "most_fields", 
    "fields": ["title", "title.stemmed", "title.exact"]  // Multiple analyzers
  },
  
  // cross_fields: Treat all fields as one big field
  // Good for: Queries that span fields (person's first + last name)
  crossFields: {
    "type": "cross_fields",
    "fields": ["first_name", "last_name"],
    "operator": "and"  // All terms must be found across fields
  },
  
  // phrase: Match as phrase in each field
  // Good for: Exact phrase matching with field preferences
  phrase: {
    "type": "phrase",
    "fields": ["title^2", "content"],
    "slop": 2  // Allow 2 words between query terms
  }
};
 
// =============== FUNCTION SCORE FOR COMPLEX BOOSTING ===============
 
const functionScoreQuery = {
  "query": {
    "function_score": {
      "query": {
        "multi_match": {
          "query": "python tutorial",
          "fields": ["title^2", "content"]
        }
      },
      "functions": [
        // Boost newer documents
        {
          "filter": { "range": { "date": { "gte": "now-1M" } } },
          "weight": 1.5
        },
        
        // Boost popular documents (log scale)
        {
          "field_value_factor": {
            "field": "view_count",
            "factor": 1.2,
            "modifier": "log1p",  // log(1 + view_count * 1.2)
            "missing": 1
          }
        },
        
        // Boost verified authors
        {
          "filter": { "term": { "author.verified": true } },
          "weight": 1.2
        },
        
        // Decay by age (documents get less relevant over time)
        {
          "gauss": {
            "date": {
              "origin": "now",
              "scale": "30d",
              "offset": "7d",
              "decay": 0.5
            }
          }
        }
      ],
      "score_mode": "multiply",  // How to combine function scores
      "boost_mode": "multiply"   // How to combine with query score
    }
  }
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

// Debugging relevance with Elasticsearch explain API
 
// =============== GET SCORE EXPLANATION ===============
 
// Add "explain": true to your search
const explainRequest = {
  "query": {
    "match": {
      "content": "python tutorial"
    }
  },
  "explain": true  // Returns score breakdown
};
 
// Response includes explanation for each hit:
const explainResponse = {
  "hits": {
    "hits": [{
      "_id": "doc1",
      "_score": 12.5,
      "_explanation": {
        "value": 12.5,
        "description": "sum of:",
        "details": [
          {
            "value": 8.3,
            "description": "weight(content:python) [BM25]",
            "details": [
              { "value": 2.1, "description": "idf, log(1 + (N - n + 0.5) / (n + 0.5))" },
              { "value": 3.95, "description": "tf, freq=10.0, with k1=1.2, b=0.75" }
            ]
          },
          {
            "value": 4.2,
            "description": "weight(content:tutorial) [BM25]",
            "details": [/* ... */]
          }
        ]
      }
    }]
  }
};
 
// =============== EXPLAIN A SPECIFIC DOCUMENT ===============
 
// Why did this document score what it did?
const explainDocRequest = {
  // POST /my_index/_explain/doc_id
  "query": {
    "match": { "content": "python tutorial" }
  }
};
 
// =============== COMMON DEBUGGING SCENARIOS ===============
 
const debuggingGuide = {
  // Issue: Document with more term occurrences ranks lower
  unexpectedlyLowScore: {
    possibleCauses: [
      "Document is very long (length normalization)",
      "Terms are common (low IDF)",
      "Field boost is lower than expected",
      "Terms match a stemmed version (different IDF)"
    ],
    debugSteps: [
      "Check explain output for IDF values",
      "Compare document lengths",
      "Verify field boosts in query",
      "Look at tf normalization factor"
    ]
  },
  
  // Issue: Irrelevant document ranks high
  unexpectedlyHighScore: {
    possibleCauses: [
      "Document is very short (benefits from length norm)",
      "Term appears in boosted field (title)",
      "Query terms are accidentally common in this doc",
      "Synonym expansion matched unintended terms"
    ],
    debugSteps: [
      "Check which field matched (explain shows field)",
      "Verify analyzer behavior with _analyze",
      "Check for synonym expansion issues",
      "Review field boosting configuration"
    ]
  },
  
  // Issue: Perfect document doesn't appear at all
  missingDocument: {
    possibleCauses: [
      "Analyzer processes query differently than document",
      "Stopword removal eliminated query",
      "Stemmer created mismatch",
      "Document was filtered (not a scoring issue)"
    ],
    debugSteps: [
      "Use _analyze API on query terms",
      "Use _termvectors to see how doc was indexed",
      "Check if filters are excluding the document",
      "Verify index contains the document"
    ]
  }
};
 
// =============== ANALYZER DEBUGGING ===============
 
// Check how a term is analyzed
const analyzeRequest = {
  // POST /my_index/_analyze
  "analyzer": "my_analyzer",
  "text": "running pythons"
};
 
// Response: ["run", "python"] (after stemming)
// If document was indexed with different analyzer, mismatch occurs!
 
// Check how a document was actually indexed
const termVectorsRequest = {
  // GET /my_index/_termvectors/doc_id
  "fields": ["content"],
  "term_statistics": true,
  "field_statistics": true
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

// Modern relevance approaches
 
// =============== LEARNING TO RANK (LTR) ===============
// Use ML to combine signals: BM25, popularity, freshness, etc.
 
interface LTRFeatures {
  bm25_score: number;
  title_match: boolean;
  doc_length: number;
  page_views: number;
  days_since_publish: number;
  author_reputation: number;
  has_images: boolean;
}
 
// Training data: [features, relevance_label]
// Model learns optimal combination of signals
 
const ltrPluginQuery = {
  "query": {
    "sltr": {
      "_name": "logged_featureset",
      "featureset": "my_features",
      "params": { "keywords": "python tutorial" },
      "model": "my_trained_model"
    }
  }
};
 
// =============== SEMANTIC / VECTOR SEARCH ===============
// Encode text as dense vectors; find similar vectors
 
// First, encode documents and queries using an embedding model
// (e.g., Sentence-BERT, OpenAI embeddings)
 
const vectorSearchQuery = {
  // Elasticsearch 8.x+ with dense_vector field
  "knn": {
    "field": "content_vector",
    "query_vector": [0.12, -0.34, 0.56, /* ... 768 dims */],
    "k": 10,
    "num_candidates": 100
  }
};
 
// =============== HYBRID SEARCH (BM25 + VECTOR) ===============
// Best practice: combine lexical and semantic signals
 
const hybridSearchQuery = {
  "query": {
    "bool": {
      "should": [
        // Lexical matching (BM25)
        {
          "multi_match": {
            "query": "python async programming",
            "fields": ["title^2", "content"],
            "boost": 1.0
          }
        }
      ]
    }
  },
  // Semantic matching (KNN)
  "knn": {
    "field": "content_vector",
    "query_vector": embedQuery("python async programming"),
    "k": 10,
    "boost": 0.5  // Weight relative to BM25
  }
};
 
// Reciprocal Rank Fusion (RRF) for combining rankings
function reciprocalRankFusion(
  bm25Ranking: string[],
  vectorRanking: string[],
  k: number = 60  // Tunable parameter
): Map<string, number> {
  const scores = new Map<string, number>();
  
  // Score = Σ 1 / (k + rank)
  bm25Ranking.forEach((docId, rank) => {
    const current = scores.get(docId) || 0;
    scores.set(docId, current + 1 / (k + rank + 1));
  });
  
  vectorRanking.forEach((docId, rank) => {
    const current = scores.get(docId) || 0;
    scores.set(docId, current + 1 / (k + rank + 1));
  });
  
  return scores;
}
 
// =============== QUERY EXPANSION ===============
// Improve BM25 input by expanding the query
 
interface ExpandedQuery {
  original: string;
  synonyms: string[];
  spellingCorrections: string[];
  relatedTerms: string[];
}
 
async function expandQuery(query: string): Promise<ExpandedQuery> {
  return {
    original: "python asyncio",
    synonyms: ["python async io", "python asynchronous"],
    spellingCorrections: [],
    relatedTerms: ["coroutines", "event loop", "await"]
  };
}
 
// Build expanded query
const expandedQueryDSL = {
  "query": {
    "bool": {
      "should": [
        { "match": { "content": { "query": "python asyncio", "boost": 2.0 } } },
        { "match": { "content": { "query": "coroutines event loop", "boost": 0.5 } } }
      ]
    }
  }
};