Stemming and Lemmatization: Linguistic Normalization for Search

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

// Examples of morphological variations in English
 
const variations = {
  // Inflectional variations (same part of speech)
  verb_conjugation: {
    base: "run",
    forms: ["run", "runs", "running", "ran"]
  },
  
  noun_pluralization: {
    base: "box",
    forms: ["box", "boxes"]
  },
  
  adjective_comparison: {
    base: "fast",
    forms: ["fast", "faster", "fastest"]
  },
  
  // Derivational variations (different parts of speech)
  derivations: {
    base: "connect",
    forms: {
      verb: "connect",
      noun: "connection",
      adjective: "connected",
      gerund: "connecting"
    }
  },
  
  // Irregular forms that break patterns
  irregulars: {
    "go": ["go", "goes", "going", "went", "gone"],
    "be": ["be", "am", "is", "are", "was", "were", "been", "being"],
    "good": ["good", "better", "best"],
    "mouse": ["mouse", "mice"],  // Irregular plural
    "child": ["child", "children"]
  }
};
 
// The challenge: how does a search engine know these are related?
// Without processing: "running shoes" does NOT match "shoes for runners"
// With stemming: "run shoe" matches "shoe runner" → both stem to similar forms
// With lemmatization: "run shoe" matches "shoe runner" → proper dictionary forms

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

// The Porter Stemmer: Most widely used English stemmer
// Developed by Martin Porter in 1980
 
// Example transformations
const porterStemmerExamples = {
  // Suffix removal rules in action
  
  // -ing removal (with consonant doubling rules)
  "running": "run",      // Double 'n' reduced
  "singing": "sing",
  "processing": "process",
  
  // -tion/-sion removal
  "connection": "connect",
  "discussion": "discuss",
  "creation": "creat",    // Note: not "create" - stems aren't real words!
  
  // -ly removal
  "quickly": "quick",
  "happily": "happili",   // Irregular result
  
  // -ed removal
  "connected": "connect",
  "jumped": "jump",
  "agreed": "agre",       // Imperfect result
  
  // Plural rules
  "boxes": "box",
  "cats": "cat",
  "libraries": "librari",  // Note the imperfect stem
  
  // Porter's five-step algorithm produces aggressive stems:
  "argue": "argu",
  "argued": "argu",
  "arguing": "argu",
  "argues": "argu",
  "argument": "argument",  // Different stem! (a problem)
};
 
// Porter Algorithm Steps (simplified):
// Step 1: Plurals and -ed, -ing suffixes
// Step 2: Map double suffixes to single ones (-ational → -ate)
// Step 3: Remove -ful, -ness, etc.
// Step 4: Remove -ant, -ence, -ment, etc.
// Step 5: Final cleanup (-e removal, double letter reduction)
 
// The algorithm has ~60 rules across these steps

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

// Lemmatization vs Stemming: Head-to-head comparison
 
const comparisonExamples = [
  // Format: [word, stem, lemma]
  
  // Cases where both work well
  ["running", "run", "run"],
  ["dogs", "dog", "dog"],
  ["quickly", "quick", "quickly"],  // Lemma keeps adverbs intact
  
  // Cases where lemmatization excels
  ["went", "went", "go"],           // Stem fails on irregular
  ["mice", "mice", "mouse"],        // Stem fails on irregular plural
  ["better", "better", "good"],     // Stem fails on irregular comparative
  ["was", "wa", "be"],              // Stem produces nonsense
  ["feet", "feet", "foot"],         // Stem can't recognize irregular
  
  // Cases where stemming works but lemmatization might not
  ["argued", "argu", "argue"],      // Stem is shorter, lemma is accurate
  ["electricity", "electr", "electricity"],  // Lemma may not reduce derivations
  
  // Cases requiring part-of-speech disambiguation for lemmatization
  ["meeting", "meet", "???"],  
  // As noun: "meeting" (the meeting was long)
  // As verb: "meet" (they are meeting now)
  // Stemmer doesn't care; lemmatizer may need POS context
  
  ["saw", "saw", "???"],
  // As noun: "saw" (a cutting tool)
  // As verb: "see" (I saw the movie)
];
 
// Lemmatization with POS tagging
interface LemmaLookup {
  word: string;
  pos: "noun" | "verb" | "adjective" | "adverb";
  lemma: string;
}
 
const lemmaExamples: LemmaLookup[] = [
  { word: "meeting", pos: "noun", lemma: "meeting" },
  { word: "meeting", pos: "verb", lemma: "meet" },
  { word: "saw", pos: "noun", lemma: "saw" },
  { word: "saw", pos: "verb", lemma: "see" },
  { word: "better", pos: "adjective", lemma: "good" },
  { word: "better", pos: "adverb", lemma: "well" },  // "He did better"
];

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

// Practical impact: Search quality metrics
 
interface SearchQualityMetrics {
  // Recall: proportion of relevant documents retrieved
  // Precision: proportion of retrieved documents that are relevant
  // F1: harmonic mean of precision and recall
  
  recall: number;
  precision: number;
  f1Score: number;
}
 
// Hypothetical results from an e-commerce product search benchmark
// Query: "running shoes for women"
 
const noNormalization: SearchQualityMetrics = {
  recall: 0.42,      // Misses "runner", "run", etc.
  precision: 0.91,   // But what it finds is very relevant
  f1Score: 0.57
};
 
const withPorterStemming: SearchQualityMetrics = {
  recall: 0.84,      // Finds "runner", "run", etc.
  precision: 0.76,   // Some noise from over-stemming
  f1Score: 0.80
};
 
const withLightStemming: SearchQualityMetrics = {
  recall: 0.71,      // Less aggressive matching
  precision: 0.85,   // Better precision than Porter
  f1Score: 0.78
};
 
const withLemmatization: SearchQualityMetrics = {
  recall: 0.79,      // Handles irregulars well
  precision: 0.88,   // High precision, real words
  f1Score: 0.83
};
 
const withHybridApproach: SearchQualityMetrics = {
  // Lemmatization for verbs, light stemming for nouns
  recall: 0.82,
  precision: 0.87,
  f1Score: 0.85      // Best overall balance
};
 
// The "right" choice depends on business priorities:
// - Recall-focused: Porter stemming (miss fewer relevant docs)
// - Precision-focused: Lemmatization or light stemming (less noise)
// - Balanced: Hybrid approach

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

// Elasticsearch/OpenSearch stemmer configuration
 
const stemmingAnalyzerConfig = {
  "settings": {
    "analysis": {
      "filter": {
        // Porter stemmer - the classic choice
        "porter_stem": {
          "type": "porter_stem"
        },
        
        // Snowball (Porter2) - improved Porter
        "snowball_english": {
          "type": "snowball",
          "language": "English"
        },
        
        // Light stemmer - conservative approach
        "light_english": {
          "type": "stemmer",
          "language": "light_english"
        },
        
        // Minimal stemmer - only plurals
        "minimal_english": {
          "type": "stemmer",
          "language": "minimal_english"
        },
        
        // Hunspell stemmer - dictionary-based (closer to lemmatization)
        "hunspell": {
          "type": "hunspell",
          "locale": "en_US",
          "dedup": true
        }
      },
      
      "analyzer": {
        // Standard with Porter stemming
        "english_porter": {
          "tokenizer": "standard",
          "filter": ["lowercase", "porter_stem"]
        },
        
        // With light stemming for product search
        "product_search": {
          "tokenizer": "standard",
          "filter": ["lowercase", "light_english"]
        },
        
        // Multi-lingual with Snowball
        "multilingual": {
          "tokenizer": "standard",
          "filter": ["lowercase", "snowball_english"]
        }
      }
    }
  }
};
 
// Testing stemmer behavior
// POST /my_index/_analyze
const testRequest = {
  "analyzer": "english_porter",
  "text": "The runners were running faster than yesterday"
};
 
// Response tokens:
// ["runner", "were", "run", "faster", "than", "yesterday"]
// Note: "were" not stemmed (stopword removal would handle this)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

// Lemmatization typically requires a preprocessing pipeline
// before indexing into the search engine
 
import spacy
 
const lemmatizationPipeline = {
  // Option 1: Preprocess documents with spaCy before indexing
  preprocessWithSpacy: async (document: string) => {
    // Python spaCy code (called via API or subprocess)
    // nlp = spacy.load("en_core_web_sm")
    // doc = nlp(document)
    // lemmas = [token.lemma_ for token in doc if not token.is_stop]
    // return " ".join(lemmas)
    
    // Returns lemmatized text to index
  },
  
  // Option 2: Use Elasticsearch's Hunspell (dictionary-based)
  hunspellConfig: {
    "settings": {
      "analysis": {
        "filter": {
          "en_US": {
            "type": "hunspell",
            "locale": "en_US",
            "dedup": true,
            "longest_only": true  // Only return the longest lemma
          }
        },
        "analyzer": {
          "english_hunspell": {
            "tokenizer": "standard",
            "filter": ["lowercase", "en_US"]
          }
        }
      }
    }
  },
  
  // Option 3: Solr's Lemmatization with dictionary
  solrLemmatizer: `
    <analyzer>
      <tokenizer class="solr.StandardTokenizerFactory"/>
      <filter class="solr.LowerCaseFilterFactory"/>
      <filter class="solr.LematizationFilterFactory"
              dictionary="lemmas.txt"
              ignoreCase="true"/>
    </analyzer>
  `
};
 
// Hybrid architecture: preprocess for quality, stem for speed
interface IndexDocument {
  // Original text for display
  title: string;
  description: string;
  
  // Lemmatized versions for high-precision search
  title_lemmatized: string;
  description_lemmatized: string;
  
  // Stemmed versions are created by ES analyzer automatically
}
 
// At query time:
// 1. Primary search: query against lemmatized fields (precision)
// 2. Fallback: query against stemmed fields (recall)
// 3. Combine results with appropriate boosting

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

// Multi-language analyzer configuration in Elasticsearch
 
const multilingualConfig = {
  "settings": {
    "analysis": {
      "filter": {
        // Language-specific stemmers
        "english_stem": { "type": "stemmer", "language": "english" },
        "german_stem": { "type": "stemmer", "language": "german" },
        "french_stem": { "type": "stemmer", "language": "french" },
        "spanish_stem": { "type": "stemmer", "language": "spanish" },
        
        // German compound splitting
        "german_decompounder": {
          "type": "hyphenation_decompounder",
          "word_list_path": "analysis/german_dictionary.txt",
          "hyphenation_patterns_path": "analysis/de_DR.xml",
          "only_longest_match": true,
          "min_subword_size": 4
        },
        
        // Arabic specific normalization
        "arabic_normalization": {
          "type": "arabic_normalization"
        },
        "arabic_stem": {
          "type": "stemmer",
          "language": "arabic"
        }
      },
      
      "analyzer": {
        "english": {
          "tokenizer": "standard",
          "filter": ["lowercase", "english_stem"]
        },
        
        "german": {
          "tokenizer": "standard",
          "filter": ["lowercase", "german_decompounder", "german_stem"]
        },
        
        "arabic": {
          "tokenizer": "standard",
          "filter": ["arabic_normalization", "lowercase", "arabic_stem"]
        },
        
        // CJK (Chinese, Japanese, Korean) analyzer
        "cjk": {
          "tokenizer": "standard",  // or "kuromoji" for Japanese
          "filter": ["cjk_width", "lowercase", "cjk_bigram"]
        }
      }
    }
  },
  
  "mappings": {
    "properties": {
      "content": {
        "type": "text",
        "fields": {
          "english": { "type": "text", "analyzer": "english" },
          "german": { "type": "text", "analyzer": "german" },
          "french": { "type": "text", "analyzer": "french" },
          "arabic": { "type": "text", "analyzer": "arabic" },
          "cjk": { "type": "text", "analyzer": "cjk" }
        }
      }
    }
  }
};
 
// At query time, determine language and query appropriate field:
async function multilingualSearch(query: string, language: string) {
  const fieldToSearch = language === "auto" 
    ? await detectLanguage(query)
    : language;
    
  return elasticClient.search({
    index: "content",
    body: {
      query: {
        match: {
          [`content.${fieldToSearch}`]: query
        }
      }
    }
  });
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

// Medical search: conservative stemming with protected terms
 
const medicalSearchConfig = {
  "settings": {
    "analysis": {
      "filter": {
        // Protect medical terms from stemming
        "protected_words": {
          "type": "keyword_marker",
          "keywords_path": "analysis/medical_terms.txt"
          // Contains: appendicitis, tonsillitis, gastroenteritis, etc.
        },
        
        // Very light stemming after protection
        "medical_stem": {
          "type": "stemmer",
          "language": "minimal_english"  // Only plurals
        },
        
        // Medical synonyms
        "medical_synonyms": {
          "type": "synonym_graph",
          "synonyms_path": "analysis/medical_synonyms.txt"
          // heart attack => myocardial infarction
          // high blood pressure => hypertension
        }
      },
      
      "analyzer": {
        "medical_analyzer": {
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "protected_words",
            "medical_synonyms",
            "medical_stem"
          ]
        }
      }
    }
  }
};
 
// E-commerce: brand protection with aggressive matching elsewhere
 
const ecommerceConfig = {
  "settings": {
    "analysis": {
      "filter": {
        // Never stem brand names
        "brand_protection": {
          "type": "keyword_marker",
          "keywords_path": "analysis/brand_names.txt"
          // iPhone, MacBook, PlayStation, Nintendo, etc.
        },
        
        // Stem everything else
        "product_stem": {
          "type": "stemmer",
          "language": "light_english"
        }
      },
      
      "analyzer": {
        "product_analyzer": {
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "brand_protection",
            "product_stem"
          ]
        }
      }
    }
  }
};
 
// Code/technical documentation: preserve code constructs
 
const technicalDocConfig = {
  "settings": {
    "analysis": {
      "char_filter": {
        // Preserve common code patterns
        "code_patterns": {
          "type": "mapping",
          "mappings": [
            "HashMap => HashMap",
            "ArrayList => ArrayList",
            "async/await => async_await"
          ]
        }
      },
      
      "tokenizer": {
        // Split on camelCase but keep original
        "code_tokenizer": {
          "type": "pattern",
          "pattern": "([A-Z][a-z]+)|([a-z]+)|([A-Z]+)",
          "group": 0
        }
      },
      
      "analyzer": {
        "code_analyzer": {
          "char_filter": ["code_patterns"],
          "tokenizer": "code_tokenizer",
          "filter": ["lowercase"]  // No stemming for code
        }
      }
    }
  }
};