1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

import nltk
from nltk.corpus import stopwords
import spacy
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
 
# Download NLTK stopwords (run once)
# nltk.download('stopwords')
 
# NLTK English stopwords
nltk_stops = set(stopwords.words('english'))
print(f"NLTK stopwords count: {len(nltk_stops)}")
print(f"Sample: {sorted(list(nltk_stops))[:20]}")
 
# spaCy English stopwords
nlp = spacy.load("en_core_web_sm")
spacy_stops = nlp.Defaults.stop_words
print(f"\nspaCy stopwords count: {len(spacy_stops)}")
print(f"Sample: {sorted(list(spacy_stops))[:20]}")
 
# scikit-learn stopwords
sklearn_stops = set(ENGLISH_STOP_WORDS)
print(f"\nscikit-learn stopwords count: {len(sklearn_stops)}")
print(f"Sample: {sorted(list(sklearn_stops))[:20]}")
 
# Analyze overlap and differences
all_lists = [nltk_stops, spacy_stops, sklearn_stops]
names = ["NLTK", "spaCy", "sklearn"]
 
print("\n=== Overlap Analysis ===")
common_to_all = nltk_stops & spacy_stops & sklearn_stops
print(f"Common to all three: {len(common_to_all)} words")
 
# Words unique to each list
print("\nWords in spaCy but not NLTK or sklearn:")
spacy_unique = spacy_stops - nltk_stops - sklearn_stops
print(f"  {sorted(list(spacy_unique))[:15]}...")
 
print("\nWords in NLTK but not spaCy or sklearn:")
nltk_unique = nltk_stops - spacy_stops - sklearn_stops
print(f"  {sorted(list(nltk_unique))[:15]}...")
 
# Important words that might be controversial as stopwords
potentially_meaningful = ['not', 'no', 'never', 'always', 'very', 'more', 
                          'most', 'few', 'many', 'much', 'all', 'some']
print("\n=== Potentially Meaningful 'Stop Words' ===")
for word in potentially_meaningful:
    in_nltk = word in nltk_stops
    in_spacy = word in spacy_stops
    in_sklearn = word in sklearn_stops
    print(f"  '{word}': NLTK={in_nltk}, spaCy={in_spacy}, sklearn={in_sklearn}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

from typing import List, Set, Callable
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import re
 
class StopWordRemover:
    """
    Comprehensive stop-word removal with multiple configuration options.
    """
    
    def __init__(
        self,
        language: str = "english",
        custom_stops: Set[str] = None,
        preserve_negation: bool = False,
        preserve_question_words: bool = False,
        case_sensitive: bool = False,
        min_word_length: int = 0
    ):
        """
        Initialize stop-word remover with configuration.
        
        Args:
            language: Language for NLTK stopwords
            custom_stops: Additional words to add as stop words
            preserve_negation: Keep negation words (not, no, never, etc.)
            preserve_question_words: Keep question words (who, what, where, etc.)
            case_sensitive: Whether stop-word matching is case-sensitive
            min_word_length: Remove words shorter than this (0 = no minimum)
        """
        # Load base stop words
        self.stops = set(stopwords.words(language))
        
        # Add custom stop words
        if custom_stops:
            self.stops.update(custom_stops)
        
        # Remove negation words if preserving
        if preserve_negation:
            negations = {'not', 'no', 'never', 'neither', 'nor', "n't", 
                        'none', 'nothing', 'nowhere', 'nobody'}
            self.stops -= negations
        
        # Remove question words if preserving
        if preserve_question_words:
            question_words = {'who', 'what', 'where', 'when', 'why', 'how',
                             'which', 'whom', 'whose'}
            self.stops -= question_words
        
        self.case_sensitive = case_sensitive
        self.min_word_length = min_word_length
        
        # If case-insensitive, lowercase all stop words
        if not case_sensitive:
            self.stops = {w.lower() for w in self.stops}
    
    def is_stopword(self, word: str) -> bool:
        """Check if a word is a stop word."""
        check_word = word if self.case_sensitive else word.lower()
        
        # Check minimum length
        if len(word) < self.min_word_length:
            return True
        
        return check_word in self.stops
    
    def remove_stopwords(self, tokens: List[str]) -> List[str]:
        """Remove stop words from a list of tokens."""
        return [t for t in tokens if not self.is_stopword(t)]
    
    def process_text(self, text: str) -> List[str]:
        """Tokenize and remove stop words from raw text."""
        tokens = word_tokenize(text)
        return self.remove_stopwords(tokens)
    
    def get_stopwords(self) -> Set[str]:
        """Return the current stop-word set."""
        return self.stops.copy()
 
# Demonstration
remover = StopWordRemover(preserve_negation=True)
text = "I am not happy with the service, but I have no complaints about the product."
 
print(f"Original text: {text}")
print(f"Tokens: {word_tokenize(text)}")
print(f"After stop-word removal: {remover.process_text(text)}")
print(f"Negation preserved: 'not' in result = {'not' in remover.process_text(text)}")
 
# Compare with negation removed
remover_strict = StopWordRemover(preserve_negation=False)
print(f"\nStrict removal: {remover_strict.process_text(text)}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

from collections import Counter
from typing import List, Set
import math
 
class DynamicStopWordIdentifier:
    """
    Identify stop words based on corpus statistics rather than fixed lists.
    Uses multiple criteria: frequency, document frequency, and TF-IDF.
    """
    
    def __init__(self, documents: List[List[str]]):
        """
        Analyze corpus to identify potential stop words.
        
        Args:
            documents: List of tokenized documents
        """
        self.documents = documents
        self.n_docs = len(documents)
        
        # Compute statistics
        self._compute_frequencies()
    
    def _compute_frequencies(self):
        """Compute term and document frequencies."""
        # Term frequency across entire corpus
        self.term_freq = Counter()
        for doc in self.documents:
            self.term_freq.update(doc)
        
        # Document frequency (in how many docs does each term appear?)
        self.doc_freq = Counter()
        for doc in self.documents:
            unique_terms = set(doc)
            self.doc_freq.update(unique_terms)
        
        # Total tokens
        self.total_tokens = sum(self.term_freq.values())
        
        # Compute IDF for each term
        self.idf = {}
        for term in self.doc_freq:
            # Smoothed IDF to avoid division by zero
            self.idf[term] = math.log((self.n_docs + 1) / (self.doc_freq[term] + 1)) + 1
    
    def get_stopwords_by_frequency(
        self, 
        top_n: int = None,
        min_frequency: float = None
    ) -> Set[str]:
        """
        Get stop words as the most frequent terms.
        
        Args:
            top_n: Return top N most frequent words
            min_frequency: Return words appearing in > min_frequency proportion
        """
        if top_n:
            return set([word for word, _ in self.term_freq.most_common(top_n)])
        
        if min_frequency:
            threshold = self.total_tokens * min_frequency
            return set([word for word, freq in self.term_freq.items() 
                       if freq > threshold])
        
        return set()
    
    def get_stopwords_by_document_frequency(
        self,
        min_doc_ratio: float = 0.8
    ) -> Set[str]:
        """
        Get stop words as terms appearing in many documents.
        Terms in > min_doc_ratio of documents are considered stop words.
        """
        threshold = self.n_docs * min_doc_ratio
        return set([word for word, freq in self.doc_freq.items()
                   if freq > threshold])
    
    def get_stopwords_by_idf(
        self,
        max_idf: float = 1.5
    ) -> Set[str]:
        """
        Get stop words as terms with low IDF (appear in many documents).
        Low IDF = high document frequency = likely stop word.
        """
        return set([word for word, idf in self.idf.items() if idf < max_idf])
    
    def get_combined_stopwords(
        self,
        top_n: int = 50,
        min_doc_ratio: float = 0.5
    ) -> Set[str]:
        """
        Combine multiple criteria for robust stop-word identification.
        A word is a stop word if it meets BOTH criteria.
        """
        freq_stops = self.get_stopwords_by_frequency(top_n=top_n)
        doc_freq_stops = self.get_stopwords_by_document_frequency(min_doc_ratio)
        
        # Intersection: must be both frequent AND appear in many documents
        return freq_stops & doc_freq_stops
    
    def analyze_word(self, word: str) -> dict:
        """Get detailed statistics for a specific word."""
        return {
            "word": word,
            "term_frequency": self.term_freq.get(word, 0),
            "document_frequency": self.doc_freq.get(word, 0),
            "doc_frequency_ratio": self.doc_freq.get(word, 0) / self.n_docs,
            "idf": self.idf.get(word, 0),
            "frequency_ratio": self.term_freq.get(word, 0) / self.total_tokens,
        }
 
# Example usage with sample corpus
documents = [
    ["the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"],
    ["the", "dog", "barks", "at", "the", "fox", "in", "the", "garden"],
    ["a", "fox", "and", "a", "dog", "are", "in", "the", "park"],
    ["the", "lazy", "cat", "sleeps", "on", "the", "sofa"],
    ["my", "dog", "loves", "to", "run", "in", "the", "park"],
]
 
identifier = DynamicStopWordIdentifier(documents)
 
print("=== Dynamic Stop-word Analysis ===\n")
 
print("Top 10 most frequent words:")
for word, freq in identifier.term_freq.most_common(10):
    stats = identifier.analyze_word(word)
    print(f"  {word:10} freq={freq:3} doc_ratio={stats['doc_frequency_ratio']:.2f} idf={stats['idf']:.3f}")
 
print("\nStop words by frequency (top 5):")
print(f"  {identifier.get_stopwords_by_frequency(top_n=5)}")
 
print("\nStop words by document frequency (>60% of docs):")
print(f"  {identifier.get_stopwords_by_document_frequency(min_doc_ratio=0.6)}")
 
print("\nCombined stop words (top 10 AND >40% docs):")
print(f"  {identifier.get_combined_stopwords(top_n=10, min_doc_ratio=0.4)}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

from typing import Set, List
from dataclasses import dataclass
 
@dataclass
class DomainStopWords:
    """
    Domain-specific stop-word configurations.
    """
    name: str
    additions: Set[str]  # Words to ADD as stop words
    preservations: Set[str]  # Standard stops to PRESERVE (not remove)
    
# Pre-defined domain configurations
MEDICAL_CONFIG = DomainStopWords(
    name="medical",
    additions={
        "patient", "patients", "treatment", "hospital", "clinical",
        "study", "mg", "ml", "daily", "administered", "presented",
        "history", "examination", "diagnosis", "prognosis"
    },
    preservations={
        "not", "no", "without", "negative", "none", "never",
        "decreased", "increased", "absent", "present"
    }
)
 
LEGAL_CONFIG = DomainStopWords(
    name="legal",
    additions={
        "hereby", "herein", "hereof", "thereof", "whereas",
        "aforesaid", "heretofore", "notwithstanding", "pursuant",
        "plaintiff", "defendant", "court", "filed", "pursuant"
    },
    preservations={
        "shall", "may", "must", "will", "should",  # Modal distinctions matter
        "not", "no", "neither", "nor"  # Negation critical
    }
)
 
ECOMMERCE_CONFIG = DomainStopWords(
    name="ecommerce",
    additions={
        "product", "item", "bought", "ordered", "received",
        "shipping", "delivered", "package", "seller", "store",
        "price", "purchase", "return", "customer"
    },
    preservations={
        "not", "never", "very", "really", "extremely", "highly",
        "worst", "best", "great", "terrible", "amazing", "awful",
        "love", "hate", "recommend"  # Sentiment-carrying words
    }
)
 
class DomainAwareStopWordRemover:
    """
    Stop-word remover that adapts to specific domains.
    """
    
    def __init__(self, base_stops: Set[str], domain_config: DomainStopWords = None):
        """
        Initialize with base stops and optional domain configuration.
        """
        self.stops = base_stops.copy()
        
        if domain_config:
            # Add domain-specific stop words
            self.stops.update(domain_config.additions)
            
            # Preserve domain-critical words (remove from stop list)
            self.stops -= domain_config.preservations
            
            self.domain = domain_config.name
        else:
            self.domain = "generic"
    
    def remove_stopwords(self, tokens: List[str]) -> List[str]:
        """Remove stop words from tokenized text."""
        return [t for t in tokens if t.lower() not in self.stops]
    
    def explain_removal(self, tokens: List[str]) -> dict:
        """Explain what was removed and why."""
        removed = []
        kept = []
        
        for token in tokens:
            if token.lower() in self.stops:
                removed.append(token)
            else:
                kept.append(token)
        
        return {
            "domain": self.domain,
            "original_tokens": len(tokens),
            "removed_count": len(removed),
            "kept_count": len(kept),
            "removed_words": removed,
            "result": kept
        }
 
# Example usage
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
 
base_stops = set(stopwords.words('english'))
 
# Generic remover
generic_remover = DomainAwareStopWordRemover(base_stops)
 
# Medical remover
medical_remover = DomainAwareStopWordRemover(base_stops, MEDICAL_CONFIG)
 
# Test sentences
sentences = [
    "The patient did not respond to treatment.",
    "Patient presented with no signs of infection.",
    "This product is not what I expected but I love it.",
]
 
print("=== Domain-Aware Stop-word Removal ===\n")
 
for sentence in sentences:
    tokens = word_tokenize(sentence.lower())
    
    print(f"Input: {sentence}")
    print(f"Tokens: {tokens}")
    
    generic_result = generic_remover.explain_removal(tokens)
    medical_result = medical_remover.explain_removal(tokens)
    
    print(f"\nGeneric removal:")
    print(f"  Removed: {generic_result['removed_words']}")
    print(f"  Result: {generic_result['result']}")
    
    print(f"\nMedical domain removal:")
    print(f"  Removed: {medical_result['removed_words']}")
    print(f"  Result: {medical_result['result']}")
    print("---\n")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

"""
Demonstration of how stop-word removal affects different NLP tasks.
"""
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
 
stops = set(stopwords.words('english'))
 
def remove_stops(text: str) -> str:
    tokens = word_tokenize(text.lower())
    filtered = [t for t in tokens if t not in stops]
    return " ".join(filtered)
 
# === Sentiment Analysis Example ===
print("=== SENTIMENT ANALYSIS ===\n")
 
sentiment_examples = [
    ("I love this movie!", "Positive"),
    ("I do not love this movie.", "Negative"),
    ("This is not a bad product.", "Positive (double negative)"),
    ("I have never been so disappointed.", "Negative"),
]
 
for text, true_sentiment in sentiment_examples:
    cleaned = remove_stops(text)
    print(f"Original: {text}")
    print(f"True sentiment: {true_sentiment}")
    print(f"After stop removal: {cleaned}")
    
    # Analyze what's lost
    original_tokens = set(word_tokenize(text.lower()))
    cleaned_tokens = set(word_tokenize(cleaned))
    removed = original_tokens - cleaned_tokens
    print(f"Removed words: {removed}")
    print()
 
# === Question Answering Example ===
print("\n=== QUESTION ANSWERING ===\n")
 
questions = [
    "Who is the president of the United States?",
    "What is machine learning?",
    "Where can I find good restaurants?",
    "When was Python created?",
    "Why is the sky blue?",
    "How does neural network work?",
]
 
for q in questions:
    cleaned = remove_stops(q)
    original_words = set(word_tokenize(q.lower()))
    cleaned_words = set(word_tokenize(cleaned))
    removed = original_words - cleaned_words
    
    print(f"Original: {q}")
    print(f"Cleaned:  {cleaned}")
    print(f"Lost question context: {'who' in removed or 'what' in removed or 'where' in removed or 'when' in removed or 'why' in removed or 'how' in removed}")
    print()
 
# === Phrase Meaning Changes ===
print("\n=== PHRASE MEANING CHANGES ===\n")
 
phrases = [
    "to be or not to be",
    "I think therefore I am",
    "The more you know",
    "Less is more",
    "It is what it is",
]
 
for phrase in phrases:
    cleaned = remove_stops(phrase)
    print(f"Original: '{phrase}'")
    print(f"Cleaned:  '{cleaned}'")
    print()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

from nltk.corpus import stopwords
import spacy
 
# NLTK supports 21 languages
print("NLTK supported languages:")
print(stopwords.fileids())
 
# Compare stop-word list sizes
languages = ['english', 'spanish', 'french', 'german', 'russian', 
             'portuguese', 'italian', 'dutch', 'arabic']
 
print("\n=== Stop-word List Sizes by Language ===\n")
for lang in languages:
    try:
        stops = stopwords.words(lang)
        print(f"{lang:12} {len(stops):4} stop words")
    except:
        print(f"{lang:12} Not available")
 
# Examples of language-specific stops
print("\n=== Language-Specific Stop-word Examples ===\n")
 
lang_examples = {
    'english': ['the', 'a', 'is', 'are', 'was', 'were'],
    'spanish': ['el', 'la', 'los', 'las', 'es', 'son'],
    'french': ['le', 'la', 'les', 'est', 'sont', 'un', 'une'],
    'german': ['der', 'die', 'das', 'ist', 'sind', 'ein', 'eine'],
    'arabic': ['من', 'في', 'على', 'إلى', 'هو', 'هي'],
}
 
for lang, examples in lang_examples.items():
    all_stops = set(stopwords.words(lang)) if lang in stopwords.fileids() else set()
    in_list = [w for w in examples if w in all_stops]
    print(f"{lang}: {examples}")
    print(f"  In NLTK stops: {in_list}\n")
 
# Using spaCy for multilingual stop words
print("\n=== spaCy Multilingual Stop Words ===\n")
 
# Load different language models (requires separate installation)
# python -m spacy download es_core_news_sm
# python -m spacy download de_core_news_sm
 
try:
    nlp_en = spacy.load("en_core_web_sm")
    nlp_es = spacy.load("es_core_news_sm")
    nlp_de = spacy.load("de_core_news_sm")
    
    print(f"English spaCy stops: {len(nlp_en.Defaults.stop_words)}")
    print(f"Spanish spaCy stops: {len(nlp_es.Defaults.stop_words)}")
    print(f"German spaCy stops: {len(nlp_de.Defaults.stop_words)}")
except OSError as e:
    print(f"Some language models not installed: {e}")
    print("Install with: python -m spacy download <model_name>")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

import time
from typing import List, Set
from nltk.corpus import stopwords
 
# Prepare test data
stops_list = stopwords.words('english')  # List
stops_set = set(stops_list)  # Set
stops_frozenset = frozenset(stops_list)  # Frozenset
 
# Generate test tokens
test_tokens = ["the", "quick", "brown", "fox", "jumps"] * 10000
 
def benchmark(func, tokens, n_iterations=100):
    """Benchmark a function."""
    start = time.perf_counter()
    for _ in range(n_iterations):
        result = func(tokens)
    elapsed = time.perf_counter() - start
    return elapsed / n_iterations
 
# Different implementations to benchmark
 
def remove_stops_list(tokens: List[str]) -> List[str]:
    """Using list membership (slow)."""
    return [t for t in tokens if t not in stops_list]
 
def remove_stops_set(tokens: List[str]) -> List[str]:
    """Using set membership (fast)."""
    return [t for t in tokens if t not in stops_set]
 
def remove_stops_frozenset(tokens: List[str]) -> List[str]:
    """Using frozenset membership (fast, immutable)."""
    return [t for t in tokens if t not in stops_frozenset]
 
def remove_stops_explicit_loop(tokens: List[str]) -> List[str]:
    """Using explicit for loop (slower than comprehension)."""
    result = []
    for t in tokens:
        if t not in stops_set:
            result.append(t)
    return result
 
def remove_stops_filter(tokens: List[str]) -> List[str]:
    """Using filter function."""
    return list(filter(lambda t: t not in stops_set, tokens))
 
# Run benchmarks
print("=== Stop-word Removal Performance Benchmark ===\n")
print(f"Test data: {len(test_tokens)} tokens\n")
 
implementations = [
    ("List membership", remove_stops_list),
    ("Set membership", remove_stops_set),
    ("Frozenset membership", remove_stops_frozenset),
    ("Explicit loop + set", remove_stops_explicit_loop),
    ("Filter + lambda + set", remove_stops_filter),
]
 
times = []
for name, func in implementations:
    elapsed = benchmark(func, test_tokens)
    times.append((name, elapsed))
    print(f"{name:25} {elapsed*1000:8.3f} ms")
 
# Calculate speedups relative to slowest
base_time = max(t[1] for t in times)
print("\nSpeedup vs slowest:")
for name, elapsed in times:
    speedup = base_time / elapsed
    print(f"  {name:25} {speedup:5.1f}x")
 
# Memory considerations
import sys
print(f"\n=== Memory Usage ===")
print(f"List size:      {sys.getsizeof(stops_list):6} bytes")
print(f"Set size:       {sys.getsizeof(stops_set):6} bytes")
print(f"Frozenset size: {sys.getsizeof(stops_frozenset):6} bytes")