src/services/Tokenizer.java
BeforeProposed
+1−1
1717 while(offset<text.length() && word(text.codePointAt(offset))) {
1818 offset+=Character.charCount(text.codePointAt(offset));points++;
1919 }
20 String term=text.substring(first,offset).toLowerCase(Locale.ROOT);
20 String term=Normalizer.normalize(text.substring(first,offset),Normalizer.Form.NFKC).toLowerCase(Locale.ROOT);
2121 result.add(new Token(term,result.size(),start,points));
2222 }
2323 return result;