Merge pull request #32385 from vespa-engine/bratseth/symmetric-normal…

…izing

Bratseth/symmetric normalizing

container-core/src/main/java/com/yahoo/component/chain/Chain.java

@@ -129,4 +129,5 @@ public int hashCode() {
         result = 31 * result + id.hashCode();
         return result;
     }
+
 }

container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java

@@ -198,7 +198,7 @@ private Substring getOffsets(BlockItem b) {
     private Item stem(BlockItem current, StemContext context, Index index) {
         Item blockAsItem = (Item)current;
         CompositeItem composite;
-        List<StemList> segments = linguistics.getStemmer().stem(current.stringValue(), index.getStemMode(), context.language);
+        List<StemList> segments = linguistics.getStemmer().stem(current.stringValue(), context.language, index.getStemMode(), index.getNormalize());
         if (segments.isEmpty()) return blockAsItem;
 
         String indexName = current.getIndexName();

container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java

@@ -45,7 +45,7 @@ public RuleBaseLinguistics withLanguage(Language language) {
     /** Processes this term according to the linguistics of this rule base */
     public String process(String term) {
         if (stemMode == StemMode.NONE) return term;
-        List<StemList> stems = linguistics.getStemmer().stem(term, StemMode.BEST, language);
+        List<StemList> stems = linguistics.getStemmer().stem(term, language, StemMode.SHORTEST, true);
         if (stems.isEmpty()) return term;
         if (stems.get(0).isEmpty()) return term;
         return stems.get(0).get(0);

container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java

@@ -18,8 +18,11 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
-
+/**
+ * @author baldersheim
+ */
 public class OpportunisticWeakAndSearcherTestCase {
+
     private static Item buildQueryItem(CompositeItem root, CompositeItem injectAtLevel2) {
         root.addItem(new WordItem("text"));
         injectAtLevel2.addItem(new WordItem("a"));
@@ -47,9 +50,9 @@ public void requireThatWeakAndIsDetected() {
     @Test
     public void requireThatWeakAndIsReplacedWithAnd() {
         assertEquals(buildQueryItem(new OrItem(), new AndItem()),
-                weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem())));
+                     weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem())));
         assertEquals(buildQueryItem(new AndItem(), new AndItem()),
-                weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem())));
+                     weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem())));
     }
 
     private static WeakAndItem try2Adjust(WeakAndItem item, int hits) {
@@ -71,6 +74,7 @@ public void requireThatDefaultWeakAndHeapIsAdjustedUpToHits() {
         assertEquals(1000, try2Adjust(new WeakAndItem(), new OrItem(), 1000).getN());
         assertFalse(try2Adjust(new WeakAndItem(), new OrItem(), 10).nIsExplicit());
     }
+
     @Test
     public void requireThatNonDefaultWeakAndHeapIsNotAdjustedUpToHits() {
         assertEquals(33, try2Adjust(new WeakAndItem(33), 1000).getN());

linguistics/abi-spec.json

@@ -605,6 +605,7 @@
       "abstract"
     ],
     "methods" : [
+      "public java.util.List stem(java.lang.String, com.yahoo.language.Language, com.yahoo.language.process.StemMode, boolean)",
       "public abstract java.util.List stem(java.lang.String, com.yahoo.language.process.StemMode, com.yahoo.language.Language)"
     ],
     "fields" : [ ]
@@ -619,6 +620,7 @@
     ],
     "methods" : [
       "public void <init>(com.yahoo.language.process.Tokenizer)",
+      "public java.util.List stem(java.lang.String, com.yahoo.language.Language, com.yahoo.language.process.StemMode, boolean)",
       "public java.util.List stem(java.lang.String, com.yahoo.language.process.StemMode, com.yahoo.language.Language)"
     ],
     "fields" : [ ]

linguistics/src/main/java/com/yahoo/language/process/Stemmer.java

@@ -12,6 +12,21 @@
  */
 public interface Stemmer {
 
+    /**
+     * Stem input according to specified stemming mode.
+     * This default implementation invokes stem(input, mode, language) and so ignores the removeAccents argument.
+     *
+     * @param input    the string to stem.
+     * @param language the language to use for stemming
+     * @param mode     the stemming mode
+     * @param removeAccents whether to normalize accents and similar
+     * @return a list of possible stems. Empty if none.
+     * @throws ProcessingException thrown if there is an exception stemming this input
+     */
+    default List<StemList> stem(String input, Language language, StemMode mode, boolean removeAccents) {
+        return stem(input, mode, language);
+    }
+
     /**
      * Stem input according to specified stemming mode.
      *

linguistics/src/main/java/com/yahoo/language/process/StemmerImpl.java

@@ -18,14 +18,19 @@ public StemmerImpl(Tokenizer tokenizer) {
     }
 
     @Override
-    public List<StemList> stem(String input, StemMode stemMode, Language language) {
+    public List<StemList> stem(String input, Language language, StemMode stemMode, boolean removeAccents) {
         List<StemList> stems = new ArrayList<>();
-        for (Token token : tokenizer.tokenize(input, language, stemMode, false)) {
+        for (Token token : tokenizer.tokenize(input, language, stemMode, removeAccents)) {
             findStems(token, stems);
         }
         return stems;
     }
 
+    @Override
+    public List<StemList> stem(String input, StemMode stemMode, Language language) {
+        return stem(input, language, stemMode, false);
+    }
+
     private void findStems(Token token, List<StemList> out) {
         int len;
         if (token.isSpecialToken() || (len = token.getNumComponents()) == 0) {

linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java

@@ -16,8 +16,8 @@ public interface Tokenizer {
      * @param input the string to tokenize. May be arbitrarily large.
      * @param language the language of the input string.
      * @param stemMode the stem mode applied on the returned tokens
-     * @param removeAccents if true accents and similar are removed from the returned tokens
-     * @return the tokens of the input String.
+     * @param removeAccents whether to normalize accents and similar
+     * @return the tokens of the input String
      * @throws ProcessingException If the underlying library throws an Exception.
      */
     Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents);

linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java

@@ -149,7 +149,7 @@ public int hashCode() {
 
     @Override
     public String toString() {
-        return "token '" + original + "'";
+        return "token '" + tokenString + "'" + ( ! tokenString.equals(original) ? " (original: " + original + ")" : "");
     }
 
     public String toDetailString() {

linguistics/src/test/java/com/yahoo/language/process/StemmerImplTestCase.java

@@ -20,13 +20,13 @@ public class StemmerImplTestCase {
 
     @Test
     public void requireThatStemIsNormalizedAndLowerCased() {
-        assertStem("FOO", List.of("foo"));
-        assertStem("a\u030A", List.of("\u00E5"));
+        assertStem("FOO", List.of("foo"), true);
+        assertStem("a\u030A", List.of("\u00E5"), false);
     }
 
     @Test
     public void requireThatOnlyIndexableTokensAreReturned() {
-        assertStem("foo. (bar)!", List.of("foo", "bar"));
+        assertStem("foo. (bar)!", List.of("foo", "bar"), true);
     }
 
     @Test
@@ -49,17 +49,17 @@ public void requireThatSpecialTokensAreNotDecompounded() {
         assertEquals(List.of(new StemList("c"),
                                    new StemList("p"),
                                    new StemList("p")),
-                     stemmer.stem("c++", StemMode.SHORTEST, Language.ENGLISH));
+                     stemmer.stem("c++", Language.ENGLISH,StemMode.SHORTEST, true));
 
         token.setSpecialToken(true);
         assertEquals(List.of(new StemList("c++")),
-                     stemmer.stem("c++", StemMode.SHORTEST, Language.ENGLISH));
+                     stemmer.stem("c++", Language.ENGLISH, StemMode.SHORTEST, true));
     }
 
-    private static void assertStem(String input, List<String> expectedStems) {
+    private static void assertStem(String input, List<String> expectedStems, boolean removeAccents) {
         Stemmer stemmer = new StemmerImpl(new SimpleTokenizer(new SimpleNormalizer()));
         List<String> got = new ArrayList<>();
-        for (StemList word : stemmer.stem(input, StemMode.ALL, Language.ENGLISH)) {
+        for (StemList word : stemmer.stem(input, Language.ENGLISH, StemMode.ALL, removeAccents)) {
             got.add(word.get(0));
         }
         assertEquals(expectedStems, got);

lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java

@@ -43,7 +43,7 @@ public void testLithuanianTokenizer() {
     @Test
     public void testStemming() {
         String text = "mūšio";
-        List<StemList> tokens = luceneLinguistics().getStemmer().stem(text, StemMode.ALL, Language.LITHUANIAN);
+        List<StemList> tokens = luceneLinguistics().getStemmer().stem(text, Language.LITHUANIAN, StemMode.ALL, true);
         assertEquals(1, tokens.size());
         assertEquals("mūš", tokens.get(0).get(0));
     }

opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java

@@ -50,7 +50,9 @@ public Stemmer getStemmer() {
     }
 
     @Override
-    public Segmenter getSegmenter() { return new SegmenterImpl(forQuerying(getTokenizer())); }
+    public Segmenter getSegmenter() {
+        return new SegmenterImpl(forQuerying(getTokenizer()));
+    }
 
     @Override
     public Detector getDetector() { return detector; }

opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java

@@ -122,8 +122,7 @@ private Iterable<Token> segmentChinese(String input) {
         return tokens;
     }
 
-    private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents,
-                                Stemmer stemmer) {
+    private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) {
         token = normalizer.normalize(token);
         token = LinguisticsCase.toLowerCase(token);
         if (removeAccents)

opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/ChineseSegmentationTest.java

@@ -35,7 +35,7 @@ public void testChineseSegmentation() {
         assertEquals(7, tokens.size());
         assertEquals("[是, 一个, 展示, 雅, 目前, 在, 测试阶段]", tokens.toString());
 
-        var stems = tester.stemmer().stem(text, StemMode.ALL, Language.CHINESE_SIMPLIFIED);
+        var stems = tester.stemmer().stem(text, Language.CHINESE_SIMPLIFIED, StemMode.ALL, true);
         assertEquals(7, tokens.size());
         assertEquals("[是, 一个, 展示, 雅, 目前, 在, 测试阶段]", tokens.toString());
     }
@@ -57,7 +57,7 @@ public void testChineseSegmentationWithoutGrams() {
         assertEquals(7, tokens.size());
         assertEquals("[是, 一个, 展示, 雅, 目前, 在, 测试阶段]", tokens.toString());
 
-        var stems = tester.stemmer().stem(text, StemMode.ALL, Language.CHINESE_SIMPLIFIED);
+        var stems = tester.stemmer().stem(text, Language.CHINESE_SIMPLIFIED, StemMode.ALL, true);
         assertEquals(7, tokens.size());
         assertEquals("[是, 一个, 展示, 雅, 目前, 在, 测试阶段]", tokens.toString());
     }

opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpLinguisticsSubclassingTest.java

@@ -23,7 +23,7 @@ public void testOpenNlpLinguisticsSubclassing() {
         var subclass = new OpenNlpLinguisticsSubclass();
         assertEquals("the only token", subclass.getTokenizer().tokenize("whatever", Language.ENGLISH, StemMode.ALL, false).iterator().next().getTokenString());
         assertEquals("the only token", subclass.getSegmenter().segment("whatever", Language.ENGLISH).iterator().next());
-        assertEquals("the only token", subclass.getStemmer().stem("whatever", StemMode.ALL, Language.ENGLISH).get(0).get(0));
+        assertEquals("the only token", subclass.getStemmer().stem("whatever", Language.ENGLISH, StemMode.ALL, true).get(0).get(0));
     }
 
     static class OpenNlpLinguisticsSubclass extends OpenNlpLinguistics {

opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpLinguisticsTester.java

@@ -3,6 +3,7 @@
 
 import ai.vespa.opennlp.OpenNlpConfig;
 import com.yahoo.language.Language;
+import com.yahoo.language.process.Normalizer;
 import com.yahoo.language.process.Segmenter;
 import com.yahoo.language.process.StemMode;
 import com.yahoo.language.process.Stemmer;
@@ -23,6 +24,7 @@ public class OpenNlpLinguisticsTester {
     private final Tokenizer tokenizer;
     private final Segmenter segmenter;
     private final Stemmer stemmer;
+    private final Normalizer normalizer;
 
     public OpenNlpLinguisticsTester() {
         this(new OpenNlpLinguistics());
@@ -36,16 +38,27 @@ public OpenNlpLinguisticsTester(OpenNlpLinguistics linguistics) {
         this.tokenizer = linguistics.getTokenizer();
         this.segmenter = linguistics.getSegmenter();
         this.stemmer = linguistics.getStemmer();
+        this.normalizer = linguistics.getNormalizer();
     }
 
     Tokenizer tokenizer() { return tokenizer; }
     Segmenter segmenter() { return segmenter; }
     Stemmer stemmer() { return stemmer; }
+    Normalizer normalizer() { return normalizer; }
 
     Iterable<Token> tokenize(String input, Language language) {
         return tokenizer.tokenize(input, language, StemMode.SHORTEST, true);
     }
 
+    String tokenizeToString(String input, Language language) {
+        return tokenize(input, language).iterator().next().getTokenString();
+    }
+
+    String stemAndNormalize(String input, Language language) {
+        var stemListList = stemmer.stem(input, language, StemMode.SHORTEST, true);
+        return normalizer.normalize(stemListList.get(0).get(0));
+    }
+
     void recurseDecompose(Token t) {
         assertTrue(t.getOffset() >= 0);
         assertTrue(t.getOrig().length() >= 0);

opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpNormalizerTestCase.java

@@ -0,0 +1,21 @@
+package com.yahoo.language.opennlp;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class OpenNlpNormalizerTestCase {
+
+    @Test
+    public void testNormalizing() {
+        var normalizer = new OpenNlpLinguisticsTester().normalizer();
+        assertEquals("cafe", normalizer.normalize("cafe"));
+        // TODO: Accent normalize
+        // assertEquals("cafe", normalizer.normalize("café"));
+        // assertEquals("cafe", normalizer.normalize("cafè"));
+    }
+
+}

opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpProcessingSymmetryTestCase.java

@@ -0,0 +1,47 @@
+package com.yahoo.language.opennlp;
+
+import ai.vespa.opennlp.OpenNlpConfig;
+import com.yahoo.language.Language;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class OpenNlpProcessingSymmetryTestCase {
+
+    @Test
+    public void testSymmetricTransformation() {
+        var tester = new OpenNlpLinguisticsTester();
+        var input = "conges";
+        String indexed = tester.tokenizeToString(input, Language.ENGLISH);
+        String queried = tester.stemAndNormalize(input, Language.ENGLISH);
+        assertEquals("Expected that the actual query token equals the indexed", indexed, queried);
+    }
+
+    @Test
+    public void testSymmetricTransformationWithAccentsEnglishKStem() {
+        var tester = new OpenNlpLinguisticsTester();
+        var input = "congés";
+        String indexed = tester.tokenizeToString(input, Language.ENGLISH);
+        String queried = tester.stemAndNormalize(input, Language.ENGLISH);
+        assertEquals("Expected that the actual query token equals the indexed", indexed, queried);
+    }
+
+    @Test
+    public void testSymmetricTransformationWithAccentsEnglishSnowball() {
+        var tester = new OpenNlpLinguisticsTester(new OpenNlpConfig.Builder().snowballStemmingForEnglish(true).build());
+        var input = "congés";
+        String indexed = tester.tokenizeToString(input, Language.ENGLISH);
+        String queried = tester.stemAndNormalize(input, Language.ENGLISH);
+        assertEquals("Expected that the actual query token equals the indexed", indexed, queried);
+    }
+
+    @Test
+    public void testSymmetricTransformationWithAccentsSpanish() {
+        var tester = new OpenNlpLinguisticsTester();
+        var input = "congés";
+        String indexed = tester.tokenizeToString(input, Language.SPANISH);
+        String queried = tester.stemAndNormalize(input, Language.SPANISH);
+        assertEquals("Expected that the actual query token equals the indexed", indexed, queried);
+    }
+
+}

opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java

@@ -204,7 +204,7 @@ public void testTokenizeEmojis() {
     public void testStemEmojis() {
         var stemmer = new OpenNlpLinguistics().getStemmer();
         String emoji = "\uD83D\uDD2A"; // 🔪
-        List<StemList> stems = stemmer.stem(emoji, StemMode.ALL, Language.ENGLISH);
+        List<StemList> stems = stemmer.stem(emoji, Language.ENGLISH, StemMode.ALL, true);
         assertEquals(1, stems.size());
         var stemList = stems.get(0);
         assertEquals(1, stemList.size());