Merge pull request #862 from takahi-i/fix-misssing-neologd-tokenizer-…

…lib-failure Increase the version of elasticsearch-analysis-kuromoji-neologd to 6.5.1
redpen-cc · May 15, 2019 · 42a24bc · 42a24bc
2 parents ed62150 + 237d4c0
commit 42a24bc
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/redpen-core/pom.xml b/redpen-core/pom.xml
@@ -170,7 +170,7 @@
         <dependency>
             <groupId>org.codelibs</groupId>
             <artifactId>elasticsearch-analysis-kuromoji-neologd</artifactId>
-            <version>1.6.0</version>
+            <version>6.5.1</version>
         </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>

diff --git a/redpen-core/src/main/java/cc/redpen/tokenizer/NeologdJapaneseTokenizer.java b/redpen-core/src/main/java/cc/redpen/tokenizer/NeologdJapaneseTokenizer.java
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.util.AttributeFactory;
 import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer;
 import org.codelibs.neologd.ipadic.lucene.analysis.ja.tokenattributes.BaseFormAttribute;
 import org.codelibs.neologd.ipadic.lucene.analysis.ja.tokenattributes.InflectionAttribute;
@@ -36,7 +37,10 @@ public class NeologdJapaneseTokenizer implements RedPenTokenizer {
     private JapaneseTokenizer tokenizer;
 
     public NeologdJapaneseTokenizer() {
-        this.tokenizer = new JapaneseTokenizer(new StringReader(""), null, false, JapaneseTokenizer.Mode.NORMAL);
+        this.tokenizer = new JapaneseTokenizer(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY,
+                null,
+                false,
+                JapaneseTokenizer.Mode.NORMAL);
     }
 
     @Override

diff --git a/redpen-core/src/main/java/cc/redpen/validator/sentence/OkuriganaValidator.java b/redpen-core/src/main/java/cc/redpen/validator/sentence/OkuriganaValidator.java
@@ -92,6 +92,7 @@ public class OkuriganaValidator extends Validator {
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("進れ", asList("動詞", "自立"), 0)));
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("進ろ", asList("動詞", "自立"), 0)));
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("進よ", asList("動詞", "自立"), 0)));
+        invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("進てい", asList("名詞", "左辺接続"), 0)));
 
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("勧", asList("動詞", "自立"), 0)));
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("勧る", asList("動詞", "自立"), 0)));
@@ -169,6 +170,7 @@ public class OkuriganaValidator extends Validator {
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("買る", asList("動詞", "自立"), 0)));
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("買れ", asList("動詞", "自立"), 0)));
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("買よ", asList("動詞", "自立"), 0)));
+        invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("買て", asList("名詞", "一般"), 0)));
 
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("上ら", asList("動詞", "自立"), 0)));
         invalidOkuriganaTokens.add(new ExpressionRule().addElement(new TokenElement("上り", asList("動詞", "自立"), 0)));