daixque commented on code in PR #12915:
URL: https://github.com/apache/lucene/pull/12915#discussion_r1423277326
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaUppercaseFilter.java:
##########
@@ -0,0 +1,83 @@
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * A {@link TokenFilter} that normalizes small letters (捨て仮名) in katakana into
normal letters. For
+ * instance, "ストップウォッチ" will be translated to "ストツプウオツチ".
+ *
+ * <p>This filter is useful if you want to search against old style Japanese
text such as patents,
+ * legal, contract policies, etc.
+ */
+public final class JapaneseKatakanaUppercaseFilter extends TokenFilter {
+ private static final Map<Character, Character> s2l;
+
+ static {
+ // supported characters are:
+ // ァ ィ ゥ ェ ォ ヵ ㇰ ヶ ㇱ ㇲ ッ ㇳ ㇴ ㇵ ㇶ ㇷ ㇷ゚ ㇸ ㇹ ㇺ ャ ュ ョ ㇻ ㇼ ㇽ ㇾ ㇿ ヮ
+ s2l =
+ Map.ofEntries(
+ Map.entry('ァ', 'ア'),
+ Map.entry('ィ', 'イ'),
+ Map.entry('ゥ', 'ウ'),
+ Map.entry('ェ', 'エ'),
+ Map.entry('ォ', 'オ'),
+ Map.entry('ヵ', 'カ'),
+ Map.entry('ㇰ', 'ク'),
+ Map.entry('ヶ', 'ケ'),
+ Map.entry('ㇱ', 'シ'),
+ Map.entry('ㇲ', 'ス'),
+ Map.entry('ッ', 'ツ'),
+ Map.entry('ㇳ', 'ト'),
+ Map.entry('ㇴ', 'ヌ'),
+ Map.entry('ㇵ', 'ハ'),
+ Map.entry('ㇶ', 'ヒ'),
+ Map.entry('ㇷ', 'フ'),
+ Map.entry('ㇸ', 'ヘ'),
+ Map.entry('ㇹ', 'ホ'),
+ Map.entry('ㇺ', 'ム'),
+ Map.entry('ャ', 'ヤ'),
+ Map.entry('ュ', 'ユ'),
+ Map.entry('ョ', 'ヨ'),
+ Map.entry('ㇻ', 'ラ'),
+ Map.entry('ㇼ', 'リ'),
+ Map.entry('ㇽ', 'ル'),
+ Map.entry('ㇾ', 'レ'),
+ Map.entry('ㇿ', 'ロ'),
+ Map.entry('ヮ', 'ワ'));
+ }
+
+ private final CharTermAttribute termAttr =
addAttribute(CharTermAttribute.class);
+
+ public JapaneseKatakanaUppercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAttr.toString();
+ // Small letter "ㇷ゚" is not single character, so it should be converted
to "プ" as String
+ term = term.replace("ㇷ゚", "プ");
+ char[] src = term.toCharArray();
Review Comment:
Thanks, but it will affect length of result character array and break the
tests. So let me keep current implementation.
Here is the example of test result.
```
term 0 expected:<ちよつと[]> but was:<ちよつと[sTerm��������������]>
Expected :ちよつと
Actual :ちよつとsTerm��������������
```
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaUppercaseFilter.java:
##########
@@ -0,0 +1,83 @@
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * A {@link TokenFilter} that normalizes small letters (捨て仮名) in katakana into
normal letters. For
+ * instance, "ストップウォッチ" will be translated to "ストツプウオツチ".
+ *
+ * <p>This filter is useful if you want to search against old style Japanese
text such as patents,
+ * legal, contract policies, etc.
+ */
+public final class JapaneseKatakanaUppercaseFilter extends TokenFilter {
+ private static final Map<Character, Character> s2l;
+
+ static {
+ // supported characters are:
+ // ァ ィ ゥ ェ ォ ヵ ㇰ ヶ ㇱ ㇲ ッ ㇳ ㇴ ㇵ ㇶ ㇷ ㇷ゚ ㇸ ㇹ ㇺ ャ ュ ョ ㇻ ㇼ ㇽ ㇾ ㇿ ヮ
+ s2l =
+ Map.ofEntries(
+ Map.entry('ァ', 'ア'),
+ Map.entry('ィ', 'イ'),
+ Map.entry('ゥ', 'ウ'),
+ Map.entry('ェ', 'エ'),
+ Map.entry('ォ', 'オ'),
+ Map.entry('ヵ', 'カ'),
+ Map.entry('ㇰ', 'ク'),
+ Map.entry('ヶ', 'ケ'),
+ Map.entry('ㇱ', 'シ'),
+ Map.entry('ㇲ', 'ス'),
+ Map.entry('ッ', 'ツ'),
+ Map.entry('ㇳ', 'ト'),
+ Map.entry('ㇴ', 'ヌ'),
+ Map.entry('ㇵ', 'ハ'),
+ Map.entry('ㇶ', 'ヒ'),
+ Map.entry('ㇷ', 'フ'),
+ Map.entry('ㇸ', 'ヘ'),
+ Map.entry('ㇹ', 'ホ'),
+ Map.entry('ㇺ', 'ム'),
+ Map.entry('ャ', 'ヤ'),
+ Map.entry('ュ', 'ユ'),
+ Map.entry('ョ', 'ヨ'),
+ Map.entry('ㇻ', 'ラ'),
+ Map.entry('ㇼ', 'リ'),
+ Map.entry('ㇽ', 'ル'),
+ Map.entry('ㇾ', 'レ'),
+ Map.entry('ㇿ', 'ロ'),
+ Map.entry('ヮ', 'ワ'));
+ }
+
+ private final CharTermAttribute termAttr =
addAttribute(CharTermAttribute.class);
+
+ public JapaneseKatakanaUppercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAttr.toString();
+ // Small letter "ㇷ゚" is not single character, so it should be converted
to "プ" as String
+ term = term.replace("ㇷ゚", "プ");
+ char[] src = term.toCharArray();
Review Comment:
Thanks, but it will affect length of result character array and break the
tests. So let me keep current implementation.
Here is the example of test result.
```
term 0 expected:<ちよつと[]> but was:<ちよつと[sTerm��������������]>
Expected :ちよつと
Actual :ちよつとsTerm��������������
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
