This is an automated email from the ASF dual-hosted git repository. airborne12 pushed a commit to branch feature/ik-custom-tokenizers in repository https://gitbox.apache.org/repos/asf/doris.git
commit 79715d0b1b29933c8956ea3be962bd4349723d4a Author: airborne12 <[email protected]> AuthorDate: Wed Sep 23 03:14:00 2026 +0800 [fix](inverted-index) Keep built-in normalizer bindings and collapse equivalent analyzer identities Canonicalizing a top-level name to its built-in spelling rebound the index to an exact policy of that name, because BE checks an exact policy before the built-in normalizer. Return the spelling that still reaches the built-in; built-in analyzers keep the canonical name, which BE dispatches before consulting any policy. Also collapse identities that describe the same pipeline, so the duplicate index fence sees them as one: a normalizer is the keyword analyzer with the same filters, a char_replace filter restating the factory defaults is the bare built-in reference, adjacent duplicate lowercase filters fold, and a pinyin tokenizer that can only emit dictionary pinyin ignores lowercase. --- .../invertedindex/AnalyzerIdentityBuilder.java | 111 +++++++++++--- .../apache/doris/indexpolicy/IndexPolicyMgr.java | 29 ++-- .../doris/alter/SchemaChangeHandlerTest.java | 36 +++++ .../analysis/InvertedIndexPropertiesTest.java | 46 ++++++ .../invertedindex/AnalyzerIdentityBuilderTest.java | 164 ++++++++++++++++++++- 5 files changed, 347 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java index 82a4069b22d..691f3f32cb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilder.java @@ -42,6 +42,15 @@ import java.util.regex.Pattern; public final class AnalyzerIdentityBuilder { private static final String PROP_MAX_NGRAM_DIFF = "max_ngram_diff"; + private static final String KEYWORD_TOKENIZER = "keyword"; + private static final String CHAR_REPLACE_FILTER = "char_replace"; + private static final String PROP_PATTERN = "pattern"; + private static final String PROP_REPLACEMENT = "replacement"; + // Defaults CharReplaceCharFilterFactory applies to a bare built-in reference. + private static final String CHAR_REPLACE_DEFAULT_PATTERN = ",._"; + private static final String CHAR_REPLACE_DEFAULT_REPLACEMENT = " "; + // Token filters that emit the same terms, offsets and provenance when applied twice in a row. + private static final Set<String> IDEMPOTENT_TOKEN_FILTERS = ImmutableSet.of("lowercase"); // Same separator BE uses between bracketed list entries. private static final Pattern ENTRY_SEPARATOR = Pattern.compile("(?<=\\])\\s*,\\s*(?=\\[)"); private static final Set<String> WORD_DELIMITER_TYPES = ImmutableSet.of( @@ -222,24 +231,26 @@ public final class AnalyzerIdentityBuilder { /** Whether BE builds the built-in normalizer for this name; an exact legacy policy shadows it. */ private static boolean isBuiltinNormalizerBinding(String name) { - if (!IndexPolicy.BUILTIN_NORMALIZERS.contains(name)) { - return false; - } try { Env env = Env.getCurrentEnv(); - return env == null || env.getIndexPolicyMgr() == null - || env.getIndexPolicyMgr().getPolicyByExactName(name) == null; + if (env != null && env.getIndexPolicyMgr() != null) { + return env.getIndexPolicyMgr().getTopLevelBuiltin( + name, IndexPolicy.BUILTIN_NORMALIZERS) != null; + } } catch (RuntimeException e) { - return true; + // Fall through to the name-only answer. } + return IndexPolicy.BUILTIN_NORMALIZERS.contains( + Strings.nullToEmpty(name).trim().toLowerCase(Locale.ROOT)); } /** * BE builds a built-in normalizer as the keyword tokenizer plus the built-in token filter of - * the same name, so it shares the identity of that custom pipeline. + * the canonical name, so it shares the identity of that custom pipeline. */ private static String builtinNormalizerIdentity(String name) { - return IndexPolicyTypeEnum.NORMALIZER.name() + ":" + IndexPolicy.PROP_TOKEN_FILTER + "=" + name + ";"; + return buildIdentityFromPolicyProperties(IndexPolicyTypeEnum.NORMALIZER, + Map.of(IndexPolicy.PROP_TOKEN_FILTER, name.trim().toLowerCase(Locale.ROOT))); } /** @@ -254,8 +265,17 @@ public final class AnalyzerIdentityBuilder { properties.get(IndexPolicy.PROP_TOKENIZER), IndexPolicyTypeEnum.TOKENIZER); FoldContext downstreamFold = foldsAsciiCaseAfterCharFilters(type, properties, tokenizerIdentity); + IndexPolicyTypeEnum identityType = type; + if (type == IndexPolicyTypeEnum.NORMALIZER) { + // BE's CustomNormalizer is the keyword tokenizer plus the configured char and token + // filters, so it emits what the equivalent analyzer emits and shares its identity. + identityType = IndexPolicyTypeEnum.ANALYZER; + tokenizerIdentity = KEYWORD_TOKENIZER; + sortedProps.put(IndexPolicy.PROP_TOKENIZER, KEYWORD_TOKENIZER); + } + StringBuilder sb = new StringBuilder(); - sb.append(type.name()).append(":"); + sb.append(identityType.name()).append(":"); for (Map.Entry<String, String> entry : sortedProps.entrySet()) { String key = entry.getKey(); @@ -323,15 +343,23 @@ public final class AnalyzerIdentityBuilder { sortedProps.remove(PROP_MAX_NGRAM_DIFF); } if (expectedType == IndexPolicyTypeEnum.CHAR_FILTER - && "char_replace".equals(sortedProps.get(IndexPolicy.PROP_TYPE))) { - String replacement = sortedProps.getOrDefault("replacement", " "); + && CHAR_REPLACE_FILTER.equals(sortedProps.get(IndexPolicy.PROP_TYPE))) { + String replacement = sortedProps.getOrDefault( + PROP_REPLACEMENT, CHAR_REPLACE_DEFAULT_REPLACEMENT); String pattern = canonicalizeCharReplacePattern( - sortedProps.get("pattern"), replacement, fold); + sortedProps.getOrDefault(PROP_PATTERN, CHAR_REPLACE_DEFAULT_PATTERN), + replacement, fold); if (pattern.isEmpty()) { return ""; } - sortedProps.put("pattern", pattern); - sortedProps.put("replacement", replacement); + if (isCharReplaceDefault(pattern, replacement, fold)) { + // Restating the factory defaults is the bare built-in reference. + sortedProps.remove(PROP_PATTERN); + sortedProps.remove(PROP_REPLACEMENT); + } else { + sortedProps.put(PROP_PATTERN, pattern); + sortedProps.put(PROP_REPLACEMENT, replacement); + } } if (normalizedType != null && sortedProps.size() == 1) { return normalizedType; @@ -348,6 +376,13 @@ public final class AnalyzerIdentityBuilder { return "empty".equals(normalizedName) ? "" : normalizedName == null ? name : normalizedName; } + /** Whether this canonical char_replace configuration is what a bare built-in reference gets. */ + private static boolean isCharReplaceDefault(String pattern, String replacement, FoldContext fold) { + return CHAR_REPLACE_DEFAULT_REPLACEMENT.equals(replacement) + && canonicalizeCharReplacePattern( + CHAR_REPLACE_DEFAULT_PATTERN, replacement, fold).equals(pattern); + } + private static void canonicalizeEffectiveComponentProperties( TreeMap<String, String> properties, String type, IndexPolicyTypeEnum expectedType) { if ("pinyin".equals(type)) { @@ -800,6 +835,18 @@ public final class AnalyzerIdentityBuilder { if (Boolean.FALSE.equals(keepJoinedFullPinyin) && !tokenizerReadsJoinedSetting) { properties.remove("keep_none_chinese_in_joined_full_pinyin"); } + + // With the original, ASCII, first-letter and joined outputs all disabled, every candidate + // the tokenizer emits comes from the pinyin dictionary, which is already lower case. The + // token filter keeps the setting: it falls back to the original token when nothing else + // would be emitted. + if (expectedType == IndexPolicyTypeEnum.TOKENIZER + && Boolean.FALSE.equals(keepFirstLetter) + && Boolean.FALSE.equals(keepNoneChinese) + && Boolean.FALSE.equals(keepOriginal) + && Boolean.FALSE.equals(keepJoinedFullPinyin)) { + properties.remove("lowercase"); + } } private static Boolean effectiveBoolean( @@ -853,15 +900,21 @@ public final class AnalyzerIdentityBuilder { String[] filters = filterList.split(",\\s*"); // DO NOT sort - filter order is semantically significant + String previous = null; for (String filterName : filters) { String filter = resolveComponentIdentity(filterName.trim(), IndexPolicyTypeEnum.TOKEN_FILTER); if (Strings.isNullOrEmpty(filter)) { continue; } + // Repeating an idempotent filter leaves the terms, offsets and provenance unchanged. + if (filter.equals(previous) && IDEMPOTENT_TOKEN_FILTERS.contains(filter)) { + continue; + } if (sb.length() > 0) { sb.append(","); } sb.append(filter); + previous = filter; } return sb.toString(); } @@ -926,21 +979,31 @@ public final class AnalyzerIdentityBuilder { return fold; } - /** Bytes a named char_replace filter rewrites, or null for any other filter. */ + /** + * Bytes a char_replace filter rewrites, or null for any other filter. A bare built-in reference + * is instantiated with the factory defaults. + */ private static boolean[] charReplaceSourceBytes(String filterName) { + String pattern = CHAR_REPLACE_DEFAULT_PATTERN; + String replacement = CHAR_REPLACE_DEFAULT_REPLACEMENT; IndexPolicy policy = findPolicy(filterName, IndexPolicyTypeEnum.CHAR_FILTER); - if (policy == null || policy.isInvalid() || policy.getProperties() == null) { - return null; - } - Map<String, String> properties = policy.getProperties(); - String type = normalizeBuiltinComponentName( - properties.get(IndexPolicy.PROP_TYPE), IndexPolicyTypeEnum.CHAR_FILTER); - String pattern = properties.get("pattern"); - if (!"char_replace".equals(type) || pattern == null) { + if (policy != null) { + if (policy.isInvalid() || policy.getProperties() == null) { + return null; + } + Map<String, String> properties = policy.getProperties(); + String type = normalizeBuiltinComponentName( + properties.get(IndexPolicy.PROP_TYPE), IndexPolicyTypeEnum.CHAR_FILTER); + if (!CHAR_REPLACE_FILTER.equals(type)) { + return null; + } + pattern = properties.getOrDefault(PROP_PATTERN, CHAR_REPLACE_DEFAULT_PATTERN); + replacement = properties.getOrDefault(PROP_REPLACEMENT, CHAR_REPLACE_DEFAULT_REPLACEMENT); + } else if (!CHAR_REPLACE_FILTER.equals( + normalizeBuiltinComponentName(filterName, IndexPolicyTypeEnum.CHAR_FILTER))) { return null; } // Replacing the single replacement byte with itself leaves the stream unchanged. - String replacement = properties.getOrDefault("replacement", " "); int replacementByte = replacement.length() == 1 && replacement.charAt(0) < 128 ? replacement.charAt(0) : -1; boolean[] sourceBytes = new boolean[256]; for (int i = 0; i < pattern.length(); ++i) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java index 33382d4b122..3c5cb837f5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java @@ -80,7 +80,8 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable { } // Callers hold either the read or write lock. BE dispatches a canonical built-in analyzer, then an - // exact policy, then a built-in by normalized name; return that built-in, or null for a policy. + // exact policy, then a built-in by normalized name; return a spelling that reaches that built-in, + // or null for a policy. private String resolveTopLevelBuiltinLocked(String name, Set<String> builtins) { String exactName = exactKey(name); if (IndexPolicy.BUILTIN_ANALYZERS.contains(exactName) && builtins.contains(exactName)) { @@ -90,12 +91,22 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable { return null; } String normalizedName = normalizeKey(name); - return builtins.contains(normalizedName) ? normalizedName : null; + if (!builtins.contains(normalizedName)) { + return null; + } + // BE checks an exact policy before the built-in normalizer, so the canonical spelling binds + // that policy instead; only the spelling given here still reaches the built-in. + if (!IndexPolicy.BUILTIN_ANALYZERS.contains(normalizedName) + && exactNameToIndexPolicy.containsKey(normalizedName)) { + return exactName; + } + return normalizedName; } /** - * The built-in from {@code builtins} that an index's analyzer or normalizer name binds, or null - * when {@link #getPolicyByName} gives its binding. Validation uses the same order. + * The spelling that makes an index's analyzer or normalizer name bind a built-in from + * {@code builtins}, or null when {@link #getPolicyByName} gives its binding. Validation uses the + * same order. */ public String getTopLevelBuiltin(String name, Set<String> builtins) { readLock(); @@ -106,16 +117,6 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable { } } - /** The policy with exactly this name, without the case-insensitive fallback. */ - public IndexPolicy getPolicyByExactName(String name) { - readLock(); - try { - return exactNameToIndexPolicy.get(exactKey(name)); - } finally { - readUnlock(); - } - } - private void writeLock() { lock.writeLock().lock(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index b1b6f712d6a..07d0f723dcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -1542,6 +1542,42 @@ public class SchemaChangeHandlerTest extends TestWithFeService { } } + @Test + public void testMixedCaseNormalizerKeepsBuiltinBindingWhenExactPolicyShadowsIt() throws Exception { + createAnalyzerAliasTable("sc_shadowed_lowercase"); + IndexPolicyMgr policyMgr = Env.getCurrentEnv().getIndexPolicyMgr(); + List<IndexPolicy> replayed = List.of( + replayAliasPolicy(policyMgr, "lowercase", IndexPolicyTypeEnum.NORMALIZER, + Map.of("token_filter", "asciifolding")), + replayAliasPolicy(policyMgr, "alter_shadow_norm_lower", IndexPolicyTypeEnum.NORMALIZER, + Map.of("token_filter", "lowercase"))); + try { + alterTable("alter table test.sc_shadowed_lowercase add index idx_mixed(c1) using inverted " + + "properties(\"normalizer\"=\"LowerCase\")", connectContext); + jobSize++; + waitAlterJobDone(Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2()); + // Canonicalizing to "lowercase" would make BE pick the shadowing policy instead. + Assertions.assertEquals("LowerCase", + storedIndexProperty("sc_shadowed_lowercase", "idx_mixed", "normalizer")); + expectException("alter table test.sc_shadowed_lowercase add index idx_equivalent(c1) " + + "using inverted properties(\"normalizer\"=\"alter_shadow_norm_lower\")", "already exists"); + + executeNereidsSql("CREATE TABLE test.sc_shadowed_lowercase_create (k INT, c1 VARCHAR(100),\n" + + "INDEX idx_mixed(c1) USING INVERTED PROPERTIES('normalizer' = 'LowerCase'),\n" + + "INDEX idx_legacy(c1) USING INVERTED PROPERTIES('normalizer' = 'lowercase'))\n" + + "DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1')"); + Assertions.assertEquals("LowerCase", + storedIndexProperty("sc_shadowed_lowercase_create", "idx_mixed", "normalizer")); + Assertions.assertEquals("lowercase", + storedIndexProperty("sc_shadowed_lowercase_create", "idx_legacy", "normalizer")); + } finally { + for (IndexPolicy policy : replayed) { + policyMgr.replayDropIndexPolicy(new DropIndexPolicyLog(policy.getId())); + } + } + } + @Test public void testAddInvertedIndexRejectsRedundantTokenCharAndReverseCaseAliases() throws Exception { createAnalyzerAliasTable("sc_fold4_alias"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java index 36b0d49252e..1478bd36f23 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java @@ -697,6 +697,52 @@ public class InvertedIndexPropertiesTest { normalizerIndexDefinition("idx_lower", "norm_lower")))))); } + @Test + public void testExactLowercasePolicyKeepsMixedCaseBuiltinNormalizerBinding() { + IndexPolicyMgr policyMgr = new IndexPolicyMgr(); + policyMgr.replayCreateIndexPolicy(new IndexPolicy( + 90, "lowercase", IndexPolicyTypeEnum.NORMALIZER, Map.of("token_filter", "asciifolding"))); + policyMgr.replayCreateIndexPolicy(new IndexPolicy( + 91, "standard", IndexPolicyTypeEnum.ANALYZER, Map.of("tokenizer", "keyword"))); + policyMgr.replayCreateIndexPolicy(new IndexPolicy( + 92, "norm_lower", IndexPolicyTypeEnum.NORMALIZER, Map.of("token_filter", "lowercase"))); + Map<String, String> mixedNormalizer = new HashMap<>(Map.of("normalizer", "LowerCase")); + Map<String, String> mixedAnalyzer = new HashMap<>(Map.of("analyzer", "Standard")); + + withIndexPolicyManager(policyMgr, () -> { + for (Map<String, String> properties : List.of(mixedNormalizer, mixedAnalyzer)) { + Assertions.assertDoesNotThrow(() -> InvertedIndexUtil.checkInvertedIndexParser("c", + PrimitiveType.VARCHAR, properties, TInvertedIndexFileStorageFormat.V3)); + } + Assertions.assertAll( + // The exact policy shadows the canonical name, so only the mixed spelling still + // reaches the built-in normalizer on BE. + () -> Assertions.assertEquals("LowerCase", mixedNormalizer.get("normalizer")), + () -> Assertions.assertEquals("LowerCase", + InvertedIndexUtil.resolveAnalyzerName("LowerCase")), + () -> Assertions.assertEquals("lowercase", + InvertedIndexUtil.resolveAnalyzerName("lowercase")), + // BE dispatches a built-in analyzer before any policy, so it stays canonical. + () -> Assertions.assertEquals("standard", mixedAnalyzer.get("analyzer")), + () -> Assertions.assertEquals("standard", + InvertedIndexUtil.resolveAnalyzerName("Standard")), + () -> Assertions.assertTrue(InvertedIndexUtil.isAnalyzerMatched( + Map.of("normalizer", "LowerCase"), "LowerCase")), + () -> Assertions.assertFalse(InvertedIndexUtil.isAnalyzerMatched( + Map.of("normalizer", "LowerCase"), "lowercase")), + () -> Assertions.assertTrue(InvertedIndexUtil.isAnalyzerMatched( + Map.of("normalizer", "lowercase"), "lowercase")), + () -> Assertions.assertTrue(InvertedIndexUtil.canHaveMultipleInvertedIndexes( + StringType.INSTANCE, List.of( + normalizerIndexDefinition("idx_builtin", "LowerCase"), + normalizerIndexDefinition("idx_legacy", "lowercase")))), + () -> Assertions.assertFalse(InvertedIndexUtil.canHaveMultipleInvertedIndexes( + StringType.INSTANCE, List.of( + normalizerIndexDefinition("idx_builtin", "LowerCase"), + normalizerIndexDefinition("idx_equivalent", "norm_lower"))))); + }); + } + @Test public void testCreateTableRejectsRedundantTokenCharAndReverseCaseAliases() { IndexPolicyMgr policyMgr = new IndexPolicyMgr(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilderTest.java index 36295fed802..0c2a26bcd07 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/invertedindex/AnalyzerIdentityBuilderTest.java @@ -84,7 +84,8 @@ public class AnalyzerIdentityBuilderTest { null); // BE builds the built-in as a keyword tokenizer with the built-in filter of the same name. Assertions.assertEquals( - IndexPolicyTypeEnum.NORMALIZER.name() + ":token_filter=" + normalizer + ";", identity); + IndexPolicyTypeEnum.ANALYZER.name() + ":token_filter=" + normalizer + + ";tokenizer=keyword;", identity); } @Test @@ -1780,4 +1781,165 @@ public class AnalyzerIdentityBuilderTest { namedAnalyzerIdentity("upper_lower_az_fold"))); } } + + @Test + public void testNormalizerIdentityMatchesEquivalentKeywordAnalyzer() { + IndexPolicyMgr policyMgr = new IndexPolicyMgr(); + replayComponent(policyMgr, 1, "lower_a", IndexPolicyTypeEnum.CHAR_FILTER, + Map.of("type", "char_replace", "pattern", "A", "replacement", "a")); + replayComponent(policyMgr, 2, "norm_lower", IndexPolicyTypeEnum.NORMALIZER, + Map.of("token_filter", "lowercase")); + replayComponent(policyMgr, 3, "keyword_lower", IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "keyword", "token_filter", "lowercase")); + replayComponent(policyMgr, 4, "standard_lower", IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "standard", "token_filter", "lowercase")); + replayComponent(policyMgr, 5, "norm_char_ascii", IndexPolicyTypeEnum.NORMALIZER, + Map.of("char_filter", "lower_a", "token_filter", "asciifolding")); + replayComponent(policyMgr, 6, "keyword_char_ascii", IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "keyword", "char_filter", "lower_a", "token_filter", "asciifolding")); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getIndexPolicyMgr()).thenReturn(policyMgr); + + try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertAll( + () -> Assertions.assertEquals(namedAnalyzerIdentity("keyword_lower"), + namedNormalizerIdentity("norm_lower")), + () -> Assertions.assertEquals(namedAnalyzerIdentity("keyword_lower"), + namedNormalizerIdentity("lowercase")), + () -> Assertions.assertEquals(namedAnalyzerIdentity("keyword_char_ascii"), + namedNormalizerIdentity("norm_char_ascii")), + // A normalizer is a keyword pipeline, so any other tokenizer stays distinct. + () -> Assertions.assertNotEquals(namedAnalyzerIdentity("standard_lower"), + namedNormalizerIdentity("norm_lower"))); + } + } + + @Test + public void testExplicitCharReplaceDefaultsMatchBuiltinReference() { + IndexPolicyMgr policyMgr = new IndexPolicyMgr(); + replayComponent(policyMgr, 1, "default_replace", IndexPolicyTypeEnum.CHAR_FILTER, + Map.of("type", "char_replace", "pattern", ",._")); + replayComponent(policyMgr, 2, "default_replace_explicit", IndexPolicyTypeEnum.CHAR_FILTER, + Map.of("type", "char_replace", "pattern", "._,", "replacement", " ")); + replayComponent(policyMgr, 3, "other_replacement", IndexPolicyTypeEnum.CHAR_FILTER, + Map.of("type", "char_replace", "pattern", ",._", "replacement", "-")); + replayComponent(policyMgr, 4, "other_pattern", IndexPolicyTypeEnum.CHAR_FILTER, + Map.of("type", "char_replace", "pattern", ",.")); + String[] filters = {"char_replace", "default_replace", "default_replace_explicit", + "other_replacement", "other_pattern"}; + long id = 10; + for (String filter : filters) { + replayComponent(policyMgr, id++, filter + "_analyzer", IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "keyword", "char_filter", filter)); + } + Env env = Mockito.mock(Env.class); + Mockito.when(env.getIndexPolicyMgr()).thenReturn(policyMgr); + + try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + String builtin = namedAnalyzerIdentity("char_replace_analyzer"); + Assertions.assertAll( + () -> Assertions.assertEquals(builtin, namedAnalyzerIdentity("default_replace_analyzer")), + () -> Assertions.assertEquals(builtin, + namedAnalyzerIdentity("default_replace_explicit_analyzer")), + () -> Assertions.assertNotEquals(builtin, + namedAnalyzerIdentity("other_replacement_analyzer")), + () -> Assertions.assertNotEquals(builtin, namedAnalyzerIdentity("other_pattern_analyzer"))); + } + } + + @Test + public void testAdjacentDuplicateLowercaseFiltersCollapse() { + IndexPolicyMgr policyMgr = new IndexPolicyMgr(); + replayComponent(policyMgr, 1, "named_lower", IndexPolicyTypeEnum.TOKEN_FILTER, + Map.of("type", "lowercase")); + String[][] analyzers = { + {"lower_once", "lowercase"}, + {"lower_twice", "lowercase,lowercase"}, + {"lower_thrice", "lowercase,lowercase,lowercase"}, + {"lower_named_lower", "lowercase,named_lower"}, + {"lower_ascii", "lowercase,asciifolding"}, + {"lower_ascii_lower", "lowercase,asciifolding,lowercase"}, + {"pinyin_once", "pinyin"}, + {"pinyin_twice", "pinyin,pinyin"}}; + long id = 10; + for (String[] analyzer : analyzers) { + replayComponent(policyMgr, id++, analyzer[0], IndexPolicyTypeEnum.ANALYZER, + Map.of("tokenizer", "keyword", "token_filter", analyzer[1])); + } + Env env = Mockito.mock(Env.class); + Mockito.when(env.getIndexPolicyMgr()).thenReturn(policyMgr); + + try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + String once = namedAnalyzerIdentity("lower_once"); + Assertions.assertAll( + () -> Assertions.assertEquals(once, namedAnalyzerIdentity("lower_twice")), + () -> Assertions.assertEquals(once, namedAnalyzerIdentity("lower_thrice")), + () -> Assertions.assertEquals(once, namedAnalyzerIdentity("lower_named_lower")), + // Only adjacent duplicates collapse, and only for a filter proven idempotent. + () -> Assertions.assertNotEquals(namedAnalyzerIdentity("lower_ascii"), + namedAnalyzerIdentity("lower_ascii_lower")), + () -> Assertions.assertNotEquals(namedAnalyzerIdentity("pinyin_once"), + namedAnalyzerIdentity("pinyin_twice"))); + } + } + + @Test + public void testPinyinLowercaseIgnoredWhenOnlyFullPinyinIsEmitted() throws Exception { + IndexPolicyMgr policyMgr = new IndexPolicyMgr(); + Map<String, String> pinyinOnly = Map.of("keep_first_letter", "false", + "keep_none_chinese", "false", "keep_original", "false"); + replayPinyinPair(policyMgr, 1, "pinyin_only", pinyinOnly); + replayPinyinPair(policyMgr, 2, "pinyin_only_cased", casedPinyin(pinyinOnly)); + Map<String, String> withOriginal = Map.of("keep_first_letter", "false", + "keep_none_chinese", "false", "keep_original", "true"); + replayPinyinPair(policyMgr, 3, "pinyin_original", withOriginal); + replayPinyinPair(policyMgr, 4, "pinyin_original_cased", casedPinyin(withOriginal)); + Map<String, String> withAscii = Map.of("keep_first_letter", "false", "keep_original", "false"); + replayPinyinPair(policyMgr, 5, "pinyin_ascii", withAscii); + replayPinyinPair(policyMgr, 6, "pinyin_ascii_cased", casedPinyin(withAscii)); + Map<String, String> withFirstLetter = Map.of("keep_none_chinese", "false", "keep_original", "false"); + replayPinyinPair(policyMgr, 7, "pinyin_first_letter", withFirstLetter); + replayPinyinPair(policyMgr, 8, "pinyin_first_letter_cased", casedPinyin(withFirstLetter)); + Map<String, String> withJoined = Map.of("keep_first_letter", "false", + "keep_none_chinese", "false", "keep_original", "false", "keep_joined_full_pinyin", "true"); + replayPinyinPair(policyMgr, 9, "pinyin_joined", withJoined); + replayPinyinPair(policyMgr, 10, "pinyin_joined_cased", casedPinyin(withJoined)); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getIndexPolicyMgr()).thenReturn(policyMgr); + Method resolve = resolveComponentIdentityMethod(); + + try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertAll( + () -> Assertions.assertEquals( + pinyinIdentity(resolve, "pinyin_only", IndexPolicyTypeEnum.TOKENIZER), + pinyinIdentity(resolve, "pinyin_only_cased", IndexPolicyTypeEnum.TOKENIZER)), + // Any output that can carry the source case keeps lower_case significant. + () -> Assertions.assertNotEquals( + pinyinIdentity(resolve, "pinyin_original", IndexPolicyTypeEnum.TOKENIZER), + pinyinIdentity(resolve, "pinyin_original_cased", IndexPolicyTypeEnum.TOKENIZER)), + () -> Assertions.assertNotEquals( + pinyinIdentity(resolve, "pinyin_ascii", IndexPolicyTypeEnum.TOKENIZER), + pinyinIdentity(resolve, "pinyin_ascii_cased", IndexPolicyTypeEnum.TOKENIZER)), + () -> Assertions.assertNotEquals( + pinyinIdentity(resolve, "pinyin_first_letter", IndexPolicyTypeEnum.TOKENIZER), + pinyinIdentity(resolve, "pinyin_first_letter_cased", IndexPolicyTypeEnum.TOKENIZER)), + () -> Assertions.assertNotEquals( + pinyinIdentity(resolve, "pinyin_joined", IndexPolicyTypeEnum.TOKENIZER), + pinyinIdentity(resolve, "pinyin_joined_cased", IndexPolicyTypeEnum.TOKENIZER)), + // The token filter has its own candidate sources, so it keeps the setting. + () -> Assertions.assertNotEquals( + pinyinIdentity(resolve, "pinyin_only", IndexPolicyTypeEnum.TOKEN_FILTER), + pinyinIdentity(resolve, "pinyin_only_cased", IndexPolicyTypeEnum.TOKEN_FILTER))); + } + } + + private static Map<String, String> casedPinyin(Map<String, String> properties) { + Map<String, String> cased = new HashMap<>(properties); + cased.put("lowercase", "false"); + return cased; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
