This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7832338fbaf [fix](fe) Pick the first matching Variant field pattern
when choosing an index, like the BE (#69730)
7832338fbaf is described below
commit 7832338fbaf72c8627c7d8d558527c36a3dad68e
Author: Jack <[email protected]>
AuthorDate: Thu Oct 29 10:11:28 2026 +0800
[fix](fe) Pick the first matching Variant field pattern when choosing an
index, like the BE (#69730)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: A VARIANT column can declare predefined fields whose
patterns overlap, for example the glob 'a.*' and the name 'a.b'. The BE
writer (variant_util.cpp) routes a subcolumn to the first predefined
field whose pattern matches its path and builds the index of that field.
`OlapTable.getInvertedIndex` looped over all the fields without stopping
and kept the last match, so a query on the same path chose the index of
a different field than the one the data was written with, and fell back
to another index or to a scan.
The loop now stops at the first match, the same order the BE uses, so a
write and a query on one path land on the same index.
### Release note
Fix the index chosen for a Variant subpath when the column declares
overlapping field patterns.
### Compatibility
- FE only, and nothing is persisted: `getInvertedIndex` computes its
answer from the table schema on every call.
- A table whose predefined field patterns do not overlap sees no change,
since a path with a single match has the same first and last match.
- A table with overlapping patterns now resolves a path to the first
matching field. The BE already built the index of the first matching
field (`variant_util.cpp`), so the index that holds the data is the one
a query now looks for, and no re-index is needed. Queries on such paths
that missed the index and fell back to another index or a scan can use
it.
### Performance
The loop over the predefined fields now stops at the first match, so it
never does more work than before.
---
.../java/org/apache/doris/catalog/OlapTable.java | 7 +++++-
.../analysis/InvertedIndexPropertiesTest.java | 25 ++++++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 09420f3fabf..f36aafd1b32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -4149,7 +4149,10 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
:
filteredInvertedIndexes.stream().filter(Index::isAnalyzedInvertedIndex).findFirst().orElse(null);
}
- // subPath is not empty, means it is a variant column, find the field
pattern from children
+ // subPath is not empty, means it is a variant column, find the field
pattern from children.
+ // The first match wins, the same as the BE writer that picks the
physical index of a subcolumn
+ // (variant_util.cpp), so a write and a query on one path land on the
same index even when
+ // the predefined patterns overlap.
String subPathString = String.join(".", subPath);
String fieldPattern = "";
if (column.getChildren() != null) {
@@ -4161,6 +4164,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
.getPathMatcher("glob:" + childName);
if
(matcher.matches(java.nio.file.Paths.get(subPathString))) {
fieldPattern = childName;
+ break;
}
} catch (Exception e) {
continue;
@@ -4168,6 +4172,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
} else if (child.getFieldPatternType() ==
PatternType.MATCH_NAME) {
if (childName.equals(subPathString)) {
fieldPattern = childName;
+ break;
}
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
index 54481181aa8..7e074643796 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
@@ -22,8 +22,11 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Function.NullableMode;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PatternType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
+import org.apache.doris.catalog.VariantField;
+import org.apache.doris.catalog.VariantType;
import org.apache.doris.catalog.info.IndexType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.indexpolicy.IndexPolicy;
@@ -682,6 +685,28 @@ public class InvertedIndexPropertiesTest {
Map.of("analyzer", "ik", "lower_case", "false"), "ik"));
}
+ @Test
+ public void
testOverlappingFieldPatternsPickTheFirstPredefinedFieldLikeTheBe() {
+ // The BE writer routes a subcolumn to the first predefined field
whose pattern matches its path,
+ // so the query side has to pick the index of that same field when the
patterns overlap.
+ Index globIndex = new Index(30, "idx_glob", List.of("v"),
IndexType.INVERTED,
+ Map.of("field_pattern", "a.*"), "");
+ Index exactIndex = new Index(31, "idx_exact", List.of("v"),
IndexType.INVERTED,
+ Map.of("field_pattern", "a.b"), "");
+ VariantField glob = new VariantField("a.*", Type.STRING, "",
PatternType.MATCH_NAME_GLOB);
+ VariantField exact = new VariantField("a.b", Type.STRING, "",
PatternType.MATCH_NAME);
+ OlapTable table = new OlapTable();
+ table.setIndexes(List.of(globIndex, exactIndex));
+
+ Column globFirst = new Column("v", new VariantType(new
ArrayList<>(List.of(glob, exact))));
+ Assertions.assertSame(globIndex, table.getInvertedIndex(globFirst,
List.of("a", "b")));
+
+ Column exactFirst = new Column("v", new VariantType(new
ArrayList<>(List.of(exact, glob))));
+ Assertions.assertSame(exactIndex, table.getInvertedIndex(exactFirst,
List.of("a", "b")));
+ // A path only the glob covers still resolves to the glob's index.
+ Assertions.assertSame(globIndex, table.getInvertedIndex(exactFirst,
List.of("a", "c")));
+ }
+
@Test
public void testExplicitBuiltinIkFallsBackToTheOnlyLegacyIndex() {
Column column = new Column("content", PrimitiveType.STRING);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
