This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch PHOENIX-7876-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-7876-feature by this
push:
new 3af242090a PHOENIX-7903 Functional index match rule EXPLAIN disclosure
(#2523)
3af242090a is described below
commit 3af242090a47d07b597cce36754c493f7f8c1958
Author: Andrew Purtell <[email protected]>
AuthorDate: Fri Jun 12 11:21:40 2026 -0700
PHOENIX-7903 Functional index match rule EXPLAIN disclosure (#2523)
Co-authored-by: Claude Opus 4.8[1m] <[email protected]>
---
.../apache/phoenix/compile/StatementContext.java | 48 +++++++++++
.../apache/phoenix/optimize/QueryOptimizer.java | 96 ++++++++++++++++++----
.../parse/IndexExpressionParseNodeRewriter.java | 52 +++++++++++-
.../apache/phoenix/compile/QueryOptimizerTest.java | 42 ++++++++++
4 files changed, 221 insertions(+), 17 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
index d5eac400e1..c2ad404027 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -21,6 +21,7 @@ import java.sql.SQLException;
import java.text.Format;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.Iterator;
@@ -106,6 +107,8 @@ public class StatementContext {
private List<String> appliedRewrites;
private int derivedTableFlattenCount;
private List<Pair<ParseNode, String>> indexExpressionSubstitutions;
+ private Map<String, List<String>> appliedIndexExpressionMatches;
+ private Set<String> functionalIndexNames;
private Set<Pair<String, String>> partialIndexCheckedSet;
private Map<String, List<Expression>> serverParsedProjections;
private StatementContext parentContext;
@@ -148,6 +151,8 @@ public class StatementContext {
this.appliedRewrites = context.appliedRewrites;
this.derivedTableFlattenCount = context.derivedTableFlattenCount;
this.indexExpressionSubstitutions = context.indexExpressionSubstitutions;
+ this.appliedIndexExpressionMatches = context.appliedIndexExpressionMatches;
+ this.functionalIndexNames = context.functionalIndexNames;
this.partialIndexCheckedSet = context.partialIndexCheckedSet;
this.serverParsedProjections = context.serverParsedProjections;
this.parentContext = context.parentContext;
@@ -216,6 +221,8 @@ public class StatementContext {
this.appliedRewrites = new ArrayList<>();
this.derivedTableFlattenCount = 0;
this.indexExpressionSubstitutions = new ArrayList<>();
+ this.appliedIndexExpressionMatches = Maps.newLinkedHashMap();
+ this.functionalIndexNames = Sets.newHashSet();
this.partialIndexCheckedSet = Sets.newHashSet();
this.serverParsedProjections = null;
this.parentContext = null;
@@ -509,6 +516,8 @@ public class StatementContext {
this.appliedRewrites = source.appliedRewrites;
this.derivedTableFlattenCount = source.derivedTableFlattenCount;
this.indexExpressionSubstitutions = source.indexExpressionSubstitutions;
+ this.appliedIndexExpressionMatches = source.appliedIndexExpressionMatches;
+ this.functionalIndexNames = source.functionalIndexNames;
this.partialIndexCheckedSet = source.partialIndexCheckedSet;
this.serverParsedProjections = source.serverParsedProjections;
}
@@ -530,6 +539,45 @@ public class StatementContext {
indexExpressionSubstitutions.add(new Pair<>(source, indexColumnName));
}
+ /**
+ * Records the path expressions that actually substituted against this query
for the given
+ * functional index. Used by the optimizer to label the chosen index's rule
as
+ * {@code matches <expr>} and to distinguish functional index rejections
that matched no query
+ * expression. Deduplicated per index, preserving first seen order.
+ */
+ public void recordAppliedIndexExpressionMatches(String indexName,
+ Collection<String> expressions) {
+ if (expressions == null || expressions.isEmpty()) {
+ return;
+ }
+ List<String> matches =
+ appliedIndexExpressionMatches.computeIfAbsent(indexName, k -> new
ArrayList<>());
+ for (String expression : expressions) {
+ if (!matches.contains(expression)) {
+ matches.add(expression);
+ }
+ }
+ }
+
+ /**
+ * Returns the path expressions that actually substituted against this query
for the given
+ * functional index, or an empty list if none were recorded.
+ */
+ public List<String> getAppliedIndexExpressionMatches(String indexName) {
+ List<String> matches = appliedIndexExpressionMatches.get(indexName);
+ return matches == null ? Collections.emptyList() : matches;
+ }
+
+ /** Marks the given index as a functional index. */
+ public void recordFunctionalIndex(String indexName) {
+ functionalIndexNames.add(indexName);
+ }
+
+ /** Returns whether the given index was recorded as a functional index
during rewriting. */
+ public boolean isFunctionalIndex(String indexName) {
+ return functionalIndexNames.contains(indexName);
+ }
+
/**
* Dedup set used by the optimizer so a partial index applicability
breadcrumb is recorded only
* once per (table, index) pair even when the same index is scored across
multiple paths.
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index b2cea9c761..b16a0ed22e 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -495,6 +495,7 @@ public class QueryOptimizer {
isHinted, indexSelect, resolver);
}
+ @SuppressWarnings("rawtypes")
private AddPlanResult addPlan(PhoenixStatement statement, SelectStatement
select, PTable index,
List<? extends PDatum> targetColumns, ParallelIteratorFactory
parallelIteratorFactory,
QueryPlan dataPlan, boolean isHinted, SelectStatement indexSelect,
ColumnResolver resolver)
@@ -531,12 +532,22 @@ public class QueryOptimizer {
}
// translate nodes that match expressions that are indexed to the
// associated column parse node
+ IndexExpressionParseNodeRewriter indexExpressionRewriter =
+ new IndexExpressionParseNodeRewriter(index, null,
statement.getConnection(),
+ indexSelect.getUdfParseNodes());
SelectStatement rewrittenIndexSelect =
- ParseNodeRewriter.rewrite(indexSelect, new
IndexExpressionParseNodeRewriter(index, null,
- statement.getConnection(), indexSelect.getUdfParseNodes()));
+ ParseNodeRewriter.rewrite(indexSelect, indexExpressionRewriter);
+ // Record which functional index expressions actually matched this
query, so the
+ // optimizer can label the chosen index's rule and distinguish
functional index rejections
+ // that matched nothing.
+ if (indexExpressionRewriter.hasFunctionalColumns()) {
+
dataPlan.getContext().recordFunctionalIndex(index.getTableName().getString());
+ }
+
dataPlan.getContext().recordAppliedIndexExpressionMatches(index.getTableName().getString(),
+
indexExpressionRewriter.getAppliedFunctionalSubstitutions().values());
QueryCompiler compiler = new QueryCompiler(statement,
rewrittenIndexSelect, resolver,
targetColumns, parallelIteratorFactory,
dataPlan.getContext().getSequenceManager(),
- isProjected, true, dataPlans);
+ isProjected, true,
dataPlans).withRewriteContext(dataPlan.getContext());
QueryPlan plan = compiler.compile();
if (indexTable.getIndexType() == IndexType.UNCOVERED_GLOBAL) {
@@ -579,10 +590,8 @@ public class QueryOptimizer {
maintainer.getIndexedColumnInfo();
for (org.apache.hadoop.hbase.util.Pair<String, String> pair :
indexedColumns) {
// The first member of the pair is the column family. For the data
table PK columns, the
- // column
- // family is set to null. The data PK columns should not be added
to the set of data
- // columns
- // to join back to index rows
+ // column family is set to null. The data PK columns should not be
added to the set of
+ // data columns to join back to index rows
if (pair.getFirst() != null) {
PColumn pColumn =
dataTable.getColumnForColumnName(pair.getSecond());
// The following adds the column to the set
@@ -598,8 +607,7 @@ public class QueryOptimizer {
indexTable = indexTableRef.getTable();
indexState = indexTable.getIndexState();
// Checking number of columns handles the wildcard cases correctly, as
in that case the
- // index
- // must contain all columns from the data table to be able to be used.
+ // index must contain all columns from the data table to be able to be
used.
if (
indexState == PIndexState.ACTIVE || indexState ==
PIndexState.PENDING_ACTIVE
|| (indexState == PIndexState.PENDING_DISABLE &&
isUnderPendingDisableThreshold(
@@ -687,7 +695,8 @@ public class QueryOptimizer {
return AddPlanResult.rejected(index,
OptimizerReasons.REASON_DEGENERATE_RANGE);
}
}
- return AddPlanResult.rejected(index,
OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION);
+ return AddPlanResult.rejected(index, adjustReasonForFunctionalIndex(index,
+ OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION,
dataPlan.getContext()));
}
// returns true if we can still use the index
@@ -935,12 +944,59 @@ public class QueryOptimizer {
* can use this inline at a {@code return} site.
*/
private static QueryPlan recordDecision(QueryPlan winner, String rule,
DecisionState state) {
+ String functionalRule = functionalIndexRule(winner);
+ if (functionalRule != null) {
+ rule = functionalRule;
+ }
winner.setOptimizerDecision(
new
OptimizerDecision(winner.getTableRef().getTable().getTableName().getString(),
rule,
state == null ? null : state.getRejections()));
return winner;
}
+ /**
+ * Returns {@link OptimizerReasons#matches(String)} for {@code winner} when
it is a functional
+ * index whose indexed expression actually substituted a path expression in
the user's query, or
+ * {@code null} when the functional index rule override does not apply. The
applied matches are
+ * only ever recorded for genuine expression substitutions, so a non-empty
match list implies the
+ * chosen plan is a functional index that matched.
+ */
+ private static String functionalIndexRule(QueryPlan winner) {
+ PTable table = winner.getTableRef().getTable();
+ List<String> matches =
+
winner.getContext().getAppliedIndexExpressionMatches(table.getTableName().getString());
+ if (matches.isEmpty()) {
+ return null;
+ }
+ return OptimizerReasons.matches(matches.get(0));
+ }
+
+ /**
+ * Swaps a generic coverage driven rejection reason for
+ * {@link OptimizerReasons#REASON_PATH_EXPRESSION_DOES_NOT_MATCH} when
{@code index} is a
+ * functional index for which no query expression matched its indexed
expression. Other reasons,
+ * and functional indexes that did match an expression, keep their original
reason.
+ */
+ private static String adjustReasonForFunctionalIndex(PTable index, String
reason,
+ StatementContext rewriteContext) {
+ if (!rewriteContext.isFunctionalIndex(index.getTableName().getString())) {
+ return reason;
+ }
+ if (
+ !OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION.equals(reason)
+ && !OptimizerReasons.REASON_NO_PK_PREFIX_BOUND.equals(reason)
+ &&
!OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE.equals(reason)
+ ) {
+ return reason;
+ }
+ if (
+
!rewriteContext.getAppliedIndexExpressionMatches(index.getTableName().getString()).isEmpty()
+ ) {
+ return reason;
+ }
+ return OptimizerReasons.REASON_PATH_EXPRESSION_DOES_NOT_MATCH;
+ }
+
/** First plan in {@code plans} that is not {@code exclude}, or {@code null}
if there is none. */
private static QueryPlan firstOther(List<QueryPlan> plans, QueryPlan
exclude) {
for (QueryPlan plan : plans) {
@@ -998,9 +1054,11 @@ public class QueryOptimizer {
continue;
}
if (adjustedBoundCount(plan, boundRanges) < winnerBound) {
- state.reject(table, OptimizerReasons.REASON_NO_PK_PREFIX_BOUND);
+ state.reject(table, adjustReasonForFunctionalIndex(table,
+ OptimizerReasons.REASON_NO_PK_PREFIX_BOUND, winner.getContext()));
} else if (table.getIndexType() == IndexType.LOCAL && winnerNonLocal) {
- state.reject(table,
OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE);
+ state.reject(table, adjustReasonForFunctionalIndex(table,
+ OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE,
winner.getContext()));
}
}
}
@@ -1055,10 +1113,20 @@ public class QueryOptimizer {
IndexStatementRewriter.translate(FACTORY.select(select, newFrom),
resolver, replacement);
for (TableRef indexTableRef : replacement.values()) {
// replace expressions with corresponding matching columns for
functional indexes
- indexSelect = ParseNodeRewriter.rewrite(indexSelect,
+ IndexExpressionParseNodeRewriter indexExpressionRewriter =
new IndexExpressionParseNodeRewriter(indexTableRef.getTable(),
indexTableRef.getTableAlias(), connection,
indexSelect.getUdfParseNodes(),
- breadcrumbContext));
+ breadcrumbContext);
+ indexSelect = ParseNodeRewriter.rewrite(indexSelect,
indexExpressionRewriter);
+ // Surface which functional index expressions actually matched so the
optimizer can label
+ // the chosen index's rule and distinguish functional index rejections
that matched nothing.
+ if (indexExpressionRewriter.hasFunctionalColumns()) {
+ breadcrumbContext
+
.recordFunctionalIndex(indexTableRef.getTable().getTableName().getString());
+ }
+ breadcrumbContext.recordAppliedIndexExpressionMatches(
+ indexTableRef.getTable().getTableName().getString(),
+ indexExpressionRewriter.getAppliedFunctionalSubstitutions().values());
}
return indexSelect;
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
index 8d93f6415f..de3faef662 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.parse;
import java.sql.SQLException;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.phoenix.compile.ColumnResolver;
@@ -44,6 +45,19 @@ public class IndexExpressionParseNodeRewriter extends
ParseNodeRewriter {
private final Map<ParseNode, ParseNode> indexedParseNodeToColumnParseNodeMap;
+ /**
+ * Maps the rewritten parse node of a functional index column to a [colName,
expressionStr] pair.
+ * Plain PK columns are excluded.
+ */
+ private final Map<ParseNode, String[]> indexedParseNodeToFunctionalColumn;
+
+ /**
+ * Records the functional index substitutions that actually fired against
the query, mapping the
+ * index column name to the expression string. Insertion order is preserved
so callers can report
+ * matches in first-seen order.
+ */
+ private final Map<String, String> appliedFunctionalSubstitutions = new
LinkedHashMap<>();
+
public IndexExpressionParseNodeRewriter(PTable index, String alias,
PhoenixConnection connection,
Map<String, UDFParseNode> udfParseNodes) throws SQLException {
this(index, alias, connection, udfParseNodes, null);
@@ -54,6 +68,7 @@ public class IndexExpressionParseNodeRewriter extends
ParseNodeRewriter {
throws SQLException {
indexedParseNodeToColumnParseNodeMap =
Maps.newHashMapWithExpectedSize(index.getColumns().size());
+ indexedParseNodeToFunctionalColumn =
Maps.newHashMapWithExpectedSize(index.getColumns().size());
NamedTableNode tableNode =
NamedTableNode.create(alias,
TableName.create(index.getParentSchemaName().getString(),
index.getParentTableName().getString()), Collections.<ColumnDef>
emptyList());
@@ -80,6 +95,14 @@ public class IndexExpressionParseNodeRewriter extends
ParseNodeRewriter {
columnParseNode = NODE_FACTORY.cast(columnParseNode,
expressionDataType, null, null);
}
indexedParseNodeToColumnParseNodeMap.put(indexedParseNode,
columnParseNode);
+ // Only true functional columns defined over an expression get an
applied match entry. A
+ // plain indexed or PK column's expression string parses to a bare
column reference. An
+ // expression column (e.g. UPPER(NAME)) parses to a compound node.
+ if (!(expressionParseNode instanceof ColumnParseNode)) {
+ // Trim leading/trailing whitespace
+ indexedParseNodeToFunctionalColumn.put(indexedParseNode,
+ new String[] { colName, expressionStr.trim() });
+ }
if (breadcrumbContext != null) {
breadcrumbContext.addAppliedRewrite("INDEX EXPRESSION " +
expressionStr + " AS " + colName);
breadcrumbContext.addIndexExpressionSubstitution(indexedParseNode,
colName);
@@ -90,9 +113,32 @@ public class IndexExpressionParseNodeRewriter extends
ParseNodeRewriter {
@Override
protected ParseNode leaveCompoundNode(CompoundParseNode node,
List<ParseNode> children,
CompoundNodeFactory factory) {
- return indexedParseNodeToColumnParseNodeMap.containsKey(node)
- ? indexedParseNodeToColumnParseNodeMap.get(node)
- : super.leaveCompoundNode(node, children, factory);
+ ParseNode replacement = indexedParseNodeToColumnParseNodeMap.get(node);
+ if (replacement == null) {
+ return super.leaveCompoundNode(node, children, factory);
+ }
+ // A functional index substitution actually fired for this query node.
+ String[] functionalColumn = indexedParseNodeToFunctionalColumn.get(node);
+ if (functionalColumn != null) {
+ appliedFunctionalSubstitutions.put(functionalColumn[0],
functionalColumn[1]);
+ }
+ return replacement;
+ }
+
+ /**
+ * Returns the functional index substitutions that actually fired against
the query, keyed by
+ * index column name with the expression string as the value, in first-seen
order.
+ */
+ public Map<String, String> getAppliedFunctionalSubstitutions() {
+ return Collections.unmodifiableMap(appliedFunctionalSubstitutions);
+ }
+
+ /**
+ * Returns {@code true} if the index this rewriter was built for has at
least one functional
+ * column, regardless of whether any such column matched the query.
+ */
+ public boolean hasFunctionalColumns() {
+ return !indexedParseNodeToFunctionalColumn.isEmpty();
}
}
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
index 9ee22f54f7..abd027ca1a 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
@@ -511,6 +511,48 @@ public class QueryOptimizerTest extends
BaseConnectionlessQueryTest {
.indexRejected(0, "INDEX_TEST_TABLE_INDEX_D",
OptimizerReasons.REASON_NO_PK_PREFIX_BOUND);
}
+ @Test
+ public void testFunctionalIndexChosenRuleMatchesExpression() throws
Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute(
+ "create table fe_match (id varchar not null primary key, name varchar,
val varchar)");
+ conn.createStatement().execute("create index fe_match_upper_idx on
fe_match (UPPER(name))");
+ // The functional index's indexed expression matches the query's
UPPER(NAME) path expression,
+ // so the chosen index's rule is overridden to the "matches <expr>" form.
Assert on the rule
+ // prefix.
+ assertPlan(conn, "select id from fe_match where UPPER(name) = 'ABC'")
+
.table("FE_MATCH_UPPER_IDX").indexRule(OptimizerReasons.matches("UPPER(NAME)"));
+ }
+
+ @Test
+ public void testFunctionalIndexRejectedPathExpressionDoesNotMatch() throws
Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute(
+ "create table fe_nomatch (id varchar not null primary key, name varchar,
val varchar, other varchar)");
+ conn.createStatement().execute("create index fe_nomatch_upper_idx on
fe_nomatch (UPPER(name))");
+ conn.createStatement()
+ .execute("create index fe_nomatch_val_idx on fe_nomatch (val) include
(other)");
+ // The query never references UPPER(NAME), so the functional index matched
no path expression
+ // and its generic "does not cover projection" rejection is swapped for
the more specific
+ // "path expression does not match".
+ assertPlan(conn, "select val, other from fe_nomatch where val = 'x'")
+
.table("FE_NOMATCH_VAL_IDX").indexRejectedContains("FE_NOMATCH_UPPER_IDX",
+ OptimizerReasons.REASON_PATH_EXPRESSION_DOES_NOT_MATCH);
+ }
+
+ @Test
+ public void testFunctionalIndexRejectedDoesNotCoverProjection() throws
Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute(
+ "create table fe_cover (id varchar not null primary key, name varchar,
val varchar)");
+ conn.createStatement().execute("create index fe_cover_upper_idx on
fe_cover (UPPER(name))");
+ // The functional index's expression DOES match UPPER(NAME) in the query,
but VAL is not in the
+ // index, so the rejection stays "does not cover projection" rather than
being swapped.
+ assertPlan(conn, "select UPPER(name), val from fe_cover where UPPER(name)
= 'ABC'")
+ .table("FE_COVER").indexRejectedContains("FE_COVER_UPPER_IDX",
+ OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION);
+ }
+
@Test
public void testCharArrayLength() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
