github-actions[bot] commented on code in PR #68499: URL: https://github.com/apache/doris/pull/68499#discussion_r4637458231
########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/SPMPlanner.java: ########## @@ -0,0 +1,1503 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.spm.builder.SPMPlan2SQLBuilder; +import org.apache.doris.nereids.spm.manager.BaselineManager; +import org.apache.doris.nereids.spm.manager.SessionBaselineStore; +import org.apache.doris.nereids.spm.matcher.SPMFrozenTreeReplacer; +import org.apache.doris.nereids.spm.matcher.SPMPlaceholderReplacer; +import org.apache.doris.nereids.spm.placeholder.SPMPlaceholderBuilder; +import org.apache.doris.nereids.spm.placeholder.SpmConstList; +import org.apache.doris.nereids.spm.placeholder.SpmConstVar; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.SqlModeHelper; +import org.apache.doris.qe.VariableMgr; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Creates baseline metadata and attempts query replay. Creation parameterizes supported literal positions in bind + * and plan trees with one builder, optimizes the plan, and stores either decompiled frozen SQL or the original SQL + * for fallback planning. + * + * Replay uses a structural hash, exact canonical digest and tree comparison to select candidates and collect + * replacements. It tries frozen SQL first, then the parameterized fallback tree when available. Output labels, row + * limits, ordering and metadata are checked before the caller plans the returned unbound tree. No usable match + * returns null so the caller can plan the original query. + */ +public class SPMPlanner { + + private static final Logger LOG = LogManager.getLogger(SPMPlanner.class); + + /** Id of the baseline used by the last successful rewrite (-1 when none). */ + private long usedBaselineId = -1; + + /** + * The matched baseline OBJECT (see verifyReplayMetadata): the post-plan + * validation must keep the fingerprint of the SAME incarnation the match used, so a + * concurrent DROP / refresh cannot make it silently skip the check. + */ + private BaselinePlan usedBaseline; + + /** + * Returns the id of the baseline used by the last successful rewrite. + * + * Returns: the baseline id, or -1 when the last rewrite did not hit a baseline + */ + public long getUsedBaselineId() { + return usedBaselineId; + } + + /** The matched baseline object (see verifyReplayMetadata). */ + public BaselinePlan getUsedBaseline() { + return usedBaseline; + } + + // query rewrite (called from StmtExecutor / EXPLAIN). + + /** + * Finds SESSION candidates before GLOBAL candidates and attempts replay using extracted marker values. Returns + * a usable unbound plan with caller labels, limits and SET_VAR hints, or null if no candidate can be used. The + * epoch-millisecond deadline is checked before matching each candidate; it is not a hard interruption of + * parsing, matching or substitution. + */ + public LogicalPlan tryRewritePlan(LogicalPlan userPlan, long deadline) { + if (System.currentTimeMillis() > deadline) { + LOG.info("SPM tryRewritePlan: timeout before matching, degrade to the original plan"); + return null; + } + ConnectContext ctx = ConnectContext.get(); + SessionBaselineStore sessionStore = ctx == null ? null : ctx.getSessionBaselineStore(); + // Fast path: with no baseline anywhere (global or session) there is nothing to + // match, so skip the whole-tree digest rendering below - it walks every literal + // of the whole query and is the most expensive part of a rewrite attempt. + if (!BaselineManager.getInstance().hasBaselines() + && (sessionStore == null || sessionStore.isEmpty())) { + return null; + } + // SELECT... INTO OUTFILE writes to a destination that lives OUTSIDE the plan + // expressions: the generic match cannot see a different path / format and the + // rewritten tree keeps the CAPTURED sink fields, so a replay would export to the + // baseline's destination. SPM refuses to rewrite such statements (the original + // query runs normally, which is always correct). + if (SPMPlanTreeSupport.containsFileSink(userPlan)) { + LOG.info("SPM tryRewritePlan: the statement writes to a file sink (OUTFILE); keeping" + + " the original plan"); + return null; + } + // Matching includes the caller's hints against bindSql. Replay uses planSql, whose SET_VAR hints + // can differ or be absent after capture. Remove captured settings and attach the caller's own ones + // at both return sites so planning the replacement keeps the query's statement-scoped assignments. + // Level 1/2 use the FULL-QUERY value-free digest (Plan.toSpmDigest renders every + // literal as "?", so the digest is value-independent). The digest is computed on a + // namespace-qualified COPY: "FROM t" is keyed as the CURRENT catalog/db's t, so a + // baseline captured under db1 can never match the same text executed under db2 + // (the frozen planSql is fully qualified and would silently keep running against + // db1.t). Explicitly qualified references stay verbatim on both sides, and so do + // references to a CTE alias: those bind inside the query's own WITH clause and are + // therefore namespace-independent. + LogicalPlan matchPlan = SPMPlanTreeSupport.namespaceQualified(userPlan, + captureCatalogName(ctx), captureDatabaseName(ctx)); + String queryDigest = SPMPlanTreeSupport.canonicalSpmDigest(matchPlan.toSpmDigest()); + long queryHash = SPMUtils.hashOf(queryDigest); + // SESSION-scope baselines of the current connection are consulted BEFORE the + // global ones, so a session baseline can override a global baseline for this + // session only; each candidate list is already priority-ordered. + List<BaselinePlan> candidates = new ArrayList<>(); + if (sessionStore != null) { + candidates.addAll(sessionStore.findCandidateBaselines(queryDigest, queryHash)); + } + candidates.addAll( + BaselineManager.getInstance().findCandidateBaselines(queryDigest, queryHash, deadline)); + if (candidates.isEmpty()) { + return null; + } + // View guard (computed lazily on the first structural match, so the per-query + // hot path pays nothing): the replay of a matched baseline is planned before the + // normal authorization pass and resolves a view to its base tables, so + // authorization would check the base tables instead of the view - a view-only + // user is denied on the base tables, while a base-table user passes the same + // view query without any view check. View queries keep the original plan. + boolean viewChecked = false; + boolean viewReferenced = false; + // Schema-identity guard state (computed lazily, once per query, only after the + // first structural match): see the check inside the loop. + boolean schemaChecked = false; + String currentSchemaFingerprint = null; + SPMPlanTreeSupport.ReplayContracts userContracts = new SPMPlanTreeSupport.ReplayContracts(matchPlan); + for (BaselinePlan candidate : candidates) { + if (System.currentTimeMillis() > deadline) { + LOG.info("SPM tryRewritePlan: timeout before matching baseline {}, " + + "degrade to the original plan", candidate.getId()); + return null; + } + // A permissive capture can freeze an implicit ANY_VALUE. Strict callers must + // analyse their own grouping expressions before that permissive result can be used. + if ((candidate.getCreatorSqlMode() & SqlModeHelper.MODE_ONLY_FULL_GROUP_BY) == 0 + && (ctx.getSessionVariable().getSqlMode() & SqlModeHelper.MODE_ONLY_FULL_GROUP_BY) != 0) { + continue; + } + LogicalPlan bindTree = candidate.getParameterizedBindPlan(); + if (bindTree == null) { + continue; + } + // Level 3: whole-tree structural match + value extraction + Map<Long, Expression> placeholderValues = new HashMap<>(); + if (!SPMPlanTreeSupport.check(bindTree, matchPlan, placeholderValues)) { + continue; + } + // Older frozen rows may already contain a permissively folded conversion. + // A strict caller must analyze the concrete cast instead of consuming that value. + if (ctx != null && ctx.getSessionVariable().enableStrictCast + && !Boolean.FALSE.equals(candidate.getPlanFrozen()) + && SPMPlanTreeSupport.containsFoldedStrictCastPayload(matchPlan)) { + continue; + } + LOG.info("SPM tryRewritePlan: baseline {} matched, extracted {} placeholder values", + candidate.getId(), placeholderValues.size()); + if (!viewChecked) { + viewChecked = true; + viewReferenced = SPMPlanTreeSupport.referencesView(ctx, userPlan); + } + if (viewReferenced) { + LOG.info("SPM tryRewritePlan: the query references a view; keeping the original" + + " plan so view authorization is preserved"); + return null; + } + // Schema-identity guard (see BaselinePlan.schemaFingerprint): the bind key is + // built from the STILL-UNBOUND query, so SELECT * FROM t WHERE k = 1 keeps + // the same digest and Level-3 tree after ALTER TABLE t ADD COLUMN extra + // (or a DROP + CREATE of t), while the frozen plan still emits the + // creator-time output columns - the matched replay would silently return the + // old column set. A baseline whose referenced-table fingerprint no longer + // matches fails closed (the user query keeps its own plan); pre-column rows + // without a fingerprint skip the check. The current fingerprint is computed + // once per query, and only after a structural match (hot path pays nothing). + // The stored fingerprint is the BIND-SIDE tables UNION the PLAN-side tables + // of the frozen plan; here - BEFORE the query is planned - only the bind side + // can be recomputed, so the guard checks that every current bind-side entry is + // still present (a DROP + CREATE / ALTER REPLACES its entry, so drift is still + // caught). The full comparison, plan side included, runs after planning + // (see verifyReplayMetadata). + String bindFingerprint = candidate.getSchemaFingerprint(); + if (bindFingerprint != null && !bindFingerprint.isEmpty()) { + if (!schemaChecked) { + schemaChecked = true; + currentSchemaFingerprint = + SPMPlanTreeSupport.schemaFingerprint(ctx, userPlan); + } + if (!SPMPlanTreeSupport.schemaFingerprintBindSideContained( + bindFingerprint, currentSchemaFingerprint)) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: the referenced table" + + " schema changed since the baseline was created", candidate.getId()); + continue; + } + } + // The expensive part (value-free digest, candidate lookup, whole-tree check) is + // already done and a baseline has matched: finish the (cheap) value substitution + // even when the budget has been consumed. Previously this path re-checked the + // deadline here and returned null on expiry, silently throwing the completed + // match away (rewrite degraded to the original plan). + // Replay the frozen plan SQL. A baseline whose frozen planSql carries + // placeholder calls (decompiled from the SPM-optimized physical plan, structure + // + join distribution hints preserved) is re-parsed and the user values are + // substituted by placeholder id - the rewrite reproduces the frozen optimal + // structure for normal re-planning. Baselines without frozen SQL + // (in-memory engine, decompile-fallback planSql, legacy "?" frozen text) fall + // back to substituting the parameterized plan tree. + LogicalPlan rewritten = rewriteFromFrozenTree(candidate, placeholderValues); + if (rewritten != null) { + // non-expression literals (LIMIT / OFFSET) are long fields, not + // placeholders; adopt the user's values so a structurally identical query + // with a different limit is rewritten with the USER limit + // the frozen sink pinned the CAPTURED output labels: expose the caller's + // own ones (a value variant must not report the captured header). A shape + // whose labels CANNOT be aligned skips this baseline instead of exposing + // the captured header (see UnalignableOutputLabelsException). + LogicalPlan alignedReplay; + try { + alignedReplay = SPMPlanTreeSupport.alignRootOutputLabels(rewritten, + matchPlan); + } catch (SPMPlanTreeSupport.UnalignableOutputLabelsException unalignable) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: {}", + candidate.getId(), unalignable.getMessage()); + continue; + } + LogicalPlan replay = SPMPlanTreeSupport.mergeLimits(alignedReplay, matchPlan); + SPMPlanTreeSupport.ReplayContracts replayContracts = new SPMPlanTreeSupport.ReplayContracts(replay); + // The cap contract keys every row cap by the OCCURRENCE of the relation it + // truncates, and the occurrence is the relation's ordinal in the tree's walk + // order. That ordinal is only stable when the caller's tree and the FROZEN plan + // render the occurrences in the same relative order - which the optimizer need + // not do: it may place a derived table on the OTHER side of a JOIN than the user + // wrote it (a derived-table TopN joined with the same table), and the very same + // cap then keys as t#0 on one side and t#1 on the other. For a caller that runs + // the statement the frozen plan was BUILT FROM (its canonical plan digest equals + // the baseline's planSqlDigest) the cap contract cannot prove anything the text + // identity does not already prove, so the replay is kept; any other caller - a + // value variant, whose digest matches but whose placement could genuinely differ - + // still has to satisfy it (see the corresponding regression test). + // The digest alone is VALUE-FREE (every literal is masked), so it cannot carry + // this exception on its own: a caller raising only the outer + // LIMIT from 1 to 3 has the SAME digest, yet its query legitimately needs the + // optimizer-generated input caps rebuilt - skipping the cap contract there + // replayed one pair where the caller returns three. The exception therefore also + // requires the caller to carry EXACTLY the row caps the baseline was created + // the SAME cap multiset as the replayed tree, modulo the occurrence ordinal the + // optimizer's own reordering flips: any LIMIT / OFFSET the caller changed - or an + // optimizer-generated input cap the caller's tree has no counterpart for - breaks + // the equality and takes the strict path. + boolean sameFrozenStatement = candidate.getPlanSqlDigest() != null + && candidate.getPlanSqlDigest().equals(queryDigest) + && SPMPlanTreeSupport.rowLimitsEquivalentModuloOccurrence(replayContracts, userContracts); + if (!SPMPlanTreeSupport.orderContractPreserved(replayContracts, userContracts) + || (!sameFrozenStatement + && !limitContractPreserved(replayContracts, userContracts, bindTree))) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: its plan cannot carry" + + " the caller's LIMIT / OFFSET / ORDER BY contract" + + " (caller {}, replayed {}, replayedCapsWithinCaller={}," + + " callerCapsSurvive={}, callerOrder={}, replayOrder={})", + candidate.getId(), + Arrays.toString(SPMPlanTreeSupport.topLevelLimitOf(matchPlan)), + Arrays.toString(SPMPlanTreeSupport.topLevelLimitOf(replay)), + SPMPlanTreeSupport.rowLimitsWithin(replayContracts, userContracts), + SPMPlanTreeSupport.rowLimitsSurviveReplay(replayContracts, userContracts), + userContracts.order(), + replayContracts.order()); + continue; + } + usedBaselineId = candidate.getId(); + usedBaseline = candidate; + return SPMPlanTreeSupport.attachCallerSetVarHints(replay, matchPlan); + } + LOG.info("SPM tryRewritePlan: baseline {} frozen planSql replay unavailable, " + + "falling back to parameterized plan tree", candidate.getId()); + LogicalPlan planTree = SPMPlanTreeSupport.stripSetVarHints(candidate.getParameterizedPlanPlan()); + if (planTree == null) { + continue; + } + // fallback: substitute the extracted user values into the parameterized plan + // tree + SPMPlaceholderReplacer replacer = new SPMPlaceholderReplacer(); + rewritten = SPMPlanTreeSupport.transform( Review Comment: [P1] Reserve cached raw-plan IDs before replay binding Create a SESSION baseline whose bind is `SELECT * REPLACE(from_second(0) AS v) FROM (SELECT k,v,7 AS b FROM t) x`, with the identical manual plan plus one trailing space. The separate CREATE parses allocate `b#0` and `b#1`; `from_second` keeps this baseline on raw replay. A later caller parses only `b#0`, leaving its allocator at 1. This transformation retains cached `b#1`, while scan binding allocates `k#1`: ```text Project(k#1, epoch AS v, b#1) Project(k#1, v#2, 7 AS b#1) Scan(k#1, v#2) ``` `MergeProjectable`/`PlanUtils.mergeProjections` builds `b#1 -> 7`; ExprId-only slot equality also replaces `k`, so `(k=1,v=2)` can return `k=7`. CREATE's nested-context continuation does not protect this later context. Reparse/remap the cached tree or reserve all its IDs before replay analysis. ########## regression-test/suites/spm/test_spm_review_round89.groovy: ########## @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_spm_review_round89", "spm") { + + // Round-89 review #1: a SUBQUERY must never reach the frozen SQL as a function call. + // + // SPMExprSqlBuilder has no SQL rendering for a subquery - it emits the structural digest + // markers `INSUBQUERY(compare)[...]` / `EXISTS[...]` / `SCALARSUBQUERY[...]`. Stored as + // frozen SQL that text is an ordinary, UNRESERVED function call, so a user alias UDF named + // INSUBQUERY (returning an array) made capture-time validation succeed, the baseline was + // stored with planFrozen = true, and every replay evaluated that UDF instead of the original + // `IN (SELECT ...)` - returning different rows. + // + // The decompiler now refuses any residual subquery and reserves the marker names, so such a + // baseline either freezes as real SQL or stays on the parameterized-plan-tree path. Either + // way the stored text must not carry an internal marker, and the replay must return the same + // rows as the user's own query. + + sql """set enable_spm_rewrite = true""" + sql """set enable_spm_fallback = false""" + + sql """DROP TABLE IF EXISTS spm_r89_t""" + sql """DROP TABLE IF EXISTS spm_r89_dim""" + sql """ + CREATE TABLE spm_r89_t (k INT) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + sql """ + CREATE TABLE spm_r89_dim (k INT) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + sql """INSERT INTO spm_r89_t VALUES (1), (2), (3), (4)""" + sql """INSERT INTO spm_r89_dim VALUES (2), (3)""" + + def dropOwnBaselines = { + sql("""SHOW BASELINE PLANS""").findAll { it[1].toString().contains("spm_r89_") } + .each { row -> sql """DROP BASELINE PLAN ${row[0]}""" } + } + dropOwnBaselines() + + def createBaseline = { String text -> + (sql('CREATE GLOBAL BASELINE PLAN "' + text + '" WITH "' + text + '"')[0][0] as Long) + } + def planSqlOf = { long id -> + sql("""SELECT plan_sql FROM __internal_schema.spm_baselines WHERE id = ${id}""")[0][0] + .toString() + } + def assertNoInternalMarker = { String planSql -> + assertTrue(!planSql.toUpperCase().contains("INSUBQUERY"), + "the frozen SQL must not carry the INSUBQUERY marker: " + planSql) + assertTrue(!planSql.toUpperCase().contains("SCALARSUBQUERY"), + "the frozen SQL must not carry the SCALARSUBQUERY marker: " + planSql) + assertTrue(!planSql.toUpperCase().contains("EXISTS["), + "the frozen SQL must not carry the EXISTS marker: " + planSql) + } + + // ==================== #1: IN subquery ==================== + String inText = "SELECT k FROM spm_r89_t WHERE k IN (SELECT k FROM spm_r89_dim)" + + " ORDER BY k" + assertNoInternalMarker(planSqlOf(createBaseline(inText))) + order_qt_r89_in_subquery """SELECT k FROM spm_r89_t WHERE k IN (SELECT k FROM spm_r89_dim) + ORDER BY k""" + // the value variant of the SAME digest: the subquery's literal changes, the rows follow + order_qt_r89_in_subquery_variant """SELECT k FROM spm_r89_t + WHERE k IN (SELECT k FROM spm_r89_dim WHERE k > 2) ORDER BY k""" Review Comment: [P2] Exercise a matching baseline in the subquery variant check The captured `inText` has `IN (SELECT k FROM spm_r89_dim)` with no inner filter, but this alleged literal variant adds `WHERE k > 2`. That creates a new LogicalFilter, changing both full-query digest and structural match. The test therefore always misses this baseline and executes ordinary planning; `enable_spm_fallback=false` does not require a hit, so a broken subquery value substitution can still pass. Capture the same filtered shape (for example `WHERE k > 1`), vary only the literal to 2, and assert that the exact executed query hits. This is a distinct test instance from the existing LAG/ORDER BY mismatch. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/manager/BaselineManager.java: ########## @@ -0,0 +1,10109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.manager; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InternalSchema; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.spm.BaselinePlan; +import org.apache.doris.nereids.spm.BaselineScope; +import org.apache.doris.nereids.spm.BaselineSource; +import org.apache.doris.nereids.spm.BaselineStatus; +import org.apache.doris.nereids.spm.SPMPlanTreeSupport; +import org.apache.doris.nereids.spm.SPMPlanner; +import org.apache.doris.nereids.spm.SPMUtils; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.MasterOpExecutor; +import org.apache.doris.qe.QueryState; +import org.apache.doris.qe.SqlModeHelper; +import org.apache.doris.qe.VariableMgr; +import org.apache.doris.statistics.repository.ResultRow; +import org.apache.doris.statistics.util.StatisticsUtil; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +/** + * Manages GLOBAL baselines, durable mutations, snapshot publication and candidate lookup. The live store uses the + * baseline table, an id-reservation table and a high-water-mark/mutation-clock table. A bounded cache holds + * complete hot hash buckets; a copy-on-write hash map provides the coarse lookup filter. + * + * GLOBAL DDL is routed to the master. Allocation reconciles durable row and reservation watermarks and confirms + * reservations before inserting rows. Collision handling, tombstones, mutation windows and publication fences + * handle delayed writes and leadership changes. + * + * writerLock serializes local mutation protocols. stateLock protects in-memory publication, and the row cache has + * its own lock. Internal-table I/O stays outside stateLock. Matching first checks the volatile hash map and cache; + * periodic refresh publishes durable changes across FEs. + */ +public class BaselineManager { + + private static final Logger LOG = LogManager.getLogger(BaselineManager.class); + + /** Singleton. */ + private static final BaselineManager INSTANCE = new BaselineManager(); + + // internal-table persistence (__internal_schema.spm_baselines). + + /** Fully qualified internal table (FeConstants.INTERNAL_DB_NAME == "__internal_schema"). */ + private static final String SPM_BASELINES_TABLE = + FeConstants.INTERNAL_DB_NAME + "." + InternalSchema.SPM_BASELINES_TBL_NAME; + + /** The append-only id reservation table (see InternalSchema.SPM_BASELINES_SEQ_TBL_NAME). */ + private static final String SPM_BASELINES_SEQ_TABLE = + FeConstants.INTERNAL_DB_NAME + "." + InternalSchema.SPM_BASELINES_SEQ_TBL_NAME; + + /** The compact id high-water-mark table (see InternalSchema.SPM_BASELINES_HWM_TBL_NAME). */ + private static final String SPM_BASELINES_HWM_TABLE = + FeConstants.INTERNAL_DB_NAME + "." + InternalSchema.SPM_BASELINES_HWM_TBL_NAME; + + /** Column order follows InternalSchema.SPM_BASELINES_SCHEMA (unpaged selection). */ + private static final String SNAPSHOT_COLUMNS = + "SELECT `id`, `bind_sql`, `bind_sql_digest`," + + " `bind_sql_hash`, `plan_sql`, `query_id`, `cost`, `query_time_ms`, `source`," + + " `status`, " + storedMicros("`create_time`") + ", " + storedMicros("`update_time`") + + ", `sql_mode`, `plan_sql_mode`," + + " `plan_frozen`, `schema_fingerprint`, `plan_sql_digest` FROM "; + + /** + * Shared page ordering over every stored scalar column. Rows that tie on id, update time, status and SQL text + * can still differ in parser modes, fingerprints, costs or other replay metadata. Ordering all columns makes + * inclusive-id pagination stable; fully identical rows remain interchangeable. + */ + private static final String SNAPSHOT_ORDER_BY = " ORDER BY `id`, `update_time`, `status`," + + " `bind_sql_digest`, `plan_sql`, `bind_sql`, `bind_sql_hash`, `query_id`," + + " `cost`, `query_time_ms`, `source`, `create_time`, `sql_mode`, `plan_sql_mode`," + + " `plan_frozen`, `schema_fingerprint`, `plan_sql_digest`"; + + private static final String SELECT_ALL_ORDERED_SQL = + SNAPSHOT_COLUMNS + SPM_BASELINES_TABLE + SNAPSHOT_ORDER_BY; + + /** + * One continuation page of a whole-table snapshot: every row with id >= + * ${lastId}, ordered by id and SKIPPING the first ${offset} rows of that + * range. The offset is what keeps an id group larger than one page readable: a + * repeated opposite-status ALTER failure leaves one more row under the id every time, + * so the group can outgrow SNAPSHOT_PAGE_SIZE rows - a jump past it would + * omit the rows behind the first page, possibly the newest durable status. The order + * must remain a TOTAL order within one id (see SELECT_ALL_ORDERED_SQL) or a + * tie order that differs between the two queries can make the offset skip a row of + * the interrupted group (content tie-breakers added). + */ + private static final String SELECT_PAGE_SQL = SNAPSHOT_COLUMNS + SPM_BASELINES_TABLE + + " WHERE `id` >= ${lastId}" + SNAPSHOT_ORDER_BY + + " LIMIT ${pageSize} OFFSET ${offset}"; + + /** + * Rows per snapshot page (see readPersistedSnapshot). Bounds what ONE + * internal query has to return, so a growing table can no longer make the whole + * snapshot read fail against a fixed timeout. + */ + private static final int SNAPSHOT_PAGE_SIZE = 2000; + + /** + * Fence re-reads a paginated snapshot read is allowed before it fails closed (see + * readStableRawSnapshot): one DDL overlapping the loop then converges on the + * retry, while a table that never stays stable must not be published as a snapshot. + */ + private static final int SNAPSHOT_STABILITY_ATTEMPTS = 3; + + /** + * The persistence-layer id watermark (see reserveAllocatedId): read + * before every id allocation. MAX over an aggregate is a light single-row query. + */ + private static final String SELECT_MAX_ID_SQL = "SELECT MAX(`id`) FROM " + SPM_BASELINES_TABLE; + + /** + * The compact id high-water mark: a tiny append-only table whose rows carry the + * newest allocated id AND the newest MUTATION CLOCK tick (see SELECT_CLOCK_SQL). The + * append-only HISTORY table (InternalSchema.SPM_BASELINES_SEQ_TBL_NAME) grows by one + * row per create forever, so its MAX(baseline_id) - the only unbounded read on the create + * path - was replaced: every allocation also records itself here (pruning the + * superseded rows), and a pre-upgrade cluster only pays the legacy full read ONCE + * (see readPersistedWatermark). + */ + private static final String SELECT_HWM_SQL = "SELECT MAX(`watermark`) FROM " + + SPM_BASELINES_HWM_TABLE; + + /** + * Maximum age in seconds for an open mutation-window row to participate in the snapshot fence. An abandoned + * row eventually expires so a failed FE cannot block snapshots indefinitely. + * The bound covers opening, pruning and visibility probes before the row statement, plus that statement's + * write timeout. After those probes, beginRowMutation also checks that enough lifetime remains before + * dispatching the statement. Successful completion explicitly closes the writer's window. + */ + private static final long MUTATION_WINDOW_MAX_AGE_SECONDS = 120L; + + /** + * Maximum allowed lead of a stored update timestamp over wall time, in microseconds. Status writers use this + * bound when incrementing an already-ahead timestamp so stored ordering remains within the shared fence + * lifetime assumptions. + */ + private static final long MAX_STORED_UPDATE_LEAD_MICROS = 60_000_000L; + + /** + * How many low bits of a mutation tick carry the window's FE identity (see + * mutationTickSalt): enough range that two FEs writing in the same millisecond almost never + * collide, small enough that the millisecond stays in the high bits (the clock's + * cluster-wide ordering: the remaining 43 positive bits cover roughly 279 years of millis). + * + * The salt only makes a collision UNLIKELY; beginRowMutation therefore VERIFIES that the + * marker it appended is the only one carrying its tick (see MUTATION_TICK_OPEN_ATTEMPTS). + */ + private static final int MUTATION_TICK_FE_BITS = 20; + + /** + * How many fresh ticks one window open may try when the marker it appended shares its tick + * with another FE's window (see beginRowMutation): a random salt is not a guarantee, and two + * windows sharing one identity would let either writer's exact-row close delete the OTHER's + * still-open marker - turning an in-flight row statement into a quiet clock. + */ + private static final int MUTATION_TICK_OPEN_ATTEMPTS = 3; + + /** The value mask of MUTATION_TICK_FE_BITS (see nextMutationTick). */ + private static final long MUTATION_TICK_FE_MASK = (1L << MUTATION_TICK_FE_BITS) - 1; + + /** + * The bounded MUTATION CLOCK of the paginated snapshot fence (see + * readStableRawSnapshot): MAX(tick) plus the slot's row count / tick sum. Every + * spm_baselines mutation (create / delete / status flip) appends a clock row with a + * strictly increasing tick BEFORE its own statement, so the token changes whenever + * the durable table changes - without any table-wide aggregate on the read path. The + * count / sum stay as tie-breakers for an append whose tick collides with the current + * maximum (skewed clocks): an append raises MAX(tick) or the pair (count, sum), so a + * fence read around a paginated snapshot observes the mutation and retries instead of + * publishing a mixed state (a residual collision of two same-instant appends from + * different FEs converges on the next refresh, like the fence's second-precision + * predecessor). + * + * The fourth column is the OPEN-WINDOW marker: MAX(pending) over the rows written + * within MUTATION_WINDOW_MAX_AGE_SECONDS. Any unclosed window poisons the fence (that + * is the point - a window is only closed by its own completion write, see + * beginRowMutation / endRowMutation), but a window WHOSE FE DIED before its completion + * became a PERMANENT poison: pending stayed non-null forever and every snapshot read + * failed, so the store could never load again. A mutation's own WINDOW - the pre-write + * budget AND the row statement it fences, see MUTATION_WINDOW_MAX_AGE_SECONDS, whose + * bound beginRowMutation verifies before dispatching anything - fits inside the window + * bound, so a begin row older than it cannot still be mid-statement and is ignored. + */ + private static final String SELECT_CLOCK_SQL = "SELECT MAX(`tick`), COUNT(*)," + + " SUM(`tick`), MAX(CASE WHEN `update_time` > DATE_SUB(NOW(), INTERVAL " + + MUTATION_WINDOW_MAX_AGE_SECONDS + " SECOND) THEN `pending` ELSE 0 END)" + + " FROM " + SPM_BASELINES_HWM_TABLE; + + /** + * The slot's newest watermark AND clock tick; the pre-mutation build of the mutation + * window (and the completion append) reads it to keep the recorded watermark untouched + * while advancing the clock (see beginRowMutation). One bounded keyed read. + */ + private static final String SELECT_SLOT_STATE_SQL = "SELECT MAX(`watermark`)," + + " MAX(`tick`) FROM " + SPM_BASELINES_HWM_TABLE; + + /** + * Whether the OPEN mutation window just appended is READABLE: the row carrying the very + * tick this process wrote in pending (see beginRowMutation). A plain append can + * report SQL OK while its row is still unreadable - and with the window marker only + * readable LOCALLY, a successor's fence read (SELECT_CLOCK_SQL, the MATCHES path of + * updateStatus) sees a QUIET clock while this process's row statement is already in + * flight: the statement then publishes AFTER the successor accepted a matching read + * and reversed it. The bounded read-back makes the statement's dispatch + * conditional on the window being SHARED. + */ + private static final String SELECT_PENDING_TICK_SQL = "SELECT COUNT(*) FROM " + + SPM_BASELINES_HWM_TABLE + + " WHERE `tick` = ${tick} AND `pending` = ${tick}"; + + /** Append one high-water-mark / clock row (see SELECT_HWM_SQL, SELECT_CLOCK_SQL). */ + private static final String INSERT_HWM_SQL = "INSERT INTO " + SPM_BASELINES_HWM_TABLE + + " (`watermark`, `update_time`, `tick`, `pending`)" + + " VALUES (${lastId}, NOW(), ${tick}, ${pending})"; + + /** + * Best-effort pruning of completed high-watermark rows superseded by the latest watermark or tick. Higher + * watermarks and open mutation-window rows are retained. Open rows are shared evidence of in-flight mutations + * and only their own completion may close them. The pending = 0 predicate also leaves legacy NULL rows + * untouched. + */ + private static final String PRUNE_HWM_SQL = "DELETE FROM " + SPM_BASELINES_HWM_TABLE + + " WHERE `watermark` < ${lastId}" + + " AND `pending` = 0"; Review Comment: [P2] Reclaim expired abandoned mutation-window rows If `endRowMutation`'s completion read/append/probe/close fails, it has already removed local ownership and keeps no retry obligation for that ordinary completion. The durable `pending=tick` row is then stranded. Both general HWM prunes require `pending=0`, and the 120-second expiry changes only the open-window MAX(CASE): MAX(tick), COUNT(*) and SUM(tick) still scan the row forever. Repeated transient completion failures or FE crashes permanently grow the supposedly compact table and its management/snapshot queries even after recovery. Safely reclaim expired windows under the established dispatch lifetime bound, or retain completion obligations. This is separate from the existing same-id compact sequence-row leak. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/placeholder/SPMPlaceholderBuilder.java: ########## @@ -0,0 +1,1452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.placeholder; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.spm.SPMPlanTreeSupport; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; +import org.apache.doris.nereids.types.coercion.FollowToArgumentType; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Parameterizes supported literal positions with scalar or IN-list markers. One builder retains marker records and + * allocates ids across bind and plan trees. Reuse requires the same value, parent signature, child position, query + * block, projection item, expression-root ordinal and occurrence path. GROUP BY mirrors can deliberately reuse a + * transformed expression to preserve analyzer equality. + * + * Syntax-bearing literals such as LIKE escape arguments are kept concrete. Some NULL payloads are retyped for + * capture analysis so their markers survive Boolean consumers; replay still substitutes the matched query + * expression. + */ +public class SPMPlaceholderBuilder extends ExpressionVisitor<Expression, Expression> + implements SPMPlanTreeSupport.ExprTransform { + + /** + * The ARRAY scalar functions that declare retArgType(0): each one's result IS its + * first argument's array type, so an all-NULL input makes the whole result an all-NULL array. + * + * They are exactly the wrappers a projected all-NULL container can sit UNDER while the + * enclosing call is still consumed as a Boolean operand, e.g. + * k > 0 OR element_at(array_reverse_sort(array(NULL)), 1): the wrapper hands + * array(NULL)'s NullType element type to element_at, so the marker has to be + * repaired at the WRAPPER's argument position. Matching only array_sort / + * array_distinct left every other member of the family unrecognized. + * + * Matched by NAME because the capture runs on the PARSE tree, where the call is still an + * unbound Function and no signature is resolved yet; a BOUND node is classified from + * its retArgType(0) signature instead (see followsFirstArgument), so a function added + * to the family is recognized without touching this list. array_concat is NOT here: + * it derives its element type from EVERY part and is handled separately + * (isArrayConcatCall). + */ + private static final Set<String> TYPE_PRESERVING_ARRAY_CALLS = Set.of( + "array_apply", "array_compact", "array_difference", "array_distinct", + "array_except", "array_except_all", "array_filter", "array_first", + "array_intersect", "array_last", "array_popback", "array_popfront", + "array_pushback", "array_pushfront", "array_remove", "array_reverse_sort", + "array_shuffle", "array_slice", "array_sort", "array_sortby", "array_union", + "reverse", "trim_array"); + + /** + * The MAP counterpart of TYPE_PRESERVING_ARRAY_CALLS (same retArgType(0) + * declaration: map_filter returns its first argument's map type). + */ + private static final Set<String> TYPE_PRESERVING_MAP_CALLS = Set.of("map_filter"); + + /** Auto-increment placeholder id counter (starts at 1). */ + private long nextId = 1; + + private final Map<PlaceholderKey, Expression> placeholderIndex = new HashMap<>(); + private final Map<Expression, String> parentSignatures = new IdentityHashMap<>(); + + /** + * Optional GROUP BY mirror lookup, installed by SPMPlanTreeSupport.TreeTransformer for + * one clause window (see setGroupByMirrorLookup): maps an expression's + * pre-parameterization text to the ALREADY-transformed GROUP BY expression that must + * be reused - also for NESTED subexpressions (e.g. the substr(...) inside + * "HAVING substr(...) = 'x'"), because the analyzer resolves such a clause + * expression-wise and demands literal expression equality with the GROUP BY under + * ONLY_FULL_GROUP_BY. Null outside the window. + */ + private java.util.function.Function<String, Expression> groupByMirrorLookup; + + /** + * DFS stack of child positions: the top is the index of the expression currently + * being visited inside its direct parent (empty = parent-less root). Two literals + * under the same parent (e.g. the operands of "1 + 1") are different positions and + * must never reuse one placeholder id. + */ + private final Deque<Integer> childIndexStack = new ArrayDeque<>(); + + /** + * Query-block identity: incremented when entering a nested query block (the plan of + * a subquery expression). Two literals in DIFFERENT blocks never share a placeholder + * id even when value / parent / position coincide: in + * SELECT * FROM t WHERE a=1 AND EXISTS (SELECT 1 FROM u WHERE a=1) the two a=1 + * literals used to reuse one id, and a user query with outer a=2 / inner a=3 was + * rejected (one id must resolve to ONE value), so the baseline could never hit. + * The counter restarts per tree (see startNewTree): bind and plan trees traverse the + * same structure in the same order, so corresponding blocks keep corresponding + * numbers and the cross-tree id alignment is preserved. + */ + private long nextBlockId = 1; + private long currentBlockId = 0; + + /** + * Enclosing query-block ids (see enterQueryBlock): a nested block's id is + * restored on exit, so SIBLING derived tables / CTE bodies get distinct ids while the + * enclosing block numbering resumes - the same discipline the subquery-expression + * blocks already followed. + */ + private final Deque<Long> blockIdStack = new ArrayDeque<>(); + + /** + * Projection ITEM positions of the SELECT-list slots currently being visited (see + * enterProjectionItem(int)): SELECT 1 AS x, 1 AS y must give its two + * literals different placeholder ids although value, parent structure and child + * position coincide (see PlaceholderKey). + */ + private final Deque<Integer> projectItemStack = new ArrayDeque<>(); + + /** + * Ordinal of the expression ROOT currently being transformed, counted per query block in + * walk order (see beginExpressionRoot): two equal literals in DIFFERENT roots of one + * block (the ON clauses of two joins, two conjuncts of one filter,...) used to share one + * placeholder id although they are independent slots - a query that varies them + * separately (1/1 -> 2/3) was then rejected by recordPlaceholderValue and the saved + * baseline could never be used. The counting restarts + * per tree run AND per block, so the bind and plan trees of one baseline traverse the + * same structure in the same order and corresponding roots keep corresponding ordinals. + */ + private long nextRootOrdinal = 0; + private long currentRootOrdinal = -1; + + /** + * Depth of an all-NULL CONDITIONAL (CASE / IF / COALESCE) currently visited in a Boolean + * context (see markerPayload): such a node has NullType and the analyzer replaces the whole + * operand with a typed constant, so its branch NULLs are parameterized BOOLEAN. + */ + private int booleanConditionalDepth; + + /** + * Depth of an all-NULL container producer whose ELEMENTS / VALUES are the markers that carry + * the container's element type (see markerPayload): an all-NULL element list is what makes + * the constructor NullType, so its NULL elements are parameterized BOOLEAN while the + * constructor itself sits in a container position of a NullType-producing chain. + */ + private int containerElementDepth; + + /** Whether the current container dependency follows MAP_KEYS through its map input. */ + private boolean mapKeyDependency; + + /** + * Whether a GROUP BY ordinal references a parameterized SELECT item. Analysis can inline + * that expression and the physical grouping can lose its binding to the original item. + * Keep the parameterized tree so replay binds the ordinal against the caller's values. + */ + private boolean parameterizedGroupByOrdinalTarget; + + /** The enclosing block's root-ordinal counter, restored by exitQueryBlock. */ + private final Deque<Long> rootOrdinalStack = new ArrayDeque<>(); + + /** The enclosing root's ordinal, restored by exitQueryBlock (see enterQueryBlock). */ + private final Deque<Long> currentRootOrdinalStack = new ArrayDeque<>(); + + /** Installs / clears the GROUP BY mirror lookup window (null = disabled). */ + @Override + public void setGroupByMirrorLookup(java.util.function.Function<String, Expression> lookup) { + this.groupByMirrorLookup = lookup; + } + + /** Reports whether capture must keep the parameterized tree for a GROUP BY ordinal. */ + public boolean hasParameterizedGroupByOrdinalTarget() { + return parameterizedGroupByOrdinalTarget; + } + + /** Marks a GROUP BY ordinal whose SELECT item carries placeholders. */ + @Override + public void noteParameterizedGroupByOrdinalTarget() { + this.parameterizedGroupByOrdinalTarget = true; + } + + /** Starts one expression ROOT of the current block (see nextRootOrdinal). */ + private void beginExpressionRoot() { + currentRootOrdinal = nextRootOrdinal++; + } + + /** Restarts the per-tree query-block numbering (a tree's root block is 0). */ + public void startNewTree() { + nextBlockId = 1; + currentBlockId = 0; + blockIdStack.clear(); + projectItemStack.clear(); + // one tree = one root-ordinal run: corresponding roots of the bind tree and the + // (separately parsed) plan tree keep corresponding ordinals + nextRootOrdinal = 0; + currentRootOrdinal = -1; + rootOrdinalStack.clear(); + currentRootOrdinalStack.clear(); + } + + /** + * The whole-tree parameterization transform: the builder IS an + * SPMPlanTreeSupport.ExprTransform, so the plan walk can hand it the query + * BLOCK and projection ITEM scopes of the expressions it visits (see + * SPMPlanTreeSupport.ExprTransform.enterQueryBlock). Passing a bare lambda + * instead would lose those scopes and merge independent literals into one placeholder + * id. + */ + @Override + public Expression apply(Expression expr) { + beginExpressionRoot(); + return expr.accept(this, null); + } + + /** + * Transform of a SUBQUERY expression's own plan tree: every expression the walk hands + * over is a block ROOT (the child-position sentinel -1), and the query-block / + * projection-item scope hooks are forwarded to the enclosing builder so nested + * derived tables keep their own block ids. + */ + private final class SubqueryBlockTransform implements SPMPlanTreeSupport.ExprTransform { + @Override + public Expression apply(Expression expr) { + beginExpressionRoot(); + childIndexStack.push(-1); + try { + return expr.accept(SPMPlaceholderBuilder.this, null); + } finally { + childIndexStack.pop(); + } + } + + @Override + public void setGroupByMirrorLookup(java.util.function.Function<String, Expression> lookup) { + SPMPlaceholderBuilder.this.setGroupByMirrorLookup(lookup); + } + + @Override + public void noteParameterizedGroupByOrdinalTarget() { + SPMPlaceholderBuilder.this.noteParameterizedGroupByOrdinalTarget(); + } + + @Override + public void enterQueryBlock() { + SPMPlaceholderBuilder.this.enterQueryBlock(); + } + + @Override + public void exitQueryBlock() { + SPMPlaceholderBuilder.this.exitQueryBlock(); + } + + @Override + public void enterProjectionItem(int index) { + SPMPlaceholderBuilder.this.enterProjectionItem(index); + } + + @Override + public void exitProjectionItem() { + SPMPlaceholderBuilder.this.exitProjectionItem(); + } + } + + /** + * Enters a nested query block (see + * org.apache.doris.nereids.spm.SPMPlanTreeSupport.ExprTransform.enterQueryBlock): + * the block gets its own monotonic id, the enclosing one is restored by + * exitQueryBlock. + */ + @Override + public void enterQueryBlock() { + blockIdStack.push(currentBlockId); + currentBlockId = nextBlockId++; + // the root-ordinal counting is per BLOCK as well: a nested block's roots must not + // shift the enclosing block's ordinals (and vice versa), and the enclosing counters + // resume on exit + rootOrdinalStack.push(nextRootOrdinal); + nextRootOrdinal = 0; + currentRootOrdinalStack.push(currentRootOrdinal); + currentRootOrdinal = -1; + } + + /** Leaves the current nested query block and restores the enclosing block id. */ + @Override + public void exitQueryBlock() { + if (!blockIdStack.isEmpty()) { + currentBlockId = blockIdStack.pop(); + } + if (!rootOrdinalStack.isEmpty()) { + nextRootOrdinal = rootOrdinalStack.pop(); + } + if (!currentRootOrdinalStack.isEmpty()) { + currentRootOrdinal = currentRootOrdinalStack.pop(); + } + } + + /** + * Enters one projection item of a projection-like list: every literal of its subtree + * carries this position, so two equally shaped literals in different items (or in a + * projection and in a filter) never share one placeholder id. + */ + @Override + public void enterProjectionItem(int index) { + projectItemStack.push(index); + } + + /** Leaves the current projection item. */ + @Override + public void exitProjectionItem() { + if (!projectItemStack.isEmpty()) { + projectItemStack.pop(); + } + } + + /** Position of the projection item currently being visited; -1 outside a SELECT list. */ + private int currentProjectItem() { + Integer top = projectItemStack.peek(); + return top == null ? -1 : top; + } + + /** Position of the expression currently being visited inside its direct parent; -1 at a root. */ + private int currentChildIndex() { + Integer top = childIndexStack.peek(); + return top == null ? -1 : top; + } + + /** + * The occurrence path of the literal currently being visited: the child positions from the + * transformed EXPRESSION ROOT down to the literal, rendered as "0/1/..." (empty for a + * root literal). Two equal literals can sit under structurally EQUAL parents at + * different places of one root ("a = 1 AND a = 1": the parents are the two And + * children), and only the path tells those independent slots apart - see + * PlaceholderKey.occurrencePath. + */ + private String currentOccurrencePath() { + if (childIndexStack.isEmpty()) { + return ""; + } + Object[] positions = childIndexStack.toArray(); + StringBuilder sb = new StringBuilder(); + for (int i = positions.length - 1; i >= 0; i--) { + int position = (Integer) positions[i]; + if (position == -1) { + // A subquery's expression root starts a new path, independent of its caller. + sb.setLength(0); + continue; + } + if (sb.length() > 0) { + sb.append('/'); + } + sb.append(position); + } + return sb.toString(); + } + + // literal -> scalar placeholder. + + /** + * Whether the expression's TYPE is NullType. + * + * The type is derived structurally because this builder walks the UNBOUND plan, where + * getDataType throws. A node produces NullType when EVERY value it can yield is + * NULL - a NULL literal, a CASE whose every result does, an IF whose two value arguments do, + * a COALESCE / IFNULL / NVL / NULLIF / GREATEST / LEAST all of whose arguments do. Collected as + * a Boolean operand (directly, or through a projected slot) such an expression is REPLACED by + * the analyzer with a typed constant NULL, which deletes the placeholders inside it. + * + * The test is RECURSIVE on purpose: a conditional nested in the RESULT of + * another conditional (k > 0 OR (CASE WHEN k = 0 THEN ifnull(NULL, NULL) ELSE NULL END)) + * yields NullType although its result is not a NULL LITERAL, so a literal-only test left its + * inner markers NullType and the consumer erased them - a TRUE variant then replayed as NULL. + * The same recursion covers the custom-signature producers: GreatestLeast.customSignature + * returns the arguments' common type, i.e. NullType for greatest(NULL, NULL), LEAST shares the path). + * + * Parameter expr: the expression to classify + * Returns: whether every value the expression can yield is NULL + */ + private static boolean producesNullType(Expression expr) { + if (expr == null) { + return false; + } + if (expr.isNullLiteral()) { + return true; + } + List<Expression> children = expr.children(); + if (expr instanceof CaseWhen) { + CaseWhen caseWhen = (CaseWhen) expr; + for (WhenClause clause : caseWhen.getWhenClauses()) { + if (!producesNullType(clause.getResult())) { + return false; + } + } + return !caseWhen.getDefaultValue().isPresent() + || producesNullType(caseWhen.getDefaultValue().get()); + } + if (isIfConditional(expr)) { + return children.size() == 3 && producesNullType(children.get(1)) + && producesNullType(children.get(2)); + } + if (isIfNullConditional(expr) || isNullIfConditional(expr) + || isGreatestLeastConditional(expr) || isCoalesceConditional(expr)) { + // every argument contributes a value the result may equal: all-NULL arguments make + // the whole call NullType (IFNULL / NVL / NULLIF / GREATEST / LEAST / COALESCE) + if (children.isEmpty()) { + return false; + } + for (Expression child : children) { + if (!producesNullType(child)) { + return false; + } + } + return true; + } + if (expr instanceof WindowExpression) { + return producesNullType(((WindowExpression) expr).getFunction()); + } + if (isFirstArgumentTypedResult(expr)) { + return !children.isEmpty() && producesNullType(children.get(0)); + } + if (isElementAtCall(expr)) { + // element_at(array(NULL), 1) takes its type from the ARRAY's element type and + // element_at(map('x', NULL), 'x') from the MAP's VALUE type (CreateMap derives + // MAP<STRING, NULL_TYPE>), so an all-NULL container makes the WHOLE call NullType: a + // Boolean consumer replaced the two markers of + // k > 0 OR element_at(array(NULL), 1) with a typed constant and the + // element_at(array(TRUE), 1) variant had nothing to substitute, and + // the MAP spelling lost the value marker the same way. + return !children.isEmpty() && isAllNullContainer(children.get(0)); + } + if (isArrayReducerCall(expr)) { + // The SAME producer classification, for the array REDUCERS: + // ARRAY_MIN / ARRAY_MAX return FollowToAnyDataType(0), i.e. the INPUT array's + // element type, so an all-NULL input makes the whole call NullType - binding then + // replaced k > 0 OR array_min(array(NULL)) with k > ph OR NULL and the + // array_min(array(TRUE)) caller got NULL instead of TRUE. Keeping the family in + // ONE classifier is what lets the marker be retyped before the Boolean consumer + // drops it - the same defect the bare element_at trigger had, so it is carried here + // instead of in a per-function inline. + return !children.isEmpty() && isAllNullArray(children.get(0)); + } + return false; + } + + /** These aggregates and window functions derive the result type from their first argument. */ + private static boolean isFirstArgumentTypedResult(Expression expression) { + if (!(expression instanceof Function)) { + return false; + } + String name = ((Function) expression).getName(); + return "min".equalsIgnoreCase(name) || "max".equalsIgnoreCase(name) + || "first_value".equalsIgnoreCase(name) || "last_value".equalsIgnoreCase(name); + } + + /** + * Finds an array return signature whose element type follows an argument type and returns that argument index. + * A bound function with no such signature returns -1. An unbound array_repeat call falls back to index zero. + */ + private static int elementTypeArgumentIndex(Expression expr) { + if (expr instanceof BoundFunction) { + for (FunctionSignature signature : ((BoundFunction) expr).getSignatures()) { + DataType returnType = signature.returnType; + if (!(returnType instanceof ArrayType)) { + continue; + } + DataType itemType = ((ArrayType) returnType).getItemType(); + if (itemType instanceof FollowToAnyDataType) { + return ((FollowToAnyDataType) itemType).getIndex(); + } + } + return -1; + } + return isArrayRepeatCall(expr) ? 0 : -1; + } + + /** + * Whether the expression is the unbound array_repeat(...) call (see + * elementTypeArgumentIndex). + */ + private static boolean isArrayRepeatCall(Expression expr) { + return expr instanceof Function + && "array_repeat".equalsIgnoreCase(((Function) expr).getName()); + } + + /** + * Whether the expression is a container whose ELEMENT type is NullType (see producesNullType): + * an all-NULL array or an all-NULL-valued map. Both overloads of element_at return the + * container's element / value type, so either one makes the call a NullType producer. + * + * Parameter expr: the expression to classify + * Returns: whether it is an all-NULL array or map value + */ + private static boolean isAllNullContainer(Expression expr) { + return isAllNullArray(expr) || isAllNullMap(expr); + } + + /** Follows a map's value type through constructors and type-preserving wrappers. */ + private static boolean isAllNullMap(Expression expr) { + return isAllNullMapComponent(expr, 0, false); + } + + /** Tests the requested map component, optionally through further array elements. */ + private static boolean isAllNullMapComponent(Expression expr, int arrayDepth, boolean key) { + if (expr instanceof MapLiteral) { + Map<Literal, Literal> entries = ((MapLiteral) expr).getValue(); + return allNullAtArrayDepth(key ? entries.keySet() : entries.values(), arrayDepth); + } + if (isTypePreservingMapCall(expr)) { + return expr.arity() > 0 && isAllNullMapComponent(expr.child(0), arrayDepth, key); + } + if (!isMapConstructor(expr)) { + return false; + } + List<Expression> pairs = expr.children(); + if (pairs.size() < 2) { + return false; + } + for (int i = key ? 0 : 1; i < pairs.size(); i += 2) { + if (!nullAtArrayDepth(pairs.get(i), arrayDepth)) { + return false; + } + } + return true; + } + + /** Tests whether the element of a recognized array producer has NullType. */ + private static boolean isAllNullArray(Expression expr) { + return isAllNullArray(expr, 1); + } + + /** Follows the result type through the requested number of array element selections. */ + private static boolean isAllNullArray(Expression expr, int depth) { + if (expr == null) { + return false; + } + if (expr.isNullLiteral()) { + return true; + } + if (isElementAtCall(expr)) { + return expr.arity() > 0 && (isAllNullArray(expr.child(0), depth + 1) + || isAllNullMapComponent(expr.child(0), depth, false)); + } + if (isArrayMapCall(expr)) { + return !expr.children().isEmpty() && expr.child(0) instanceof Lambda + && nullAtArrayDepth(((Lambda) expr.child(0)).getLambdaFunction(), depth - 1); + } + int elementIndex = elementTypeArgumentIndex(expr); + if (elementIndex >= 0) { + return elementIndex < expr.arity() && nullAtArrayDepth(expr.child(elementIndex), depth - 1); + } + if (isArrayConcatCall(expr)) { + return !expr.children().isEmpty() + && expr.children().stream().allMatch(part -> isAllNullArray(part, depth)); + } + if (isMapValuesCall(expr) || isMapKeysCall(expr)) { + return expr.arity() == 1 && isAllNullMapComponent(expr.child(0), depth - 1, isMapKeysCall(expr)); + } + if (isTypePreservingArrayCall(expr)) { + return !expr.children().isEmpty() && isAllNullArray(expr.child(0), depth); + } + if (!isArrayConstructor(expr)) { Review Comment: [P1] Follow conditional container result dependencies before freezing With `t(k)={-1}`, capture `SELECT k,k>0 OR element_at(if(k>0,array(NULL),array(NULL)),1) AS c FROM t`, then run the version with both arrays changed to `array(TRUE)`. The parsed operand is `Or(k>phA, ElementAt(If(k>phB,Array(phC),Array(phD)),phE))`. This classifier does not follow IF/CASE/COALESCE container results, so the NULL elements keep NullType. Binding yields ARRAY<NULL_TYPE>, then `ExpressionAnalyzer.visitOr` replaces the complete ElementAt by Boolean NULL. The surviving threshold marker passes the freeze probe; the matching TRUE variant returns NULL instead of TRUE. Follow conditional value arms, or decline freezing. This is distinct from the repaired nested-ElementAt extraction case. The omitted `NTH_VALUE` result dependency has the same marker-loss symptom; include that family when generalizing the repair. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/builder/SPMExprSqlBuilder.java: ########## @@ -0,0 +1,1009 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.builder; + +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.spm.matcher.SPMFrozenTreeReplacer; +import org.apache.doris.nereids.spm.placeholder.SPMSubquerySupport; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.AggregateExpression; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.Between; +import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.BinaryOperator; +import org.apache.doris.nereids.trees.expressions.BitAnd; +import org.apache.doris.nereids.trees.expressions.BitNot; +import org.apache.doris.nereids.trees.expressions.BitOr; +import org.apache.doris.nereids.trees.expressions.BitXor; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Exists; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.IsFalse; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.IsTrue; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StringRegexPredicate; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.TryCast; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctGroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; +import org.apache.doris.nereids.trees.expressions.functions.window.Lag; +import org.apache.doris.nereids.trees.expressions.functions.window.Lead; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.DecimalV2Type; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.VarBinaryType; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Renders supported expressions using the ExprId reference mapping of a SQLRelation. Marker payloads are omitted + * in favor of id-only calls. Composite expressions recurse through the visitor; a generic toSql fallback rejects + * registered slots that require remapping. + * + * Subquery expressions have only a compact diagnostic digest rendering here, not executable subquery SQL. The + * physical-plan decompiler rejects residual subquery expressions before invoking this printer. + */ +public class SPMExprSqlBuilder extends ExpressionVisitor<String, SQLRelation> { + + /** + * The INTERNAL pseudo function names visitSubqueryExpr emits for a subquery: the + * structural DIGEST form, never SQL. + * + * They are reserved by SPM. A frozen SQL must never be readable as - or shadowed by - a user + * function of one of these names: INSUBQUERY(x)[...] is an ordinary unreserved call, so + * a compatible user alias UDF made a frozen text validate and every replay evaluated that UDF + * (and its array access) instead of the original IN (SELECT...). Subqueries are + * therefore refused before decompilation and a plan calling a function of these names is not + * freezable either (see SPMPlan2SQLBuilder.rejectSubqueryExpressions / + * #rejectReservedInternalFunctionNames). + */ + public static final java.util.Set<String> INTERNAL_PSEUDO_FUNCTION_NAMES = + java.util.Set.of("insubquery", "scalarsubquery"); + + /** + * Prints a single expression. + * + * Parameter expr: the expression to print + * Parameter relation: the SQLRelation carrying the columnNames mapping + * Returns: the SQL text + */ + public String print(Expression expr, SQLRelation relation) { + return expr.accept(this, relation); + } + + /** + * Renders one function's argument list. + * + * Parameter aggregate: whether the arguments belong to an AGGREGATE call + * Parameter arguments: the arguments to render + * Parameter context: the relation of the enclosing scope + * Returns: the comma separated argument text + */ + private String renderArguments(boolean aggregate, List<Expression> arguments, + SQLRelation context) { + return arguments.stream() + .map(argument -> renderArgument(argument, context, aggregate)) + .collect(Collectors.joining(", ")); + } + + /** + * Renders one function argument. At an aggregate argument root, preserves a recognized LARGEINT accumulator + * widening; other casts follow visitCast. This root-only rule avoids retaining ordinary nested coercions that + * would pin a captured operand type. + */ + private String renderArgument(Expression argument, SQLRelation context, boolean aggregate) { + if (aggregate && argument instanceof Cast && preservesLoweredAccumulator((Cast) argument)) { + Cast cast = (Cast) argument; + return "CAST(" + cast.child().accept(this, context) + " AS " + + cast.getDataType().toSql() + ")"; + } + return argument.accept(this, context); + } + + @Override + public String visit(Expression expr, SQLRelation context) { + // SPM placeholder: render as the id-only function call "_spm_const_var(id)" / + // "_spm_const_list(id)" (the value does not participate in the digest / planSql + // serialization). The id-only form keeps the digest value-independent AND makes + // the frozen planSql parseable at replay: the text carries the id and the + // replay substitutes the matched expression under that id. + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstVar) { + org.apache.doris.nereids.spm.placeholder.SpmConstVar ph = + (org.apache.doris.nereids.spm.placeholder.SpmConstVar) expr; + return "_spm_const_var(" + ph.getId() + ")"; + } + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstList) { + org.apache.doris.nereids.spm.placeholder.SpmConstList list = + (org.apache.doris.nereids.spm.placeholder.SpmConstList) expr; + // IN-list placeholder: render size-independently (_spm_const_list(id)) so + // the digest / planSql stays value AND list-length independent: a baseline + // "IN (1, 2, 3)" matches a user "IN (10, 20)" regardless of list length. + return "_spm_const_list(" + list.getId() + ")"; + } + // Default fallback: use toSql of the expression itself (no column mapping). + // toSql bypasses the ExprId -> column mapping, so a composite shape whose + // children need remapping (e.g. ~left.a above a join that renamed the colliding + // a to c_N / qualified it) would freeze a stale, unresolvable column and - with + // the default enable_spm_fallback = false - fail the rewritten query. Reject + // such shapes so CREATE keeps the user planSql and the rewrite degrades to the + // parameterized-tree path. + ensureNoRemappedSlots(expr, context); + return expr.toSql(); + } + + /** + * Safety net of the generic toSql fallback: rejects the expression when any slot + * below it is registered under a reference that toSql would NOT reproduce (the + * collision-renaming / quoting / qualification case), because toSql would then emit + * a stale or differently-shaped name. Only a byte-identical registered reference is + * renderable through the fallback. Public for tests. + * + * Parameter expr: the expression about to fall back to toSql + * Parameter context: the relation carrying the ExprId -> column-reference mapping + * Throws UnsupportedOperationException: when a child slot needs remapping + */ + public static void ensureNoRemappedSlots(Expression expr, SQLRelation context) { + if (expr instanceof SlotReference) { + SlotReference slot = (SlotReference) expr; + String mapped = context.getColumnNames().get(slot.getExprId()); + // toSql prints the BARE slot name, so a registered reference is only safe + // when it is byte-identical to that rendering. A special-character column + // maps to a QUOTED reference (a-b) and a disambiguated one to a QUALIFIED + // reference (l.x): both still "name" the column, but freezing their unquoted / + // unqualified toSql text produces SQL that no longer resolves after reload + // ("a-b MATCH 'foo'" parses as a subtraction; a bare x may be ambiguous). + // Reject the generic fallback for those shapes - CREATE keeps the user planSql + // and the rewrite degrades to the parameterized-tree path. + if (mapped != null && !mapped.equals(slot.getName())) { + throw new UnsupportedOperationException( + "SPM decompile: expression " + expr.toSql() + " needs column remapping (" + + mapped + " -> " + slot.getName() + ") which toSql() cannot apply"); + } + } + for (Expression child : expr.children()) { + ensureNoRemappedSlots(child, context); + } + } + + // column references. + + @Override + public String visitSlotReference(SlotReference slotReference, SQLRelation context) { + String mapped = context.getColumnNames().get(slotReference.getExprId()); + if (mapped != null) { + return mapped; + } + // Doris rollup materializes "GROUPING(col)" as an internal slot named + // GROUPING_PREFIX_<col>; when such a slot was not registered by the repeat + // decompiler (defensive fallback), render it back to GROUPING(col). + String name = slotReference.getName(); + if (name != null && name.startsWith("GROUPING_PREFIX_")) { + return "GROUPING(" + name.substring("GROUPING_PREFIX_".length()) + ")"; + } + // An unregistered slot whose name is Doris's internal mark-slot naming + // ("$c$N", e.g. the EXISTS marker of an uncorrelated subquery folded into a + // constant CROSS JOIN) can never be resolved in the decompiled SQL - emitting + // the raw name would freeze a planSql that fails on replay ("Unknown column + // '$c$1'"). Such a shape has no faithful SQL rendering here, so fail the + // decompile: CREATE BASELINE then falls back to the user-supplied planSql + // (which is exactly the original query for these constant boolean outputs). + if (name != null && name.startsWith("$c$")) { + throw new UnsupportedOperationException( + "SPM decompile: unresolvable internal mark slot " + name); + } + // Unregistered column reference (e.g. a CTE output column) degrades to the raw + // column name + return slotReference.toSql(); + } + + // binary operations (arithmetic / comparison / logic). + + @Override + public String visitComparisonPredicate(ComparisonPredicate comparisonPredicate, SQLRelation context) { + return "(" + renderValueOperand(comparisonPredicate.left(), context) + + " " + operatorSymbol(comparisonPredicate) + + " " + renderValueOperand(comparisonPredicate.right(), context) + ")"; + } + + @Override + public String visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, SQLRelation context) { + return "(" + renderValueOperand(binaryArithmetic.left(), context) + + " " + operatorSymbol(binaryArithmetic) + + " " + renderValueOperand(binaryArithmetic.right(), context) + ")"; + } + + /** + * BitNot (~x): the generic fallback would print the expression through toSql and + * bypass the column mapping; above a join whose colliding column was renamed, that + * freezes the stale slot text. Render the child recursively instead so the mapped + * column reference stays mapped. + */ + @Override + public String visitBitNot(BitNot bitNot, SQLRelation context) { + return "~(" + bitNot.child().accept(this, context) + ")"; + } + + @Override + public String visitCompoundPredicate(CompoundPredicate compoundPredicate, SQLRelation context) { + // And / Or are n-ary in Nereids: a single node can hold more than two children, + // so child(0)/child(1) alone would drop the rest. Flatten the same-type operators + // into a list of operands with ExpressionUtils.extract (see extractConjunction / + // extractDisjunction) and join them with the operator symbol. extract stops at a + // different operator type, so e.g. (a or b) and c keeps the inner Or intact. + // + // CompoundPredicate is not a BinaryOperator (And/Or extend CompoundPredicate + // directly), so resolve the symbol here instead of using operatorSymbol. + String symbol = compoundPredicate instanceof And ? "AND" : "OR"; + String joined = ExpressionUtils.extract(compoundPredicate).stream() + .map(operand -> operand.accept(this, context)) + .collect(Collectors.joining(" " + symbol + " ")); + // AND / OR need parentheses to keep the precedence + return "(" + joined + ")"; + } + + @Override + public String visitNot(Not not, SQLRelation context) { + return "(NOT (" + not.child().accept(this, context) + "))"; + } + + @Override + public String visitIsNull(IsNull isNull, SQLRelation context) { + return renderValueOperand(isNull.child(), context) + " IS NULL"; + } + + @Override + public String visitIsTrue(IsTrue isTrue, SQLRelation context) { + return renderValueOperand(isTrue.child(), context) + " IS TRUE"; + } + + @Override + public String visitIsFalse(IsFalse isFalse, SQLRelation context) { + return renderValueOperand(isFalse.child(), context) + " IS FALSE"; + } + + // predicates. + + @Override + public String visitInPredicate(InPredicate inPredicate, SQLRelation context) { + String compareExpr = renderValueOperand(inPredicate.getCompareExpr(), context); + List<String> options = inPredicate.getOptions().stream() + .map(o -> o.accept(this, context)) + .collect(Collectors.toList()); + // A literal-only IN option list may be stored in a set-backed collection whose + // iteration order is NOT stable across runs / FE restarts (observed: TPCDS q75's + // "d_year IN (2001, 2002)" decompiled as (2001, 2002) one run and (2002, 2001) + // the next, flipping the frozen planSql and failing the.out comparison). IN + // semantics are order-independent, so a literal-only list is emitted in sorted + // order to keep the decompiled planSql deterministic. + boolean allLiterals = inPredicate.getOptions().stream() + .allMatch(o -> o instanceof Literal); + if (allLiterals && options.size() > 1) { + Collections.sort(options); + } + return compareExpr + " IN (" + String.join(", ", options) + ")"; + } + + @Override + public String visitBetween(Between between, SQLRelation context) { + String compareExpr = renderValueOperand(between.getCompareExpr(), context); + String lower = renderValueOperand(between.getLowerBound(), context); + String upper = renderValueOperand(between.getUpperBound(), context); + return compareExpr + " BETWEEN " + lower + " AND " + upper; + } + + @Override + public String visitLike(Like like, SQLRelation context) { + String rendered = renderValueOperand(like.left(), context) + + " LIKE " + renderValueOperand(like.right(), context); + if (like.children().size() > 2) { + // The third child is the ESCAPE character. Dropping it froze a three-argument + // LIKE as a two-child predicate: parameterizing the escape literal fails + // LIKE's literal-only legality check, so creation fell back to the raw plan and + // persisted the ALTERED predicate - a column-pattern LIKE replayed without + // ESCAPE can match different rows. Render the child recursively. + rendered += " ESCAPE " + renderValueOperand(like.child(2), context); + } + return rendered; + } + + /** Parentheses let a predicate occupy a value-expression position in the parser. */ + private String renderValueOperand(Expression operand, SQLRelation context) { + if (operand instanceof Nullable) { + return renderValueOperand(operand.child(0), context); + } + String rendered = operand.accept(this, context); + if (operand instanceof IsNull || operand instanceof IsTrue || operand instanceof IsFalse + || operand instanceof InPredicate || operand instanceof Between + || operand instanceof StringRegexPredicate || operand instanceof Match) { + return "(" + rendered + ")"; + } + return rendered; + } + + // function calls. + + @Override + public String visitBoundFunction(BoundFunction boundFunction, SQLRelation context) { + // The Nullable marker (nullable(x) wraps a column to make it nullable, e.g. on a + // recursive CTE union branch) has no SQL equivalent: decompile the inner + // expression only. + if (boundFunction instanceof Nullable) { + return boundFunction.child(0).accept(this, context); + } + if (boundFunction instanceof GroupConcat || boundFunction instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat((AggregateFunction) boundFunction, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + return rendered; + } + String args = renderArguments(boundFunction instanceof AggregateFunction, + boundFunction.children(), context); + return functionName(boundFunction) + "(" + args + ")"; + } + + /** + * Renders a function name, quoting and qualifying UDF names by their recorded database. UDF names that collide + * with scalar or list markers are rejected so the caller can retain raw plan SQL. Builtins use their function + * name directly. + */ + public static String functionName(Function function) { + if (function instanceof Udf) { + String name = function.getName(); + if (SPMFrozenTreeReplacer.CONST_VAR_FUNC.equals(name) + || SPMFrozenTreeReplacer.CONST_LIST_FUNC.equals(name)) { + throw new UnsupportedOperationException("SPM decompile: the user function " + + name + " collides with the SPM placeholder marker; keeping the" + + " original planSql"); + } + String dbName = ((Udf) function).getDbName(); + if (dbName != null && !dbName.isEmpty()) { + // Quote each component: a legal database name like my-db is NOT a + // parser identifier. Quoting the database component preserves the call + // when the stored SQL is parsed again. + return SPMPlan2SQLBuilder.quoteIdentifier(dbName) + "." + + SPMPlan2SQLBuilder.quoteIdentifier(name); + } + // a GLOBAL UDF without a database: quote the (possibly non-plain) name + return SPMPlan2SQLBuilder.quoteIdentifier(name); + } + return function.getName(); + } + + @Override + public String visitUnboundFunction(UnboundFunction unboundFunction, SQLRelation context) { + // UnboundFunction (a raw parsed function call, e.g. substr(ca_zip, 1, 5) in a + // WHERE clause before analysis). Render the children recursively so placeholder + // values inside the arguments stay normalized (_spm_const_var(id)); the + // default toSql fallback would render them with their embedded values and make + // the digest value-dependent. + String args = unboundFunction.children().stream() + .map(a -> a.accept(this, context)) + .collect(Collectors.joining(", ")); + // Keep the database qualifier: db1.f(k + 1) must not freeze as unqualified + // f(...) and resolve to db2.f when replayed under USE db2. + String dbName = unboundFunction.getDbName(); + String qualifiedName = (dbName == null || dbName.isEmpty()) + ? unboundFunction.getName() + : SPMPlan2SQLBuilder.quoteIdentifier(dbName) + "." + + SPMPlan2SQLBuilder.quoteIdentifier(unboundFunction.getName()); + return qualifiedName + "(" + args + ")"; + } + + @Override + public String visitAggregateExpression(AggregateExpression aggregateExpression, SQLRelation context) { + AggregateFunction fn = aggregateExpression.getFunction(); + if (fn instanceof GroupConcat || fn instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat(fn, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + return rendered; + } + // Explicitly handle DISTINCT aggregates: count(distinct col) + String distinct = fn.isDistinct() ? "distinct " : ""; + String args = renderArguments(true, fn.children(), context); + return functionName(fn) + "(" + distinct + args + ")"; + } + + @Override + public String visitWindow(WindowExpression windowExpression, SQLRelation context) { + // Hand-render the OVER clause so every slot reference (partition keys, order + // keys, function arguments) goes through the column mapping - Doris's own + // toSql prints raw slot names such as "GROUPING_PREFIX_s_county" or a local + // buffer name that is not referenceable in the decompiled SQL. + Expression fn = windowExpression.getFunction(); + String fnSql; + if (fn instanceof AggregateFunction) { + AggregateFunction aggFn = (AggregateFunction) fn; + if (aggFn instanceof GroupConcat || aggFn instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat(aggFn, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + fnSql = rendered; + } else { + String distinct = aggFn.isDistinct() ? "DISTINCT " : ""; + fnSql = functionName(aggFn) + "(" + distinct + + renderArguments(true, fn.children(), context) + ")"; + } + } else if (fn instanceof BoundFunction) { + fnSql = functionName((BoundFunction) fn) + "(" + + renderArguments(false, fn.children(), context) + ")"; + } else { + fnSql = print(fn, context); + } + StringBuilder sb = new StringBuilder(fnSql).append(" OVER ("); + List<String> parts = new ArrayList<>(); + if (!windowExpression.getPartitionKeys().isEmpty()) { + parts.add("PARTITION BY " + windowExpression.getPartitionKeys().stream() + .map(p -> print(p, context)).collect(Collectors.joining(", "))); + } + if (!windowExpression.getOrderKeys().isEmpty()) { + parts.add("ORDER BY " + windowExpression.getOrderKeys().stream() + .map(o -> print(o.child(), context) + (o.isAsc() ? " ASC" : " DESC") + + (o.isNullFirst() ? " NULLS FIRST" : " NULLS LAST")) + .collect(Collectors.joining(", "))); + } + // an explicit frame is only valid together with an ORDER BY clause; without one + // the SQL default frame (RANGE UNBOUNDED PRECEDING.. CURRENT ROW) is identical. + // + // LAG / LEAD are the exception: their SQL text must NOT carry a frame + // (WindowFunctionChecker.visitLag / #visitLead REJECT an explicit one and then + // install an INTERNAL rows frame on the window expression), so every captured + // plan carries a frame that the user never wrote. Emitting it froze SQL whose + // own re-analysis throws "WindowFrame for LAG must be null" - the baseline + // could never be replayed. The user-level absence of a frame is the faithful + // rendering. + if (!isFrameForbiddenWindowFunction(fn) + && !windowExpression.getOrderKeys().isEmpty() + && windowExpression.getWindowFrame().isPresent()) { + parts.add(windowExpression.getWindowFrame().get().computeToSql()); + } + return sb.append(String.join(" ", parts)).append(")").toString(); + } + + /** + * Whether the window function's SQL text must not carry an explicit frame (see + * the frame rendering in visitWindow): LAG / LEAD only. + */ + private static boolean isFrameForbiddenWindowFunction(Expression fn) { + if (fn instanceof Lag || fn instanceof Lead) { + return true; + } + String name = fn instanceof UnboundFunction ? ((UnboundFunction) fn).getName() : null; + return "lag".equalsIgnoreCase(name) || "lead".equalsIgnoreCase(name); + } + + // other common types. + + /** + * Renders a GROUP_CONCAT / MULTI_DISTINCT_GROUP_CONCAT into the dedicated + * GROUP_CONCAT grammar. + * + * Doris's analyzer admits ONE value expression next to an optional CONSTANT separator + * (the second non-order argument must be constant - a non-constant one is rejected by + * checkLegalityBeforeTypeCoercion), with an optional trailing ORDER BY. That covers + * every SQL-reachable shape: + * + * GROUP_CONCAT(DISTINCT v ORDER BY k): the physical plan carries the execution + * shape MultiDistinctGroupConcat(v, ORDER BY k) (GroupConcat.mustUseMultiDistinctAgg + * converts it). The DISTINCT form re-analyzes to exactly that node, so it is + * rendered - previously it returned null and forced the whole decompile to fall + * back to the user-supplied plan text; + * MULTI_DISTINCT_GROUP_CONCAT(v[, sep]): the function NAME carries the dedup + * contract while its isDistinct flag is false, so the DISTINCT keyword must be + * re-emitted - GROUP_CONCAT(v) would keep the duplicates ("1,1,2" instead of + * "1,2"). + * + * Parameter fn: the aggregate function (GroupConcat or MultiDistinctGroupConcat) + * Parameter context: the relation carrying the column mapping + * Returns: the SQL text, or null when the shape has no faithful rendering + */ + public String renderGroupConcat(AggregateFunction fn, SQLRelation context) { + return renderGroupConcat(fn, context, false); + } + + /** Renders GROUP_CONCAT with DISTINCT restored from an eliminated aggregate stage. */ + public String renderGroupConcat(AggregateFunction fn, SQLRelation context, boolean restoredDistinct) { + List<Expression> children = fn.children(); + int firstOrder = children.size(); + for (int i = 0; i < children.size(); i++) { + if (children.get(i) instanceof OrderExpression) { + firstOrder = i; + break; + } + } + List<Expression> values = children.subList(0, firstOrder); + List<Expression> orders = children.subList(firstOrder, children.size()); + // the dedicated grammar carries one value expression plus an optional CONSTANT + // separator; two NON-constant value arguments cannot be written down at all (the + // analyzer rejects them: "requires separator must be a constant"), so such a + // programmatically-built shape keeps failing the decompile instead of freezing + // SQL that cannot be re-parsed + boolean separatorArgument = values.size() == 2 && values.get(1).isConstant(); + if (values.isEmpty() || (values.size() > 1 && !separatorArgument)) { + return null; + } + // MultiDistinctGroupConcat is the execution shape of GROUP_CONCAT(DISTINCT...): + // its constructor keeps isDistinct = false, but the dedup contract must survive + // the freeze + boolean distinct = restoredDistinct || fn.isDistinct() || fn instanceof MultiDistinctGroupConcat; + StringBuilder sb = new StringBuilder("GROUP_CONCAT("); + if (distinct) { + sb.append("DISTINCT "); + } + sb.append(print(values.get(0), context)); + if (!orders.isEmpty()) { + // GROUP_CONCAT(DISTINCT v ORDER BY k) re-analyzes to the very + // MultiDistinctGroupConcat(v, k) execution shape it was rendered from, so the + // ORDER BY form is faithful; a verifier may also compare it against the + // alternative spelling MULTI_DISTINCT_GROUP_CONCAT(v ORDER BY k) + sb.append(" ORDER BY "); + sb.append(orders.stream().map(order -> { + OrderExpression orderExpression = (OrderExpression) order; + return print(orderExpression.child(), context) + + (orderExpression.isAsc() ? " ASC" : " DESC") + + (orderExpression.isNullFirst() ? " NULLS FIRST" : " NULLS LAST"); + }).collect(Collectors.joining(", "))); + } + if (separatorArgument) { + sb.append(" SEPARATOR ").append(print(values.get(1), context)); + } + return sb.append(")").toString(); + } + + @Override + public String visitCast(Cast cast, SQLRelation context) { + if (!cast.isExplicitType() && !preservesTypeConstructorFallback(cast)) { + // analyzer / optimizer inserted coercion CAST (type alignment, DECIMAL scale + // adjustment for division, placeholder typing,...) - an execution + // detail that would otherwise pin the result type/scale in the frozen planSql. + // Strip it so the replayed SQL analyzes the user-level expression and derives the + // same types/coercions as the original query (e.g. DECIMAL division keeps its full + // scale: "0.0286487413" instead of the coerced "0.02864874"). Explicit user + // CAST (isExplicitType = true) is preserved verbatim. The LOWERING's accumulator + // widening is preserved by renderArgument, which is the only place that knows the + // cast is the ROOT of an aggregate argument (see preservesLoweredAccumulator). + return cast.child().accept(this, context); + } + return "CAST(" + cast.child().accept(this, context) + " AS " + cast.getDataType().toSql() + ")"; + } + + /** + * Whether an ANALYZER-inserted cast is part of the frozen plan's SEMANTICS rather than an + * execution detail that may be stripped. + * + * The AVG lowering (AVG_DISTINCT_TO_SUM_DIV_COUNT) rewrites avg(DISTINCT k) into + * sum(DISTINCT CAST(k AS LARGEINT)) / count(...), and that widening IS the LARGEINT + * accumulator AVG computes in. Stripping it made the frozen SQL re-analyze + * SUM(DISTINCT k) with a BIGINT accumulator, so two distinct values 5e18 and 6e18 - which + * the original AVG averages to 5.5e18 - overflowed the frozen replay's sum, even for the + * identical SQL (the CREATE probe only RE-PLANS the text, it never executes it). + * + * The accumulator guarantee only exists at the ROOT of an aggregate's argument + * (see renderArgument): an operand of ordinary arithmetic + * (k + 9223372036854775808) carries the same shape, but there it is plain type + * coercion - keeping it pins the CAPTURED operand type and makes a literal of another type + * replay under the wrong coercion (the + 0.25 variant truncated to an integer). + * + * Only an integral WIDENING over a REAL value qualifies: a cast over a substituted + * placeholder / literal is exactly the type pinning the stripping exists to avoid, and a + * narrowing or same-width cast carries no accumulator guarantee. + * + * Parameter cast: the analyzer-inserted cast + * Returns: true when the cast must survive into the frozen SQL + */ + private boolean preservesLoweredAccumulator(Cast cast) { + Expression child = cast.child(); + try { + DataType childType = child.getDataType(); + DataType targetType = cast.getDataType(); + // LARGEINT is the SUM / AVG accumulator: the lowering of avg(DISTINCT expr) writes + // exactly this widening, and re-deriving SUM over the SMALLER type is what + // overflows. An ordinary arithmetic widening (INT -> BIGINT and friends) is an + // execution detail the strip exists for, so it is NOT kept - it would only pin + // coercions the re-analyzed user expression derives by itself. + // + // The child must read at least one REAL column: AVG(DISTINCT k + 1) lowers to + // sum(CAST((k + ph) AS LARGEINT)), and skipping the cast just because a placeholder + // sits inside the arithmetic froze SUM over BIGINT again. A child built ONLY from + // placeholders (ph1 + ph2 widened to INT) stays stripped - there the cast pins + // the CAPTURED literals' width on the caller's values. + return childType.isIntegralType() && targetType instanceof LargeIntType + && targetType.width() > childType.width() + && child.containsType(SlotReference.class); + } catch (RuntimeException e) { + // an unbound child has no type here: keeping the strip decision is the + // historical behavior and cannot break a bound plan + return false; + } + } + + /** + * Whether the cast IS the parser's TYPE-CONSTRUCTOR fallback. + * + * DATE 'x' / TIMESTAMP 'x' folds to a typed literal when x PARSES, and to + * CAST(x AS DATE|DATETIME) when it does not (LogicalPlanBuilder.visitTypeConstructor). + * That cast carries the user's TYPE - SELECT DATE 'not-a-date' executes as a typed DATE + * NULL - so stripping it froze Project(ph1 AS d) and the replay returned the raw STRING + * 'not-a-date' (the CREATE probe only re-plans the text, so it accepted that projection). + * + * The detector is the fallback's own cause: the literal does NOT parse as the target type. + * An ordinary implicit coercion of a VALID literal (d = '2020-01-01' also becomes + * CAST('2020-01-01' AS DATE)) still parses, so it is still stripped and re-derived by the + * replay's own analysis exactly as before - which keeps the frozen text of the ordinary + * date-filter shape unchanged. + * + * Parameter cast: the parser-inserted cast + * Returns: true when the cast must survive into the frozen SQL + */ + private static boolean preservesTypeConstructorFallback(Cast cast) { + String text = constantText(cast.child()); + if (text == null) { + return false; + } + DataType target = cast.getDataType(); + if (target == null) { + return false; + } + return !parsesAs(target, text); + } + + /** + * Returns the string value of a string literal, unwrapping a scalar marker payload when present. Returns null + * for other expressions. Used to recognize capture-time date and time constructor fallbacks; a replay can + * supply a different payload value. + */ + private static String constantText(Expression expr) { + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstVar) { + return constantText( + ((org.apache.doris.nereids.spm.placeholder.SpmConstVar) expr).getValue()); + } + if (expr instanceof StringLikeLiteral) { + return ((StringLikeLiteral) expr).getStringValue(); + } + return null; + } + + /** + * Whether the text is a LEGAL value of the target date / time type, i.e. whether the + * parser would have folded the constructor instead of emitting the fallback cast. + * + * Parameter target: the cast's target type + * Parameter text: the literal text + * Returns: whether the text parses as that type (true for any other type) + */ + private static boolean parsesAs(DataType target, String text) { + try { + if (target instanceof DateV2Type) { + new DateV2Literal(text); + return true; + } + if (target instanceof DateType) { + new DateLiteral(text); + return true; + } + if (target instanceof DateTimeV2Type) { + new DateTimeV2Literal(text); + return true; + } + if (target instanceof DateTimeType) { + new DateTimeLiteral(text); + return true; + } + } catch (RuntimeException e) { + return false; + } + return true; + } + + @Override + public String visitTryCast(TryCast tryCast, SQLRelation context) { + // toSql renders a grammar-parseable type name (e.g. STRUCT<name:type>); + // toString emits a diagnostic form (STRUCT<StructField[...]>) that cannot be + // re-parsed by the frozen SQL after a reload. + return "TRY_CAST(" + tryCast.child().accept(this, context) + " AS " + + tryCast.getDataType().toSql() + ")"; + } + + @Override + public String visitOrderExpression(OrderExpression orderExpression, SQLRelation context) { + String child = orderExpression.child().accept(this, context); + return child + (orderExpression.isAsc() ? " ASC" : " DESC"); + } + + @Override + public String visitAlias(Alias alias, SQLRelation context) { + return alias.child().accept(this, context); + } + + @Override + public String visitCaseWhen(CaseWhen caseWhen, SQLRelation context) { + // CASE WHEN... THEN... ELSE... END + StringBuilder sb = new StringBuilder("CASE"); + for (WhenClause whenClause : caseWhen.getWhenClauses()) { + sb.append(" WHEN ").append(whenClause.getOperand().accept(this, context)) + .append(" THEN ").append(whenClause.getResult().accept(this, context)); + } + if (caseWhen.getDefaultValue().isPresent()) { + sb.append(" ELSE ").append(caseWhen.getDefaultValue().get().accept(this, context)); + } + sb.append(" END"); + return sb.toString(); + } + + @Override + public String visitLiteral(Literal literal, SQLRelation context) { + if (literal instanceof StringLikeLiteral) { + // Generator arguments (LATERAL VIEW / UNNEST) are intentionally NOT + // parameterized and reach this method as concrete literals. Their toSql + // (StringLikeLiteral.toString) only wraps the SEMANTIC value in quotes, so + // a delimiter such as a'b froze invalid split(t.s, 'a'b'): the stored + // baseline then had no parseable fallback tree and stopped applying after a + // reload. Render through the default-mode-safe quote helper (stored text is + // always re-parsed under MODE_DEFAULT). + return SPMPlan2SQLBuilder.quoteSqlString(((StringLikeLiteral) literal).getStringValue()); Review Comment: [P2] Preserve declared character types in the early literal branch On nonempty `t(k)`, capture and replay `SELECT * REPLACE(CAST('a' AS CHAR(10)) AS k) FROM t`. REPLACE stays concrete and `FoldConstantRuleOnFE.visitCast` produces a CHAR(10) literal, but this branch emits only `'a'`, which `visitStringLiteral` reparses as VARCHAR(1). The safe character cast passes the session-sensitive guards; the planning probe and table/label checks accept the changed output schema. VARCHAR(n) and STRING payloads similarly lose their declared type/length. Retain the captured character type around the safely quoted value and check replay result metadata. This early branch bypasses the now-typed generic UUID/decimal fallback from the existing report. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/matcher/SPMAstCheckVisitor.java: ########## @@ -0,0 +1,415 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.matcher; + +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.analyzer.UnboundStar; +import org.apache.doris.nereids.spm.SPMPlanTreeSupport; +import org.apache.doris.nereids.spm.placeholder.SpmConstList; +import org.apache.doris.nereids.spm.placeholder.SpmConstVar; +import org.apache.doris.nereids.trees.expressions.BinaryOperator; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.Exists; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.ScalarSubquery; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.WindowFrame; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Compares bind and user expressions and records user expressions under scalar or IN-list marker ids. Concrete + * expression classes and operator metadata are checked before recursively comparing children. SlotReference uses + * its name; UnboundSlot uses its digest, which preserves qualification. Unordered predicate matching restores + * captured values after a failed tentative pairing. + */ +public class SPMAstCheckVisitor extends ExpressionVisitor<Boolean, SPMAstCheckVisitor.CheckContext> { + + /** + * Match context: carries the user-side expression and the placeholder value + * extraction result. + */ + public static class CheckContext { + /** The user-side expression corresponding to the current bind-side node. */ + private final Expression userExpr; + /** Placeholder id -> actual value (extraction result). */ + private final Map<Long, Expression> placeholderValues; + + public CheckContext(Expression userExpr, Map<Long, Expression> placeholderValues) { + this.userExpr = userExpr; + this.placeholderValues = PlaceholderBindings.track(placeholderValues); + } + } + + /** + * Match entry point. + * + * Parameter bindExpr: the bindSql-side expression + * Parameter userExpr: the user-query-side expression + * Parameter placeholderValues: the placeholder value extraction result (filled in here) + * Returns: whether the structures match + */ + public boolean checkExpression(Expression bindExpr, Expression userExpr, + Map<Long, Expression> placeholderValues) { + return bindExpr.accept(this, new CheckContext(userExpr, placeholderValues)); + } + + /** Cheap necessary conditions only; keeps the existing unordered search order intact. */ + public static boolean mayMatch(Expression bind, Expression user) { + if (bind instanceof SpmConstVar) { + return true; + } + if (bind instanceof InPredicate ? !(user instanceof InPredicate) : bind.getClass() != user.getClass()) { + return false; + } + if (bind instanceof UnboundFunction) { + UnboundFunction left = (UnboundFunction) bind; + UnboundFunction right = (UnboundFunction) user; + if (!left.getName().equalsIgnoreCase(right.getName()) + || left.isDistinct() != right.isDistinct() + || !Objects.equals(left.getDbName(), right.getDbName())) { + return false; + } + } + if (bind instanceof SlotReference || bind instanceof UnboundSlot) { + return sameSlotName(bind, user); + } + if ((bind instanceof BinaryOperator && !(bind instanceof CompoundPredicate) + || bind instanceof InPredicate) && !bind.children().isEmpty() && !user.children().isEmpty()) { + Expression first = bind.child(0); + if (first instanceof SlotReference || first instanceof UnboundSlot) { + Expression other = user.child(0); + return first.getClass() == other.getClass() + && sameSlotName(first, other); + } + } + return true; + } + + @Override + public Boolean visit(Expression bindExpr, CheckContext context) { + Expression userExpr = context.userExpr; + + // placeholder: extract the value (do not compare inner children). + if (bindExpr instanceof SpmConstVar) { + SpmConstVar placeholder = (SpmConstVar) bindExpr; + return recordPlaceholderValue(placeholder.getId(), userExpr, context); + } + + // Type mismatch -> no match + if (bindExpr.getClass() != userExpr.getClass()) { + return false; + } + + // Identity fields that live OUTSIDE children must be compared explicitly: + // UnboundStar: the REPLACE payload (a leaf with no children - a baseline + // captured with "* REPLACE(k + 1 AS k)" must not replay its captured + // expression for a query using "* REPLACE(k + 2 AS k)") + // UnboundFunction: the name / DISTINCT flag / database qualifier (abs(k) must not + // match sqrt(k), count(k) must not match count(distinct k), db1.f must not + // match db2.f) + // WindowExpression: the frame (ROWS 1 PRECEDING must not replay as 2 PRECEDING; + // WindowFrame is not an expression child, so neither the placeholder + // construction nor the generic child comparison sees its bound offsets) + if (bindExpr instanceof UnboundStar) { + // The REPLACE / EXCEPT payloads are kept OUTSIDE children. Compare them + // STRUCTURALLY rather than by toSql text: the payload may own a SUBQUERY + // whose relations are namespace-qualified on the user side while the stored + // bind tree stays the raw parse (relation NAMES are deliberately not part of + // Level 3 - the digest pre-filter owns the namespace), so text equality + // rejected a valid baseline whose own text contains + // "* REPLACE((SELECT max(v) FROM u) AS k)" (the namespace rebuild qualified + // the user's payload to internal.db.u). Everything else stays EXACT: the + // qualifier, and the payload literals (they are NOT parameterized inside a + // star, so accepting a value variant would replay the captured one). + UnboundStar bindStar = (UnboundStar) bindExpr; + UnboundStar userStar = (UnboundStar) userExpr; + if (!Objects.equals(bindStar.getQualifier(), userStar.getQualifier())) { + return false; + } + if (!samePayloadExpressions(bindStar.getExceptedSlots(), userStar.getExceptedSlots(), + context)) { + return false; + } + return samePayloadExpressions(bindStar.getReplacedAlias(), + userStar.getReplacedAlias(), context); + } + if (bindExpr instanceof UnboundFunction) { + UnboundFunction bindFunction = (UnboundFunction) bindExpr; + UnboundFunction userFunction = (UnboundFunction) userExpr; + // the NAME is compared case-insensitively: the digest upper-cases it, so a + // case-only spelling difference is the SAME function and must stay + // matchable; a genuinely different name (or DISTINCT flag) must never match, + // or the replay would evaluate another function + if (!bindFunction.getName().equalsIgnoreCase(userFunction.getName()) + || bindFunction.isDistinct() != userFunction.isDistinct() + || !Objects.equals(bindFunction.getDbName(), userFunction.getDbName())) { + return false; + } + } + if (bindExpr instanceof WindowExpression) { + WindowFrame bindFrame = ((WindowExpression) bindExpr).getWindowFrame().orElse(null); + WindowFrame userFrame = ((WindowExpression) userExpr).getWindowFrame().orElse(null); + if ((bindFrame == null) != (userFrame == null)) { + return false; + } + if (bindFrame != null + && !Objects.equals(bindFrame.computeToSql(), userFrame.computeToSql())) { + return false; + } + } + // Match: the analyzer (the tokenizer named in "USING ANALYZER...") decides how the + // pattern is read and is stored outside children; BinaryOperator.toDigest + // omits it as well. Two MATCH variants with different analyzers must not match: + // the replay keeps the baseline analyzer and can return different matches. + if (bindExpr instanceof Match + && !Objects.equals(((Match) bindExpr).getAnalyzer(), + ((Match) userExpr).getAnalyzer())) { + return false; + } + + // A WHERE clause is collected into ONE n-ary AND (OR) node, whose operands are an + // UNORDERED multiset: the bind side renders placeholders while the user side + // carries the literal text, so a positional comparison rejects a valid baseline + // whenever the literal values reverse the textual order. Match the operands + // transactionally: a tentative pairing rolls its extractions back on failure. + if (bindExpr instanceof CompoundPredicate) { + List<Expression> bindChildren = bindExpr.children(); + List<Expression> userChildren = userExpr.children(); + if (bindChildren.size() != userChildren.size()) { + return false; + } + List<Expression> remaining = new ArrayList<>(userChildren); + for (Expression bindChild : bindChildren) { + int matched = -1; + // the operand pairing is an UNORDERED search: a greedy pairing can + // succeed here and still break a LATER use of the same placeholder id + // (a filter constraining the same id differently) - the retry driver + // re-runs the check with a different starting point (see MatchAttempt) + int offset = MatchAttempt.offset(remaining.size()); + for (int i = 0; i < remaining.size(); i++) { + int candidate = (offset + i) % remaining.size(); + if (!mayMatch(bindChild, remaining.get(candidate))) { + continue; + } + PlaceholderBindings bindings = (PlaceholderBindings) context.placeholderValues; + int checkpoint = bindings.checkpoint(); + if (checkExpression(bindChild, remaining.get(candidate), + context.placeholderValues)) { + matched = candidate; + break; + } + bindings.rollback(checkpoint); + } + if (matched < 0) { + return false; + } + remaining.remove(matched); + } + return true; + } + + // Column references are compared by name (a = 100 must not match b = 42) + if (bindExpr instanceof SlotReference || bindExpr instanceof UnboundSlot) { + return sameSlotName(bindExpr, userExpr); + } + + // An unparameterized bind literal, such as a LIKE escape character or a star replacement value, must + // equal the user literal. Otherwise replay would retain a captured value from a position the builder + // intentionally leaves concrete. + if (bindExpr instanceof Literal && userExpr instanceof Literal) { + return Objects.equals(((Literal) bindExpr).getValue(), ((Literal) userExpr).getValue()); Review Comment: [P1] Compare complete temporal values in concrete payloads On nonempty `t(k)`, capture `SELECT * REPLACE(TIMESTAMP '2020-01-01 00:00:00.1' AS k) FROM t`, then execute its `.2` variant. REPLACE payloads stay concrete and both literals have the same value-free digest/class, but `DateTimeV2Literal` inherits `DateTimeLiteral.getValue()`, which omits microseconds. This comparison accepts `.1` and `.2` as equal. There is no marker for this value, so both frozen and raw replay retain `.1` instead of the caller's `.2`; table/label checks do not detect it. Compare full literal identity, including fractional value and type (`DateTimeV2Literal.equals` already does), and cover concrete temporal value variants. This is a matcher-value defect, separate from the repaired temporal SQL-type rendering. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
