github-actions[bot] commented on code in PR #68499: URL: https://github.com/apache/doris/pull/68499#discussion_r4613860479
########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/SPMPlanner.java: ########## @@ -0,0 +1,1493 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.spm.builder.SPMPlan2SQLBuilder; +import org.apache.doris.nereids.spm.manager.BaselineManager; +import org.apache.doris.nereids.spm.manager.SessionBaselineStore; +import org.apache.doris.nereids.spm.matcher.SPMFrozenTreeReplacer; +import org.apache.doris.nereids.spm.matcher.SPMPlaceholderReplacer; +import org.apache.doris.nereids.spm.placeholder.SPMPlaceholderBuilder; +import org.apache.doris.nereids.spm.placeholder.SpmConstList; +import org.apache.doris.nereids.spm.placeholder.SpmConstVar; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SqlModeHelper; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Creates baseline metadata and attempts query replay. Creation parameterizes supported literal positions in bind + * and plan trees with one builder, optimizes the plan, and stores either decompiled frozen SQL or the original SQL + * for fallback planning. + * + * Replay uses a structural hash, exact canonical digest and tree comparison to select candidates and collect + * replacements. It tries frozen SQL first, then the parameterized fallback tree when available. Output labels, row + * limits, ordering and metadata are checked before the caller plans the returned unbound tree. No usable match + * returns null so the caller can plan the original query. + */ +public class SPMPlanner { + + private static final Logger LOG = LogManager.getLogger(SPMPlanner.class); + + /** Id of the baseline used by the last successful rewrite (-1 when none). */ + private long usedBaselineId = -1; + + /** + * The matched baseline OBJECT (see verifyReplayMetadata): the post-plan + * validation must keep the fingerprint of the SAME incarnation the match used, so a + * concurrent DROP / refresh cannot make it silently skip the check. + */ + private BaselinePlan usedBaseline; + + /** + * Returns the id of the baseline used by the last successful rewrite. + * + * Returns: the baseline id, or -1 when the last rewrite did not hit a baseline + */ + public long getUsedBaselineId() { + return usedBaselineId; + } + + /** The matched baseline object (see verifyReplayMetadata). */ + public BaselinePlan getUsedBaseline() { + return usedBaseline; + } + + // query rewrite (called from StmtExecutor / EXPLAIN). + + /** + * Finds SESSION candidates before GLOBAL candidates and attempts replay using extracted marker values. Returns + * a usable unbound plan with caller labels, limits and SET_VAR hints, or null if no candidate can be used. The + * epoch-millisecond deadline is checked before matching each candidate; it is not a hard interruption of + * parsing, matching or substitution. + */ + public LogicalPlan tryRewritePlan(LogicalPlan userPlan, long deadline) { + if (System.currentTimeMillis() > deadline) { + LOG.info("SPM tryRewritePlan: timeout before matching, degrade to the original plan"); + return null; + } + ConnectContext ctx = ConnectContext.get(); + SessionBaselineStore sessionStore = ctx == null ? null : ctx.getSessionBaselineStore(); + // Fast path: with no baseline anywhere (global or session) there is nothing to + // match, so skip the whole-tree digest rendering below - it walks every literal + // of the whole query and is the most expensive part of a rewrite attempt. + if (!BaselineManager.getInstance().hasBaselines() + && (sessionStore == null || sessionStore.isEmpty())) { + return null; + } + // SELECT... INTO OUTFILE writes to a destination that lives OUTSIDE the plan + // expressions: the generic match cannot see a different path / format and the + // rewritten tree keeps the CAPTURED sink fields, so a replay would export to the + // baseline's destination. SPM refuses to rewrite such statements (the original + // query runs normally, which is always correct). + if (SPMPlanTreeSupport.containsFileSink(userPlan)) { + LOG.info("SPM tryRewritePlan: the statement writes to a file sink (OUTFILE); keeping" + + " the original plan"); + return null; + } + // Matching includes the caller's hints against bindSql. Replay uses planSql, whose SET_VAR hints + // can differ or be absent after capture. Remove captured settings and attach the caller's own ones + // at both return sites so planning the replacement keeps the query's statement-scoped assignments. + // Level 1/2 use the FULL-QUERY value-free digest (Plan.toSpmDigest renders every + // literal as "?", so the digest is value-independent). The digest is computed on a + // namespace-qualified COPY: "FROM t" is keyed as the CURRENT catalog/db's t, so a + // baseline captured under db1 can never match the same text executed under db2 + // (the frozen planSql is fully qualified and would silently keep running against + // db1.t). Explicitly qualified references stay verbatim on both sides, and so do + // references to a CTE alias: those bind inside the query's own WITH clause and are + // therefore namespace-independent. + LogicalPlan matchPlan = SPMPlanTreeSupport.namespaceQualified(userPlan, + captureCatalogName(ctx), captureDatabaseName(ctx)); + String queryDigest = SPMPlanTreeSupport.canonicalSpmDigest(matchPlan.toSpmDigest()); + long queryHash = SPMUtils.hashOf(queryDigest); + // SESSION-scope baselines of the current connection are consulted BEFORE the + // global ones, so a session baseline can override a global baseline for this + // session only; each candidate list is already priority-ordered. + List<BaselinePlan> candidates = new ArrayList<>(); + if (sessionStore != null) { + candidates.addAll(sessionStore.findCandidateBaselines(queryDigest, queryHash)); + } + candidates.addAll( + BaselineManager.getInstance().findCandidateBaselines(queryDigest, queryHash, deadline)); + if (candidates.isEmpty()) { + return null; + } + // View guard (computed lazily on the first structural match, so the per-query + // hot path pays nothing): the replay of a matched baseline is planned before the + // normal authorization pass and resolves a view to its base tables, so + // authorization would check the base tables instead of the view - a view-only + // user is denied on the base tables, while a base-table user passes the same + // view query without any view check. View queries keep the original plan. + boolean viewChecked = false; + boolean viewReferenced = false; + // Schema-identity guard state (computed lazily, once per query, only after the + // first structural match): see the check inside the loop. + boolean schemaChecked = false; + String currentSchemaFingerprint = null; + SPMPlanTreeSupport.ReplayContracts userContracts = new SPMPlanTreeSupport.ReplayContracts(matchPlan); + for (BaselinePlan candidate : candidates) { + if (System.currentTimeMillis() > deadline) { + LOG.info("SPM tryRewritePlan: timeout before matching baseline {}, " + + "degrade to the original plan", candidate.getId()); + return null; + } + // A permissive capture can freeze an implicit ANY_VALUE. Strict callers must + // analyse their own grouping expressions before that permissive result can be used. + if ((candidate.getCreatorSqlMode() & SqlModeHelper.MODE_ONLY_FULL_GROUP_BY) == 0 + && (ctx.getSessionVariable().getSqlMode() & SqlModeHelper.MODE_ONLY_FULL_GROUP_BY) != 0) { + continue; + } + LogicalPlan bindTree = candidate.getParameterizedBindPlan(); + if (bindTree == null) { + continue; + } + // Level 3: whole-tree structural match + value extraction + Map<Long, Expression> placeholderValues = new HashMap<>(); + if (!SPMPlanTreeSupport.check(bindTree, matchPlan, placeholderValues)) { + continue; + } + LOG.info("SPM tryRewritePlan: baseline {} matched, extracted {} placeholder values", + candidate.getId(), placeholderValues.size()); + if (!viewChecked) { + viewChecked = true; + viewReferenced = SPMPlanTreeSupport.referencesView(ctx, userPlan); + } + if (viewReferenced) { + LOG.info("SPM tryRewritePlan: the query references a view; keeping the original" + + " plan so view authorization is preserved"); + return null; + } + // Schema-identity guard (see BaselinePlan.schemaFingerprint): the bind key is + // built from the STILL-UNBOUND query, so SELECT * FROM t WHERE k = 1 keeps + // the same digest and Level-3 tree after ALTER TABLE t ADD COLUMN extra + // (or a DROP + CREATE of t), while the frozen plan still emits the + // creator-time output columns - the matched replay would silently return the + // old column set. A baseline whose referenced-table fingerprint no longer + // matches fails closed (the user query keeps its own plan); pre-column rows + // without a fingerprint skip the check. The current fingerprint is computed + // once per query, and only after a structural match (hot path pays nothing). + // The stored fingerprint is the BIND-SIDE tables UNION the PLAN-side tables + // of the frozen plan; here - BEFORE the query is planned - only the bind side + // can be recomputed, so the guard checks that every current bind-side entry is + // still present (a DROP + CREATE / ALTER REPLACES its entry, so drift is still + // caught). The full comparison, plan side included, runs after planning + // (see verifyReplayMetadata). + String bindFingerprint = candidate.getSchemaFingerprint(); + if (bindFingerprint != null && !bindFingerprint.isEmpty()) { + if (!schemaChecked) { + schemaChecked = true; + currentSchemaFingerprint = + SPMPlanTreeSupport.schemaFingerprint(ctx, userPlan); + } + if (!SPMPlanTreeSupport.schemaFingerprintBindSideContained( + bindFingerprint, currentSchemaFingerprint)) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: the referenced table" + + " schema changed since the baseline was created", candidate.getId()); + continue; + } + } + // The expensive part (value-free digest, candidate lookup, whole-tree check) is + // already done and a baseline has matched: finish the (cheap) value substitution + // even when the budget has been consumed. Previously this path re-checked the + // deadline here and returned null on expiry, silently throwing the completed + // match away (rewrite degraded to the original plan). + // Replay the frozen plan SQL. A baseline whose frozen planSql carries + // placeholder calls (decompiled from the SPM-optimized physical plan, structure + // + join distribution hints preserved) is re-parsed and the user values are + // substituted by placeholder id - the rewrite reproduces the frozen optimal + // structure for normal re-planning. Baselines without frozen SQL + // (in-memory engine, decompile-fallback planSql, legacy "?" frozen text) fall + // back to substituting the parameterized plan tree. + LogicalPlan rewritten = rewriteFromFrozenTree(candidate, placeholderValues); + if (rewritten != null) { + // non-expression literals (LIMIT / OFFSET) are long fields, not + // placeholders; adopt the user's values so a structurally identical query + // with a different limit is rewritten with the USER limit + // the frozen sink pinned the CAPTURED output labels: expose the caller's + // own ones (a value variant must not report the captured header). A shape + // whose labels CANNOT be aligned skips this baseline instead of exposing + // the captured header (see UnalignableOutputLabelsException). + LogicalPlan alignedReplay; + try { + alignedReplay = SPMPlanTreeSupport.alignRootOutputLabels(rewritten, + matchPlan); + } catch (SPMPlanTreeSupport.UnalignableOutputLabelsException unalignable) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: {}", + candidate.getId(), unalignable.getMessage()); + continue; + } + LogicalPlan replay = SPMPlanTreeSupport.mergeLimits(alignedReplay, matchPlan); + SPMPlanTreeSupport.ReplayContracts replayContracts = new SPMPlanTreeSupport.ReplayContracts(replay); + // The cap contract keys every row cap by the OCCURRENCE of the relation it + // truncates, and the occurrence is the relation's ordinal in the tree's walk + // order. That ordinal is only stable when the caller's tree and the FROZEN plan + // render the occurrences in the same relative order - which the optimizer need + // not do: it may place a derived table on the OTHER side of a JOIN than the user + // wrote it (a derived-table TopN joined with the same table), and the very same + // cap then keys as t#0 on one side and t#1 on the other. For a caller that runs + // the statement the frozen plan was BUILT FROM (its canonical plan digest equals + // the baseline's planSqlDigest) the cap contract cannot prove anything the text + // identity does not already prove, so the replay is kept; any other caller - a + // value variant, whose digest matches but whose placement could genuinely differ - + // still has to satisfy it (see the corresponding regression test). + // The digest alone is VALUE-FREE (every literal is masked), so it cannot carry + // this exception on its own: a caller raising only the outer + // LIMIT from 1 to 3 has the SAME digest, yet its query legitimately needs the + // optimizer-generated input caps rebuilt - skipping the cap contract there + // replayed one pair where the caller returns three. The exception therefore also + // requires the caller to carry EXACTLY the row caps the baseline was created + // the SAME cap multiset as the replayed tree, modulo the occurrence ordinal the + // optimizer's own reordering flips: any LIMIT / OFFSET the caller changed - or an + // optimizer-generated input cap the caller's tree has no counterpart for - breaks + // the equality and takes the strict path. + boolean sameFrozenStatement = candidate.getPlanSqlDigest() != null + && candidate.getPlanSqlDigest().equals(queryDigest) + && SPMPlanTreeSupport.rowLimitsEquivalentModuloOccurrence(replayContracts, userContracts); + if (!SPMPlanTreeSupport.orderContractPreserved(replayContracts, userContracts) + || (!sameFrozenStatement + && !limitContractPreserved(replayContracts, userContracts, bindTree))) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: its plan cannot carry" + + " the caller's LIMIT / OFFSET / ORDER BY contract" + + " (caller {}, replayed {}, replayedCapsWithinCaller={}," + + " callerCapsSurvive={}, callerOrder={}, replayOrder={})", + candidate.getId(), + Arrays.toString(SPMPlanTreeSupport.topLevelLimitOf(matchPlan)), + Arrays.toString(SPMPlanTreeSupport.topLevelLimitOf(replay)), + SPMPlanTreeSupport.rowLimitsWithin(replayContracts, userContracts), + SPMPlanTreeSupport.rowLimitsSurviveReplay(replayContracts, userContracts), + userContracts.order(), + replayContracts.order()); + continue; + } + usedBaselineId = candidate.getId(); + usedBaseline = candidate; + return SPMPlanTreeSupport.attachCallerSetVarHints(replay, matchPlan); + } + LOG.info("SPM tryRewritePlan: baseline {} frozen planSql replay unavailable, " + + "falling back to parameterized plan tree", candidate.getId()); + LogicalPlan planTree = SPMPlanTreeSupport.stripSetVarHints(candidate.getParameterizedPlanPlan()); + if (planTree == null) { + continue; + } + // fallback: substitute the extracted user values into the parameterized plan + // tree + SPMPlaceholderReplacer replacer = new SPMPlaceholderReplacer(); + rewritten = SPMPlanTreeSupport.transform( + planTree, expr -> expr.accept(replacer, placeholderValues)); + // safety net: a plan-only placeholder (no value extracted from the user query) + // would reach the analyzer - never rewrite with this candidate; keep trying the + // remaining candidates (they are validated independently) and degrade to the + // original plan when none works + if (SPMPlanTreeSupport.containsPlaceholder(rewritten)) { + continue; + } + // same label contract as the frozen path: the caller's own headers win; an + // unalignable shape skips the candidate instead of exposing captured headers + LogicalPlan fallbackAligned; + try { + fallbackAligned = SPMPlanTreeSupport.alignRootOutputLabels(rewritten, matchPlan); + } catch (SPMPlanTreeSupport.UnalignableOutputLabelsException unalignable) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: {}", + candidate.getId(), unalignable.getMessage()); + continue; + } + LogicalPlan replay = SPMPlanTreeSupport.mergeLimits(fallbackAligned, matchPlan); + // The cap/order identity compared below is NAME-SPELLING sensitive (the table + // qualifiers AND the CTE aliases are part of the key, see + // SPMPlanTreeSupport.collectRowLimits) and the two sides spell the same query + // differently: the caller's tree was NAMESPACE-QUALIFIED for matching, while + // the stored parameterized tree keeps the names as written (unqualified, and + // with the ORIGINAL CTE aliases - the decompiled frozen text of other baselines + // carries the regenerated ones). Compare against a namespace-qualified VIEW of + // the fallback replay so an identical-text baseline is not rejected by + // spelling: qualification resolves the same session defaults the planner will + // apply to the bare names when the replayed tree is actually planned, so the + // view describes exactly what the replay will read. + LogicalPlan capView = SPMPlanTreeSupport.namespaceQualified(replay, + captureCatalogName(ctx), captureDatabaseName(ctx)); + SPMPlanTreeSupport.ReplayContracts replayContracts = new SPMPlanTreeSupport.ReplayContracts(capView); + if (!limitContractPreserved(replayContracts, userContracts, bindTree) + || !SPMPlanTreeSupport.orderContractPreserved(replayContracts, userContracts)) { + LOG.info("SPM tryRewritePlan: baseline {} skipped: its plan cannot carry" + + " the caller's LIMIT / OFFSET / ORDER BY contract" + + " (caller {}, replayed {}, replayedCapsWithinCaller={}," + + " callerCapsSurvive={}, callerOrder={}, replayOrder={})", + candidate.getId(), + Arrays.toString(SPMPlanTreeSupport.topLevelLimitOf(matchPlan)), + Arrays.toString(SPMPlanTreeSupport.topLevelLimitOf(replay)), + SPMPlanTreeSupport.rowLimitsWithin(replayContracts, userContracts), + SPMPlanTreeSupport.rowLimitsSurviveReplay(replayContracts, userContracts), + userContracts.order(), + replayContracts.order()); + continue; + } + usedBaselineId = candidate.getId(); + usedBaseline = candidate; + return SPMPlanTreeSupport.attachCallerSetVarHints(replay, matchPlan); + } + return null; + } + + /** + * Whether the CALLER's top-level LIMIT / OFFSET is honored by the replayed tree. + * + * Top-level limit VALUES are deliberately ignored by the match (a limit variant + * reuses the baseline and gets its own values transferred by mergeLimits), but the + * transfer is a POSITIONAL merge: it only replaces values at nodes that align with + * the caller's tree. A manual plan may keep its OWN limit below a node the merge + * cannot align - bind 'SELECT k FROM t ORDER BY k LIMIT 1' WITH 'SELECT + * DISTINCT k FROM (SELECT k FROM t ORDER BY k LIMIT 1) s' - and replaying it for a + * caller asking LIMIT 2 would then return the CAPTURED row count (one row instead of + * two distinct keys). Such a candidate is skipped unless the caller asks for exactly + * the captured limit / offset, in which case the plan's own placement already IS the + * caller's contract. + * + * Parameter replayed: the replayed tree AFTER the merge (its top-level limit is the one + * the caller would observe) + * Parameter userPlan: the caller's own tree (top-level limit source) + * Parameter bindTree: the stored parameterized bind tree (captured limit values) + * Returns: whether the replay may be used for this caller + */ + private static boolean limitContractPreserved(SPMPlanTreeSupport.ReplayContracts replayed, + SPMPlanTreeSupport.ReplayContracts userPlan, LogicalPlan bindTree) { + long[] userLimit = userPlan.topLimit(); + if (userLimit == null) { + // The caller asks for NO top-level cap: the capture-side LIMIT + // VALUES are NOT the caller's contract here, so the replay must carry no + // row-limiting node at all. A manual planSql like 'SELECT k FROM t LIMIT 1' + // frozen over an unbounded bind otherwise passed creation and this early + // return silently returned ONE row for every matching unbounded SELECT. + return SPMPlanTreeSupport.rowLimitsWithin(replayed, userPlan); + } + if (Arrays.equals(userLimit, replayed.topLimit())) { + // The caller's own top-level limit is in place. Every OTHER cap the replay + // still exposes must be one the caller's own tree also has: + // equal OUTER limits do not make the plans equivalent - a manual plan + // 'SELECT k FROM (SELECT k FROM t ORDER BY k ASC LIMIT 2) s ORDER BY k DESC + // LIMIT 2' passes the single-scan CREATE guard and answers a LIMIT 2 caller + // with (2,1) although the caller's own plan returns (3,2): its INNER cap + // truncates a DIFFERENT slice before the outer sort. Only the captured caps + // themselves may justify the replay's caps - the captured (bind) tree is + // what the caller's own tree structurally matched, so an identical-text + // baseline keeps passing. additionally requires the CALLER's + // caps to survive: 'SELECT k FROM (SELECT k FROM t ORDER BY k LIMIT 1) s + // ORDER BY k LIMIT 2' against a manual plan carrying only the outer cap + // passed the one-directional check (replayed caps ⊆ caller caps) although + // the replay returns TWO rows where the caller returns one. + if (SPMPlanTreeSupport.rowLimitsWithin(replayed, userPlan) + && SPMPlanTreeSupport.rowLimitsSurviveReplay(replayed, userPlan)) { + return true; + } + // The captured-cap justification the comment above promises: the frozen + // replay is the decompiled OPTIMIZED plan of the BIND tree, and the + // optimizer folds a top-level LIMIT n into the plan (one cap per input + // scan, all with the SAME n) - so the IDENTICAL query's frozen plan + // legitimately exposes caps the RAW caller tree has no node for. Rejecting + // those skipped the original query's own baseline: tpcds q28/q77 re-ran + // their own query and got NO hit although caller and replay limits are + // identical. Three guards keep the cases rejected: + // the top-level LIMIT must be UNCHANGED from the BIND tree's own: a + // variant that changed the limit transfers the value POSITIONALLY and + // the captured inner caps would truncate its different slice - unless the + // replay carries NO cap besides the (merged) top one, where nothing can + // truncate a different slice at all (tpcds q76's LIMIT 101 variant); + // every replay cap must be one the caller's own tree has, or a + // same-valued pushdown of the caller's own top-level cap, with the + // caller asking for NO order contract (an equal-valued inner cap below + // an ordering-dependent slice truncates a DIFFERENT slice); + // every caller cap must still survive in the replay. + boolean topUnchangedFromBind = Arrays.equals(userLimit, + SPMPlanTreeSupport.topLevelLimitOf(bindTree)); + if ((topUnchangedFromBind || SPMPlanTreeSupport.onlyTopCap(replayed)) + && (SPMPlanTreeSupport.topCapContractPreserved(replayed, userPlan) + || (topUnchangedFromBind + && SPMPlanTreeSupport.rowLimitsSurviveReplay( + replayed, userPlan) + && SPMPlanTreeSupport.replayCapsJustifiedByCallerTopLimit( + replayed, userPlan)))) { + return true; + } + return false; + } + // Matching the CAPTURED limit VALUE is not enough - the replayed tree + // must actually CARRY the caller's cap. A manual plan 'SELECT k FROM t' freezes a + // text with NO limit node: mergeLimits only replaces the VALUE of a cap that + // aligns positionally, it cannot add a missing one, so accepting this replay + // returned every row although the caller asked for the captured LIMIT 2 (the two + // branches above are the only proofs that the cap survived - equal top-level caps + // plus every nested cap justified). + return false; + } + + /** + * Parses frozen SQL under the default SQL mode and substitutes matched expressions by marker id. Persisted + * provenance takes precedence over legacy marker-based classification. Returns null for raw fallback rows, + * unavailable context, parse failures or unresolved markers. Successful SQL replay remains subject to ordinary + * planning; it does not force the same physical plan on every execution. + */ + private LogicalPlan rewriteFromFrozenTree(BaselinePlan candidate, + Map<Long, Expression> placeholderValues) { + // Persisted provenance first (see BaselinePlan.planFrozen): a row explicitly + // marked as NOT decompiled (raw user fallback text) must never be replayed as + // frozen SQL just because its text contains a call shaped like a placeholder - + // a real db._spm_const_var(1) UDF would otherwise be replaced by the user's + // literal and return the literal instead of evaluating the function. + Boolean persistedFrozen = candidate.getPlanFrozen(); + if (persistedFrozen != null && !persistedFrozen) { + LOG.info("SPM replay baseline {} frozen replay unavailable: the row is marked as" + + " raw planSql fallback text (the decompiler refused at creation)", + candidate.getId()); + return null; + } + String planSql = candidate.getPlanSql(); + if (planSql == null) { + return null; + } + boolean textHasPlaceholder = planSql.contains(SPMFrozenTreeReplacer.CONST_VAR_FUNC) + || planSql.contains(SPMFrozenTreeReplacer.CONST_LIST_FUNC); + if (persistedFrozen == null && !textHasPlaceholder) { + return null; // legacy row, not a frozen (placeholder-carrying) plan text + } + // re-parsing the frozen text needs the session context (join-hint / statement + // context handling); when absent (pure in-memory callers) fall back + ConnectContext ctx = ConnectContext.get(); + if (ctx == null) { + return null; + } + try { + // The frozen text is SPM's own rendering, produced for the DEFAULT sql_mode + // (backslash escapes active - see SPMPlan2SQLBuilder.quoteSqlString). Pin that + // mode for this re-parse: inheriting a NO_BACKSLASH_ESCAPES session would + // decode the doubled backslashes to a different value and could select + // another external ref. + Plan parsed = SPMPlanTreeSupport.parseStoredSqlWithoutSetVarSideEffects(planSql, + SqlModeHelper.MODE_DEFAULT); + if (!(parsed instanceof LogicalPlan) || parsed instanceof Command) { + return null; + } + // Classify on the PARSED TREE instead of the raw text: the fallback path + // stores the ORIGINAL planSql when the decompiler rejects a node (e.g. + // PhysicalAssertNumRows), and that ordinary SQL may merely CONTAIN one of + // the placeholder function names inside a string literal, identifier or + // comment. Substituting nothing and returning such a tree would replay the + // CAPTURED literals - the user's values must go through the parameterized + // fallback tree instead. + // + // The classification only guards rows WITHOUT provenance: when the row is + // EXPLICITLY marked frozen (planFrozen=true), the text is the decompiler's + // own rendering, and a literal-free optimized plan legitimately contains no + // placeholder call - it must replay as-is with an EMPTY substitution map + // (the frozen structure / hints are the point), not fall back to the + // original pre-optimization tree. + boolean parsedHasPlaceholder = + SPMPlanTreeSupport.containsFrozenPlaceholder((LogicalPlan) parsed); + if (persistedFrozen == null && !parsedHasPlaceholder) { + return null; + } + SPMFrozenTreeReplacer replacer = new SPMFrozenTreeReplacer(); + LogicalPlan rewritten = SPMPlanTreeSupport.transform( + (LogicalPlan) parsed, expr -> expr.accept(replacer, placeholderValues)); + // safety net: a plan-only placeholder (no user value extracted) would reach + // the analyzer as an unregistered function - never rewrite with it + if (SPMPlanTreeSupport.containsFrozenPlaceholder(rewritten)) { + LOG.info("SPM replay baseline {} frozen replay unavailable: placeholder ids" + + " without an extracted value remain after substitution", + candidate.getId()); + return null; + } + if (LOG.isInfoEnabled()) { + LOG.info("SPM replay baseline {} from frozen planSql (id substitution)", + candidate.getId()); + } + // The frozen text keeps its STRUCTURAL hints (INDEX pin, [BROADCAST] / + // [SHUFFLE] join distribution) - that is what the frozen replay exists for - but + // NOT the CREATOR's SET_VAR payload: a SET_VAR writes a STATEMENT-scoped session + // variable, and the caller's own hints are re-attached from the CALLER's tree by + // attachCallerSetVarHints at the return sites. An INDEX-pinned baseline + // used to replay the captured time_zone='+08:00' inside a nested scalar subquery + // and change from_unixtime for a caller that carried no hint at all; the peel + // reaches expression-owned subquery plans too (see + // SPMPlanTreeSupport.stripSetVarHints). + LogicalPlan replayable = SPMPlanTreeSupport.stripSetVarHints(rewritten); + return replayable == null ? rewritten : replayable; + } catch (Throwable t) { + // legacy "?" frozen text or any other re-parse failure -> fall back to the + // parameterized plan tree path + LOG.info("SPM replay baseline {} frozen replay unavailable: the frozen text does" + + " not re-parse ({}: {})", candidate.getId(), + t.getClass().getSimpleName(), t.getMessage()); + return null; + } + } + + /** + * The effective catalog name of the creation / matching context (null when unknown); + * part of the namespace-qualified matching key. + */ + private static String captureCatalogName(ConnectContext ctx) { + return ctx == null || ctx.getCurrentCatalog() == null + ? null : ctx.getCurrentCatalog().getName(); + } + + /** The effective database of the creation / matching context (null when unknown). */ + private static String captureDatabaseName(ConnectContext ctx) { + return ctx == null ? null : ctx.getDatabase(); + } + + /** + * Post-planning metadata revalidation of a replayed baseline (the replay-side half + * of the frozen-metadata binding, see + * SPMPlanTreeSupport.schemaFingerprintForCreate): the pre-match fingerprint guard + * runs BEFORE the query planner takes its metadata locks, so an ALTER TABLE + * committing in between would let the frozen SQL be planned against a DIFFERENT + * schema than the one it was validated against (and than the one its output slots + * were frozen with). This re-checks the stored fingerprint against the metadata + * the REPLAYED plan was actually planned with - the planned PhysicalPlan's catalog + * relations plus the baseline's own bind tree resolved in this context. A mismatch + * surfaces as a rewrite failure; the caller's enable_spm_fallback policy decides + * whether to re-plan the original query (enabled) or surface the error (disabled). + * + * Parameter ctx: the replaying session + * Parameter baselineId: the baseline the rewritten tree came from (-1 = none) + * Parameter plannedPlan: the replayed plan after planning + */ + public static void verifyReplayMetadata(ConnectContext ctx, long baselineId, + Plan plannedPlan) { + if (ctx == null || plannedPlan == null || baselineId < 0) { + return; + } + // Prefer the baseline OBJECT retained on the statement context: re-fetching by id + // can silently miss it after a concurrent DROP / refresh, and returning then + // would skip the post-plan schema check exactly when a table DDL may have + // committed between the pre-match validation and the replay planning. + BaselinePlan baseline = null; + if (ctx.getStatementContext() != null + && ctx.getStatementContext().getSpmUsedBaseline() != null + && ctx.getStatementContext().getSpmUsedBaseline().getId() == baselineId) { + baseline = ctx.getStatementContext().getSpmUsedBaseline(); + } + if (baseline == null) { + baseline = ctx.getSessionBaselineStore() == null + ? null : ctx.getSessionBaselineStore().getBaseline(baselineId); + } + if (baseline == null) { + baseline = BaselineManager.getInstance().getBaseline(baselineId); + } + if (baseline == null) { + // A recorded id whose baseline is gone (concurrent DROP / refresh): the stored + // fingerprint can no longer be revalidated and the frozen SQL may be a stale + // SELECT * output list - fail so the caller applies its fallback policy + // instead of silently keeping the replay. + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "SPM baseline " + baselineId + " disappeared before the replay" + + " validation; keeping the original query"); + } + String stored = baseline.getSchemaFingerprint(); + if (stored == null || stored.isEmpty()) { + return; + } + String current = SPMPlanTreeSupport.schemaFingerprintForReplay( + ctx, baseline.getParameterizedBindPlan(), plannedPlan, baseline.getPlanSql()); + // tolerance: a row persisted before the nullability section existed carries + // section-less entries (see SPMPlanTreeSupport.legacyEntryOf); a row written + // since must match exactly, so a nullability change between the pre-match + // validation and this post-plan check still fails closed + if (!SPMPlanTreeSupport.schemaFingerprintEquivalent(stored, current)) { + LOG.warn("SPM replay metadata mismatch for baseline {}: stored=[{}] current=[{}]", + baselineId, stored, current); + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "SPM replay metadata changed between validation and planning; keeping" + + " the original query for baseline " + baselineId); + } + } + + // baseline creation. + + /** + * Rejects statements whose frozen SQL would persist the CREATOR's session state: a + * session / user variable or current_user / session_user / database / + * connection_id survives parameterization, the creator-context optimization then + * resolves it to a LITERAL, and the frozen planSql persists that value - while + * matching still compares the original unbound bind tree, so a global baseline would + * serve every other user the creator's identity / variables. Clock functions are the + * same class of value: FE constant folding evaluates now / current_timestamp from + * the CREATE statement's start time (see DateTimeAcquire), so a baseline for + * SELECT now() AS ts FROM t returned the CREATE timestamp on every later + * matching query. + */ + private static void rejectReplayContextExpressions(LogicalPlan bindPlan, LogicalPlan planPlan, + String bindSql) throws AnalysisException { + if (SPMPlanTreeSupport.containsReplayContextExpression(bindPlan) + || SPMPlanTreeSupport.containsReplayContextExpression(planPlan)) { + throw new AnalysisException("SPM does not support replay-time context expressions" + + " (user/session variables including a parsed @v, current_user()," + + " session_user(), user(), database(), current_catalog(), connection_id()," + + " last_query_id(), version(), a clock function evaluated at the" + + " statement start - now() / current_timestamp() / current_date() /" + + " utc_timestamp() / localtime() / unix_timestamp() without arguments -" + + " and any * REPLACE payload carrying one):" + + " the frozen SQL would persist the creator's value: " + + bindSql); + } + // The CALL alone is not enough: the guard above walks the PARSED trees, but + // an ALIAS UDF's registry-owned body only enters the plan when analysis INLINES it - + // CREATE ALIAS FUNCTION f() WITH PARAMETER() AS current_user() then had that + // CurrentUser folded into a literal by the creator-context optimization (no + // NeedSessionVarGuard on CurrentUser) and frozen into the planSql, while matching still + // compared the caller's own f() text: a GLOBAL baseline answered every other user with + // the creating identity. The resolved bodies are inspected TRANSITIVELY (and in the + // same resolved scope the call will use) before anything is parameterized. + ConnectContext currentCtx = ConnectContext.get(); + if (SPMPlanTreeSupport.containsReplayContextInAliasBodies(currentCtx, bindPlan) + || SPMPlanTreeSupport.containsReplayContextInAliasBodies(currentCtx, planPlan)) { + throw new AnalysisException("SPM does not support replay-time context expressions" + + " inside a resolved ALIAS FUNCTION body (an alias whose body calls" + + " current_user() / session_user() / database() / @v / a clock function," + + " possibly through another alias): analysis INLINES the body and the" + + " creator-context folding then persists the CREATOR's value in the frozen" + + " SQL while matching compares the caller's own call: " + + bindSql); + } + } + + /** + * Builds a GLOBAL baseline candidate from bind and plan SQL. Parses both texts in the creator mode, validates + * supported correspondence, parameterizes supported literals and optimizes the plan. Stores frozen SQL when + * decompilation and replay checks allow it; otherwise retains the original plan SQL and fallback tree. The + * caller stores the returned candidate. + */ + public BaselinePlan buildBaselineFromSql(ConnectContext ctx, String bindSql, String planSql) + throws UserException { + // The 3-arg entry is the GLOBAL create / capture path; temporary tables are + // rejected there (the frozen planSql would carry the creator session's internal + // temp name). The CREATE command uses the 4-arg overload to allow a SESSION-scope + // baseline over the session's own temporary table. + return buildBaselineFromSql(ctx, bindSql, planSql, true); + } + + /** + * Builds a baseline from SQL texts with a SPM-mode optimized planSql, with an explicit + * storage scope for the mutable-constraint / temporary-table guards. + * + * Parameter ctx: the connect context (catalog + session variables) + * Parameter bindSql: the binding SQL (SELECT) + * Parameter planSql: the plan SQL (SELECT, may contain SET_VAR hints) + * Parameter globalScope: whether the baseline is stored in the shared GLOBAL store + * Returns: the constructed BaselinePlan (not yet stored) + * Throws UserException: when the SQLs cannot be parsed / planned + */ + public BaselinePlan buildBaselineFromSql(ConnectContext ctx, String bindSql, String planSql, + boolean globalScope) throws UserException { + // Save the mode before either input is parsed. Parsing applies SET_VAR to the live session, + // and planning applies it again, but neither assignment may decide the persisted parser mode. + // A later reload must build bind expressions such as a || b exactly as CREATE did. + final long creatorMode = SqlModeHelper.currentMode(); + // Pin both inputs to that mode. The bind hint can change the real session between these calls; + // without the override, identical a || b expressions could become OR in bind and CONCAT in plan. + // This controls parser reads only; SET_VAR assignments remain active until statement cleanup. + LogicalPlan bindPlan = parseSelectWithSqlMode(creatorMode, bindSql, + "SPM bindSql must be a SELECT statement: " + bindSql); + LogicalPlan planPlan = bindSql.equals(planSql) + ? bindPlan + : parseSelectWithSqlMode(creatorMode, planSql, + "SPM planSql must be a SELECT statement: " + planSql); + // The destination of SELECT... INTO OUTFILE lives outside the plan expressions: + // the match cannot compare it and the rewritten tree keeps the captured sink, so + // replay would export to the baseline's destination. Reject instead of freezing. + if (SPMPlanTreeSupport.containsFileSink(bindPlan) + || SPMPlanTreeSupport.containsFileSink(planPlan)) { + throw new AnalysisException( + "SPM does not support SELECT ... INTO OUTFILE statements: " + bindSql); + } + rejectReplayContextExpressions(bindPlan, planPlan, bindSql); + // Collect shared facts once, then validate scan selectors and logical query equivalence. + SPMPlanTreeSupport.rejectManualPlanMismatch(bindPlan, planPlan, bindSql); + // key(...) folds a named secret into a constant during optimization, and the parsed + // form of KEY db.key is an EncryptKeyRef (NOT an UnboundFunction): reject on BOTH + // parsed inputs BEFORE optimizing / storing anything. + SPMPlanTreeSupport.rejectVolatileFunctionDependencies(bindPlan, planPlan); + // Parameterize both whole trees with ONE shared builder (placeholder ids aligned + // across bind / plan), then optimize the PARAMETERIZED plan tree so the frozen + // planSql keeps the placeholder ids for the rewrite-time value substitution. + SPMPlaceholderBuilder parameterizeBuilder = new SPMPlaceholderBuilder(); + Pair<LogicalPlan, LogicalPlan> trees = parameterizeWholeTrees(bindPlan, planPlan, + parameterizeBuilder); + LogicalPlan parameterizedPlan = trees.second; + // Keep original plan SQL when either input references a view rather than storing its base-table + // expansion. The rewrite entry currently skips view queries before replay so their ordinary planning + // retains view resolution and authorization. + boolean referencesView = SPMPlanTreeSupport.referencesView(ctx, bindPlan) + || SPMPlanTreeSupport.referencesView(ctx, planPlan); + // An inline VALUES cell / * REPLACE payload lives OUTSIDE the placeholder machinery + // (its literals are matched CONCRETELY), so the creator-context optimization FOLDS + // a constant-argument zone-sensitive call there (from_unixtime(0) -> the CREATOR's + // zone literal) into the decompiled text while the caller's identical query still + // matches the stored tree: a frozen replay would serve the creator's value in every + // other session zone. Decline the freeze for such a statement - the raw planSql is + // kept and the rewrite replays the parameterized tree, whose re-planning evaluates + // the expression in the CALLER's own session (the same degradation the view guard + // uses). + boolean foldedSessionSensitivePayload = + SPMPlanTreeSupport.containsFoldedSessionSensitivePayload(bindPlan) + || SPMPlanTreeSupport.containsFoldedSessionSensitivePayload(planPlan); + // An explicit FROM t INDEX mv pin is a SEMANTIC choice (on an aggregate-key + // table a coarser rollup returns aggregated rows where the base table returns one + // per record), and the decompiled text cannot carry it: the physical scan only + // knows the SELECTED index id - which every optimizer-side rollup / MV choice sets + // as well - while the matching key DOES carry the pin (the digest renders INDEX + // <name>, so only callers with the same pin match). A frozen "FROM t" would then + // read the base table for a pinned caller, silently returning finer-grained rows; + // the replan probe does not catch it (the text is valid) and the schema + // fingerprint does not hash the selection. Decline the freeze exactly like the + // view / session-sensitive-payload guards: the raw planSql is kept and the rewrite + // replays the parameterized tree, whose own analysis re-applies the pin. + boolean pinsExplicitIndex = SPMPlanTreeSupport.pinsExplicitIndex(bindPlan) + || SPMPlanTreeSupport.pinsExplicitIndex(planPlan); + + SPMOptimizer.OptimizeResult optimizeResult; + DecompiledPlan frozen; + boolean rawPlanOptimized = false; + try { + // optimize the parameterized plan tree in SPM mode: placeholders travel + // through analyze / rewrite / CBO and survive into the physical plan. + // The tree handed to the optimizer is POLICY-FREE: a creator (non-root ADMIN) + // may carry a row filter / data mask, and resolving it here would freeze the + // CREATOR's policy into the planSql as an ordinary predicate / projection - + // every later user matching the same bind query would replay it, even without + // such a policy (valid rows disappear / values stay masked). The STORED + // parameterized trees keep their CHECK markers and a frozen text is re-parsed + // at replay, so the EXECUTING user's own policy checks still run exactly as + // for an ordinary query. + optimizeResult = SPMOptimizer.optimize(ctx, + SPMPlanTreeSupport.stripCheckPolicy(parameterizedPlan), planSql); + frozen = decompileFrozenPlan(referencesView, foldedSessionSensitivePayload, + pinsExplicitIndex, + parameterizeBuilder.hasParameterizedGroupByOrdinalTarget(), + optimizeResult, planSql); + } catch (UserException | RuntimeException e) { + // When the parameterized tree cannot be planned (e.g. a placeholder cannot + // survive some analyzer path yet), fall back to optimizing the raw planSql - + // the CREATE must not fail because of the parameterized-tree optimization. + // The fallback replays through the parameterized-plan TREE (planFrozen=false), + // so the stored text must be the ORIGINAL planSql: the decompiled rendering of + // the RAW plan carries its CONCRETE literals, and the next reload + // re-parameterizes that TRANSFORMED text (BETWEEN 1 AND 2 comes back as + // k >= 1 AND k <= 2), whose new placeholder ids the bind tree never extracted - + // every replay then failed the residue check. The original text maps its + // literals to the same ids as the bind tree. The fallback tree is stripped of + // check policy markers for the same reason as above. + LOG.warn("SPM parameterized plan optimization failed, falling back to raw planSql", + e); + rawPlanOptimized = true; + optimizeResult = SPMOptimizer.optimize(ctx, + SPMPlanTreeSupport.stripCheckPolicy( + parseSelectWithSqlMode(creatorMode, planSql, + "SPM planSql must be a SELECT statement: " + planSql)), + planSql); + frozen = new DecompiledPlan(planSql, false); + } + // A successful decompilation is not proof the text can be re-planned: the + // optimizer's output slots may use names the emitted SQL never exposes (an + // ORDER BY over columns a lower scope dropped). The builder prevents the known + // forms at the source - the ORDER BY hoist re-exports every key it moves above + // its scope (hoistableOrderBy / reExportedLabels; the step that made tpcds q78 + // legal: its hoisted "ORDER BY c_14..." now re-exports c_4 AS c_14 through the + // wrapper below) - but a future shape may still slip through, and EVERY replay + // of such a text fails as a planning error ("SPM rewritten plan failed"), which + // is worse than the documented fallback; downgrade HERE so the baseline stays + // usable through the parameterized tree. The check plans the frozen text once - + // CREATE is a one-shot operation. + if (frozen.decompiled) { + frozen = validateFrozenTextReplannable(ctx, frozen, planSql, optimizeResult.getPhysicalPlan()); + } + if (globalScope) { + // A GLOBAL baseline must never be frozen over a temporary table: the physical + // relation's catalog object carries the CREATOR session's internal name + // (<sessionId>_#TEMP._<name>). The bind key for "FROM t" is only + // catalog.db.t, so a second session running the same text matched the same + // digest, and the frozen SQL (emitting the internal name) resolved the + // creator's still-live temporary table instead of the second session's t. + rejectTemporaryRelations(optimizeResult, bindSql); + } + // Explicit provenance of the stored planSql, persisted so a reload never has to + // GUESS whether the text is SPM's decompiled frozen rendering or the user's raw + // fallback (see BaselinePlan.planFrozen / #planSqlMode). + // + // The provenance is the DECOMPILER's success over the PARAMETERIZED tree - NOT + // the mere presence of a placeholder call: a literal-free optimized join (e.g. a + // hinted broadcast shuffled join with no literal value) decompiles successfully + // yet contains no spm_const* call. Recording planFrozen=false for it made every + // replay reject the frozen text and fall back to the ORIGINAL pre-optimization + // logical tree - losing exactly the stored join / distribution choice - while + // after a GLOBAL refresh (the row is rebuilt from the same decompiled planSql) + // the behavior flipped to frozen. + // + // EXCEPTION: the RAW-PLAN fallback re-uses ordinary user text (it never + // decompiles): replaying it as frozen would skip the user-value substitution and + // return the captured literals - such a text must keep the parameterized-plan-tree + // rewrite path (planFrozen=false), exactly like a failed decompile. + Boolean planFrozen = frozen.decompiled && !rawPlanOptimized; + long planSqlMode = frozen.decompiled ? SqlModeHelper.MODE_DEFAULT : creatorMode; + BaselinePlan baseline = assembleBaseline(bindPlan, trees.first, parameterizedPlan, bindSql, + frozen.sql, optimizeResult.getCost(), + captureCatalogName(ctx), captureDatabaseName(ctx), creatorMode); Review Comment: [P2] Track the manual plan's effective analysis mode With `ONLY_FULL_GROUP_BY` enabled, create an unhinted bind `SELECT k,v,count(*) AS n FROM t WHERE k>0 GROUP BY k` and the same manual plan with `SET_VAR(sql_mode=0)`. Admission unwraps the hint, and capture analyzes the plan permissively: ```text Bind: Aggregate[k; k,v,count(*)] -> Filter -> Scan(t) Frozen: Aggregate[k; k,ANY_VALUE(v),count(*)] -> Filter -> Scan(t) ``` This call persists the original strict `creatorMode`, so the new replay guard sees its strict bit and admits the frozen `ANY_VALUE(v)` for the strict caller, returning rows instead of the required grouping error. This capture-time plan-only hint case differs from the repaired ambient-mode-transition report. Validate the bind under its own analysis settings or track the effective capture analysis mode separately from the parser-origin mode. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/builder/SPMExprSqlBuilder.java: ########## @@ -0,0 +1,902 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.builder; + +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.spm.matcher.SPMFrozenTreeReplacer; +import org.apache.doris.nereids.spm.placeholder.SPMSubquerySupport; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.AggregateExpression; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.Between; +import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.BinaryOperator; +import org.apache.doris.nereids.trees.expressions.BitAnd; +import org.apache.doris.nereids.trees.expressions.BitNot; +import org.apache.doris.nereids.trees.expressions.BitOr; +import org.apache.doris.nereids.trees.expressions.BitXor; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Exists; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.IsFalse; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.IsTrue; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StringRegexPredicate; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.TryCast; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctGroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; +import org.apache.doris.nereids.trees.expressions.functions.window.Lag; +import org.apache.doris.nereids.trees.expressions.functions.window.Lead; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Renders supported expressions using the ExprId reference mapping of a SQLRelation. Marker payloads are omitted + * in favor of id-only calls. Composite expressions recurse through the visitor; a generic toSql fallback rejects + * registered slots that require remapping. + * + * Subquery expressions have only a compact diagnostic digest rendering here, not executable subquery SQL. The + * physical-plan decompiler rejects residual subquery expressions before invoking this printer. + */ +public class SPMExprSqlBuilder extends ExpressionVisitor<String, SQLRelation> { + + /** + * The INTERNAL pseudo function names visitSubqueryExpr emits for a subquery: the + * structural DIGEST form, never SQL. + * + * They are reserved by SPM. A frozen SQL must never be readable as - or shadowed by - a user + * function of one of these names: INSUBQUERY(x)[...] is an ordinary unreserved call, so + * a compatible user alias UDF made a frozen text validate and every replay evaluated that UDF + * (and its array access) instead of the original IN (SELECT...). Subqueries are + * therefore refused before decompilation and a plan calling a function of these names is not + * freezable either (see SPMPlan2SQLBuilder.rejectSubqueryExpressions / + * #rejectReservedInternalFunctionNames). + */ + public static final java.util.Set<String> INTERNAL_PSEUDO_FUNCTION_NAMES = + java.util.Set.of("insubquery", "scalarsubquery"); + + /** + * Prints a single expression. + * + * Parameter expr: the expression to print + * Parameter relation: the SQLRelation carrying the columnNames mapping + * Returns: the SQL text + */ + public String print(Expression expr, SQLRelation relation) { + return expr.accept(this, relation); + } + + /** + * Renders one function's argument list. + * + * Parameter aggregate: whether the arguments belong to an AGGREGATE call + * Parameter arguments: the arguments to render + * Parameter context: the relation of the enclosing scope + * Returns: the comma separated argument text + */ + private String renderArguments(boolean aggregate, List<Expression> arguments, + SQLRelation context) { + return arguments.stream() + .map(argument -> renderArgument(argument, context, aggregate)) + .collect(Collectors.joining(", ")); + } + + /** + * Renders one function argument. At an aggregate argument root, preserves a recognized LARGEINT accumulator + * widening; other casts follow visitCast. This root-only rule avoids retaining ordinary nested coercions that + * would pin a captured operand type. + */ + private String renderArgument(Expression argument, SQLRelation context, boolean aggregate) { + if (aggregate && argument instanceof Cast && preservesLoweredAccumulator((Cast) argument)) { + Cast cast = (Cast) argument; + return "CAST(" + cast.child().accept(this, context) + " AS " + + cast.getDataType().toSql() + ")"; + } + return argument.accept(this, context); + } + + @Override + public String visit(Expression expr, SQLRelation context) { + // SPM placeholder: render as the id-only function call "_spm_const_var(id)" / + // "_spm_const_list(id)" (the value does not participate in the digest / planSql + // serialization). The id-only form keeps the digest value-independent AND makes + // the frozen planSql parseable at replay: the text carries the id and the + // replay substitutes the matched expression under that id. + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstVar) { + org.apache.doris.nereids.spm.placeholder.SpmConstVar ph = + (org.apache.doris.nereids.spm.placeholder.SpmConstVar) expr; + return "_spm_const_var(" + ph.getId() + ")"; + } + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstList) { + org.apache.doris.nereids.spm.placeholder.SpmConstList list = + (org.apache.doris.nereids.spm.placeholder.SpmConstList) expr; + // IN-list placeholder: render size-independently (_spm_const_list(id)) so + // the digest / planSql stays value AND list-length independent: a baseline + // "IN (1, 2, 3)" matches a user "IN (10, 20)" regardless of list length. + return "_spm_const_list(" + list.getId() + ")"; + } + // Default fallback: use toSql of the expression itself (no column mapping). + // toSql bypasses the ExprId -> column mapping, so a composite shape whose + // children need remapping (e.g. ~left.a above a join that renamed the colliding + // a to c_N / qualified it) would freeze a stale, unresolvable column and - with + // the default enable_spm_fallback = false - fail the rewritten query. Reject + // such shapes so CREATE keeps the user planSql and the rewrite degrades to the + // parameterized-tree path. + ensureNoRemappedSlots(expr, context); + return expr.toSql(); + } + + /** + * Safety net of the generic toSql fallback: rejects the expression when any slot + * below it is registered under a reference that toSql would NOT reproduce (the + * collision-renaming / quoting / qualification case), because toSql would then emit + * a stale or differently-shaped name. Only a byte-identical registered reference is + * renderable through the fallback. Public for tests. + * + * Parameter expr: the expression about to fall back to toSql + * Parameter context: the relation carrying the ExprId -> column-reference mapping + * Throws UnsupportedOperationException: when a child slot needs remapping + */ + public static void ensureNoRemappedSlots(Expression expr, SQLRelation context) { + if (expr instanceof SlotReference) { + SlotReference slot = (SlotReference) expr; + String mapped = context.getColumnNames().get(slot.getExprId()); + // toSql prints the BARE slot name, so a registered reference is only safe + // when it is byte-identical to that rendering. A special-character column + // maps to a QUOTED reference (a-b) and a disambiguated one to a QUALIFIED + // reference (l.x): both still "name" the column, but freezing their unquoted / + // unqualified toSql text produces SQL that no longer resolves after reload + // ("a-b MATCH 'foo'" parses as a subtraction; a bare x may be ambiguous). + // Reject the generic fallback for those shapes - CREATE keeps the user planSql + // and the rewrite degrades to the parameterized-tree path. + if (mapped != null && !mapped.equals(slot.getName())) { + throw new UnsupportedOperationException( + "SPM decompile: expression " + expr.toSql() + " needs column remapping (" + + mapped + " -> " + slot.getName() + ") which toSql() cannot apply"); + } + } + for (Expression child : expr.children()) { + ensureNoRemappedSlots(child, context); + } + } + + // column references. + + @Override + public String visitSlotReference(SlotReference slotReference, SQLRelation context) { + String mapped = context.getColumnNames().get(slotReference.getExprId()); + if (mapped != null) { + return mapped; + } + // Doris rollup materializes "GROUPING(col)" as an internal slot named + // GROUPING_PREFIX_<col>; when such a slot was not registered by the repeat + // decompiler (defensive fallback), render it back to GROUPING(col). + String name = slotReference.getName(); + if (name != null && name.startsWith("GROUPING_PREFIX_")) { + return "GROUPING(" + name.substring("GROUPING_PREFIX_".length()) + ")"; + } + // An unregistered slot whose name is Doris's internal mark-slot naming + // ("$c$N", e.g. the EXISTS marker of an uncorrelated subquery folded into a + // constant CROSS JOIN) can never be resolved in the decompiled SQL - emitting + // the raw name would freeze a planSql that fails on replay ("Unknown column + // '$c$1'"). Such a shape has no faithful SQL rendering here, so fail the + // decompile: CREATE BASELINE then falls back to the user-supplied planSql + // (which is exactly the original query for these constant boolean outputs). + if (name != null && name.startsWith("$c$")) { + throw new UnsupportedOperationException( + "SPM decompile: unresolvable internal mark slot " + name); + } + // Unregistered column reference (e.g. a CTE output column) degrades to the raw + // column name + return slotReference.toSql(); + } + + // binary operations (arithmetic / comparison / logic). + + @Override + public String visitComparisonPredicate(ComparisonPredicate comparisonPredicate, SQLRelation context) { + return "(" + renderValueOperand(comparisonPredicate.left(), context) + + " " + operatorSymbol(comparisonPredicate) + + " " + renderValueOperand(comparisonPredicate.right(), context) + ")"; + } + + @Override + public String visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, SQLRelation context) { + return "(" + renderValueOperand(binaryArithmetic.left(), context) + + " " + operatorSymbol(binaryArithmetic) + + " " + renderValueOperand(binaryArithmetic.right(), context) + ")"; + } + + /** + * BitNot (~x): the generic fallback would print the expression through toSql and + * bypass the column mapping; above a join whose colliding column was renamed, that + * freezes the stale slot text. Render the child recursively instead so the mapped + * column reference stays mapped. + */ + @Override + public String visitBitNot(BitNot bitNot, SQLRelation context) { + return "~(" + bitNot.child().accept(this, context) + ")"; + } + + @Override + public String visitCompoundPredicate(CompoundPredicate compoundPredicate, SQLRelation context) { + // And / Or are n-ary in Nereids: a single node can hold more than two children, + // so child(0)/child(1) alone would drop the rest. Flatten the same-type operators + // into a list of operands with ExpressionUtils.extract (see extractConjunction / + // extractDisjunction) and join them with the operator symbol. extract stops at a + // different operator type, so e.g. (a or b) and c keeps the inner Or intact. + // + // CompoundPredicate is not a BinaryOperator (And/Or extend CompoundPredicate + // directly), so resolve the symbol here instead of using operatorSymbol. + String symbol = compoundPredicate instanceof And ? "AND" : "OR"; + String joined = ExpressionUtils.extract(compoundPredicate).stream() + .map(operand -> operand.accept(this, context)) + .collect(Collectors.joining(" " + symbol + " ")); + // AND / OR need parentheses to keep the precedence + return "(" + joined + ")"; + } + + @Override + public String visitNot(Not not, SQLRelation context) { + return "(NOT (" + not.child().accept(this, context) + "))"; + } + + @Override + public String visitIsNull(IsNull isNull, SQLRelation context) { + return renderValueOperand(isNull.child(), context) + " IS NULL"; + } + + @Override + public String visitIsTrue(IsTrue isTrue, SQLRelation context) { + return renderValueOperand(isTrue.child(), context) + " IS TRUE"; + } + + @Override + public String visitIsFalse(IsFalse isFalse, SQLRelation context) { + return renderValueOperand(isFalse.child(), context) + " IS FALSE"; + } + + // predicates. + + @Override + public String visitInPredicate(InPredicate inPredicate, SQLRelation context) { + String compareExpr = renderValueOperand(inPredicate.getCompareExpr(), context); + List<String> options = inPredicate.getOptions().stream() + .map(o -> o.accept(this, context)) + .collect(Collectors.toList()); + // A literal-only IN option list may be stored in a set-backed collection whose + // iteration order is NOT stable across runs / FE restarts (observed: TPCDS q75's + // "d_year IN (2001, 2002)" decompiled as (2001, 2002) one run and (2002, 2001) + // the next, flipping the frozen planSql and failing the.out comparison). IN + // semantics are order-independent, so a literal-only list is emitted in sorted + // order to keep the decompiled planSql deterministic. + boolean allLiterals = inPredicate.getOptions().stream() + .allMatch(o -> o instanceof Literal); + if (allLiterals && options.size() > 1) { + Collections.sort(options); + } + return compareExpr + " IN (" + String.join(", ", options) + ")"; + } + + @Override + public String visitBetween(Between between, SQLRelation context) { + String compareExpr = renderValueOperand(between.getCompareExpr(), context); + String lower = renderValueOperand(between.getLowerBound(), context); + String upper = renderValueOperand(between.getUpperBound(), context); + return compareExpr + " BETWEEN " + lower + " AND " + upper; + } + + @Override + public String visitLike(Like like, SQLRelation context) { + String rendered = renderValueOperand(like.left(), context) + + " LIKE " + renderValueOperand(like.right(), context); + if (like.children().size() > 2) { + // The third child is the ESCAPE character. Dropping it froze a three-argument + // LIKE as a two-child predicate: parameterizing the escape literal fails + // LIKE's literal-only legality check, so creation fell back to the raw plan and + // persisted the ALTERED predicate - a column-pattern LIKE replayed without + // ESCAPE can match different rows. Render the child recursively. + rendered += " ESCAPE " + renderValueOperand(like.child(2), context); + } + return rendered; + } + + /** Parentheses let a predicate occupy a value-expression position in the parser. */ + private String renderValueOperand(Expression operand, SQLRelation context) { + if (operand instanceof Nullable) { + return renderValueOperand(operand.child(0), context); + } + String rendered = operand.accept(this, context); + if (operand instanceof IsNull || operand instanceof IsTrue || operand instanceof IsFalse + || operand instanceof InPredicate || operand instanceof Between + || operand instanceof StringRegexPredicate || operand instanceof Match) { + return "(" + rendered + ")"; + } + return rendered; + } + + // function calls. + + @Override + public String visitBoundFunction(BoundFunction boundFunction, SQLRelation context) { + // The Nullable marker (nullable(x) wraps a column to make it nullable, e.g. on a + // recursive CTE union branch) has no SQL equivalent: decompile the inner + // expression only. + if (boundFunction instanceof Nullable) { + return boundFunction.child(0).accept(this, context); + } + if (boundFunction instanceof GroupConcat || boundFunction instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat((AggregateFunction) boundFunction, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + return rendered; + } + String args = renderArguments(boundFunction instanceof AggregateFunction, + boundFunction.children(), context); + return functionName(boundFunction) + "(" + args + ")"; + } + + /** + * Renders a function name, quoting and qualifying UDF names by their recorded database. UDF names that collide + * with scalar or list markers are rejected so the caller can retain raw plan SQL. Builtins use their function + * name directly. + */ + public static String functionName(Function function) { + if (function instanceof Udf) { + String name = function.getName(); + if (SPMFrozenTreeReplacer.CONST_VAR_FUNC.equals(name) + || SPMFrozenTreeReplacer.CONST_LIST_FUNC.equals(name)) { + throw new UnsupportedOperationException("SPM decompile: the user function " + + name + " collides with the SPM placeholder marker; keeping the" + + " original planSql"); + } + String dbName = ((Udf) function).getDbName(); + if (dbName != null && !dbName.isEmpty()) { + // Quote each component: a legal database name like my-db is NOT a + // parser identifier. Quoting the database component preserves the call + // when the stored SQL is parsed again. + return SPMPlan2SQLBuilder.quoteIdentifier(dbName) + "." + + SPMPlan2SQLBuilder.quoteIdentifier(name); + } + // a GLOBAL UDF without a database: quote the (possibly non-plain) name + return SPMPlan2SQLBuilder.quoteIdentifier(name); + } + return function.getName(); + } + + @Override + public String visitUnboundFunction(UnboundFunction unboundFunction, SQLRelation context) { + // UnboundFunction (a raw parsed function call, e.g. substr(ca_zip, 1, 5) in a + // WHERE clause before analysis). Render the children recursively so placeholder + // values inside the arguments stay normalized (_spm_const_var(id)); the + // default toSql fallback would render them with their embedded values and make + // the digest value-dependent. + String args = unboundFunction.children().stream() + .map(a -> a.accept(this, context)) + .collect(Collectors.joining(", ")); + // Keep the database qualifier: db1.f(k + 1) must not freeze as unqualified + // f(...) and resolve to db2.f when replayed under USE db2. + String dbName = unboundFunction.getDbName(); + String qualifiedName = (dbName == null || dbName.isEmpty()) + ? unboundFunction.getName() + : SPMPlan2SQLBuilder.quoteIdentifier(dbName) + "." + + SPMPlan2SQLBuilder.quoteIdentifier(unboundFunction.getName()); + return qualifiedName + "(" + args + ")"; + } + + @Override + public String visitAggregateExpression(AggregateExpression aggregateExpression, SQLRelation context) { + AggregateFunction fn = aggregateExpression.getFunction(); + if (fn instanceof GroupConcat || fn instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat(fn, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + return rendered; + } + // Explicitly handle DISTINCT aggregates: count(distinct col) + String distinct = fn.isDistinct() ? "distinct " : ""; + String args = renderArguments(true, fn.children(), context); + return functionName(fn) + "(" + distinct + args + ")"; + } + + @Override + public String visitWindow(WindowExpression windowExpression, SQLRelation context) { + // Hand-render the OVER clause so every slot reference (partition keys, order + // keys, function arguments) goes through the column mapping - Doris's own + // toSql prints raw slot names such as "GROUPING_PREFIX_s_county" or a local + // buffer name that is not referenceable in the decompiled SQL. + Expression fn = windowExpression.getFunction(); + String fnSql; + if (fn instanceof AggregateFunction) { + AggregateFunction aggFn = (AggregateFunction) fn; + if (aggFn instanceof GroupConcat || aggFn instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat(aggFn, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + fnSql = rendered; + } else { + String distinct = aggFn.isDistinct() ? "DISTINCT " : ""; + fnSql = functionName(aggFn) + "(" + distinct + + renderArguments(true, fn.children(), context) + ")"; + } + } else if (fn instanceof BoundFunction) { + fnSql = functionName((BoundFunction) fn) + "(" + + renderArguments(false, fn.children(), context) + ")"; + } else { + fnSql = print(fn, context); + } + StringBuilder sb = new StringBuilder(fnSql).append(" OVER ("); + List<String> parts = new ArrayList<>(); + if (!windowExpression.getPartitionKeys().isEmpty()) { + parts.add("PARTITION BY " + windowExpression.getPartitionKeys().stream() + .map(p -> print(p, context)).collect(Collectors.joining(", "))); + } + if (!windowExpression.getOrderKeys().isEmpty()) { + parts.add("ORDER BY " + windowExpression.getOrderKeys().stream() + .map(o -> print(o.child(), context) + (o.isAsc() ? " ASC" : " DESC") + + (o.isNullFirst() ? " NULLS FIRST" : " NULLS LAST")) + .collect(Collectors.joining(", "))); + } + // an explicit frame is only valid together with an ORDER BY clause; without one + // the SQL default frame (RANGE UNBOUNDED PRECEDING.. CURRENT ROW) is identical. + // + // LAG / LEAD are the exception: their SQL text must NOT carry a frame + // (WindowFunctionChecker.visitLag / #visitLead REJECT an explicit one and then + // install an INTERNAL rows frame on the window expression), so every captured + // plan carries a frame that the user never wrote. Emitting it froze SQL whose + // own re-analysis throws "WindowFrame for LAG must be null" - the baseline + // could never be replayed. The user-level absence of a frame is the faithful + // rendering. + if (!isFrameForbiddenWindowFunction(fn) + && !windowExpression.getOrderKeys().isEmpty() + && windowExpression.getWindowFrame().isPresent()) { + parts.add(windowExpression.getWindowFrame().get().computeToSql()); + } + return sb.append(String.join(" ", parts)).append(")").toString(); + } + + /** + * Whether the window function's SQL text must not carry an explicit frame (see + * the frame rendering in visitWindow): LAG / LEAD only. + */ + private static boolean isFrameForbiddenWindowFunction(Expression fn) { + if (fn instanceof Lag || fn instanceof Lead) { + return true; + } + String name = fn instanceof UnboundFunction ? ((UnboundFunction) fn).getName() : null; + return "lag".equalsIgnoreCase(name) || "lead".equalsIgnoreCase(name); + } + + // other common types. + + /** + * Renders a GROUP_CONCAT / MULTI_DISTINCT_GROUP_CONCAT into the dedicated + * GROUP_CONCAT grammar. + * + * Doris's analyzer admits ONE value expression next to an optional CONSTANT separator + * (the second non-order argument must be constant - a non-constant one is rejected by + * checkLegalityBeforeTypeCoercion), with an optional trailing ORDER BY. That covers + * every SQL-reachable shape: + * + * GROUP_CONCAT(DISTINCT v ORDER BY k): the physical plan carries the execution + * shape MultiDistinctGroupConcat(v, ORDER BY k) (GroupConcat.mustUseMultiDistinctAgg + * converts it). The DISTINCT form re-analyzes to exactly that node, so it is + * rendered - previously it returned null and forced the whole decompile to fall + * back to the user-supplied plan text; + * MULTI_DISTINCT_GROUP_CONCAT(v[, sep]): the function NAME carries the dedup + * contract while its isDistinct flag is false, so the DISTINCT keyword must be + * re-emitted - GROUP_CONCAT(v) would keep the duplicates ("1,1,2" instead of + * "1,2"). + * + * Parameter fn: the aggregate function (GroupConcat or MultiDistinctGroupConcat) + * Parameter context: the relation carrying the column mapping + * Returns: the SQL text, or null when the shape has no faithful rendering + */ + public String renderGroupConcat(AggregateFunction fn, SQLRelation context) { + return renderGroupConcat(fn, context, false); + } + + /** Renders GROUP_CONCAT with DISTINCT restored from an eliminated aggregate stage. */ + public String renderGroupConcat(AggregateFunction fn, SQLRelation context, boolean restoredDistinct) { + List<Expression> children = fn.children(); + int firstOrder = children.size(); + for (int i = 0; i < children.size(); i++) { + if (children.get(i) instanceof OrderExpression) { + firstOrder = i; + break; + } + } + List<Expression> values = children.subList(0, firstOrder); + List<Expression> orders = children.subList(firstOrder, children.size()); + // the dedicated grammar carries one value expression plus an optional CONSTANT + // separator; two NON-constant value arguments cannot be written down at all (the + // analyzer rejects them: "requires separator must be a constant"), so such a + // programmatically-built shape keeps failing the decompile instead of freezing + // SQL that cannot be re-parsed + boolean separatorArgument = values.size() == 2 && values.get(1).isConstant(); + if (values.isEmpty() || (values.size() > 1 && !separatorArgument)) { + return null; + } + // MultiDistinctGroupConcat is the execution shape of GROUP_CONCAT(DISTINCT...): + // its constructor keeps isDistinct = false, but the dedup contract must survive + // the freeze + boolean distinct = restoredDistinct || fn.isDistinct() || fn instanceof MultiDistinctGroupConcat; + StringBuilder sb = new StringBuilder("GROUP_CONCAT("); + if (distinct) { + sb.append("DISTINCT "); + } + sb.append(print(values.get(0), context)); + if (!orders.isEmpty()) { + // GROUP_CONCAT(DISTINCT v ORDER BY k) re-analyzes to the very + // MultiDistinctGroupConcat(v, k) execution shape it was rendered from, so the + // ORDER BY form is faithful; a verifier may also compare it against the + // alternative spelling MULTI_DISTINCT_GROUP_CONCAT(v ORDER BY k) + sb.append(" ORDER BY "); + sb.append(orders.stream().map(order -> { + OrderExpression orderExpression = (OrderExpression) order; + return print(orderExpression.child(), context) + + (orderExpression.isAsc() ? " ASC" : " DESC") + + (orderExpression.isNullFirst() ? " NULLS FIRST" : " NULLS LAST"); + }).collect(Collectors.joining(", "))); + } + if (separatorArgument) { + sb.append(" SEPARATOR ").append(print(values.get(1), context)); + } + return sb.append(")").toString(); + } + + @Override + public String visitCast(Cast cast, SQLRelation context) { + if (!cast.isExplicitType() && !preservesTypeConstructorFallback(cast)) { + // analyzer / optimizer inserted coercion CAST (type alignment, DECIMAL scale + // adjustment for division, placeholder typing,...) - an execution + // detail that would otherwise pin the result type/scale in the frozen planSql. + // Strip it so the replayed SQL analyzes the user-level expression and derives the + // same types/coercions as the original query (e.g. DECIMAL division keeps its full + // scale: "0.0286487413" instead of the coerced "0.02864874"). Explicit user + // CAST (isExplicitType = true) is preserved verbatim. The LOWERING's accumulator + // widening is preserved by renderArgument, which is the only place that knows the + // cast is the ROOT of an aggregate argument (see preservesLoweredAccumulator). + return cast.child().accept(this, context); + } + return "CAST(" + cast.child().accept(this, context) + " AS " + cast.getDataType().toSql() + ")"; + } + + /** + * Whether an ANALYZER-inserted cast is part of the frozen plan's SEMANTICS rather than an + * execution detail that may be stripped. + * + * The AVG lowering (AVG_DISTINCT_TO_SUM_DIV_COUNT) rewrites avg(DISTINCT k) into + * sum(DISTINCT CAST(k AS LARGEINT)) / count(...), and that widening IS the LARGEINT + * accumulator AVG computes in. Stripping it made the frozen SQL re-analyze + * SUM(DISTINCT k) with a BIGINT accumulator, so two distinct values 5e18 and 6e18 - which + * the original AVG averages to 5.5e18 - overflowed the frozen replay's sum, even for the + * identical SQL (the CREATE probe only RE-PLANS the text, it never executes it). + * + * The accumulator guarantee only exists at the ROOT of an aggregate's argument + * (see renderArgument): an operand of ordinary arithmetic + * (k + 9223372036854775808) carries the same shape, but there it is plain type + * coercion - keeping it pins the CAPTURED operand type and makes a literal of another type + * replay under the wrong coercion (the + 0.25 variant truncated to an integer). + * + * Only an integral WIDENING over a REAL value qualifies: a cast over a substituted + * placeholder / literal is exactly the type pinning the stripping exists to avoid, and a + * narrowing or same-width cast carries no accumulator guarantee. + * + * Parameter cast: the analyzer-inserted cast + * Returns: true when the cast must survive into the frozen SQL + */ + private boolean preservesLoweredAccumulator(Cast cast) { + Expression child = cast.child(); + try { + DataType childType = child.getDataType(); + DataType targetType = cast.getDataType(); + // LARGEINT is the SUM / AVG accumulator: the lowering of avg(DISTINCT expr) writes + // exactly this widening, and re-deriving SUM over the SMALLER type is what + // overflows. An ordinary arithmetic widening (INT -> BIGINT and friends) is an + // execution detail the strip exists for, so it is NOT kept - it would only pin + // coercions the re-analyzed user expression derives by itself. + // + // The child must read at least one REAL column: AVG(DISTINCT k + 1) lowers to + // sum(CAST((k + ph) AS LARGEINT)), and skipping the cast just because a placeholder + // sits inside the arithmetic froze SUM over BIGINT again. A child built ONLY from + // placeholders (ph1 + ph2 widened to INT) stays stripped - there the cast pins + // the CAPTURED literals' width on the caller's values. + return childType.isIntegralType() && targetType instanceof LargeIntType + && targetType.width() > childType.width() + && child.containsType(SlotReference.class); + } catch (RuntimeException e) { + // an unbound child has no type here: keeping the strip decision is the + // historical behavior and cannot break a bound plan + return false; + } + } + + /** + * Whether the cast IS the parser's TYPE-CONSTRUCTOR fallback. + * + * DATE 'x' / TIMESTAMP 'x' folds to a typed literal when x PARSES, and to + * CAST(x AS DATE|DATETIME) when it does not (LogicalPlanBuilder.visitTypeConstructor). + * That cast carries the user's TYPE - SELECT DATE 'not-a-date' executes as a typed DATE + * NULL - so stripping it froze Project(ph1 AS d) and the replay returned the raw STRING + * 'not-a-date' (the CREATE probe only re-plans the text, so it accepted that projection). + * + * The detector is the fallback's own cause: the literal does NOT parse as the target type. + * An ordinary implicit coercion of a VALID literal (d = '2020-01-01' also becomes + * CAST('2020-01-01' AS DATE)) still parses, so it is still stripped and re-derived by the + * replay's own analysis exactly as before - which keeps the frozen text of the ordinary + * date-filter shape unchanged. + * + * Parameter cast: the parser-inserted cast + * Returns: true when the cast must survive into the frozen SQL + */ + private static boolean preservesTypeConstructorFallback(Cast cast) { + String text = constantText(cast.child()); + if (text == null) { + return false; + } + DataType target = cast.getDataType(); + if (target == null) { + return false; + } + return !parsesAs(target, text); + } + + /** + * Returns the string value of a string literal, unwrapping a scalar marker payload when present. Returns null + * for other expressions. Used to recognize capture-time date and time constructor fallbacks; a replay can + * supply a different payload value. + */ + private static String constantText(Expression expr) { + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstVar) { + return constantText( + ((org.apache.doris.nereids.spm.placeholder.SpmConstVar) expr).getValue()); + } + if (expr instanceof StringLikeLiteral) { + return ((StringLikeLiteral) expr).getStringValue(); + } + return null; + } + + /** + * Whether the text is a LEGAL value of the target date / time type, i.e. whether the + * parser would have folded the constructor instead of emitting the fallback cast. + * + * Parameter target: the cast's target type + * Parameter text: the literal text + * Returns: whether the text parses as that type (true for any other type) + */ + private static boolean parsesAs(DataType target, String text) { + try { + if (target instanceof DateV2Type) { + new DateV2Literal(text); + return true; + } + if (target instanceof DateType) { + new DateLiteral(text); + return true; + } + if (target instanceof DateTimeV2Type) { + new DateTimeV2Literal(text); + return true; + } + if (target instanceof DateTimeType) { + new DateTimeLiteral(text); + return true; + } + } catch (RuntimeException e) { + return false; + } + return true; + } + + @Override + public String visitTryCast(TryCast tryCast, SQLRelation context) { + // toSql renders a grammar-parseable type name (e.g. STRUCT<name:type>); + // toString emits a diagnostic form (STRUCT<StructField[...]>) that cannot be + // re-parsed by the frozen SQL after a reload. + return "TRY_CAST(" + tryCast.child().accept(this, context) + " AS " + + tryCast.getDataType().toSql() + ")"; + } + + @Override + public String visitOrderExpression(OrderExpression orderExpression, SQLRelation context) { + String child = orderExpression.child().accept(this, context); + return child + (orderExpression.isAsc() ? " ASC" : " DESC"); + } + + @Override + public String visitAlias(Alias alias, SQLRelation context) { + return alias.child().accept(this, context); + } + + @Override + public String visitCaseWhen(CaseWhen caseWhen, SQLRelation context) { + // CASE WHEN... THEN... ELSE... END + StringBuilder sb = new StringBuilder("CASE"); + for (WhenClause whenClause : caseWhen.getWhenClauses()) { + sb.append(" WHEN ").append(whenClause.getOperand().accept(this, context)) + .append(" THEN ").append(whenClause.getResult().accept(this, context)); + } + if (caseWhen.getDefaultValue().isPresent()) { + sb.append(" ELSE ").append(caseWhen.getDefaultValue().get().accept(this, context)); + } + sb.append(" END"); + return sb.toString(); + } + + @Override + public String visitLiteral(Literal literal, SQLRelation context) { + if (literal instanceof StringLikeLiteral) { + // Generator arguments (LATERAL VIEW / UNNEST) are intentionally NOT + // parameterized and reach this method as concrete literals. Their toSql + // (StringLikeLiteral.toString) only wraps the SEMANTIC value in quotes, so + // a delimiter such as a'b froze invalid split(t.s, 'a'b'): the stored + // baseline then had no parseable fallback tree and stopped applying after a + // reload. Render through the default-mode-safe quote helper (stored text is + // always re-parsed under MODE_DEFAULT). + return SPMPlan2SQLBuilder.quoteSqlString(((StringLikeLiteral) literal).getStringValue()); + } + if (literal instanceof DateLiteral || literal instanceof TimeV2Literal) { + // Concrete temporal literals (for example in a star REPLACE) must retain + // their SQL type and precision when the frozen statement is parsed again. + return "CAST(" + SPMPlan2SQLBuilder.quoteSqlString(literal.getStringValue()) + + " AS " + literal.getDataType().toSql() + ")"; + } + return literal.toSql(); Review Comment: [P2] Preserve nested literal types when freezing containers On `t(k INT,arr ARRAY<DATE>)`, capture and replay `SELECT * REPLACE([DATE '2020-01-01'] AS arr) FROM t`: ```text Capture: Project[k, ArrayLiteral<DATE> AS arr] -> Scan(t) Replay: Project[k, ArrayLiteral<STRING> AS arr] -> Scan(t) ``` REPLACE keeps this literal concrete. This fallback calls `ArrayLiteral.toSql()`, which invokes each inner literal's own `toSql()` and bypasses the temporal CAST branch above. Frozen `['2020-01-01']` therefore changes the result element type. The planning probe accepts it; fingerprints and label alignment do not check result types. This is the composite-literal branch, separate from the repaired direct temporal leaf. Render nested elements through the safe visitor or decline freezing the container; nested escaped strings also bypass the safe quoting branch. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
