github-actions[bot] commented on code in PR #68499: URL: https://github.com/apache/doris/pull/68499#discussion_r4638031062
########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/SPMForwardedSession.java: ########## @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.spm.manager.SessionBaselineStore; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SqlModeHelper; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Serializes enabled SESSION baselines for a forwarded statement and rebuilds them in the receiving context. Query + * statements carry candidates selected by hash and digest; other statements carry all enabled rows. Imported rows + * are request-context state, not GLOBAL baselines. + */ +public final class SPMForwardedSession { + + /** + * Character budget for the serialized baseline payload. SessionBaselineStore applies the same accounting when + * admitting rows, including disabled rows that may later be enabled. This character limit is not a byte count + * for the enclosing transport request. + */ + public static final int MAX_PAYLOAD_CHARS = 8 * 1024 * 1024; + + /** Characters used by the JSON array enclosure, included in both admission and serialization accounting. */ + public static final int PAYLOAD_ENCLOSURE_CHARS = 2; + + private static final Logger LOG = LogManager.getLogger(SPMForwardedSession.class); + + private SPMForwardedSession() { + } + + /** + * Conservative payload charge for one row: its JSON length plus one separator character. An array with rows + * needs one fewer separator than the sum of these charges. + */ + public static int payloadRowChars(BaselinePlan plan) { + return new Gson().toJson(toPayloadRow(plan)).length() + 1; + } + + /** + * Serializes the ENABLED rows of the session store. The store rejects a creation that + * would exceed MAX_PAYLOAD_CHARS (see SessionBaselineStore), so every + * enabled row is carried and the rewrite context is IDENTICAL on this FE and on the + * master that receives the forwarded statement. Reaching the budget here means the + * store invariant broke (a row was registered without the check): fail the statement + * loudly instead of silently planning it without a baseline the session owns. + * + * Parameter store: the connection's session store (may be null) + * Returns: the JSON payload, or "" when there is nothing to carry + * Throws IllegalStateException: when the enabled rows exceed the payload budget + */ + public static String serialize(SessionBaselineStore store) { + if (store == null || store.isEmpty()) { + return ""; + } + return serializeEnabled(store.getAllBaselines(), null); + } + + /** + * Serializes enabled rows with the statement match key when queryMatchKey can compute one. If no key is + * available, serializes all enabled rows. Disabled rows are always omitted, including for management + * statements. + */ + public static String serializeForStatement(SessionBaselineStore store, ConnectContext ctx, + String sql) { + if (store == null || store.isEmpty()) { + return ""; + } + Pair<String, Long> matchKey = SPMPlanner.queryMatchKey(ctx, sql); + if (matchKey == null) { + return serialize(store); + } + return serializeEnabled(store.getAllBaselines(), matchKey); + } + + /** + * The JSON of the ENABLED rows, restricted to the given match key when one is + * supplied (a row can only be matched on its (hash, digest) pair, so the restriction + * preserves the rewrite context of the statement while leaving every non-matching + * row out of the master's reconstruction). The payload budget is enforced on the + * selected rows alone: a subset of what serialize would carry can only be smaller. + */ + private static String serializeEnabled(List<BaselinePlan> plans, Pair<String, Long> matchKey) { + Gson gson = new Gson(); + List<Map<String, String>> rows = new ArrayList<>(); + long payloadChars = PAYLOAD_ENCLOSURE_CHARS; // the enclosing [] + for (BaselinePlan plan : plans) { + if (plan.getStatus() != BaselineStatus.ENABLED) { Review Comment: [P2] Forward disabled SESSION rows for management DDL. The no-match-key path is used for ALTER/DROP/SHOW, and FEOpExecutor documents that it carries the full session store, but `serializeEnabled` skips this row (and every disabled row). `importInto` first clears the receiver, so a forwarded `ALTER ... <disabled-id> ENABLE` or DROP cannot resolve the connection's baseline. Include disabled rows for management statements while keeping enabled-only filtering for query rewrites. ########## fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java: ########## @@ -210,7 +210,34 @@ protected TMasterOpRequest buildStmtForwardParams() throws AnalysisException { } // session variables - params.setSessionVariables(ctx.getSessionVariable().getForwardVariables()); + // SESSION-scope SPM baselines belong to THIS connection and live only in this + // FE's store: attach the enabled rows the forwarded statement could MATCH + // (rebuilt from the store HERE, so a client SET cannot inject rows, and filtered + // by the statement's own (hash, digest) match key) - the master otherwise plans + // the forwarded statement in a fresh context with an empty store and silently + // ignores the very baseline this connection created. Statements that are not + // plan-rewritable queries (DDL, batches) carry the full store: the management + // commands address rows by id on the master. With rewrite disabled the master can + // never consult a baseline, so nothing is shipped: serializing every enabled row + // would add unbounded (re)parse work to each forwarded statement of such a + // connection. + if (ctx.getSessionVariable().isEnableSpmRewrite()) { + ctx.getSessionVariable().setSpmForwardedSessionBaselines( + org.apache.doris.nereids.spm.SPMForwardedSession.serializeForStatement( + ctx.getSessionBaselineStore(), ctx, originStmt.originStmt)); + } else { + ctx.getSessionVariable().setSpmForwardedSessionBaselines(""); Review Comment: [P2] Preserve SESSION baselines for management forwarding when rewrite is disabled. This branch sends an empty payload whenever `enable_spm_rewrite` is false, but the receiver clears its session store before `importInto` handles that payload. An observer-side ALTER/DROP/SHOW SESSION BASELINE PLAN therefore cannot find even enabled rows. Forward the store for management statements independently of query rewrite, or execute those DDLs on the owning FE. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/builder/SPMPlan2SQLBuilder.java: ########## @@ -0,0 +1,4674 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.builder; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.DistributionSpec; +import org.apache.doris.nereids.properties.DistributionSpecReplicated; +import org.apache.doris.nereids.trees.TableSample; +import org.apache.doris.nereids.trees.expressions.AggregateExpression; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctGroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingId; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.AggPhase; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.LimitPhase; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeFileScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeTVFScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; +import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnionAnchor; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnionProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWorkTableReference; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Decompiles supported physical plans into SQL that can be parsed and planned for baseline replay. SQL blocks and + * distribution hints retain selected plan structure; replay still runs ordinary analysis and optimization. + * Unsupported operators or expression forms raise UnsupportedOperationException so baseline creation can retain + * the original plan SQL. + * + * The renderer tracks ExprId references, exported column names, CTE definitions and live output columns. + * Execution-only stages are folded where supported, while wrappers preserve semantic clause boundaries and + * assertions. + */ +public class SPMPlan2SQLBuilder extends PlanVisitor<SQLRelation, Void> { + + private static final Logger LOG = LogManager.getLogger(SPMPlan2SQLBuilder.class); + + /** Generated output names ("c_" + position) produced by re-export labels and dedupe renames. */ + private static final Pattern GENERATED_NAME_PATTERN = Pattern.compile("\\bc_\\d+\\b"); + + /** JOIN distribution HINT prefix constants. */ + private static final String HINT_JOIN_BROADCAST = "BROADCAST"; + private static final String HINT_JOIN_SHUFFLE = "SHUFFLE"; + + /** + * Re-export labels a hoist step created one level below ("X AS c_N" items): maps the + * generated label back to the ExprId of the item it re-exports, so a later hoist step + * can re-export the SAME label through one more projection level (tpcds q78: the + * ORDER BY hoisted out of the 16-item projection references c_14/c_15/c_16, and the + * 11-item projection above it must therefore re-export them). + */ + private final Map<String, ExprId> reExportedLabels = new HashMap<>(); + + /** Expression printer (carries the columnNames mapping of SQLRelation). */ + private final SPMExprSqlBuilder exprSqlBuilder = new SPMExprSqlBuilder(); + + /** + * Per-decompile alias sequence for LATERAL VIEW clauses whose output slot has no + * user qualifier (reset in toSQL). + */ + private int lateralViewSeq = 0; + + /** + * CTE body relations registered by the producer and referenced through generated WITH aliases by consumers. + */ + private final Map<CTEId, SQLRelation> cteBodies = new HashMap<>(); + + /** CTEId -> the generated alias (t_N) that references the WITH definition. */ + private final Map<CTEId, String> cteAliases = new HashMap<>(); + + /** + * The WITH entries (alias AS (body)) in producer visit order - the definition + * order of the emitted WITH clause. They are attached to the ROOT relation in + * toSQL: every consumer lies inside the statement, so the definitions are + * visible everywhere regardless of how deeply the CTE anchors are nested in the + * physical tree (attaching at each anchor instead would place a definition inside + * one join branch while a consumer sits in another branch). Producers are visited + * in dependency order - a consumer requires its producer to be registered already - + * so the list is a valid SQL definition order (a CTE used by another CTE is + * defined before it). + */ + private final List<String> cteDefinitions = new ArrayList<>(); + + /** + * Recursive CTE output column names keyed by CTE name. Registered from the anchor + * branch of a PhysicalRecursiveUnion; used to name the recursive work-table self + * reference and the outer reference so the decompiled WITH RECURSIVE stays + * re-parseable. + */ + private final Map<String, List<String>> recursiveCteColumns = new HashMap<>(); + + /** + * Local (partial) aggregate output column -> the aggregate input expression it + * aggregates, e.g. local partial_sum(x)#N registers N -> x. A global + * aggregate references such a column as sum(partial_sum(x)#N); the global + * decompile rewrites it back to sum(x) so the decompiled SQL stays a single + * logical aggregate (no partial_ intermediate functions). + */ + private final Map<ExprId, Expression> localAggParams = new HashMap<>(); + + /** + * Buffer columns produced by the distinct-dedup branch: their defining partial + * aggregate consumed a DATA column (e.g. DISTINCT_LOCAL's partial_count(key)), + * not another partial buffer. In a DISTINCT_GLOBAL merge stage a count(...) over + * such a buffer is the user's count(DISTINCT key). + * + * A plan may mix plain aggregates with a distinct one (TPCDS q28: + * avg(x), count(x), count(DISTINCT x)): the plain count rides along the same + * DISTINCT_GLOBAL stage, but its buffer is a merge chain (its defining partial + * aggregated another partial buffer, e.g. partial_count(partial_count(x))), so it + * must NOT be rendered with DISTINCT. Both chains otherwise resolve to the same + * data column, which is why the provenance has to be recorded while the + * intermediate stages are eliminated. + */ + private final Set<ExprId> distinctMergeBuffers = new HashSet<>(); + + /** + * Table id to (partition id to the name the CAPTURE resolved), snapshotted while the + * planning locks were held (see SPMOptimizer.OptimizeResult.getPartitionNames). Empty when + * the caller has none (a direct / test-side decompile). + */ + private final Map<Long, Map<Long, org.apache.doris.nereids.spm.SPMPlanTreeSupport.PartitionPin>> + capturedPartitionNames; + + /** + * Outermost projection reached through unary nodes, stopping at an aggregate or branching node. It emits the + * final project list. Intermediate projections carry child references and their own expressions, filtered by + * live-column analysis when available. + */ + private final Set<Plan> outputProjects = + Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + + /** + * Identity map: physical plan node -> the ExprIds of its output columns that upper + * layers actually consume (the final result columns plus every column referenced by + * a decompiled expression above). An entry ABSENT, or mapped to null, means "no + * pruning for this node and everything below it" (the pre-pruning behaviour). + * + * The decompiled planSql inflates when every intermediate projection re-emits the + * whole child column set (a deep TPCDS join stack re-lists hundreds of c_N columns + * per level). This map drives live-column pruning: each SELECT list is filtered to + * the live columns only, so a column that is never referenced above and is not part + * of the final result disappears from every intermediate projection. Equivalence is + * preserved because only dead columns are dropped. + */ + private final Map<Plan, Set<ExprId>> neededOutputs = + new java.util.IdentityHashMap<>(); + + /** + * CTEId -> the CTE body output columns referenced by every inlined consumer (the + * consumer output slots mapped back to their producer slots). Filled while the main + * query is propagated top-down; the CTE producer subtree is propagated afterwards + * with exactly this column set, so a big CTE body (a deep TPCDS join stack) only + * keeps the columns its consumers actually read. + */ + private final Map<CTEId, Set<ExprId>> cteConsumerNeeds = new HashMap<>(); + /** + * CTE ids with at least one consumer that must keep every body column (see + * propagateNeed): a consumer under a NON-prunable node (e.g. a UNION branch) receives + * need = null, and without this obligation the producer was pruned to the OTHER + * consumer's columns alone while the union branch still emitted its y reference - + * an invalid frozen text that the creation probe accepted whenever an IN-list marker + * skipped validation. + */ + private final Set<CTEId> cteKeepAllConsumers = new HashSet<>(); + + /** + * Per-decompile generated-column-name state: when a decompiled output column has no + * clean user alias it is exported under a generated c_N name (c_1, c_2,...), + * assigned in decompile order and memoized by the output ExprId so every reference + * to the same column prints the same alias. A fresh per-decompile sequence (instead + * of embedding the analyzer's ExprId) keeps the generated names small, readable and + * independent of how many internal ExprIds the optimizer allocated. + */ + private final Map<ExprId, String> generatedColumnNames = new HashMap<>(); + private int generatedColumnSeq = 0; + + /** + * Normalized names already visible in the CURRENT decompile: user aliases and the + * preserved source columns of the relations a generated name gets exported next to. + * A generated c_N must avoid them - the counter alone only keeps the + * GENERATED names apart, so a source column literally named c_1 (or a group-by + * item / user alias of that name) could end up next to sum(v) AS c_1 or + * MARK_SLOT c_1: two ExprIds registered under one visible name make the + * enclosing projection / result sink read an AMBIGUOUS column from the derived + * relation after reload. Project / window outputs repair duplicates afterwards + * (dedupeSelectOutputNames); the join (MARK_SLOT / explicit projection) and aggregate + * exports do not, so they reserve here instead. + */ + private final Set<String> reservedOutputNames = new HashSet<>(); + + /** + * Builds a renderer WITHOUT an under-lock partition-name snapshot (direct / test-side + * decompile): a pinned partition is then named from the live catalog. + */ + public SPMPlan2SQLBuilder() { + this(java.util.Map.of()); + } + + /** + * Builds a renderer carrying the partition names the CAPTURE resolved while the planning locks + * were held (see SPMOptimizer.OptimizeResult.getPartitionNames). + * + * Parameter partitionNames: table id to (partition id to the ADMITTED pin), or null + */ + public SPMPlan2SQLBuilder( + Map<Long, Map<Long, org.apache.doris.nereids.spm.SPMPlanTreeSupport.PartitionPin>> + partitionNames) { + this.capturedPartitionNames = partitionNames == null + ? java.util.Map.of() : partitionNames; + } + + /** Fuses freeze guards while retaining session, subquery, reserved-name error precedence. */ + private static void rejectUnsupportedExpressions(Plan plan) { + Expression[] unsupported = new Expression[2]; + org.apache.doris.nereids.spm.SPMPlanTreeSupport.<RuntimeException>walkPlans(plan, node -> { + for (Expression expr : node.getExpressions()) { + inspectFreezeExpression(expr, expr, unsupported); + } + }); + if (unsupported[0] != null) { + throw new UnsupportedOperationException("SPM cannot freeze an expression" + + " carrying a subquery (its SQL has no subquery rendering): " + unsupported[0]); + } + if (unsupported[1] != null) { + throw new UnsupportedOperationException("SPM cannot freeze an expression" + + " that calls the reserved internal name of a subquery marker: " + unsupported[1]); + } + } + + private static void inspectFreezeExpression(Expression expression, Expression root, + Expression[] unsupported) { + if (expression instanceof org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr) { + throw new UnsupportedOperationException("SPM cannot freeze an expression" + + " carrying a session-variable guard: " + root); + } + if (unsupported[0] == null + && expression instanceof org.apache.doris.nereids.trees.expressions.SubqueryExpr) { + unsupported[0] = root; + } + if (unsupported[1] == null && expression instanceof Function + && SPMExprSqlBuilder.INTERNAL_PSEUDO_FUNCTION_NAMES.contains( + ((Function) expression).getName().toLowerCase(java.util.Locale.ROOT))) { + unsupported[1] = root; + } + for (Expression child : expression.children()) { + inspectFreezeExpression(child, root, unsupported); + } + } + + /** + * Marks the OUTERMOST projection of the decompiled tree: the first PhysicalProject + * reached from the root along single-child pass-through nodes (ResultSink / Sort / + * Distribute /...). Walking stops at an aggregate - an aggregate that feeds the + * result carries the final SELECT list itself, so there is no outermost projection + * and every projection below it is an intermediate one (full child output). + */ + private void markOutputProject(Plan plan) { + Plan current = plan; + while (current != null) { + if (current instanceof PhysicalProject) { + outputProjects.add(current); + return; + } + if (current instanceof PhysicalHashAggregate || current.arity() != 1) { + return; + } + current = current.child(0); + } + } + + // live-column analysis (dead-column pruning). + + /** + * Whether live-column requirements can be propagated through this node. Supported linear, join, aggregate and + * CTE-anchor shapes propagate requirements; other nodes retain all descendant columns. CTE consumers record + * producer-column requirements separately. + */ + private static boolean isPrunableNode(Plan node) { + return node instanceof PhysicalProject + || node instanceof PhysicalWindow + || node instanceof PhysicalHashJoin + || node instanceof PhysicalNestedLoopJoin + || node instanceof PhysicalHashAggregate + || node instanceof PhysicalFilter + || node instanceof PhysicalTopN + || node instanceof PhysicalQuickSort + || node instanceof PhysicalLimit + || node instanceof PhysicalDistribute + || node instanceof PhysicalLazyMaterialize + || node instanceof PhysicalLazyMaterializeOlapScan + || node instanceof PhysicalPartitionTopN + || node instanceof PhysicalAssertNumRows + || node instanceof PhysicalResultSink + || node instanceof PhysicalCTEAnchor; + } + + /** Pre-analysis entry: fills neededOutputs top-down from the root. */ + private void computeNeeded(Plan root) { + neededOutputs.clear(); + cteConsumerNeeds.clear(); + cteKeepAllConsumers.clear(); + Set<ExprId> rootNeed = new HashSet<>(); + for (Slot slot : root.getOutput()) { + rootNeed.add(slot.getExprId()); + } + propagateNeed(root, rootNeed); + } + + /** + * Top-down live-column propagation. need is the set of output ExprIds of + * node that upper layers consume; null means "keep everything" (never prune + * below). Every parent hands each child the child's output columns that must stay + * alive: the columns the parent passes through and the columns the parent's own + * decompiled expressions reference. + */ + private void propagateNeed(Plan node, Set<ExprId> need) { + if (node == null) { + return; + } + if (node instanceof PhysicalCTEConsumer) { + // a CTE consumer has no children; record which body columns its live output + // columns map back to, so the producer subtree can be pruned to exactly them. + // need = null means "keep everything": the consumer sits under a non-prunable + // node (a UNION branch, a recursive union), so it must record a KEEP-ALL + // obligation - otherwise the producer was pruned to the OTHER consumers' + // columns while this consumer still referenced its own (see + // cteKeepAllConsumers). + PhysicalCTEConsumer consumer = (PhysicalCTEConsumer) node; + if (need == null) { + cteKeepAllConsumers.add(consumer.getCteId()); + return; + } + Set<ExprId> bodyNeeds = cteConsumerNeeds.computeIfAbsent( + consumer.getCteId(), k -> new HashSet<>()); + for (Slot slot : consumer.getOutput()) { + if (need.contains(slot.getExprId())) { + bodyNeeds.add(consumer.getProducerSlot(slot).getExprId()); + } + } + return; + } + if (need == null || !isPrunableNode(node)) { + // keep everything on this node and below (no pruning boundary) + for (Plan child : node.children()) { + propagateNeed(child, null); + } + return; + } + Set<ExprId> nodeNeed = new HashSet<>(need); + neededOutputs.put(node, nodeNeed); + + if (node instanceof PhysicalCTEAnchor) { + // child(0) is the CTE producer whose body every consumer inlines. The main + // query (child(1)) is propagated FIRST so every consumer records which body + // columns it reads; the body subtree is then pruned to exactly those columns. + // A body with no consumer (or one whose consumers never resolve) keeps every + // column. + PhysicalCTEAnchor<? extends Plan, ? extends Plan> anchor = + (PhysicalCTEAnchor<? extends Plan, ? extends Plan>) node; + if (anchor.child(1) != null) { + propagateNeed(anchor.child(1), nodeNeed); + } + Plan producer = anchor.child(0); + if (producer == null) { + return; + } + if (producer instanceof PhysicalCTEProducer) { + PhysicalCTEProducer<? extends Plan> cteProducer = + (PhysicalCTEProducer<? extends Plan>) producer; + Set<ExprId> bodyNeed = cteConsumerNeeds.get(cteProducer.getCteId()); + if (cteKeepAllConsumers.contains(cteProducer.getCteId())) { + // at least one consumer keeps every column: the body must NOT be + // pruned (a UNION branch's y reference would otherwise dangle) + propagateNeed(cteProducer.child(0), null); + } else if (bodyNeed != null && !bodyNeed.isEmpty()) { + propagateNeed(cteProducer.child(0), bodyNeed); + } else { + propagateNeed(cteProducer.child(0), null); + } + } else { + propagateNeed(producer, null); + } + return; + } + if (node instanceof PhysicalProject) { + propagateProjectNeed((PhysicalProject<? extends Plan>) node, nodeNeed); + } else if (node instanceof PhysicalHashJoin || node instanceof PhysicalNestedLoopJoin) { + propagateJoinNeed((AbstractPhysicalJoin<? extends Plan, ? extends Plan>) node, nodeNeed); + } else if (node instanceof PhysicalHashAggregate) { + propagateAggregateNeed((PhysicalHashAggregate<? extends Plan>) node, nodeNeed); + } else if (node instanceof PhysicalWindow) { + propagateWindowNeed((PhysicalWindow<? extends Plan>) node, nodeNeed); + } else if (node instanceof PhysicalFilter) { + PhysicalFilter<? extends Plan> filter = (PhysicalFilter<? extends Plan>) node; + Set<ExprId> childNeed = new HashSet<>(nodeNeed); + addExprSlots(filter.getPredicate(), childNeed); + propagateNeed(filter.child(0), childNeed); + } else if (node instanceof PhysicalTopN) { + PhysicalTopN<? extends Plan> topN = (PhysicalTopN<? extends Plan>) node; + Set<ExprId> childNeed = new HashSet<>(nodeNeed); + for (org.apache.doris.nereids.properties.OrderKey key : topN.getOrderKeys()) { + addExprSlots(key.getExpr(), childNeed); + } + propagateNeed(topN.child(0), childNeed); + } else if (node instanceof PhysicalQuickSort) { + PhysicalQuickSort<? extends Plan> sort = (PhysicalQuickSort<? extends Plan>) node; + Set<ExprId> childNeed = new HashSet<>(nodeNeed); + for (org.apache.doris.nereids.properties.OrderKey key : sort.getOrderKeys()) { + addExprSlots(key.getExpr(), childNeed); + } + propagateNeed(sort.child(0), childNeed); + } else { + // pure pass-through nodes (limit / distribute / lazy materialize /...): + // their output slots are the child slots, so the same need flows down + for (Plan child : node.children()) { + propagateNeed(child, nodeNeed); + } + } + } + + /** + * PhysicalProject: live pass-through columns plus the columns referenced by the + * project's own expressions whose output is itself live. + */ + private void propagateProjectNeed(PhysicalProject<? extends Plan> project, Set<ExprId> need) { + Plan child = project.child(0); + if (child == null) { + return; + } + Set<ExprId> childNeed = new HashSet<>(); + boolean finalProject = outputProjects.contains(project); + if (finalProject) { + // the outermost projection emits its full projection list: every referenced + // child column must stay alive + for (NamedExpression projectExpr : project.getProjects()) { + addExprSlots(projectExpr, childNeed); + } + } else { + // pass-through: the child columns that are still live above this projection + Set<ExprId> childOut = outputIdSet(child); + for (ExprId id : need) { + if (childOut.contains(id)) { + childNeed.add(id); + } + } + // plus the columns referenced by this projection's own live expressions + for (NamedExpression projectExpr : project.getProjects()) { + if (need.contains(projectExpr.getExprId())) { + addExprSlots(projectExpr, childNeed); + } + } + } + propagateNeed(child, childNeed); + } + + /** + * Join: the preserved output columns flow to the side that produces them; the + * hash / other / mark conjuncts are always decompiled, so every column they + * reference stays alive on its own side. + */ + private void propagateJoinNeed(AbstractPhysicalJoin<? extends Plan, ? extends Plan> join, + Set<ExprId> need) { + Plan left = join.left(); + Plan right = join.right(); + if (left == null || right == null) { + return; + } + Set<ExprId> leftOut = outputIdSet(left); + Set<ExprId> rightOut = outputIdSet(right); + Set<ExprId> conjRefs = new HashSet<>(); + for (Expression conjunct : join.getHashJoinConjuncts()) { + addExprSlots(conjunct, conjRefs); + } + for (Expression conjunct : join.getOtherJoinConjuncts()) { + addExprSlots(conjunct, conjRefs); + } + for (Expression conjunct : join.getMarkJoinConjuncts()) { + addExprSlots(conjunct, conjRefs); + } + Set<ExprId> leftNeed = new HashSet<>(); + Set<ExprId> rightNeed = new HashSet<>(); + for (ExprId id : need) { + if (leftOut.contains(id)) { + leftNeed.add(id); + } + if (rightOut.contains(id)) { + rightNeed.add(id); + } + } + for (ExprId id : conjRefs) { + if (leftOut.contains(id)) { + leftNeed.add(id); + } + if (rightOut.contains(id)) { + rightNeed.add(id); + } + } + propagateNeed(left, leftNeed); + propagateNeed(right, rightNeed); + } + + /** + * Aggregate: the global stage keeps its whole output list (GROUP BY keys and + * aggregate functions), but the input columns its expressions reference must stay + * alive below. Local / intermediate execution stages are pass-through nodes. + */ + private void propagateAggregateNeed(PhysicalHashAggregate<? extends Plan> agg, Set<ExprId> need) { + AggPhase phase = agg.getAggPhase(); + if (phase.isLocal() || isIntermediateAggStage(agg)) { + for (Plan child : agg.children()) { + propagateNeed(child, need); + } + return; + } + Plan child = agg.child(0); + if (child == null) { + return; + } + Set<ExprId> childNeed = new HashSet<>(); + for (Expression groupBy : agg.getGroupByExpressions()) { + addExprSlots(groupBy, childNeed); + } + for (NamedExpression output : agg.getOutputExpressions()) { + addAggregateOutputRefs(output, childNeed); + } + propagateNeed(child, childNeed); + } + + /** + * Window: live pass-through columns plus the input columns of the live window + * expressions. + */ + private void propagateWindowNeed(PhysicalWindow<? extends Plan> window, Set<ExprId> need) { + Plan child = window.child(0); + if (child == null) { + return; + } + Set<ExprId> childNeed = new HashSet<>(); + Set<ExprId> childOut = outputIdSet(child); + for (ExprId id : need) { + if (childOut.contains(id)) { + childNeed.add(id); + } + } + for (NamedExpression windowExpr : window.getWindowExpressions()) { + if (need.contains(windowExpr.getExprId())) { + addExprSlots(windowExpr, childNeed); + } + } + propagateNeed(child, childNeed); + } + + /** + * The columns referenced by one global-aggregate output expression, with every + * local partial-buffer reference resolved down to its data columns (mirrors + * appendAggSelect / resolveBufferSlots so the pruned SELECT lists keep exactly the + * columns the decompiled aggregate prints). + */ + private void addAggregateOutputRefs(NamedExpression output, Set<ExprId> out) { + Expression inner = output instanceof Alias ? ((Alias) output).child() : output; + if (inner instanceof SlotReference) { + // group-by key pass-through + out.add(((SlotReference) inner).getExprId()); + return; + } + if (inner instanceof AggregateExpression) { + AggregateExpression aggExpr = (AggregateExpression) inner; + List<Expression> args = aggExpr.getFunction().children().isEmpty() + ? new ArrayList<>(aggExpr.children()) : aggExpr.getFunction().children(); + for (Expression arg : args) { + addResolvedExprSlots(arg, out); + } + return; + } + addExprSlots(inner, out); + } + + /** + * Collects every slot of expr, resolving partial-buffer slots through + * localAggParams down to their data columns (mirrors resolveBufferSlots). + */ + private void addResolvedExprSlots(Expression expr, Set<ExprId> out) { + if (expr instanceof SlotReference) { + Expression param = localAggParams.get(((SlotReference) expr).getExprId()); + if (param != null) { + addResolvedExprSlots(param, out); + return; + } + out.add(((SlotReference) expr).getExprId()); + return; + } + for (Expression childExpr : expr.children()) { + addResolvedExprSlots(childExpr, out); + } + } + + /** Output ExprId set of a plan node. */ + private static Set<ExprId> outputIdSet(Plan node) { + Set<ExprId> ids = new HashSet<>(); + for (Slot slot : node.getOutput()) { + ids.add(slot.getExprId()); + } + return ids; + } + + /** Adds every slot ExprId used by an expression. */ + private static void addExprSlots(Expression expr, Set<ExprId> out) { + if (expr == null) { + return; + } + collectAllSlotIds(expr, out); + } + + /** + * Pre-walk that fills localAggParams bottom-up so the live-column analysis + * (which runs before the decompile walk) can resolve partial-buffer references. The + * decompile walk re-fills the same map while it descends (harmless duplicate). + */ + private void collectLocalAggParams(Plan node) { + for (Plan child : node.children()) { + collectLocalAggParams(child); + } + if (node instanceof PhysicalHashAggregate) { + PhysicalHashAggregate<? extends Plan> agg = (PhysicalHashAggregate<? extends Plan>) node; + if (agg.getAggPhase().isLocal() || isIntermediateAggStage(agg)) { + recordLocalAggStage(agg); + } + } + } + + /** + * Registers one local (partial) aggregate stage's buffer outputs (see the field + * comment of localAggParams). + */ + private void recordLocalAggStage(PhysicalHashAggregate<? extends Plan> agg) { + for (NamedExpression output : agg.getOutputExpressions()) { + Expression inner = output instanceof Alias ? ((Alias) output).child() : output; + if (inner instanceof AggregateExpression && isPartialAggregate((AggregateExpression) inner)) { + Expression rawParam = extractPartialParam((AggregateExpression) inner); + if (agg.getGroupByExpressions().isEmpty() + && isDistinctMergeContribution(((AggregateExpression) inner).children())) { + distinctMergeBuffers.add(output.getExprId()); + } + Expression param = rawParam; + if (param == null) { + // count(*) has no data argument (extractPartialParam -> null). Map the + // buffer to the partial expression itself so the enclosing + // merge-finalize count(*) resolves its argument back to the star, which + // appendAggSelect collapses into count(*) (isNestedNoArgCount) - without + // this the raw buffer slot (named e.g. "partial_count(*)") would leak + // into the decompiled SQL as the invalid count(partial_count(*)). + param = inner; + } + while (param instanceof SlotReference) { + Expression resolved = localAggParams.get(((SlotReference) param).getExprId()); + if (resolved == null) { + break; + } + param = resolved; + } + localAggParams.put(output.getExprId(), param); + } + } + } + + /** + * Whether an output of an eliminated DISTINCT_LOCAL stage is the distinct-dedup + * contribution (see distinctMergeBuffers). + * + * The distinction has to be read from the aggregate expression's own children - + * the physical input of the buffer - because the merge function's argument is + * normalized to the data column for EVERY buffer: + * + * merge-chain buffer (plain aggregate riding along): its input is another + * partial buffer, already recorded in localAggParams, or a count-star function + * with no argument at all; + * distinct-dedup buffer: its input is the dedup KEY data column, either as the + * bare slot (partial_count(key)) or wrapped as the partial function's argument + * (count(key)). + */ + private boolean isDistinctMergeContribution(List<Expression> exprChildren) { + if (exprChildren.isEmpty()) { + return false; + } + Expression bufferArg = exprChildren.get(0); + if (bufferArg instanceof SlotReference) { + return !localAggParams.containsKey(((SlotReference) bufferArg).getExprId()); + } + List<Expression> argChildren = bufferArg.children(); + if (argChildren.isEmpty()) { + // e.g. the count(*) star buffer: raw-row counting, never the distinct merge + return false; + } + for (Expression child : argChildren) { + if (!(child instanceof SlotReference) + || localAggParams.containsKey(((SlotReference) child).getExprId())) { + return false; + } + } + return true; + } + + /** + * The live column filter for one explicit SELECT list: the list a node emits is + * pruned to the node's needed output columns (no entry in the map, or a null need, + * keeps the whole list - e.g. mock plan trees and unpruned subtrees). + */ + private void filterLiveSelects(Plan node, List<Pair<ExprId, String>> selects) { + Set<ExprId> need = neededOutputs.get(node); + if (need == null) { + return; + } + selects.removeIf(p -> !need.contains(p.key())); + } + + /** + * Decompile entry: physical plan -> planSql. + * + * Parameter plan: the optimal physical plan + * Returns: planSql (standard SQL text) + */ + public String toSQL(Plan plan) { + // An alias-UDF expansion computed under the DEFINITION's stored session + // variables (decimalOverflowScale / enable_decimal256 /...) carries a + // SessionVarGuardExpr; the SQL text has no way to express that guard, so + // printing only its child would replan the arithmetic under the LATER caller's + // variables and could change its type, scale or value while the same bind SQL + // still matches. Reject freezing instead (CREATE keeps the user planSql / + // falls back to the parameterized-plan-tree path). + // A residual SUBQUERY has no SQL rendering here (SPMExprSqlBuilder emits the digest + // pseudo form, which a user function of the same name could legally satisfy), and SPM's + // pseudo names are reserved - both are refusals, so the baseline stays on the + // parameterized-plan-tree path (see rejectUnsupportedExpressions). + rejectUnsupportedExpressions(plan); + // reset the per-decompile alias / generated-name sequences: t_N and c_N only + // need to be unique WITHIN the one produced SQL, so numbering restarts here and + // the decompiled text stays compact across calls + SQLRelation.resetAliasCounter(); + cteBodies.clear(); + cteAliases.clear(); + cteDefinitions.clear(); + generatedColumnNames.clear(); + generatedColumnSeq = 0; + reservedOutputNames.clear(); + lateralViewSeq = 0; + outputProjects.clear(); + localAggParams.clear(); + distinctMergeBuffers.clear(); + markOutputProject(plan); + collectLocalAggParams(plan); + computeNeeded(plan); + SQLRelation relation = plan.accept(this, null); + // attach the collected CTE definitions (WITH) to the outermost relation: they + // are collected in producer dependency order and are visible to the whole + // statement, which is exactly the scope a CTE anchor binds + if (!cteDefinitions.isEmpty()) { + List<String> merged = relation.getCte() == null + ? new ArrayList<>() : new ArrayList<>(relation.getCte()); + merged.addAll(cteDefinitions); + relation.setCte(merged); + } + // PhysicalAssertNumRows carries no node of its own in the frozen text: it renders + // as the ASSERT_ROWS relation prefix wherever its parent references the input + // relation (visitPhysicalAssertNumRows), and the parser rebuilds the assertion. + // + // At the ROOT there is no parent to consume that prefix: toSQL of the + // relation never renders assertRows, so SELECT * FROM ASSERT_ROWS (SELECT...) a + // froze as an ordinary filtered SELECT - CREATE's probe planned it happily and an + // identical replay returned BOTH rows where native execution raises the row-count + // error. The boundary is therefore materialized HERE, in the only statement form the + // grammar has for it: the ASSERT_ROWS relation under a plain SELECT *. + return renderStatement(relation); + } + + /** Materializes a pending assertion when a relation is emitted as a standalone SELECT. */ + private String renderStatement(SQLRelation relation) { + return relation.isAssertRows() ? "SELECT * FROM " + relation.toRelationSQL() : relation.toSQL(); + } + + /** + * Handles a child node. + * + * Parameter plan: child physical plan + * Returns: SQLRelation of the child plan + */ + private SQLRelation process(Plan plan) { + return plan.accept(this, null); + } + + // default: unsupported operators. + + @Override + public SQLRelation visit(Plan plan, Void context) { + throw new UnsupportedOperationException( + "SPMPlan2SQLBuilder does not support plan: " + plan.getClass().getSimpleName()); + } + + // pass-through mode. + /** + * PhysicalStorageLayerAggregate: a scan-shaped shortcut that returns COUNT / MIN / MAX + * for the wrapped scan from table or footer metadata instead of reading the data. + * AggregateStrategies keeps the enclosing aggregate (or the constant-only project) on + * TOP of the shortcut and keeps every expression in terms of the wrapped relation's + * slots, so decompiling the wrapped relation publishes exactly those slots and the + * enclosing operators render the same SQL aggregate (count(*) / min(x) /...) over the + * same table. Replaying that SQL re-derives an equivalent plan; there is no clause that + * describes the shortcut itself and none is needed, because the shortcut is an + * execution strategy of the aggregate rather than a different result. + */ + @Override + public SQLRelation visitPhysicalStorageLayerAggregate( + PhysicalStorageLayerAggregate storageLayerAggregate, Void context) { + return visitPhysicalRelation(storageLayerAggregate.getRelation(), context); + } + + /** + * PhysicalDistribute: a data exchange node with no SQL equivalent; returns the child. + */ + @Override + public SQLRelation visitPhysicalDistribute(PhysicalDistribute<? extends Plan> distribute, Void context) { + return process(distribute.child(0)); + } + + /** + * PhysicalLazyMaterialize: an optimizer lazy-column-materialization wrapper with no + * SQL equivalent; returns the child. + */ + @Override + public SQLRelation visitPhysicalLazyMaterialize( + PhysicalLazyMaterialize<? extends Plan> lazy, Void context) { + return process(lazy.child(0)); + } + + /** + * PhysicalLazyMaterializeOlapScan: an OlapScan wrapped with lazy column + * materialization; decompiled as a normal scan. + */ + @Override + public SQLRelation visitPhysicalLazyMaterializeOlapScan( + PhysicalLazyMaterializeOlapScan scan, Void context) { + return visitPhysicalRelation(scan, context); + } + + /** + * PhysicalLazyMaterializeFileScan: a file scan wrapped with lazy column + * materialization; decompiled as a normal scan (including its scan modifiers - + * the wrapper subclasses PhysicalFileScan, so the TABLESAMPLE / snapshot / scan + * parameter rendering applies unchanged). + */ + @Override + public SQLRelation visitPhysicalLazyMaterializeFileScan( + PhysicalLazyMaterializeFileScan scan, Void context) { + return visitPhysicalRelation(scan, context); + } + + /** + * PhysicalResultSink (and sinks in general): result collection nodes with no SQL + * equivalent; passed through. For the RESULT sink the final output column ORDER is + * forced to the sink's output list (the user SELECT order) - the physical aggregate + * / projection below may emit columns in a different order (e.g. group-by keys + * before aggregates), which would reorder the user-visible result columns. + */ + @Override + public SQLRelation visitPhysicalSink(PhysicalSink<? extends Plan> sink, Void context) { + if (sink instanceof PhysicalResultSink) { + SQLRelation child = process(sink.child(0)); + List<Slot> outputs = sink.getOutput(); + if (outputs.isEmpty()) { + return child; + } + // Final output columns in the USER SELECT order. Each output column is + // emitted as its in-scope reference (child.getColumnNames), and - when the + // reference is a decompile-internal alias (c_N) that hides the original + // column label - re-labelled with the ResultSink output slot's name + // (output.getName): that label is exactly what a normal execution of the + // original query shows in the result header (a plain alias such as + // "d_week_seq1", a table column name, or the expression text of an + // un-aliased expression column), so the frozen planSql keeps the SAME output + // labels from the result sink rather than internal generated references. + List<Pair<ExprId, String>> ordered = new ArrayList<>(); + boolean anyRenamed = false; + for (Slot output : outputs) { + String ref = child.getColumnNames().get(output.getExprId()); + if (ref == null) { + ref = output.getName(); + } + String display = output.getName(); + if (display == null || display.isEmpty() || display.equals(ref)) { + ordered.add(Pair.of(output.getExprId(), ref)); + continue; + } + // Re-expose the column under its original label: "c_3 AS d_week_seq1" + // (or "AS round(...)" for an un-aliased expression column, whose header + // text is the expression itself). The quoted name never participates in + // name resolution of the frozen SQL body, so quoting is always safe. + ordered.add(Pair.of(output.getExprId(), ref + " AS " + quoteIdentifier(display))); + anyRenamed = true; + } + if (child.getSelects().isEmpty()) { + // Without an explicit projection, the FROM columns are referenceable. + // A scan emits star columns in schema order even when the sink requests a + // different projection. Apply the sink order in place, preserving ORDER BY / LIMIT, + // UNLESS a re-labelled alias would SHADOW a name the ORDER BY references + // "c_3 AS b" next to "ORDER BY b" rebinds the sort to the + // alias although the clause was rendered against the base column b. Then + // the relabel moves to the wrapper below, where the clause keeps its own + // query block and the aliases cannot capture it. + String inPlaceOrderBy = child.getOrderBy(); + if (inPlaceOrderBy.isEmpty() + || !orderByShadowedByRelabel(child, inPlaceOrderBy, ordered)) { + child.setSelects(ordered); + return child; + } + } + // Explicit projection already emitted (e.g. a bare aggregate + // "sum(...) AS revenue" or a pass-through projection). Overwriting it in + // place would replace the expression with a bare name that is not resolvable + // in the FROM scope, so: + // if the emitted columns already carry the original labels in the user + // SELECT order, keep the projection untouched; + // otherwise wrap the child in a subquery and select the ordered + // (re-labelled) columns from it. + if (!anyRenamed && sameExprIdOrder(child.getSelects(), ordered)) { + return child; + } + SQLRelation outer = new SQLRelation(); + String alias = outer.newAlias(); + // A TOP-LEVEL ORDER BY / LIMIT pair is a semantic user clause: matching + // deliberately ignores its LIMIT value, and mergeLimits adopts the user's value + // by position - but a limit left INSIDE the derived table can never be reached + // by that positional merge (the frozen root would have no Limit node next to + // the user's Limit), so a matched LIMIT 200 query kept the captured 100-row + // cap. This wrapper is a PURE output relabelling (it neither filters nor + // aggregates), so hoisting the pair onto it is semantically identical and + // makes the frozen root a Limit node again. + // The ORDER BY moves WITH the LIMIT: a derived-table ORDER BY WITHOUT its + // LIMIT is only a hint the optimizer is free to drop, and the outer SELECT + // would then return an arbitrary LIMIT slice (a q02-style query replayed as + // unordered rows). Clear both BEFORE rendering the child text - toSQL + // captures them into the string. + String sinkOrderBy = child.getOrderBy(); + String sinkLimit = child.getLimit(); + // The order and cap belong to the input checked by ASSERT_ROWS. + if (!child.isAssertRows() && (!sinkOrderBy.isEmpty() || !sinkLimit.isEmpty())) { + // Independently of the LIMIT: "SELECT a FROM t ORDER BY b + 1" + // (no LIMIT at all) arrives here as a Sort under the final relabel wrapper, + // and leaving the ORDER BY inside the derived table was NOT safe - the + // frozen text is re-planned at replay, and a sort that the user asked for + // but that sits one level down came back as unordered rows (Nereids drops + // an inner sort it considers redundant). The wrapper is a pure output + // relabelling, so the user clause belongs on it. Hidden sort keys the + // child's SELECT list does not export are added to it first (see + // hoistableOrderBy); a key that cannot be exported, or that a relabelled + // wrapper alias would capture, keeps the clause inside the child - its own + // query block still resolves it. ORDER BY and LIMIT + // always move TOGETHER: a limit without its order picks arbitrary rows. + String hoisted = sinkOrderBy.isEmpty() ? "" + : hoistableOrderBy(child, sinkOrderBy, ordered, true); + if (hoisted != null) { + outer.setOrderBy(hoisted); + outer.setLimit(sinkLimit); + child.setOrderBy(""); + child.setLimit(""); + } + } + // A relabelled ASSERT_ROWS child keeps its assertion: toSQL of the + // relation never renders the relation PREFIX (only toRelationSQL does), so the + // wrapper used to drop the single-row assertion of + // SELECT * FROM ASSERT_ROWS (SELECT k+1 FROM t WHERE k>0) a - CREATE's probe planned + // the plain SELECT happily and an identical replay returned BOTH rows where native + // execution raises the row-count error. The relation prefix is the only statement + // form the grammar has for it, so it is rendered HERE with this wrapper's alias. + outer.setFrom((child.isAssertRows() + ? "ASSERT_ROWS (" + child.toSQL() + ") " + : "(" + child.toSQL() + ") ") + alias); + outer.setSelects(ordered); + return outer; + } + return visit((Plan) sink, context); + } + + /** Whether the projection ExprIds appear in the same order as ordered. */ + private static boolean sameExprIdOrder(List<Pair<ExprId, String>> projection, + List<Pair<ExprId, String>> ordered) { + if (projection.size() != ordered.size()) { + return false; + } + for (int i = 0; i < projection.size(); i++) { + if (projection.get(i).key() != ordered.get(i).key()) { + return false; + } + } + return true; + } + + /** + * Returns a valid unquoted identifier unchanged and otherwise backtick-quotes the whole name, doubling + * embedded backticks. A name containing a minus sign or a dot must remain one identifier rather than becoming + * an expression or multiple components. + */ + public static String quoteIdentifier(String name) { + if (name == null) { + return null; + } + if (name.matches("[A-Za-z_][A-Za-z0-9_]*") && NereidsParser.isValidUnquotedIdentifier(name)) { + return name; + } + return "`" + name.replace("`", "``") + "`"; + } + + /** + * Fully-qualified name of a table with every COMPONENT quoted separately + * (catalog.db.table). getNameWithFullQualifiers FLATTENS a legal quoted + * component such as t.a into "internal.db.t.a", and splitting that on every dot + * would emit FOUR identifiers instead of the intended three-part name with t.a + * quoted as ONE component - a manually created frozen baseline carrying that broken + * text then fails re-analysis after a reload, and no raw fallback tree exists for a + * frozen row. The metadata components are taken structurally, never re-split. + */ + static String quoteQualifiedTableName(org.apache.doris.catalog.TableIf table) { + StringBuilder sb = new StringBuilder(); + for (String component : table.getFullQualifiers()) { + if (sb.length() > 0) { + sb.append('.'); + } + sb.append(quoteIdentifier(component)); + } + return sb.toString(); + } + + // Scan (wrapped as subquery or inline). + + /** + * Renders a catalog relation with a qualified table name, supported scan modifiers and registered output + * references. Temporary table names are refused so creation can retain user SQL for session-local resolution. + */ + @Override + public SQLRelation visitPhysicalRelation(PhysicalRelation relation, Void context) { + if (!(relation instanceof PhysicalCatalogRelation)) { + throw new UnsupportedOperationException( + "SPMPlan2SQLBuilder does not support relation: " + relation.getClass().getSimpleName()); + } + PhysicalCatalogRelation catalogRelation = (PhysicalCatalogRelation) relation; + // A TEMPORARY table's catalog object carries the CREATOR session's internal name + // (<sessionId>_#TEMP._<name>), not the text the user typed. Emitting it into the + // frozen planSql would persist a session-scoped identifier: another session + // running the same "FROM t" text matches the same bind key (catalog.db.t), and + // Database.getTableNullable accepts the already-marked internal name unchanged, + // so the replay would read the CREATOR's (possibly still live) temporary table + // instead of its own t. Never render it - the GLOBAL create / capture path + // rejects temporary relations before freezing (SPMPlanner), and a SESSION-scope + // freeze (where the session's own temp table is the intended target) falls back + // to the raw user text instead. + if (catalogRelation.getTable().isTemporary()) { + throw new UnsupportedOperationException( + "SPMPlan2SQLBuilder does not support temporary tables: the physical relation" + + " carries the creator session's internal name"); + } + SQLRelation sqlRelation = new SQLRelation(); + // Emit the fully qualified name (catalog.db.table) so the frozen planSql resolves + // the same table when it is replayed from a session whose current database (or + // catalog) differs from the one used at CREATE time (cross-db queries, + // information_schema,...). Tables without a database (e.g. FunctionGenTable) + // keep the bare name. Every COMPONENT is backtick-quoted separately when it is + // not a plain identifier, so a metadata name containing operators is re-parsed + // as an identifier instead of an expression, and a legal quoted component such + // as t.a keeps its boundary (see quoteQualifiedTableName). Scan modifiers + // (partition selection, TABLESAMPLE, snapshot, scan parameters) follow the name + // in grammar order. + String table = catalogRelation.getTable().getDatabase() == null + ? quoteIdentifier(catalogRelation.getTable().getName()) + : quoteQualifiedTableName(catalogRelation.getTable()); + sqlRelation.setFrom(table + renderScanModifiers(relation)); + // Remember WHICH table this scan reads: two occurrences of one table are a self + // join even when their FROM texts differ (each occurrence may carry its own + // PARTITION / TABLESAMPLE pin), and the join must wrap both sides then. + sqlRelation.setRelationIdentity(table); + // Register output columns: ExprId -> real column name. Internal system columns + // (e.g. rowid columns a join may request from the scan) are execution details + // and are never registered so they cannot leak into projections / ON clauses. + for (Slot slot : relation.getOutput()) { + if (isSystemColumnName(slot.getName())) { + continue; + } + // registered as executable SQL text: a special-character column (a-b) + // must be backtick-quoted, otherwise a frozen projection SELECT a-b + // re-parses as the subtraction a - b and returns a different value + sqlRelation.registerRef(slot.getExprId(), quoteIdentifier(slot.getName())); + } + return sqlRelation; + } + + // table-valued functions. + + /** + * PhysicalTVFRelation: a table-valued function in FROM (e.g. + * numbers('number' = '5')). The function's own SQL text is used because a TVF + * argument list is a property list, not a normal expression list; the output + * columns keep their names. + */ + @Override + public SQLRelation visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, Void context) { + SQLRelation relation = new SQLRelation(); + relation.setFrom(renderTableValuedFunction(tvfRelation.getFunction())); + for (Slot slot : tvfRelation.getOutput()) { + relation.registerRef(slot.getExprId(), quoteIdentifier(slot.getName())); + } + return relation; + } + + /** + * Renders one TVF call safely. TableValuedFunction.computeToSql concatenates each + * concrete key/value between apostrophes WITHOUT escaping, so a valid property such + * as an S3 object key containing an apostrophe or backslash would freeze malformed + * SQL: with a placeholder-bearing predicate the row is classified as frozen, no + * fallback tree is rebuilt, and after refresh/restart the baseline silently stops + * applying. The property map is serialized in a deterministic key order with the + * same default-mode-safe quoting as every other SPM-emitted string literal (the + * stored text is always re-parsed under MODE_DEFAULT). + */ + private static String renderTableValuedFunction(TableValuedFunction function) { + String args = new TreeMap<>(function.getTVFProperties().getMap()).entrySet().stream() + .map(kv -> quoteSqlString(kv.getKey()) + " = " + quoteSqlString(kv.getValue())) + .collect(Collectors.joining(", ")); + return quoteIdentifier(function.getName()) + "(" + args + ")"; + } + + /** PhysicalLazyMaterializeTVFScan: a TVF scan wrapped by lazy materialization. */ + @Override + public SQLRelation visitPhysicalLazyMaterializeTVFScan( + PhysicalLazyMaterializeTVFScan scan, Void context) { + return visitPhysicalTVFRelation(scan, context); + } + + /** + * Renders the scan modifiers that the frozen SQL must carry, in the order the grammar + * accepts them after the table name + * (optScanParams, materializedViewName, tableSnapshot, specifiedPartition, sample): + * + * olap scans: @paramType(...) parameters (binlog reads), the partition list + * when the scan reads a strict non-empty subset of the table partitions, and + * TABLESAMPLE; + * file scans: @paramType(...) parameters, FOR VERSION/TIME AS OF + * snapshots and TABLESAMPLE. + * + * Dropping any of them would silently change what the frozen planSql reads: a + * "FROM t PARTITION(p1)" baseline would replay over every partition, a snapshot read + * would run against the moving head of the table, and a sampled scan would return the + * full table. + * + * The partition list is emitted only when the user pinned it (a pruned scan re-derives + * its selection by replay); a frozen text without the pin simply does not match the + * pinned user query (a safe miss, never a replay over the wrong partitions). + * + * Two scan states are deliberately NOT emitted. The selected index is an optimizer + * choice (a rollup is a consistent copy, so the choice carries no semantics) that + * cannot be told apart from a user-written INDEX clause; freezing it would pin the + * optimization and stop the plain user query from matching. A user-written INDEX + * pin only survives in the user tree, so such queries never match a frozen text that + * dropped the pin. + * + * A user-written INDEX pin is NOT lost to this asymmetry: the pin is a + * SEMANTIC choice (on an aggregate-key table the forced rollup returns one aggregated + * row per rollup key while the base table returns one per record), the matching key + * carries it (UnboundRelation.toDigest renders "INDEX index_name", so only callers + * with the same pin match), and this method cannot reproduce it. The CREATE therefore + * declines the FREEZE for a statement that pins an index at all + * (SPMPlanTreeSupport.pinsExplicitIndex): the user planSql is kept, plan_frozen stays + * false, and the rewrite replays the parameterized tree, whose own analysis re-applies + * the pin - so the frozen-text path below is never asked to render one. + */ + private String renderScanModifiers(PhysicalRelation relation) { + StringBuilder modifiers = new StringBuilder(); + if (relation instanceof PhysicalOlapScan) { + PhysicalOlapScan scan = (PhysicalOlapScan) relation; + modifiers.append(renderScanParams(scan.getScanParams())); + modifiers.append(renderPartitionSelection(scan)); + modifiers.append(renderTabletSelection(scan)); + modifiers.append(renderTableSample(scan.getTableSample())); + } else if (relation instanceof PhysicalFileScan) { + PhysicalFileScan scan = (PhysicalFileScan) relation; + modifiers.append(renderScanParams(scan.getScanParams())); + modifiers.append(renderTableSnapshot(scan.getTableSnapshot())); + modifiers.append(renderTableSample(scan.getTableSample())); + } + return modifiers.toString(); + } + + /** + * Whether the CAPTURE admitted one pinned partition in the TEMPORARY namespace (see + * recordPartitionNames), and whether the live table still agrees. + * + * A pin without a snapshot (a table the capture never described) keeps the live answer, so + * an old baseline is unaffected. A snapshot that DISAGREES with the live namespace declines + * the freeze - the pin can no longer be described by the caller's own text. + * + * Parameter table: the scanned OLAP table + * Parameter pins: the captured pins of that table, or null + * Parameter partitionId: the pinned partition + * Returns: whether the pinned partition is TEMPORARY + */ + private static boolean snapshotTemporary(OlapTable table, + Map<Long, org.apache.doris.nereids.spm.SPMPlanTreeSupport.PartitionPin> pins, + long partitionId) { + boolean live = table.isTemporaryPartition(partitionId); + org.apache.doris.nereids.spm.SPMPlanTreeSupport.PartitionPin pin = + pins == null ? null : pins.get(partitionId); + if (pin == null) { + return live; + } + if (pin.isTemporary() != live) { + throw new UnsupportedOperationException( + "SPM decompile: the pinned partition " + partitionId + " of table " + + table.getName() + " moved between the TEMPORARY and the formal" + + " namespace after the capture (the admitted pin was " + + (pin.isTemporary() ? "TEMPORARY" : "formal") + "); keeping the" + + " user planSql so the replay re-resolves the pin"); + } + return pin.isTemporary(); + } + + /** + * OLAP partition selection: the ids of a user-written PARTITION(...) / + * TEMPORARY PARTITION(...) list are frozen as that very clause - including + * when the pin happens to cover every partition that exists right now (a cardinality + * test would drop the clause and a later ADD PARTITION would let the pinned query + * silently read the new partition) and including the temporary namespace (a temp pin + * replayed as a formal PARTITION(name) binds the wrong partition or fails to bind). + * Partition pruning also shrinks selectedPartitionIds without any user pin, + * so only the manual provenance is ever rendered; a pruned scan re-derives its + * selection by replay. Ids are sorted so the frozen text is deterministic (the + * matcher compares the selection as a set). + */ + private String renderPartitionSelection(PhysicalOlapScan scan) { + List<Long> pinnedIds = scan.getManuallySpecifiedPartitions(); + if (pinnedIds.isEmpty()) { + return ""; + } + OlapTable table = scan.getTable(); + List<Long> sortedIds = new ArrayList<>(pinnedIds); + Collections.sort(sortedIds); + Map<Long, org.apache.doris.nereids.spm.SPMPlanTreeSupport.PartitionPin> tablePins = + capturedPartitionNames.get(table.getId()); + // The NAMESPACE comes from the CAPTURE: a temporary pin promoted to a + // formal partition (use_temp_partition_name=true) between the unlock and this render made + // the live check say "formal", the frozen text said PARTITION(name), and an identical + // replay bound the formal partition - or nothing - while the caller's own query reads + // the re-created TEMPORARY one. + boolean temporary = snapshotTemporary(table, tablePins, sortedIds.get(0)); + StringBuilder partition = new StringBuilder(temporary ? " TEMPORARY PARTITION(" : " PARTITION("); + for (int i = 0; i < sortedIds.size(); i++) { + if (i > 0) { + partition.append(", "); + } + if (snapshotTemporary(table, tablePins, sortedIds.get(i)) != temporary) { + throw new UnsupportedOperationException( + "SPM decompile: a partition pin mixing temporary and formal partitions" + + " has no single-clause rendering"); + } + long partitionId = sortedIds.get(i); + Partition partitionMeta = table.getPartition(partitionId); + org.apache.doris.nereids.spm.SPMPlanTreeSupport.PartitionPin pin = + tablePins == null ? null : tablePins.get(partitionId); + String capturedName = pin == null ? null : pin.getName(); + if (capturedName != null) { + // The ADMITTED name wins: the applied pin is a SET OF IDS, while the + // stored bind text (and every later caller matching it) names the partition it + // wrote. Reading the LIVE name here would freeze a RENAME PARTITION that landed + // between the capture and this render, so an identical caller pinning the OLD + // name - now a different, newly added partition - would be answered from the + // renamed one. A partition whose live name already moved makes the CREATE decline + // the freeze instead of freezing a text the bind cannot describe. + if (partitionMeta == null || !capturedName.equals(partitionMeta.getName())) { + throw new UnsupportedOperationException( + "SPM decompile: the pinned partition " + partitionId + " of table " + + table.getName() + " was renamed / dropped after the capture" + + " ('" + capturedName + "' is the admitted name); keeping" + + " the user planSql so the replay re-resolves the pin"); + } + partition.append(quoteIdentifier(capturedName)); + continue; + } + if (partitionMeta == null) { + throw new UnsupportedOperationException( + "SPM decompile: the pinned partition " + partitionId + " of table " + + table.getName() + " no longer exists and the capture did not" + + " snapshot its name"); + } + partition.append(quoteIdentifier(partitionMeta.getName())); + } + return partition.append(')').toString(); + } + + /** + * OLAP tablet pin: TABLET(id,...) is frozen when the user wrote it. Bucket + * pruning also fills selectedTabletIds, so only the manual provenance is + * rendered; without the clause a replayed scan would read every tablet of the + * selected partitions and could return rows the captured query excluded. Ids are + * sorted (the matcher compares the tablet list as a set). + */ + private static String renderTabletSelection(PhysicalOlapScan scan) { + List<Long> pinnedIds = scan.getManuallySpecifiedTabletIds(); + if (pinnedIds.isEmpty()) { + return ""; + } + List<Long> sortedIds = new ArrayList<>(pinnedIds); + Collections.sort(sortedIds); + StringBuilder tablet = new StringBuilder(" TABLET("); + for (int i = 0; i < sortedIds.size(); i++) { + if (i > 0) { + tablet.append(", "); + } + tablet.append(sortedIds.get(i)); + } + return tablet.append(')').toString(); + } + + /** + * TABLESAMPLE(n PERCENT | n ROWS) [REPEATABLE seed]: both olap and file scans keep the + * user's sample. Dropping it would replay over the full table and return rows the + * captured plan never sampled in. + */ + private static String renderTableSample(Optional<TableSample> tableSample) { + if (!tableSample.isPresent()) { + return ""; + } + TableSample sample = tableSample.get(); + StringBuilder modifiers = new StringBuilder(" TABLESAMPLE(") + .append(sample.sampleValue) + .append(sample.isPercent ? " PERCENT)" : " ROWS)"); + if (sample.seek >= 0) { + modifiers.append(" REPEATABLE ").append(sample.seek); + } + return modifiers.toString(); + } + + /** + * FOR VERSION AS OF / FOR TIME AS OF: a time-travel read must stay pinned to the + * captured version, otherwise the replay reads the current table contents. A numeric + * version is emitted as a literal, every other value (and all times) as a string. + */ + private static String renderTableSnapshot(Optional<TableSnapshot> tableSnapshot) { + if (!tableSnapshot.isPresent()) { + return ""; + } + TableSnapshot snapshot = tableSnapshot.get(); + if (snapshot.getType() == TableSnapshot.VersionType.TIME) { + return " FOR TIME AS OF " + quoteSqlString(snapshot.getValue()); + } + String value = snapshot.getValue(); + if (value.matches("[0-9]+")) { + return " FOR VERSION AS OF " + value; + } + return " FOR VERSION AS OF " + quoteSqlString(value); + } + + /** + * The @paramType(...) read parameters (incremental / branch / tag / options / + * snapshot / reset): the map form when the parameters carry key/value pairs, otherwise + * the bare identifier list form. Dropping them would replay a different (e.g. + * non-incremental) read than the captured one. + */ + private static String renderScanParams(Optional<TableScanParams> scanParams) { + if (!scanParams.isPresent()) { + return ""; + } + TableScanParams params = scanParams.get(); + StringBuilder modifiers = new StringBuilder(" @").append(params.getParamType()).append('('); + Map<String, String> mapParams = params.getMapParams(); + if (!mapParams.isEmpty()) { + boolean first = true; + for (Map.Entry<String, String> param : mapParams.entrySet()) { + if (!first) { + modifiers.append(", "); + } + first = false; + modifiers.append(quoteIdentifier(param.getKey())) + .append(" = ") + .append(quoteSqlString(param.getValue())); + } + } else { + List<String> listParams = params.getListParams(); + for (int i = 0; i < listParams.size(); i++) { + if (i > 0) { + modifiers.append(", "); + } + modifiers.append(quoteIdentifier(listParams.get(i))); + } + } + return modifiers.append(')').toString(); + } + + /** + * A single-quoted SQL string literal: embedded single quotes are doubled AND + * backslashes are doubled, so the value cannot terminate the literal, change the frozen + * SQL structure, or decode to a DIFFERENT value. The DEFAULT sql_mode treats a backslash + * as an escape introducer (a semantic value such as release\next would otherwise + * reparse as release + newline + ext under that mode and select another external ref), + * so the literal doubles it; the SPM re-parses pin the DEFAULT mode (see + * SPMPlanner.parseStoredSelect), which makes the round trip exact under default AND + * NO_BACKSLASH_ESCAPES sessions alike. + */ + static String quoteSqlString(String value) { + return "'" + value.replace("\\", "\\\\").replace("'", "''") + "'"; + } + + // Generate (LATERAL VIEW). + + /** + * PhysicalGenerate: LATERAL VIEW clauses over the child relation - one clause per + * generator. The parser wraps every user-written LATERAL VIEW... into its + * own single-generator node, while MergeGenerates (for two independent + * views) folds stacked nodes into one node carrying several generators; the + * executor rolls several functions over each child row, i.e. a cartesian expansion, + * which is exactly what stacked LATERAL VIEWs express (MergeGenerates only merges + * when the upper view does not reference the lower view's output). LATERAL VIEW + * clauses attach to the child's COMPLETE query block: the child's WHERE / GROUP BY / + * HAVING / ORDER BY / LIMIT clauses must stay inside the lateral-view input (a + * derived table with LIMIT 10 limits the input, not the exploded rows). The + * generator output columns are registered on the same relation so parent operators + * can reference them; the alias is taken from the output slot's qualifier when it + * survived analysis, otherwise a per-decompile lv_N alias is used. Generate + * conjuncts (if any) are appended to the relation's WHERE. + */ + @Override + public SQLRelation visitPhysicalGenerate(PhysicalGenerate<? extends Plan> generate, Void context) { + SQLRelation childRelation = process(generate.child(0)); + List<Function> generators = generate.getGenerators(); + List<Slot> outputs = generate.getGeneratorOutput(); + if (generators.isEmpty() || generators.size() != outputs.size()) { + // post-binding plans keep exactly one output slot per generator (multi-column + // generators carry their columns in the expand alias project above) + throw new UnsupportedOperationException("SPM decompile generate: generator/output arity mismatch " + + generators.size() + "/" + outputs.size()); + } + // The LATERAL VIEW must attach to the child's COMPLETE query block. Attaching it + // to the bare FROM fragment (getFrom) would keep the child's WHERE / GROUP BY / + // HAVING / ORDER BY / LIMIT clauses on the OUTER relation, where they apply AFTER + // the explode: a derived table with LIMIT 10 would limit the exploded rows + // instead of the lateral-view input, a GROUP BY would regroup the generator + // output, and frozen replay could return rows the captured plan filtered out. + // A FROM-less child needs a wrapper, and an ASSERT_ROWS child must assert + // the input cardinality before the generator changes it. + SQLRelation relation; + String baseSql; + if (childRelation.isAssertRows() || childRelation.getFrom().isEmpty() || childRelation.hasOwnBlock()) { + if (childRelation.getRelationName() == null) { + childRelation.newAlias(); + } + baseSql = childRelation.toRelationSQL(); + // the wrapper carries the child's column mapping only; the child's clauses + // stay inside baseSql (moving them onto this relation would re-apply them + // after the lateral view) + relation = new SQLRelation(); + relation.getColumnNames().putAll(childRelation.getColumnNames()); + } else { + relation = childRelation; + baseSql = relation.getFrom(); + } + List<String> aliases = new ArrayList<>(outputs.size()); + // The analyzer may name a generator output with an internal column name + // ("$c$N"); such a name cannot be referenced in SQL, so give it a generated + // visible name and register the slot under that name for the parent operators. + // A generator output may also COLLIDE with a name the child already exports + // (SELECT t.x, lv.x FROM t LATERAL VIEW explode(t.arr) lv AS x is valid: the two + // slots carry different qualifiers, but the frozen SQL references a derived + // relation's columns by name) - registering both as bare x made the parent emit + // SELECT x, x..., which fails binding as ambiguous after reload. Rename the + // generator output to a unique visible name and register THAT for the slot. + // + // The comparison follows the ANALYZER's identifier-case rules: Doris resolves + // column names case-insensitively, so a case-SENSITIVE set missed the collision + // of a source column X and a generator output x (the frozen projection emitted + // X AS a, x AS b, exactly the ambiguous pair the rename exists to avoid, and + // the create-time probe skipped this text because of its IN-list marker). + Set<String> takenNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String existing : childRelation.getColumnNames().values()) { + if (existing != null) { + takenNames.add(existing.replace("`", "")); + } + } + List<String> columnNames = new ArrayList<>(outputs.size()); + for (Slot slot : outputs) { + String alias = slot.getQualifier().isEmpty() + ? "" : slot.getQualifier().get(slot.getQualifier().size() - 1); + aliases.add(alias.isEmpty() ? "lv_" + (lateralViewSeq++) : alias); + String name = slot.getName(); + String visible = name == null || name.startsWith("$c$") + ? "lv_col_" + (lateralViewSeq++) : name; + while (takenNames.contains(visible)) { + visible = visible + "_"; + } + takenNames.add(visible); + columnNames.add(visible); + } + StringBuilder from = new StringBuilder(baseSql); + for (int i = 0; i < generators.size(); i++) { + from.append(" LATERAL VIEW ") + .append(exprSqlBuilder.print(generators.get(i), relation)) + .append(' ') + .append(quoteIdentifier(aliases.get(i))) + .append(" AS ") + .append(quoteIdentifier(columnNames.get(i))); + } + relation.setFrom(from.toString()); + for (int i = 0; i < outputs.size(); i++) { + relation.registerRef(outputs.get(i).getExprId(), quoteIdentifier(columnNames.get(i))); + } + // When this relation is wrapped as a subquery by its parent, its SELECT list is + // the subquery output: the generator columns must be part of it, otherwise the + // parent's reference to a generated column fails to resolve ("Unknown column + // ... in table list"). An empty SELECT list means "*" and already covers them. + if (!relation.getSelects().isEmpty()) { + List<Pair<ExprId, String>> selects = new ArrayList<>(relation.getSelects()); + for (int i = 0; i < outputs.size(); i++) { + selects.add(Pair.of(outputs.get(i).getExprId(), quoteIdentifier(columnNames.get(i)))); + } + relation.setSelects(selects); + } + if (!generate.getConjuncts().isEmpty()) { + // The conjuncts are the generate's original join ON predicates (they reach + // the TableFunctionNode as expandConjuncts). On an OUTER generator the BE + // keeps ONE NULL-extended left row when every generated value fails them + // (LEFT JOIN UNNEST(arr) AS u(x) ON u.x > 0 over arr = [-1] keeps the left + // row), while the LATERAL VIEW + WHERE rendering below would filter that row + // away. The frozen text has no rendering for that ON semantics, so decline + // and let the rewrite replay the parameterized tree. + for (Function generator : generators) { + if (isOuterGenerator(generator)) { + throw new UnsupportedOperationException("SPM decompile generate: the join ON" + + " conjuncts of the outer generator " + generator + " rely on" + + " expandConjuncts keeping the NULL-extended row when every generated" + + " value fails them; replay the parameterized tree instead"); + } + } + String conjuncts = generate.getConjuncts().stream() + .map(expr -> exprSqlBuilder.print(expr, relation)) + .collect(Collectors.joining(" AND ")); + relation.setWhere(relation.getWhere().isEmpty() + ? conjuncts + : relation.getWhere() + " AND " + conjuncts); + } + return relation; + } + + /** + * Whether one generator renders its OUTER form - the "xxx_outer" function family + * (explode_outer / posexplode_outer /...; the catalog derives the same name for + * outer lookup) or an Unnest still carrying the outer flag before convertUnnest. + * Only the outer form keeps the NULL-extended row when the expandConjuncts fail + * (see visitPhysicalGenerate). + */ + private static boolean isOuterGenerator(Function generator) { + if (generator instanceof Unnest) { + return ((Unnest) generator).isOuter(); + } + return generator.getName().endsWith("_outer"); + } + + // CTE (anchor / producer / consumer). + + /** + * PhysicalCTEAnchor: two children - child(0) is the PhysicalCTEProducer (the CTE + * body, processed first: it registers the WITH definition and the alias consumers + * reference) and child(1) is the main query. The WITH entry itself is attached to + * the root relation in toSQL, not here (see cteDefinitions). + */ + @Override + public SQLRelation visitPhysicalCTEAnchor( + PhysicalCTEAnchor<? extends Plan, ? extends Plan> anchor, Void context) { + process(anchor.child(0)); + return process(anchor.child(1)); + } + + /** + * PhysicalCTEProducer: processes the CTE body and registers it (plus a fresh alias + * and the WITH entry) under the CTEId, so every PhysicalCTEConsumer references the + * single shared definition instead of inlining a body copy. The body subtree is + * already pruned to the union of all consumer needs by the live-column analysis. + */ + @Override + public SQLRelation visitPhysicalCTEProducer(PhysicalCTEProducer<? extends Plan> producer, Void context) { + SQLRelation body = process(producer.child(0)); + String alias = SQLRelation.newTableAlias(); + cteBodies.put(producer.getCteId(), body); + cteAliases.put(producer.getCteId(), alias); + // A pending ROW ASSERTION must render INSIDE the WITH definition: + // visitPhysicalAssertNumRows only sets body.assertRows, which body.toSQL does not + // consume (only toRelationSQL does), so the definition became a plain filtered SELECT + // and the identical replay returned the rows the caller's ASSERT_ROWS rejects. + String definition = renderStatement(body); + cteDefinitions.add(alias + " AS (" + definition + ")"); + return body; + } + + /** + * PhysicalCTEConsumer: emits FROM the CTE alias - a reference to the WITH + * definition registered by the producer - and maps the consumer output slots onto + * the body's registered column references. Each reference gets its own subquery + * alias (t_N), so two consumers of one CTE can appear side by side (self join) + * without a duplicate relation name. + */ + @Override + public SQLRelation visitPhysicalCTEConsumer(PhysicalCTEConsumer consumer, Void context) { + SQLRelation body = cteBodies.get(consumer.getCteId()); + String alias = cteAliases.get(consumer.getCteId()); + if (body == null || alias == null) { + throw new UnsupportedOperationException( + "SPMPlan2SQLBuilder cannot find CTE body for " + consumer.getCteId()); + } + SQLRelation relation = new SQLRelation(); + relation.setFrom(alias); + relation.newAlias(); + for (Slot slot : consumer.getOutput()) { + Slot producerSlot = consumer.getProducerSlot(slot); + String col = body.getColumnNames().get(producerSlot.getExprId()); + relation.registerRef(slot.getExprId(), + col != null ? col : quoteIdentifier(producerSlot.getName())); + } + return relation; + } + + // Recursive CTE. + + /** + * Renders anchor and recursive member queries as a WITH RECURSIVE definition. Branch SELECT lists are aligned + * to the CTE output columns where possible. The anchor names are registered while visiting the recursive + * member so its work-table references use those names. + */ + @Override + public SQLRelation visitPhysicalRecursiveUnion( + PhysicalRecursiveUnion<? extends Plan, ? extends Plan> recCte, Void context) { + String cteName = recCte.getCteName(); + // anchor first (child 0): its output column names become the CTE's visible + // column names, needed to name the recursive work-table self reference + SQLRelation anchor = process(recCte.child(0)); + List<String> anchorColNames = new ArrayList<>(); + for (Slot slot : recCte.getRegularChildOutput(0)) { + anchorColNames.add(slot.getName()); + } + // The physical anchor / recursive subtrees may carry optimizer-added + // intermediate columns that are not part of the CTE (e.g. a GROUP BY key of a + // "SELECT 3, 4 FROM t GROUP BY k1" recursive member). Doris requires the anchor + // and the recursive member to expose the SAME column arity, so each branch SQL + // is aligned onto exactly the CTE columns (getRegularChildOutput(i), CTE order). + alignCteBranchSelects(anchor, recCte.getRegularChildOutput(0)); + recursiveCteColumns.put(cteName, anchorColNames); + // recursive member (child 1) + SQLRelation recursive = process(recCte.child(1)); + alignCteBranchSelects(recursive, recCte.getRegularChildOutput(1)); + recursiveCteColumns.remove(cteName); + + // body = (anchor) UNION [ALL] (recursive) + String op = recCte.isUnionAll() ? "UNION ALL" : "UNION"; + String bodySql = "(" + renderStatement(anchor) + ") " + op + " (" + renderStatement(recursive) + ")"; + + // outer reference: WITH RECURSIVE cte(cols) AS (body) SELECT cols FROM cte + String colList = anchorColNames.stream() + .map(SPMPlan2SQLBuilder::quoteIdentifier) + .collect(Collectors.joining(", ")); + SQLRelation relation = new SQLRelation(); + relation.setCte(Collections.singletonList( + "RECURSIVE " + quoteIdentifier(cteName) + "(" + colList + ") AS (" + bodySql + ")")); + relation.setFrom(quoteIdentifier(cteName)); + List<Pair<ExprId, String>> selects = new ArrayList<>(); + List<Slot> cteOutput = recCte.getOutput(); + for (int i = 0; i < cteOutput.size(); i++) { + String col = i < anchorColNames.size() ? anchorColNames.get(i) : cteOutput.get(i).getName(); + String ref = quoteIdentifier(col); + relation.registerRef(cteOutput.get(i).getExprId(), ref); + selects.add(Pair.of(cteOutput.get(i).getExprId(), ref)); + } + relation.setSelects(selects); + relation.newAlias(); + return relation; + } + + /** PhysicalRecursiveUnionAnchor: a sentinel on the anchor branch; passes through. */ + @Override + public SQLRelation visitPhysicalRecursiveUnionAnchor( + PhysicalRecursiveUnionAnchor<? extends Plan> anchor, Void context) { + return process(anchor.child()); + } + + /** + * Aligns one recursive-CTE branch (anchor or recursive member) onto the CTE's + * exposed columns. The decompiled branch subtree can carry optimizer-added + * intermediate columns that do not belong to the CTE - e.g. a "SELECT 3, 4 FROM t + * GROUP BY k1" recursive member keeps its GROUP BY key k1 in the physical plan, so + * the decompiled branch emits three columns while the anchor emits two. AnalyzeCTE + * rejects a recursive CTE whose anchor and member have different output arities + * ("anchor and recursive child's output size must be same"), so the branch SQL is + * re-projected onto exactly the CTE columns (cteCols, in CTE order). When a + * CTE column cannot be resolved on the branch the branch is left untouched (the + * arity is already consistent in that case). + */ + private void alignCteBranchSelects(SQLRelation branch, List<SlotReference> cteCols) { + List<Pair<ExprId, String>> selects = branch.getSelects(); + if (cteCols == null || cteCols.isEmpty() || selects == null || selects.isEmpty() + || selects.size() <= cteCols.size()) { + return; + } + List<Pair<ExprId, String>> aligned = new ArrayList<>(); + for (SlotReference cteCol : cteCols) { + Pair<ExprId, String> match = null; + for (Pair<ExprId, String> select : selects) { + if (select.key().equals(cteCol.getExprId())) { + match = select; + break; + } + } + if (match == null) { + // CTE column not emitted under its own exprId: match the exported alias + String name = cteCol.getName(); + for (Pair<ExprId, String> select : selects) { + // top-level AS only: a quoted name may itself contain " as " + if (selectOutputName(select.value()).equals(name)) { + match = select; + break; + } + } + } + if (match == null) { + return; // cannot resolve every CTE column: leave the branch untouched + } + aligned.add(match); + } + if (aligned.size() == cteCols.size()) { + branch.setSelects(aligned); + } + } + + /** PhysicalRecursiveUnionProducer: a sentinel on the recursive branch; passes through. */ + @Override + public SQLRelation visitPhysicalRecursiveUnionProducer( + PhysicalRecursiveUnionProducer<? extends Plan> producer, Void context) { + return process(producer.child()); + } + + /** + * PhysicalWorkTableReference: the recursive member's self reference to the CTE + * (FROM cte inside the recursive SELECT). Its output columns map onto the + * CTE's visible column names (registered from the anchor branch); a slot without a + * mapped CTE name falls back to its own name so a parent operator can never + * reference an unresolved work-table column. + */ + @Override + public SQLRelation visitPhysicalWorkTableReference(PhysicalWorkTableReference reference, Void context) { + SQLRelation relation = new SQLRelation(); + String tableName = reference.getTableName(); + relation.setFrom(quoteIdentifier(tableName)); + List<String> cols = recursiveCteColumns.get(reference.getNameParts().isEmpty() + ? tableName : reference.getNameParts().get(reference.getNameParts().size() - 1)); + List<Slot> outputs = reference.getOutput(); + for (int i = 0; i < outputs.size(); i++) { + String col = cols != null && i < cols.size() ? cols.get(i) : outputs.get(i).getName(); + relation.registerRef(outputs.get(i).getExprId(), quoteIdentifier(col)); + } + return relation; + } + + /** + * PhysicalOneRowRelation: a relation that produces one row from a projection list + * with no FROM (e.g. the anchor "SELECT 1" of a recursive CTE). Rendered as a bare + * SELECT list. + */ + @Override + public SQLRelation visitPhysicalOneRowRelation(PhysicalOneRowRelation oneRow, Void context) { + SQLRelation relation = new SQLRelation(); + List<Pair<ExprId, String>> selects = new ArrayList<>(); + for (NamedExpression project : oneRow.getProjects()) { + String sql = exprSqlBuilder.print(project, relation); + String ref = sql; + // A FROM-less projection that is the query's result (SELECT 1 AS a) never + // reaches a projection layer that could relabel the output: the alias must + // be emitted HERE, otherwise the frozen SQL is "SELECT _spm_const_var(1)" + // and the replay exposes an expression-derived header instead of a. Only + // explicit aliases are emitted; nameFromChild names are the parser's + // fallback from the expression text and are not identifier-safe. + if (project instanceof Alias && !((Alias) project).isNameFromChild()) { + ref = quoteIdentifier(((Alias) project).getName()); + sql = sql + " AS " + ref; + } + relation.registerRef(project.getExprId(), ref); + selects.add(Pair.of(project.getExprId(), sql)); + } + relation.setSelects(selects); + return relation; + } + + // Empty relation. + + /** + * PhysicalEmptyRelation: a relation that is statically known to return no rows. + * Rendered as a NULL value cast to the column's type (SELECT... FROM (SELECT 1) WHERE FALSE) + * so parent + * operators can still reference its output columns AND each column keeps its DECLARED + * type: defaulting every column to INT 1 (a) makes the frozen text fail to analyze once + * the placeholder substitution types another branch (e.g. DATEV2), and (b) lets a + * reanalyzed placeholder-bearing UNION widen the branch (INT + DATEV2 -> DATETIMEV2) + * and change the result metadata. WHERE FALSE still filters the row out, and CAST(NULL + * AS type) is grammar-parseable for every type. + */ + @Override + public SQLRelation visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, Void context) { + SQLRelation relation = new SQLRelation(); + List<Pair<ExprId, String>> selects = Lists.newArrayList(); + for (NamedExpression project : emptyRelation.getProjects()) { + ExprId id = project.getExprId(); + String name = generatedColumnName(id); + selects.add(Pair.of(id, "CAST(NULL AS " + project.getDataType().toSql() + ") AS " + name)); + relation.registerRef(id, name); + } + relation.setSelects(selects); + relation.setFrom("(SELECT 1)"); Review Comment: [P1] Keep NULL-typed empty relations parseable. `visitPhysicalEmptyRelation` always emits `CAST(NULL AS <project type>)`; a `NullType` project (for example `SELECT NULL WHERE FALSE` or an empty set branch) makes this `CAST(NULL AS NULL)`, which is invalid SQL type syntax. The decompiled text then cannot be analyzed on replay. Emit a bare `NULL`/concrete inferred type for NullType or decline the frozen rendering. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/manager/SessionBaselineStore.java: ########## @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.manager; + +import org.apache.doris.nereids.spm.BaselinePlan; +import org.apache.doris.nereids.spm.BaselineScope; +import org.apache.doris.nereids.spm.BaselineStatus; +import org.apache.doris.nereids.spm.SPMForwardedSession; +import org.apache.doris.nereids.spm.SPMUtils; + +import com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +/** + * Connection-owned SESSION baseline storage. Rows are kept in memory and can be serialized into a forwarded + * request context. Matching checks this store before GLOBAL candidates. The hash index is followed by exact digest + * and active-status filtering, with the same candidate ordering as BaselineManager. + * + * Locally generated ids use a process-wide counter in the SESSION range. Imports can preserve ids from another FE; + * numeric ids need only be unambiguous within the owning store. Synchronized methods protect the maps, but + * returned BaselinePlan objects are not deep copies. + */ +public class SessionBaselineStore { + + /** + * Process-wide counter for locally created SESSION ids. Imported rows may retain ids allocated by another FE. + */ + private static final AtomicLong SESSION_ID_GENERATOR = + new AtomicLong(BaselineScope.SESSION_ID_BASE); + + /** id -> baseline. */ + private final Map<Long, BaselinePlan> baselines = new HashMap<>(); + + /** bindSqlHash -> baseline id list (Level 1 coarse filter index). */ + private final Map<Long, List<Long>> hashIndex = new HashMap<>(); + + /** + * Payload characters the CURRENT rows take when this connection's baselines are + * forwarded (see SPMForwardedSession.payloadRowChars). The bound is enforced at + * CREATION - a store that accepts more than the forwarded payload can carry forced + * serialize to drop rows, and a dropped row still participates in LOCAL + * matching: the same statement was rewritten here but planned without that baseline on + * the master (a silently different rewrite context). Every accepted row (ENABLED or + * not: a later ALTER ENABLE must stay forwardable) therefore always fits. + */ + private long forwardedPayloadChars = 0; + + /** + * Creates a session baseline. + * + * Duplicate detection mirrors BaselineManager: identical (bindSqlHash, + * bindSqlDigest, planSql, schemaFingerprint) returns the existing id + * ("IF NOT EXISTS" semantics). A row with the same bind key but an OLD fingerprint + * (the referenced schema changed, e.g. ALTER TABLE after the baseline was created) + * is NOT a duplicate: SPM matching skips it, so returning its id would leave the + * session unable to recreate a usable baseline - it is retired instead. + * + * Parameter plan: the baseline (id is assigned here from the session id range) + * Returns: the id of the created baseline (or the existing id when duplicated) + */ + public synchronized long createBaseline(BaselinePlan plan) { + long duplicate = retireStaleAndFindDuplicate(plan); + if (duplicate > 0) { + return duplicate; + } + return registerLocked(plan, SESSION_ID_GENERATOR.getAndIncrement()); Review Comment: [P1] Reserve imported SESSION ids before local allocation. `importBaseline` keeps an incoming id at `SESSION_ID_BASE` but never advances or checks the static generator, while `createBaseline` always takes `getAndIncrement()`. A forwarded CREATE on a fresh receiving FE can allocate the imported row's id and `registerLocked` replace that existing session baseline. Advance/skip the generator and occupied ids when importing so forwarding cannot discard earlier session baselines. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
