englefly commented on code in PR #67549: URL: https://github.com/apache/doris/pull/67549#discussion_r4238199189
########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommand.java: ########## @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.PinnedHboStatistics; +import org.apache.doris.nereids.stats.HboStructFreshness; +import org.apache.doris.nereids.stats.SimpleStructInfo; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatisticsEntry; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * Manual HBO statistics inspection: + * <pre> + * HBO SHOW STATISTICS [LIKE '<pattern>'] -- pinned + learned, simplified struct info + * HBO SHOW STATISTICS FULL [LIKE '<pattern>'] -- canonical struct info + * HBO SHOW PINNED STATISTICS [FULL] [LIKE '<pattern>'] -- injected entries only + * HBO SHOW LEARNED STATISTICS [FULL] [LIKE '<pattern>'] -- learned entries only + * </pre> + * {@code FULL} decides both which struct info is printed and which one {@code LIKE} matches (the + * canonical form, i.e. the fingerprint input, or its simplified rendering). The simplified form is + * a pure function of the canonical form (see {@link SimpleStructInfo}), so it is never persisted and + * can never be stale. + * + * <p>Pinned entries come from the in-memory (authoritative) pinned cache of this FE; learned entries + * come from the recent-runs cache keyed by hbo fingerprint. The learned row count is the output + * row count of the latest recorded run of that fingerprint (see the Detail column for the number of + * recorded runs), so it is a summary, not a single current value. + * + * <p><b>The State column is judged in catalog scope.</b> This statement cannot run the query an + * entry was injected for, so it compares the recorded data state with the tables as a whole + * ({@code HboStructFreshness.of}): an entry whose struct prunes partitions reports {@code unknown} + * here even while the read side keeps applying it for a query which reads the partitions it was + * measured on. The state the read side actually saw is the {@code last=} part of the Detail column + * (filled when this FE applied the entry) and the {@code used=} / {@code skipped=} marker of the + * EXPLAIN annotation. {@code HBO DELETE STALE STATISTICS} removes on this catalog scope as well, so + * it can only remove an entry which is stale for the tables as a whole. + */ +public class HboShowStatisticsCommand extends ShowCommand { + private static final String SCOPE_PINNED = "pinned"; + private static final String SCOPE_LEARNED = "learned"; + + private final String scope; + private final boolean fullStructInfo; + private final String likePattern; + private final ShowResultSetMetaData metaData; + + /** + * HboShowStatisticsCommand + * @param scope PINNED / LEARNED, null or empty means both + * @param fullStructInfo print (and match against) the canonical struct info instead of the + * simplified one + * @param likePattern optional SQL LIKE pattern applied to the printed struct info column + */ + public HboShowStatisticsCommand(String scope, boolean fullStructInfo, String likePattern) { + super(PlanType.HBO_SHOW_STATISTICS_COMMAND); + this.scope = scope == null || scope.isEmpty() ? null : scope.toLowerCase(Locale.ROOT); + this.fullStructInfo = fullStructInfo; + this.likePattern = likePattern == null || likePattern.isEmpty() ? null : likePattern; + this.metaData = ShowResultSetMetaData.builder() + .addColumn(new Column("Kind", ScalarType.createVarchar(16))) + .addColumn(new Column("Fingerprint", ScalarType.createVarchar(64))) + .addColumn(new Column("LiteralMode", ScalarType.createVarchar(16))) + .addColumn(new Column("Type", ScalarType.createVarchar(16))) + .addColumn(new Column("Value", ScalarType.createVarchar(32))) + .addColumn(new Column(fullStructInfo ? "StructInfo" : "SimpleStruct", + ScalarType.createVarchar(65533))) + .addColumn(new Column("Baseline", ScalarType.createVarchar(512))) + .addColumn(new Column("State", ScalarType.createVarchar(16))) + .addColumn(new Column("Detail", ScalarType.createVarchar(512))) + .build(); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager() + .checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + throw new AnalysisException("Access denied: HBO statistics management requires ADMIN privilege"); + } + if (scope != null && !SCOPE_PINNED.equals(scope) && !SCOPE_LEARNED.equals(scope)) { + throw new AnalysisException("invalid hbo show scope, expect PINNED or LEARNED: " + scope); + } + Pattern pattern = compileLikePattern(likePattern); + HboPlanStatisticsManager hboManager = Env.getCurrentEnv().getHboPlanStatisticsManager(); + List<List<String>> rows = new ArrayList<>(); + if (scope == null || SCOPE_PINNED.equals(scope)) { + for (PinnedHboStatistics pinned : hboManager.getAllPinnedPlanStatistics().values()) { + String structInfo = displayedStructInfo(pinned.getStructCanonical()); + if (!matches(pattern, pinned.getStructCanonical(), structInfo)) { + continue; + } + // an expansion entry is keyed by join conditions, so it is not tied to any data + // state and can not go stale + HboStructFreshness freshness = pinned.isExpansion() + ? null : HboStructFreshness.of(pinned.getStructCanonical()); + List<String> row = new ArrayList<>(); + row.add(SCOPE_PINNED); + row.add(pinned.getFingerprint()); + row.add(pinned.getLiteralMode().name().toLowerCase(Locale.ROOT)); + row.add(pinned.getType().name().toLowerCase(Locale.ROOT)); + // a JOIN_EXPANSION entry carries a fan-out factor instead of a row count; the x + // suffix keeps the two readable in one column + row.add(pinned.isExpansion() + ? trimDouble(pinned.getExpansion()) + "x" : String.valueOf(pinned.getRows())); + row.add(structInfo); + row.add(freshness == null ? "-" : freshness.getRecorded()); + row.add(freshness == null ? "-" : freshness.getState()); + row.add(pinnedDetail(pinned.getCreateTime(), + freshness == null ? null : freshness.getLiveDetail(), pinned.getLastUse())); + rows.add(row); + } + } + if (scope == null || SCOPE_LEARNED.equals(scope)) { + for (Map.Entry<String, RecentRunsPlanStatistics> entry : hboManager.getHboPlanStatisticsProvider() + .getAllHboPlanStats().entrySet()) { + Optional<String> canonical = hboManager.getLearnedStructCanonical(entry.getKey()); + // a learned key is generated internally, so it only has a struct info when it was + // injected together with a struct literal + String structInfo = canonical.map(this::displayedStructInfo).orElse(""); + if (!matches(pattern, canonical.orElse(null), structInfo)) { + continue; + } + List<RecentRunsPlanStatisticsEntry> recentRuns = entry.getValue().getRecentRunsStatistics(); + String rowsText = recentRuns.isEmpty() ? "" + : String.valueOf(recentRuns.get(recentRuns.size() - 1).getPlanStatistics().getOutputRows()); + List<String> row = new ArrayList<>(); + row.add(SCOPE_LEARNED); + row.add(entry.getKey()); + // learned keys are constant agnostic for join / aggregation and the scan token for + // scans, so no user facing granularity can be reported + row.add("-"); + row.add("-"); + row.add(rowsText); + row.add(structInfo.isEmpty() ? "-" : structInfo); + HboStructFreshness freshness = canonical.map(HboStructFreshness::of).orElse(null); + row.add(freshness == null ? "-" : freshness.getRecorded()); + row.add(freshness == null ? HboStructFreshness.STATE_UNKNOWN : freshness.getState()); + row.add(freshness == null || freshness.getLive().isEmpty() + ? "runs=" + recentRuns.size() + : "runs=" + recentRuns.size() + " now=" + freshness.getLiveDetail()); + rows.add(row); + } + } + rows.sort(Comparator.comparing((List<String> row) -> row.get(0)) + .thenComparing(row -> row.get(1))); + return new ShowResultSet(metaData, rows); + } + + @Override + public ShowResultSetMetaData getMetaData() { + return metaData; + } + + @Override + public RedirectStatus toRedirectStatus() { + return RedirectStatus.FORWARD_NO_SYNC; Review Comment: 认可这是不一致：SHOW 用 FORWARD_NO_SYNC 转发 master，而 SET/DELETE 各 FE 本地改自己的 manager。两个方向（统一本地 / 统一转发）需要先定产品语义——HBO 条目是 FE 本地资源还是集群资源；定了我就改。 ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java: ########## @@ -38,4 +187,408 @@ public HboPlanStatisticsProvider getHboPlanStatisticsProvider() { public HboPlanInfoProvider getHboPlanInfoProvider() { return hboPlanInfoProvider; } + + /** + * Inject (or overwrite) a pinned statistics entry applied unconditionally. + * @param fingerprint hbo fingerprint (simplified group struct info sha256) + * @param rows output row count that overrides the optimizer estimation + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + putPinnedPlanStatistics(fingerprint, rows, PinnedType.EXACT, structCanonical, LiteralMode.NO_LITERAL); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * @param type how the injected row count is applied (unconditional vs filter-small guard) + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + LiteralMode literalMode) { + putPinnedStatistics(fingerprint, rows, type, structCanonical, 0, literalMode); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * + * @param expansion fan-out factor, only for {@link PinnedType#JOIN_EXPANSION} entries + */ + private void putPinnedStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + double expansion, LiteralMode literalMode) { + // LRU bounded by Config.hbo_pinned_stats_cache_num; pinned entries are otherwise never + // expired automatically and are only removed by HBO DELETE STATISTICS or eviction + long createTimeMs = System.currentTimeMillis(); + synchronized (pinnedLoadLock) { + pinnedPlanStatistics.put(fingerprint, + new PinnedHboStatistics(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs)); + registerRelationKey(structCanonical); + // a SET after a failed DELETE re-creates the entry: drop the deletion intent so a + // pending load does not skip the re-created row + pendingLoadTombstones.remove(fingerprint); + // persist under the same lock as DELETE's DB removal, so concurrent same-key + // SET/DELETE serialize their DB writes in the same order as their memory updates + // and, when the best-effort writes succeed, the stored row matches the last memory + // writer (a failed write only logs a warning and may reappear after a FE restart) + if (persistenceEnabled()) { + HboStatisticsStore.persist(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs); + } + } + } + + /** + * The pinned row-count entry of a sub tree. Entries of another kind (a join expansion factor + * keyed by the join conditions) are never returned here, so the two read paths can not pick up + * each other's entry. + * + * <p>This is the planning hot path, so it only reads the in-memory cache: the persisted entries + * of this FE are loaded by {@link HboPinnedStatisticsLoader} in the background. Doing it here + * would run a DDL and a full table scan on a user query's planning thread, and the internal + * statement it runs would re-enter this manager (see the class javadoc). + */ + public Optional<PinnedHboStatistics> getPinnedPlanStatistics(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(pinned -> !pinned.isExpansion()); + } + + /** + * Whether this FE holds any pinned entry at all (of any kind). The planning path looks an entry + * up by the hbo fingerprint of a plan node, and that fingerprint is expensive to build (a whole + * memo subtree traversal); an empty cache means every such lookup is a guaranteed miss, so the + * caller can skip building the fingerprint. Like {@link #getPinnedPlanStatistics(String)} this + * only reads the in-memory cache and never triggers the background loader. + */ + public boolean hasAnyPinnedStatistics() { + return !pinnedPlanStatistics.asMap().isEmpty(); + } + + /** + * Whether a pinned entry could possibly apply to a sub tree which reads exactly {@code + * relationKey} (see {@link GroupStructInfo.HboStructSummary#getRelationKey()}). + * + * <p>An entry is keyed by the fingerprint of its struct info, and two equal struct infos + * necessarily read the same tables, so an entry whose relation key differs can not match: the + * caller can skip building the (expensive) fingerprint. The index is an over approximation - + * keys are only ever added, so a removed or evicted entry may keep its key - which can only make + * a lookup happen, never suppress one. + */ + public boolean mayHavePinnedEntryForRelations(String relationKey) { + return pinnedEntriesWithoutRelationKey.get() > 0 || pinnedRelationKeys.contains(relationKey); + } + + /** Register the relation key of an entry's struct info in the lookup index. */ + private void registerRelationKey(String structCanonical) { + List<HboScanDescriptor> scans = HboScanDescriptor.parseAll(structCanonical); + if (scans.isEmpty()) { + // an entry which records no scan at all can not be filtered by a relation key + pinnedEntriesWithoutRelationKey.incrementAndGet(); + return; + } + List<String> tables = new ArrayList<>(scans.size()); + for (HboScanDescriptor scan : scans) { + tables.add(scan.getTable()); + } + Collections.sort(tables); + pinnedRelationKeys.add(String.join(",", tables)); + } + + /** + * Remove a pinned statistics entry. Learned entries of the same key (if any) are left + * untouched here; callers may invalidate them explicitly. + */ + public void removePinnedPlanStatistics(String fingerprint) { + synchronized (pinnedLoadLock) { + pinnedPlanStatistics.invalidate(fingerprint); + if (persistenceEnabled()) { + // the DB removal runs under the lock so a lazy load can never SELECT the row in + // the window between the memory invalidation and its DB removal (M-1 R1) + if (!HboStatisticsStore.delete(fingerprint) && !hboPinnedLoaded) { + // the row is still stored and a pending/retrying load would resurrect it + // into memory: remember the deletion intent (M-1 R2) + pendingLoadTombstones.add(fingerprint); + } + } + } + } + + /** + * Inject (or overwrite) the fan-out factor of a set of join equality conditions + * ({@code HBO SET STATISTICS ... TYPE=JOIN_EXPANSION}). It is a pinned entry like every other + * hbo statistics entry - same table, same lifecycle - only its value is a factor instead of a + * row count and its key is the join condition fingerprint. + * + * @param fingerprint sha256 of the canonical equality-condition set + * @param expansion measured fan-out factor relative to the larger input of the join (0.1 keeps + * 10% of the larger input, 1000 fans out to 1000 times it), greater than 0 + * @param condCanonical the canonical equality-condition set the factor was measured for + */ + public void putPinnedExpansionStatistics(String fingerprint, double expansion, String condCanonical) { + // a condition key never carries a literal by itself: it is the constant agnostic form + putPinnedStatistics(fingerprint, 0, PinnedType.JOIN_EXPANSION, condCanonical, expansion, + LiteralMode.NO_LITERAL); + } + + /** + * The pinned join expansion entry of a set of equality conditions, if one was injected. + */ + public Optional<PinnedHboStatistics> getPinnedExpansion(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(PinnedHboStatistics::isExpansion); + } + + /** + * Inject a learned entry (used by {@code HBO SET LEARNED STATISTICS}) so that the learned + * lookup path can be exercised without waiting for a real profile publish. The injected entry + * carries no input table statistics and therefore matches by fingerprint alone. + */ + public void putLearnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + PlanStatistics planStatistics = new PlanStatistics(0, rows, rows, rows, rows, + rows, rows, rows, rows, 0, 0, 1); + RecentRunsPlanStatistics runs = new RecentRunsPlanStatistics( + Lists.newArrayList(new RecentRunsPlanStatisticsEntry(planStatistics, Lists.newArrayList()))); + hboPlanStatisticsProvider.putHboPlanStatsByFingerprint(fingerprint, runs); + if (structCanonical != null && !structCanonical.isEmpty()) { + learnedStructCanonical.put(fingerprint, structCanonical); + } + } + + /** The struct info a learned entry was injected with, when it was injected with one. */ + public Optional<String> getLearnedStructCanonical(String fingerprint) { + return Optional.ofNullable(learnedStructCanonical.getIfPresent(fingerprint)); + } + + /** Remove a learned entry by fingerprint. */ + public void removeLearnedPlanStatistics(String fingerprint) { + hboPlanStatisticsProvider.removeHboPlanStats(fingerprint); + learnedStructCanonical.invalidate(fingerprint); + } + + /** + * Remove every pinned entry which can never be applied again: an entry records the data state it + * was measured in (the baseline of the scans of its struct info), and once a table moved beyond + * {@code Config.hbo_row_count_change_ratio} (or was dropped) the entry is not applied any more + * (see {@link HboStructFreshness}); it only keeps a cache slot until it is removed here. Entries + * which are merely {@code drifted} (the data moved, but within the tolerance) are kept: they are + * still applied. + * + * @param olderThanMillis when non negative, entries whose state cannot be determined (their + * struct info records no comparable state, e.g. it prunes partitions or + * does not resolve to existing tables) are removed as well, but only when + * they are older than this many milliseconds; a negative value keeps them + * @return the number of removed entries + */ + public int deleteStalePinnedPlanStatistics(long olderThanMillis) { + long now = System.currentTimeMillis(); + int removed = 0; + for (PinnedHboStatistics pinned : getAllPinnedPlanStatistics().values()) { + // an expansion entry is keyed by join conditions and carries no data state, so it can + // neither go stale nor be judged by age: OLDER_THAN must not silently drop it + if (pinned.isExpansion()) { + continue; + } + HboStructFreshness freshness = HboStructFreshness.of(pinned.getStructCanonical()); + boolean obsolete = freshness.isStale() + || (olderThanMillis >= 0 && freshness.isUnknown() + && now - pinned.getCreateTime() > olderThanMillis); + if (obsolete) { + removePinnedPlanStatistics(pinned.getFingerprint()); Review Comment: 认可。deleteStalePinnedPlanStatistics 在锁外算 freshness、锁内直接删 fingerprint，并发 SET 可能刚换成新基线却被误删。修法：删除阶段在 pinnedLoadLock 内重新取 entry 并按 (fingerprint, createTime) 比对后再删，DB 删除同临界区。我补提交。 ########## fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java: ########## @@ -7182,6 +7185,146 @@ public LogicalPlan visitAdminSetFrontendConfig(DorisParser.AdminSetFrontendConfi applyToAll); } + @Override + public LogicalPlan visitHboSetStatistics(DorisParser.HboSetStatisticsContext ctx) { + checkHboStatementWords(ctx.hbo, ctx.statistics); + Double value = null; + String typeName = null; + String fingerprint = null; + String structCanonical = null; + String literalModeName = null; + for (DorisParser.HboSetParamContext param : ctx.hboSetParam()) { + if (param instanceof DorisParser.HboSetValueContext) { + if (value != null) { + throw new ParseException("VALUE is given twice in hbo set statistics statement"); + } + try { + value = Double.parseDouble(((DorisParser.HboSetValueContext) param).value.getText()); Review Comment: 认可。Double.parseDouble 会把 9007199254740993 变成 …992，几百位整数还会变成 Infinity 绕过 JOIN_EXPANSION 的 value>0。修法：无小数点/指数的 token 用 Long.parseLong（溢出报错）并统一加 Double.isFinite。不涉及 key，我马上补提交。 ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/HboShowStatisticsCommand.java: ########## @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager; +import org.apache.doris.nereids.stats.HboPlanStatisticsManager.PinnedHboStatistics; +import org.apache.doris.nereids.stats.HboStructFreshness; +import org.apache.doris.nereids.stats.SimpleStructInfo; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatistics; +import org.apache.doris.statistics.hbo.RecentRunsPlanStatisticsEntry; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * Manual HBO statistics inspection: + * <pre> + * HBO SHOW STATISTICS [LIKE '<pattern>'] -- pinned + learned, simplified struct info + * HBO SHOW STATISTICS FULL [LIKE '<pattern>'] -- canonical struct info + * HBO SHOW PINNED STATISTICS [FULL] [LIKE '<pattern>'] -- injected entries only + * HBO SHOW LEARNED STATISTICS [FULL] [LIKE '<pattern>'] -- learned entries only + * </pre> + * {@code FULL} decides both which struct info is printed and which one {@code LIKE} matches (the + * canonical form, i.e. the fingerprint input, or its simplified rendering). The simplified form is + * a pure function of the canonical form (see {@link SimpleStructInfo}), so it is never persisted and + * can never be stale. + * + * <p>Pinned entries come from the in-memory (authoritative) pinned cache of this FE; learned entries + * come from the recent-runs cache keyed by hbo fingerprint. The learned row count is the output + * row count of the latest recorded run of that fingerprint (see the Detail column for the number of + * recorded runs), so it is a summary, not a single current value. + * + * <p><b>The State column is judged in catalog scope.</b> This statement cannot run the query an + * entry was injected for, so it compares the recorded data state with the tables as a whole + * ({@code HboStructFreshness.of}): an entry whose struct prunes partitions reports {@code unknown} + * here even while the read side keeps applying it for a query which reads the partitions it was + * measured on. The state the read side actually saw is the {@code last=} part of the Detail column + * (filled when this FE applied the entry) and the {@code used=} / {@code skipped=} marker of the + * EXPLAIN annotation. {@code HBO DELETE STALE STATISTICS} removes on this catalog scope as well, so + * it can only remove an entry which is stale for the tables as a whole. + */ +public class HboShowStatisticsCommand extends ShowCommand { + private static final String SCOPE_PINNED = "pinned"; + private static final String SCOPE_LEARNED = "learned"; + + private final String scope; + private final boolean fullStructInfo; + private final String likePattern; + private final ShowResultSetMetaData metaData; + + /** + * HboShowStatisticsCommand + * @param scope PINNED / LEARNED, null or empty means both + * @param fullStructInfo print (and match against) the canonical struct info instead of the + * simplified one + * @param likePattern optional SQL LIKE pattern applied to the printed struct info column + */ + public HboShowStatisticsCommand(String scope, boolean fullStructInfo, String likePattern) { + super(PlanType.HBO_SHOW_STATISTICS_COMMAND); + this.scope = scope == null || scope.isEmpty() ? null : scope.toLowerCase(Locale.ROOT); + this.fullStructInfo = fullStructInfo; + this.likePattern = likePattern == null || likePattern.isEmpty() ? null : likePattern; Review Comment: 认可。构造函数把空串归一成 null，于是 LIKE '' 变成不过滤、返回全部条目；SQL 语义下只应匹配空串。修法：区分“未给 LIKE”与“给了空串”。小改动，我补提交。 ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java: ########## @@ -38,4 +187,408 @@ public HboPlanStatisticsProvider getHboPlanStatisticsProvider() { public HboPlanInfoProvider getHboPlanInfoProvider() { return hboPlanInfoProvider; } + + /** + * Inject (or overwrite) a pinned statistics entry applied unconditionally. + * @param fingerprint hbo fingerprint (simplified group struct info sha256) + * @param rows output row count that overrides the optimizer estimation + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + putPinnedPlanStatistics(fingerprint, rows, PinnedType.EXACT, structCanonical, LiteralMode.NO_LITERAL); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * @param type how the injected row count is applied (unconditional vs filter-small guard) + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + LiteralMode literalMode) { + putPinnedStatistics(fingerprint, rows, type, structCanonical, 0, literalMode); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * + * @param expansion fan-out factor, only for {@link PinnedType#JOIN_EXPANSION} entries + */ + private void putPinnedStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + double expansion, LiteralMode literalMode) { + // LRU bounded by Config.hbo_pinned_stats_cache_num; pinned entries are otherwise never + // expired automatically and are only removed by HBO DELETE STATISTICS or eviction + long createTimeMs = System.currentTimeMillis(); + synchronized (pinnedLoadLock) { + pinnedPlanStatistics.put(fingerprint, + new PinnedHboStatistics(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs)); + registerRelationKey(structCanonical); + // a SET after a failed DELETE re-creates the entry: drop the deletion intent so a + // pending load does not skip the re-created row + pendingLoadTombstones.remove(fingerprint); + // persist under the same lock as DELETE's DB removal, so concurrent same-key + // SET/DELETE serialize their DB writes in the same order as their memory updates + // and, when the best-effort writes succeed, the stored row matches the last memory + // writer (a failed write only logs a warning and may reappear after a FE restart) + if (persistenceEnabled()) { + HboStatisticsStore.persist(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs); + } + } + } + + /** + * The pinned row-count entry of a sub tree. Entries of another kind (a join expansion factor + * keyed by the join conditions) are never returned here, so the two read paths can not pick up + * each other's entry. + * + * <p>This is the planning hot path, so it only reads the in-memory cache: the persisted entries + * of this FE are loaded by {@link HboPinnedStatisticsLoader} in the background. Doing it here + * would run a DDL and a full table scan on a user query's planning thread, and the internal + * statement it runs would re-enter this manager (see the class javadoc). + */ + public Optional<PinnedHboStatistics> getPinnedPlanStatistics(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(pinned -> !pinned.isExpansion()); + } + + /** + * Whether this FE holds any pinned entry at all (of any kind). The planning path looks an entry + * up by the hbo fingerprint of a plan node, and that fingerprint is expensive to build (a whole + * memo subtree traversal); an empty cache means every such lookup is a guaranteed miss, so the + * caller can skip building the fingerprint. Like {@link #getPinnedPlanStatistics(String)} this + * only reads the in-memory cache and never triggers the background loader. + */ + public boolean hasAnyPinnedStatistics() { + return !pinnedPlanStatistics.asMap().isEmpty(); + } + + /** + * Whether a pinned entry could possibly apply to a sub tree which reads exactly {@code + * relationKey} (see {@link GroupStructInfo.HboStructSummary#getRelationKey()}). + * + * <p>An entry is keyed by the fingerprint of its struct info, and two equal struct infos + * necessarily read the same tables, so an entry whose relation key differs can not match: the + * caller can skip building the (expensive) fingerprint. The index is an over approximation - + * keys are only ever added, so a removed or evicted entry may keep its key - which can only make + * a lookup happen, never suppress one. + */ + public boolean mayHavePinnedEntryForRelations(String relationKey) { + return pinnedEntriesWithoutRelationKey.get() > 0 || pinnedRelationKeys.contains(relationKey); + } + + /** Register the relation key of an entry's struct info in the lookup index. */ + private void registerRelationKey(String structCanonical) { + List<HboScanDescriptor> scans = HboScanDescriptor.parseAll(structCanonical); + if (scans.isEmpty()) { + // an entry which records no scan at all can not be filtered by a relation key + pinnedEntriesWithoutRelationKey.incrementAndGet(); + return; + } + List<String> tables = new ArrayList<>(scans.size()); + for (HboScanDescriptor scan : scans) { + tables.add(scan.getTable()); + } + Collections.sort(tables); + pinnedRelationKeys.add(String.join(",", tables)); + } + + /** + * Remove a pinned statistics entry. Learned entries of the same key (if any) are left + * untouched here; callers may invalidate them explicitly. + */ + public void removePinnedPlanStatistics(String fingerprint) { + synchronized (pinnedLoadLock) { + pinnedPlanStatistics.invalidate(fingerprint); + if (persistenceEnabled()) { + // the DB removal runs under the lock so a lazy load can never SELECT the row in + // the window between the memory invalidation and its DB removal (M-1 R1) + if (!HboStatisticsStore.delete(fingerprint) && !hboPinnedLoaded) { + // the row is still stored and a pending/retrying load would resurrect it + // into memory: remember the deletion intent (M-1 R2) + pendingLoadTombstones.add(fingerprint); + } + } + } + } + + /** + * Inject (or overwrite) the fan-out factor of a set of join equality conditions + * ({@code HBO SET STATISTICS ... TYPE=JOIN_EXPANSION}). It is a pinned entry like every other + * hbo statistics entry - same table, same lifecycle - only its value is a factor instead of a + * row count and its key is the join condition fingerprint. + * + * @param fingerprint sha256 of the canonical equality-condition set + * @param expansion measured fan-out factor relative to the larger input of the join (0.1 keeps + * 10% of the larger input, 1000 fans out to 1000 times it), greater than 0 + * @param condCanonical the canonical equality-condition set the factor was measured for + */ + public void putPinnedExpansionStatistics(String fingerprint, double expansion, String condCanonical) { + // a condition key never carries a literal by itself: it is the constant agnostic form + putPinnedStatistics(fingerprint, 0, PinnedType.JOIN_EXPANSION, condCanonical, expansion, + LiteralMode.NO_LITERAL); + } + + /** + * The pinned join expansion entry of a set of equality conditions, if one was injected. + */ + public Optional<PinnedHboStatistics> getPinnedExpansion(String fingerprint) { + return Optional.ofNullable(pinnedPlanStatistics.getIfPresent(fingerprint)) + .filter(PinnedHboStatistics::isExpansion); + } + + /** + * Inject a learned entry (used by {@code HBO SET LEARNED STATISTICS}) so that the learned + * lookup path can be exercised without waiting for a real profile publish. The injected entry + * carries no input table statistics and therefore matches by fingerprint alone. + */ + public void putLearnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + PlanStatistics planStatistics = new PlanStatistics(0, rows, rows, rows, rows, + rows, rows, rows, rows, 0, 0, 1); + RecentRunsPlanStatistics runs = new RecentRunsPlanStatistics( + Lists.newArrayList(new RecentRunsPlanStatisticsEntry(planStatistics, Lists.newArrayList()))); + hboPlanStatisticsProvider.putHboPlanStatsByFingerprint(fingerprint, runs); + if (structCanonical != null && !structCanonical.isEmpty()) { + learnedStructCanonical.put(fingerprint, structCanonical); + } + } + + /** The struct info a learned entry was injected with, when it was injected with one. */ + public Optional<String> getLearnedStructCanonical(String fingerprint) { + return Optional.ofNullable(learnedStructCanonical.getIfPresent(fingerprint)); + } + + /** Remove a learned entry by fingerprint. */ + public void removeLearnedPlanStatistics(String fingerprint) { + hboPlanStatisticsProvider.removeHboPlanStats(fingerprint); + learnedStructCanonical.invalidate(fingerprint); + } + + /** + * Remove every pinned entry which can never be applied again: an entry records the data state it + * was measured in (the baseline of the scans of its struct info), and once a table moved beyond + * {@code Config.hbo_row_count_change_ratio} (or was dropped) the entry is not applied any more + * (see {@link HboStructFreshness}); it only keeps a cache slot until it is removed here. Entries + * which are merely {@code drifted} (the data moved, but within the tolerance) are kept: they are + * still applied. + * + * @param olderThanMillis when non negative, entries whose state cannot be determined (their + * struct info records no comparable state, e.g. it prunes partitions or + * does not resolve to existing tables) are removed as well, but only when + * they are older than this many milliseconds; a negative value keeps them + * @return the number of removed entries + */ + public int deleteStalePinnedPlanStatistics(long olderThanMillis) { + long now = System.currentTimeMillis(); + int removed = 0; + for (PinnedHboStatistics pinned : getAllPinnedPlanStatistics().values()) { + // an expansion entry is keyed by join conditions and carries no data state, so it can + // neither go stale nor be judged by age: OLDER_THAN must not silently drop it + if (pinned.isExpansion()) { + continue; + } + HboStructFreshness freshness = HboStructFreshness.of(pinned.getStructCanonical()); + boolean obsolete = freshness.isStale() + || (olderThanMillis >= 0 && freshness.isUnknown() + && now - pinned.getCreateTime() > olderThanMillis); + if (obsolete) { + removePinnedPlanStatistics(pinned.getFingerprint()); + removed++; + } + } + return removed; + } + + /** + * Every pinned entry of this FE, after making sure the persisted ones have been loaded. + * + * <p>Unlike the planning path ({@link #getPinnedPlanStatistics}) this is only used by the admin + * statements ({@code HBO SHOW} / {@code HBO DELETE STALE STATISTICS}), which are expected to wait + * for the entries of the internal table and to report what is stored. Loading here cannot + * re-enter the way it could before the load moved to {@link HboPinnedStatisticsLoader}: the + * internal statement a load runs is planned by this FE, and its planning reads the pinned cache + * without triggering a load any more. + */ + public Map<String, PinnedHboStatistics> getAllPinnedPlanStatistics() { + ensurePinnedLoaded(); + return Collections.unmodifiableMap(pinnedPlanStatistics.asMap()); + } + + /** + * Start loading the persisted pinned statistics of this FE in the background. Called once by + * {@code Env} while the daemon threads are started; a FE which never loads them (the internal + * schema is not ready, or persistence is turned off) simply plans without the persisted entries. + */ + public void startPinnedStatisticsLoader() { + synchronized (pinnedLoadLock) { + if (loader != null && loader.isAlive()) { + // the daemon threads are started again when a FE is promoted; a second loader would + // only double the retry attempts + return; + } + loader = new HboPinnedStatisticsLoader(this); + loader.start(); + } + LOG.info("started the hbo pinned statistics loader"); + } + + /** Whether the persisted pinned statistics of this FE have been loaded (or are not needed). */ + public boolean isPinnedStatisticsLoaded() { + return hboPinnedLoaded || !FeConstants.enableInternalSchemaDb; + } + + /** Whether the loader has nothing left to do. */ + boolean needsPinnedStatisticsLoad() { + return FeConstants.enableInternalSchemaDb && !hboPinnedLoaded; + } + + /** Load the persisted pinned statistics once; a failure is retried by the loader. */ + void loadPinnedStatistics() { + ensurePinnedLoaded(); + } + + private void ensurePinnedLoaded() { + if (hboPinnedLoaded || !persistenceEnabled()) { + return; + } + synchronized (pinnedLoadLock) { + if (hboPinnedLoaded || !persistenceEnabled()) { + return; + } + long now = System.currentTimeMillis(); + if (now - lastLoadFailedMs < LOAD_RETRY_INTERVAL_MS) { + return; + } + List<PinnedHboStatistics> loaded = HboStatisticsStore.loadAll(); + if (loaded == null) { + lastLoadFailedMs = now; Review Comment: 认可现象，但这是有意取舍：读侧热路径不等加载（否则用户查询的规划线程会跑 DDL + 全表扫描），代价就是 admin 语句在加载完成前可能只看到内存里的条目。修法不必动读侧：在 SHOW / DELETE STALE 里当 persistenceEnabled() && !hboPinnedLoaded 时明确提示结果可能不完整（或等一次加载）。我补提交。 ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboPlanStatisticsManager.java: ########## @@ -38,4 +187,408 @@ public HboPlanStatisticsProvider getHboPlanStatisticsProvider() { public HboPlanInfoProvider getHboPlanInfoProvider() { return hboPlanInfoProvider; } + + /** + * Inject (or overwrite) a pinned statistics entry applied unconditionally. + * @param fingerprint hbo fingerprint (simplified group struct info sha256) + * @param rows output row count that overrides the optimizer estimation + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, String structCanonical) { + putPinnedPlanStatistics(fingerprint, rows, PinnedType.EXACT, structCanonical, LiteralMode.NO_LITERAL); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * @param type how the injected row count is applied (unconditional vs filter-small guard) + * @param structCanonical optional human-readable simplified struct info canonical string + */ + public void putPinnedPlanStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + LiteralMode literalMode) { + putPinnedStatistics(fingerprint, rows, type, structCanonical, 0, literalMode); + } + + /** + * Inject (or overwrite) a pinned statistics entry. + * + * @param expansion fan-out factor, only for {@link PinnedType#JOIN_EXPANSION} entries + */ + private void putPinnedStatistics(String fingerprint, long rows, PinnedType type, String structCanonical, + double expansion, LiteralMode literalMode) { + // LRU bounded by Config.hbo_pinned_stats_cache_num; pinned entries are otherwise never + // expired automatically and are only removed by HBO DELETE STATISTICS or eviction + long createTimeMs = System.currentTimeMillis(); + synchronized (pinnedLoadLock) { + pinnedPlanStatistics.put(fingerprint, + new PinnedHboStatistics(fingerprint, rows, type, structCanonical, expansion, literalMode, + createTimeMs)); + registerRelationKey(structCanonical); Review Comment: 已在本轮修复：registerRelationKey 只对非 JOIN_EXPANSION 的行数条目登记（注入与加载两条路径都改了），JE{...} 不会再让预筛对所有 group 失效。关于“键/计数删除时不回收”：那是有意的过近似（只增不减），最坏多做一次注定 miss 的查找，不会漏查，已在方法注释里写明。 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
