This is an automated email from the ASF dual-hosted git repository. lijibing pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new 297809ed58e [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 (#40982) 297809ed58e is described below commit 297809ed58e02b08a97472737d58ce5dfbfb4972 Author: Jibing-Li <64681310+jibing...@users.noreply.github.com> AuthorDate: Thu Sep 26 14:04:17 2024 +0800 [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 (#40982) backport: https://github.com/apache/doris/pull/40811 --- .../doris/statistics/AnalysisInfoBuilder.java | 2 +- .../apache/doris/statistics/TableStatsMeta.java | 12 +++--- .../doris/statistics/util/StatisticsUtil.java | 35 ++++++---------- .../statistics/test_drop_stats_and_truncate.groovy | 48 ++++++++++++++++++++++ 4 files changed, 68 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index ddef30ee4de..17a32900be3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -63,7 +63,7 @@ public class AnalysisInfoBuilder { private boolean usingSqlForPartitionColumn; private long tblUpdateTime; private boolean emptyJob; - private boolean userInject; + private boolean userInject = false; private long rowCount; public AnalysisInfoBuilder() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index a7701f06ef1..d0673998b54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -24,7 +24,6 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.util.StatisticsUtil; @@ -167,7 +166,9 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { public void update(AnalysisInfo analyzedJob, TableIf tableIf) { updatedTime = analyzedJob.tblUpdateTime; - userInjected = analyzedJob.userInject; + if (analyzedJob.userInject) { + userInjected = true; + } String colNameStr = analyzedJob.colName; // colName field AnalyzeJob's format likes: "[col1, col2]", we need to remove brackets here // TODO: Refactor this later @@ -195,9 +196,6 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { indexesRowCount.putAll(analyzedJob.indexesRowCount); clearStaleIndexRowCount((OlapTable) tableIf); } - if (analyzedJob.emptyJob && AnalysisMethod.SAMPLE.equals(analyzedJob.analysisMethod)) { - return; - } if (analyzedJob.colToPartitions.keySet() .containsAll(tableIf.getBaseSchema().stream() .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) @@ -205,6 +203,10 @@ public class TableStatsMeta implements Writable, GsonPostProcessable { updatedRows.set(0); newPartitionLoaded.set(false); } + // Set userInject back to false after manual analyze. + if (JobType.MANUAL.equals(jobType) && !analyzedJob.userInject) { + userInjected = false; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e0eef39a217..fe32755be45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -976,37 +976,26 @@ public class StatisticsUtil { } public static boolean isEmptyTable(TableIf table, AnalysisInfo.AnalysisMethod method) { - int waitRowCountReportedTime = 90; + int waitRowCountReportedTime = 120; if (!(table instanceof OlapTable) || method.equals(AnalysisInfo.AnalysisMethod.FULL)) { return false; } OlapTable olapTable = (OlapTable) table; + long rowCount = 0; for (int i = 0; i < waitRowCountReportedTime; i++) { - if (olapTable.getRowCount() > 0) { - return false; - } - boolean allInitVersion = true; - // If all partitions' visible version are PARTITION_INIT_VERSION, return true. - // If any partition's visible version is greater than 2, return true. - // Otherwise, wait row count to be reported. - for (Partition p : olapTable.getPartitions()) { - if (p.getVisibleVersion() != Partition.PARTITION_INIT_VERSION) { - allInitVersion = false; + rowCount = olapTable.getRowCountForIndex(olapTable.getBaseIndexId(), true); + // rowCount == -1 means new table or first load row count not fully reported, need to wait. + if (rowCount == -1) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.info("Sleep interrupted."); } - if (p.getVisibleVersion() > Partition.PARTITION_INIT_VERSION + 1) { - return true; - } - } - if (allInitVersion) { - return true; - } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - LOG.info("Sleep interrupted.", e); + continue; } + break; } - return true; + return rowCount == 0; } } diff --git a/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy b/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy index 85d04aba993..abeb754724b 100644 --- a/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy +++ b/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy @@ -174,6 +174,54 @@ suite("test_drop_stats_and_truncate") { columns = all_columns.split(","); assertEquals(9, columns.size()) + sql """drop table part""" + sql """CREATE TABLE `part` ( + `id` INT NULL, + `colint` INT NULL, + `coltinyint` tinyint NULL, + `colsmallint` smallINT NULL, + `colbigint` bigINT NULL, + `collargeint` largeINT NULL, + `colfloat` float NULL, + `coldouble` double NULL, + `coldecimal` decimal(27, 9) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + PARTITION BY RANGE(`id`) + ( + PARTITION p1 VALUES [("-2147483648"), ("10000")), + PARTITION p2 VALUES [("10000"), ("20000")), + PARTITION p3 VALUES [("20000"), ("30000")) + ) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ + sql """analyze table part with sync""" + sql """Insert into part values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1)""" + result = sql """show table stats part""" + assertEquals("true", result[0][6]) + sql """truncate table part partition(p1)""" + result = sql """show table stats part""" + assertEquals("true", result[0][6]) + sql """analyze table part with sample rows 100 with sync""" + result = sql """show table stats part""" + if (result[0][6].equals("true")) { + result = """show index stats part part""" + logger.info("Report not ready. index stats: " + result) + sql """analyze table part with sample rows 100 with sync""" + result = sql """show table stats part""" + } + if (result[0][6].equals("true")) { + result = """show index stats part part""" + logger.info("Report not ready. index stats: " + result) + sql """analyze table part with sample rows 100 with sync""" + result = sql """show table stats part""" + } + assertEquals("false", result[0][6]) + sql """drop database if exists test_drop_stats_and_truncate""" } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org