This is an automated email from the ASF dual-hosted git repository.

lijibing pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
     new 297809ed58e [fix](statistics)Fix empty table keep auto analyze bug. 
(#40811) #40982 (#40982)
297809ed58e is described below

commit 297809ed58e02b08a97472737d58ce5dfbfb4972
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Thu Sep 26 14:04:17 2024 +0800

    [fix](statistics)Fix empty table keep auto analyze bug. (#40811) #40982 
(#40982)
    
    backport: https://github.com/apache/doris/pull/40811
---
 .../doris/statistics/AnalysisInfoBuilder.java      |  2 +-
 .../apache/doris/statistics/TableStatsMeta.java    | 12 +++---
 .../doris/statistics/util/StatisticsUtil.java      | 35 ++++++----------
 .../statistics/test_drop_stats_and_truncate.groovy | 48 ++++++++++++++++++++++
 4 files changed, 68 insertions(+), 29 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index ddef30ee4de..17a32900be3 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -63,7 +63,7 @@ public class AnalysisInfoBuilder {
     private boolean usingSqlForPartitionColumn;
     private long tblUpdateTime;
     private boolean emptyJob;
-    private boolean userInject;
+    private boolean userInject = false;
     private long rowCount;
 
     public AnalysisInfoBuilder() {
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index a7701f06ef1..d0673998b54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -24,7 +24,6 @@ import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
-import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
 import org.apache.doris.statistics.AnalysisInfo.JobType;
 import org.apache.doris.statistics.util.StatisticsUtil;
 
@@ -167,7 +166,9 @@ public class TableStatsMeta implements Writable, 
GsonPostProcessable {
 
     public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
         updatedTime = analyzedJob.tblUpdateTime;
-        userInjected = analyzedJob.userInject;
+        if (analyzedJob.userInject) {
+            userInjected = true;
+        }
         String colNameStr = analyzedJob.colName;
         // colName field AnalyzeJob's format likes: "[col1, col2]", we need to 
remove brackets here
         // TODO: Refactor this later
@@ -195,9 +196,6 @@ public class TableStatsMeta implements Writable, 
GsonPostProcessable {
                 indexesRowCount.putAll(analyzedJob.indexesRowCount);
                 clearStaleIndexRowCount((OlapTable) tableIf);
             }
-            if (analyzedJob.emptyJob && 
AnalysisMethod.SAMPLE.equals(analyzedJob.analysisMethod)) {
-                return;
-            }
             if (analyzedJob.colToPartitions.keySet()
                     .containsAll(tableIf.getBaseSchema().stream()
                             .filter(c -> 
!StatisticsUtil.isUnsupportedType(c.getType()))
@@ -205,6 +203,10 @@ public class TableStatsMeta implements Writable, 
GsonPostProcessable {
                 updatedRows.set(0);
                 newPartitionLoaded.set(false);
             }
+            // Set userInject back to false after manual analyze.
+            if (JobType.MANUAL.equals(jobType) && !analyzedJob.userInject) {
+                userInjected = false;
+            }
         }
     }
 
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index e0eef39a217..fe32755be45 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -976,37 +976,26 @@ public class StatisticsUtil {
     }
 
     public static boolean isEmptyTable(TableIf table, 
AnalysisInfo.AnalysisMethod method) {
-        int waitRowCountReportedTime = 90;
+        int waitRowCountReportedTime = 120;
         if (!(table instanceof OlapTable) || 
method.equals(AnalysisInfo.AnalysisMethod.FULL)) {
             return false;
         }
         OlapTable olapTable = (OlapTable) table;
+        long rowCount = 0;
         for (int i = 0; i < waitRowCountReportedTime; i++) {
-            if (olapTable.getRowCount() > 0) {
-                return false;
-            }
-            boolean allInitVersion = true;
-            // If all partitions' visible version are PARTITION_INIT_VERSION, 
return true.
-            // If any partition's visible version is greater than 2, return 
true.
-            // Otherwise, wait row count to be reported.
-            for (Partition p : olapTable.getPartitions()) {
-                if (p.getVisibleVersion() != Partition.PARTITION_INIT_VERSION) 
{
-                    allInitVersion = false;
+            rowCount = 
olapTable.getRowCountForIndex(olapTable.getBaseIndexId(), true);
+            // rowCount == -1 means new table or first load row count not 
fully reported, need to wait.
+            if (rowCount == -1) {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    LOG.info("Sleep interrupted.");
                 }
-                if (p.getVisibleVersion() > Partition.PARTITION_INIT_VERSION + 
1) {
-                    return true;
-                }
-            }
-            if (allInitVersion) {
-                return true;
-            }
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-                LOG.info("Sleep interrupted.", e);
+                continue;
             }
+            break;
         }
-        return true;
+        return rowCount == 0;
     }
 
 }
diff --git 
a/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy 
b/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy
index 85d04aba993..abeb754724b 100644
--- a/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy
+++ b/regression-test/suites/statistics/test_drop_stats_and_truncate.groovy
@@ -174,6 +174,54 @@ suite("test_drop_stats_and_truncate") {
     columns = all_columns.split(",");
     assertEquals(9, columns.size())
 
+    sql """drop table part"""
+    sql """CREATE TABLE `part` (
+            `id` INT NULL,
+            `colint` INT NULL,
+            `coltinyint` tinyint NULL,
+            `colsmallint` smallINT NULL,
+            `colbigint` bigINT NULL,
+            `collargeint` largeINT NULL,
+            `colfloat` float NULL,
+            `coldouble` double NULL,
+            `coldecimal` decimal(27, 9) NULL
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        PARTITION BY RANGE(`id`)
+        (
+            PARTITION p1 VALUES [("-2147483648"), ("10000")),
+            PARTITION p2 VALUES [("10000"), ("20000")),
+            PARTITION p3 VALUES [("20000"), ("30000"))
+        )
+        DISTRIBUTED BY HASH(`id`) BUCKETS 3
+        PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+        )
+    """
+    sql """analyze table part with sync"""
+    sql """Insert into part values (1, 1, 1, 1, 1, 1, 1.1, 1.1, 1.1)"""
+    result = sql """show table stats part"""
+    assertEquals("true", result[0][6])
+    sql """truncate table part partition(p1)"""
+    result = sql """show table stats part"""
+    assertEquals("true", result[0][6])
+    sql """analyze table part with sample rows 100 with sync"""
+    result = sql """show table stats part"""
+    if (result[0][6].equals("true")) {
+        result = """show index stats part part"""
+        logger.info("Report not ready. index stats: " + result)
+        sql """analyze table part with sample rows 100 with sync"""
+        result = sql """show table stats part"""
+    }
+    if (result[0][6].equals("true")) {
+        result = """show index stats part part"""
+        logger.info("Report not ready. index stats: " + result)
+        sql """analyze table part with sample rows 100 with sync"""
+        result = sql """show table stats part"""
+    }
+    assertEquals("false", result[0][6])
+
     sql """drop database if exists test_drop_stats_and_truncate"""
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Reply via email to