This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new efc06217ee6 [improvement](statistics)Skip auto analyze when table row
count is not fully reported. (return -1) (#42209)
efc06217ee6 is described below
commit efc06217ee664595b66a52b4b30595dc5ca11503
Author: Jibing-Li <[email protected]>
AuthorDate: Wed Oct 23 16:51:59 2024 +0800
[improvement](statistics)Skip auto analyze when table row count is not
fully reported. (return -1) (#42209)
Skip auto analyze when table row count is not fully reported. Not fully
reported means row count is -1.
---
.../java/org/apache/doris/catalog/OlapTable.java | 6 ++--
.../doris/statistics/StatisticsAutoCollector.java | 10 ++++++
.../statistics/StatisticsAutoCollectorTest.java | 30 ++++++++++++++++++
.../test_auto_analyze_black_white_list.groovy | 37 ++++++++++++++++++++++
4 files changed, 81 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 2d6f13f85bb..1ad1df536d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -138,6 +138,8 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
WAITING_STABLE
}
+ public static long ROW_COUNT_BEFORE_REPORT = -1;
+
@SerializedName(value = "tst", alternate = {"state"})
private volatile OlapTableState state;
@@ -1615,10 +1617,10 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
if (index == null) {
LOG.warn("Index {} not exist in partition {}, table {}, {}",
indexId, entry.getValue().getName(), id, name);
- return -1;
+ return ROW_COUNT_BEFORE_REPORT;
}
if (strict && !index.getRowCountReported()) {
- return -1;
+ return ROW_COUNT_BEFORE_REPORT;
}
rowCount += index.getRowCount() == -1 ? 0 : index.getRowCount();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 2ca0ab1c265..9ba52169605 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -149,6 +149,9 @@ public class StatisticsAutoCollector extends MasterDaemon {
return;
}
AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns,
priority);
+ if (analyzeJob == null) {
+ return;
+ }
LOG.debug("Auto analyze job : {}", analyzeJob.toString());
try {
executeSystemAnalysisJob(analyzeJob);
@@ -203,6 +206,13 @@ public class StatisticsAutoCollector extends MasterDaemon {
if (StatisticsUtil.enablePartitionAnalyze() &&
table.isPartitionedTable()) {
analysisMethod = AnalysisMethod.FULL;
}
+ if (table instanceof OlapTable &&
analysisMethod.equals(AnalysisMethod.SAMPLE)) {
+ OlapTable ot = (OlapTable) table;
+ if (ot.getRowCountForIndex(ot.getBaseIndexId(), true) ==
OlapTable.ROW_COUNT_BEFORE_REPORT) {
+ LOG.info("Table {} row count is not fully reported, skip auto
analyzing this time.", ot.getName());
+ return null;
+ }
+ }
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0
:
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 6324624abac..9eb2004ec25 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -138,4 +138,34 @@ public class StatisticsAutoCollectorTest {
ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable",
"hmsDb", null);
Assertions.assertTrue(collector.supportAutoAnalyze(hiveExternalTable));
}
+
+ @Test
+ public void testCreateAnalyzeJobForTbl() {
+ StatisticsAutoCollector collector = new StatisticsAutoCollector();
+ OlapTable table = new OlapTable();
+ new MockUp<OlapTable>() {
+ @Mock
+ public long getDataSize(boolean singleReplica) {
+ return 100;
+ }
+
+ @Mock
+ public long getRowCountForIndex(long indexId, boolean strict) {
+ return -1;
+ }
+
+ @Mock
+ public boolean isPartitionedTable() {
+ return false;
+ }
+ };
+ Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null,
null));
+ new MockUp<OlapTable>() {
+ @Mock
+ public long getRowCountForIndex(long indexId, boolean strict) {
+ return 100;
+ }
+ };
+ Assertions.assertThrows(NullPointerException.class, () ->
collector.createAnalyzeJobForTbl(table, null, null));
+ }
}
diff --git
a/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
b/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
index 8ba453b81f5..89da74d8702 100644
---
a/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
+++
b/regression-test/suites/statistics/test_auto_analyze_black_white_list.groovy
@@ -17,6 +17,36 @@
suite("test_auto_analyze_black_white_list") {
+ def wait_row_count_reported = { db, table, row, column, expected ->
+ def result = sql """show frontends;"""
+ logger.info("show frontends result origin: " + result)
+ def host
+ def port
+ for (int i = 0; i < result.size(); i++) {
+ if (result[i][8] == "true") {
+ host = result[i][1]
+ port = result[i][4]
+ }
+ }
+ def tokens = context.config.jdbcUrl.split('/')
+ def url=tokens[0] + "//" + host + ":" + port
+ logger.info("Master url is " + url)
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url) {
+ sql """use ${db}"""
+ result = sql """show frontends;"""
+ logger.info("show frontends result master: " + result)
+ for (int i = 0; i < 120; i++) {
+ Thread.sleep(5000)
+ result = sql """show index stats ${table} ${table};"""
+ logger.info("result " + result)
+ if (result[row][column] == expected) {
+ return;
+ }
+ }
+ throw new Exception("Row count report timeout.")
+ }
+ }
+
sql """drop database if exists test_auto_analyze_black_white_list"""
sql """create database test_auto_analyze_black_white_list"""
sql """use test_auto_analyze_black_white_list"""
@@ -38,6 +68,13 @@ suite("test_auto_analyze_black_white_list") {
)
"""
+ try {
+ wait_row_count_reported("test_auto_analyze_black_white_list",
"test_bw", 0, 4, "0")
+ } catch (Exception e) {
+ logger.info(e.getMessage());
+ return;
+ }
+
// Test show index row count
def result = sql """show table stats test_bw"""
assertEquals(1, result.size())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
