This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 96f5891be84 branch-4.1: [fix](statistics) Use one algorithm decision
to pick both params and template in sample analyze #66578 (#66657)
96f5891be84 is described below
commit 96f5891be8457cd19e7c9d6c6fd0386a2f1170b2
Author: yujun <[email protected]>
AuthorDate: Wed Aug 12 10:12:22 2026 +0800
branch-4.1: [fix](statistics) Use one algorithm decision to pick both
params and template in sample analyze #66578 (#66657)
cherry-pick: #66578
---
.../apache/doris/statistics/BaseAnalysisTask.java | 15 ++
.../apache/doris/statistics/OlapAnalysisTask.java | 157 +++++++++++++--------
.../doris/statistics/OlapAnalysisTaskTest.java | 132 ++++++++++++++---
3 files changed, 224 insertions(+), 80 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index f382cb8350c..47a2cb98f05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -65,6 +65,21 @@ public abstract class BaseAnalysisTask {
public static final long LIMIT_SIZE = 1024 * 1024 * 1024; // 1GB
public static final double LIMIT_FACTOR = 1.2;
+ /**
+ * The statistics collection algorithm chosen for a sampled analyze task.
It determines
+ * both the SQL template used to collect stats and the params rendered
into it, so callers
+ * must derive the template and the params from the same algorithm
decision.
+ */
+ public enum AnalyzeSampleAlgorithm {
+ // Full table scan without sampling, used when the table is small or
the sample
+ // tablets contain too few rows. Statistics are computed with the
LINEAR template.
+ FULL,
+ // Linear estimator, used for single unique key column / single
distribution column.
+ LINEAR,
+ // DUJ1 estimator, based on the PostgreSQL analyze algorithm.
+ DUJ1
+ }
+
/**
* Marker string embedded in {@code assert_true} inside statistics
collection SQL.
* When any row's string column length exceeds the configured limit, BE
throws an
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index bd0c5d94f12..0e856c9787e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -68,7 +68,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
private boolean keyColumnSampleTooManyRows = false;
private boolean partitionColumnSampleTooManyRows = false;
- private boolean scanFullTable = false;
private static final long MAXIMUM_SAMPLE_ROWS = 1_000_000_000;
public static final long NO_SKIP_TABLET_ID = -1;
@@ -123,16 +122,17 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
long tableRowCount = info.indexId == -1
? tbl.getRowCount()
: ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
- getSampleParams(params, tableRowCount);
+ SampleCollectInfo collectInfo = getSampleCollectInfo(tableRowCount);
+ getSampleParams(params, tableRowCount, collectInfo);
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql;
- if (useLinearAnalyzeTemplate()) {
- sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
- } else {
+ if (collectInfo.algorithm == AnalyzeSampleAlgorithm.DUJ1) {
sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
+ } else {
+ sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
}
- LOG.info("Analyze param: scanFullTable {}, partitionColumnTooMany {},
keyColumnTooMany {}",
- scanFullTable, partitionColumnSampleTooManyRows,
keyColumnSampleTooManyRows);
+ LOG.info("Analyze param: algorithm {}, partitionColumnTooMany {},
keyColumnTooMany {}",
+ collectInfo.algorithm, partitionColumnSampleTooManyRows,
keyColumnSampleTooManyRows);
LOG.debug(sql);
runQuery(sql);
}
@@ -234,9 +234,8 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
LOG.info("Add large tablet {} in table {} back, with rows {}",
largeTabletId, tbl.getName(), largeTabletRows);
}
- if (selectedRows < targetSampleRows) {
- scanFullTable = true;
- } else if (forPartitionColumn && selectedRows > MAXIMUM_SAMPLE_ROWS) {
+ // If sampled rows are not enough, the caller falls back to a full
table scan.
+ if (forPartitionColumn && selectedRows > MAXIMUM_SAMPLE_ROWS) {
// If the selected tablets for partition column contain too many
rows, change to linear sample.
partitionColumnSampleTooManyRows = true;
sampleTabletIds.clear();
@@ -251,12 +250,58 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
return Pair.of(sampleTabletIds, selectedRows);
}
+ /**
+ * Result of the sample analyze algorithm decision: the chosen algorithm
and, when
+ * sampling is actually performed, the picked tablets used to build the
{@code TABLET(...)}
+ * hint. Kept local to the sampling flow so the long-lived task holds no
sampling state.
+ */
+ protected static class SampleCollectInfo {
+ public final AnalyzeSampleAlgorithm algorithm;
+ public final Pair<List<Long>, Long> sampleTablets;
+
+ public SampleCollectInfo(AnalyzeSampleAlgorithm algorithm,
Pair<List<Long>, Long> sampleTablets) {
+ this.algorithm = algorithm;
+ this.sampleTablets = sampleTablets;
+ }
+ }
+
+ /**
+ * Decide which statistics algorithm to use and, when sampling is actually
performed,
+ * collect the sample tablets. Call this before
+ * {@link #getSampleParams(Map, long, SampleCollectInfo)} so the params
are always filled
+ * according to the algorithm used to pick the template.
+ */
+ protected SampleCollectInfo getSampleCollectInfo(long tableRowCount) {
+ // Debug point used by tests to force the DUJ1 algorithm. Check it
before the
+ // row-count based FULL decision so tests are not affected by BE row
count report
+ // timing (e.g. a small table whose row count is not fully reported
yet).
+ if (DebugPointUtil.isEnable("OlapAnalysisTask.useDUJ1Template")) {
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
getSampleTablets());
+ }
+ long targetSampleRows = getSampleRows();
+ // If table row count is less than the target sample row count, simple
scan the full table.
+ if (tableRowCount <= targetSampleRows) {
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null);
+ }
+ Pair<List<Long>, Long> sampleTablets = getSampleTablets();
+ if (sampleTablets.second < targetSampleRows) {
+ // Sampled tablets contain too few rows, fall back to a full table
scan.
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null);
+ }
+ if (useLinearAnalyzeTemplate()) {
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
sampleTablets);
+ }
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
sampleTablets);
+ }
+
/**
* Get the sql params for this sample task.
* @param params Sql params to use in analyze task.
* @param tableRowCount BE reported table/index row count.
+ * @param info The analyze algorithm decision from {@link
#getSampleCollectInfo(long)}.
*/
- protected void getSampleParams(Map<String, String> params, long
tableRowCount) {
+ protected void getSampleParams(Map<String, String> params, long
tableRowCount,
+ SampleCollectInfo info) {
long targetSampleRows = getSampleRows();
params.put("rowCount", String.valueOf(tableRowCount));
params.put("type", col.getType().toString());
@@ -270,44 +315,51 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("preAggHint", "/*+PREAGGOPEN*/");
}
- // If table row count is less than the target sample row count, simple
scan the full table.
- if (tableRowCount <= targetSampleRows) {
+ // If the algorithm is FULL (table too small or sampled rows not
enough), scan the full table.
+ if (info.algorithm == AnalyzeSampleAlgorithm.FULL) {
params.put("scaleFactor", "1");
params.put("sampleHints", "");
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
// For full table scan, use COUNT(1) for table row count.
params.put("rowCount", "COUNT(1)");
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
- scanFullTable = true;
- return;
- }
- Pair<List<Long>, Long> sampleTabletsInfo = getSampleTablets();
- String tabletStr = sampleTabletsInfo.first.stream()
- .map(Object::toString)
- .collect(Collectors.joining(", "));
- String sampleHints = scanFullTable ? "" : String.format("TABLET(%s)",
tabletStr);
- params.put("sampleHints", sampleHints);
- long selectedRows = sampleTabletsInfo.second;
- long finalScanRows = selectedRows;
- double scaleFactor = scanFullTable ? 1 : (double) tableRowCount /
finalScanRows;
- params.put("scaleFactor", String.valueOf(scaleFactor));
-
- // If the tablets to be sampled are too large, use limit to control
the rows to read, and re-calculate
- // the scaleFactor.
- if (needLimit()) {
- finalScanRows = Math.min(targetSampleRows, selectedRows);
- if (col.isKey() && keyColumnSampleTooManyRows) {
- finalScanRows = MAXIMUM_SAMPLE_ROWS;
- }
- // Empty table doesn't need to limit.
- if (finalScanRows > 0) {
- scaleFactor = (double) tableRowCount / finalScanRows;
- params.put("limit", "limit " + finalScanRows);
- params.put("scaleFactor", String.valueOf(scaleFactor));
+ } else {
+ String tabletStr = info.sampleTablets.first.stream()
+ .map(Object::toString)
+ .collect(Collectors.joining(", "));
+ params.put("sampleHints", String.format("TABLET(%s)", tabletStr));
+ long selectedRows = info.sampleTablets.second;
+ long finalScanRows = selectedRows;
+ double scaleFactor = (double) tableRowCount / finalScanRows;
+ params.put("scaleFactor", String.valueOf(scaleFactor));
+
+ // If the tablets to be sampled are too large, use limit to
control the rows to read, and re-calculate
+ // the scaleFactor.
+ if (needLimit()) {
+ finalScanRows = Math.min(targetSampleRows, selectedRows);
+ if (col.isKey() && keyColumnSampleTooManyRows) {
+ finalScanRows = MAXIMUM_SAMPLE_ROWS;
+ }
+ // Empty table doesn't need to limit.
+ if (finalScanRows > 0) {
+ scaleFactor = (double) tableRowCount / finalScanRows;
+ params.put("limit", "limit " + finalScanRows);
+ params.put("scaleFactor", String.valueOf(scaleFactor));
+ }
}
}
- // Set algorithm related params.
- if (useLinearAnalyzeTemplate()) {
+ setSampleParamsByAlgorithm(params, tableRowCount, info.algorithm);
+ }
+
+ /**
+ * Set the algorithm related params. Must be called with the same
algorithm that the
+ * caller uses to pick the template, so the params always match the
template.
+ */
+ protected void setSampleParamsByAlgorithm(Map<String, String> params, long
tableRowCount,
+ AnalyzeSampleAlgorithm algorithm) {
+ if (algorithm == AnalyzeSampleAlgorithm.DUJ1) {
+ params.put("ndvFunction",
getNdvFunction(String.valueOf(tableRowCount)));
+ params.put("dataSizeFunction", getDataSizeFunction(col, true));
+ params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)");
+ } else {
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
// For single unique key, use count as ndv.
if (isSingleUniqueKey()) {
@@ -315,10 +367,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
} else {
params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
}
- } else {
- params.put("ndvFunction",
getNdvFunction(String.valueOf(tableRowCount)));
- params.put("dataSizeFunction", getDataSizeFunction(col, true));
- params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)");
}
}
@@ -496,9 +544,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
* @return Return true when need to limit.
*/
protected boolean needLimit() {
- if (scanFullTable) {
- return false;
- }
// Key column is sorted, use limit will cause the ndv not accurate
enough, so skip key columns.
if (col.isKey() && !keyColumnSampleTooManyRows) {
return false;
@@ -529,7 +574,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
if (DebugPointUtil.isEnable("OlapAnalysisTask.useDUJ1Template")) {
return false;
}
- if (partitionColumnSampleTooManyRows || scanFullTable) {
+ if (partitionColumnSampleTooManyRows) {
return true;
}
if (isSingleUniqueKey()) {
@@ -582,14 +627,4 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
public void setPartitionColumnSampleTooManyRows(boolean value) {
partitionColumnSampleTooManyRows = value;
}
-
- @VisibleForTesting
- public void setScanFullTable(boolean value) {
- scanFullTable = value;
- }
-
- @VisibleForTesting
- public boolean scanFullTable() {
- return scanFullTable;
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index 4d25654afb7..b63816348b4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -39,9 +39,11 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
+import org.apache.doris.statistics.BaseAnalysisTask.AnalyzeSampleAlgorithm;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.TStorageMedium;
@@ -135,7 +137,13 @@ public class OlapAnalysisTaskTest {
}
@Mock
- void getSampleParams(Map<String, String> params, long
tableRowCount) {}
+ void getSampleParams(Map<String, String> params, long
tableRowCount, OlapAnalysisTask.SampleCollectInfo info) {}
+
+ @Mock
+ OlapAnalysisTask.SampleCollectInfo getSampleCollectInfo(long
tableRowCount) {
+ return new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L));
+ }
@Mock
boolean useLinearAnalyzeTemplate() {
@@ -193,8 +201,9 @@ public class OlapAnalysisTaskTest {
}
@Mock
- boolean useLinearAnalyzeTemplate() {
- return false;
+ OlapAnalysisTask.SampleCollectInfo getSampleCollectInfo(long
tableRowCount) {
+ return new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L));
}
};
olapAnalysisTask.doSample();
@@ -318,11 +327,6 @@ public class OlapAnalysisTaskTest {
task.setPartitionColumnSampleTooManyRows(true);
Assertions.assertTrue(task.useLinearAnalyzeTemplate());
- task.setPartitionColumnSampleTooManyRows(false);
- task.setScanFullTable(true);
- Assertions.assertTrue(task.useLinearAnalyzeTemplate());
-
- task.setScanFullTable(false);
task.setPartitionColumnSampleTooManyRows(false);
new MockUp<OlapAnalysisTask>() {
@Mock
@@ -360,6 +364,11 @@ public class OlapAnalysisTaskTest {
protected boolean useLinearAnalyzeTemplate() {
return false;
}
+
+ @Mock
+ protected boolean isSingleUniqueKey() {
+ return false;
+ }
};
new MockUp<OlapTable>() {
@@ -370,17 +379,39 @@ public class OlapAnalysisTaskTest {
};
task.col = new Column("testColumn", Type.INT, true, null, null, "");
task.setTable(new OlapTable());
- task.getSampleParams(params, 10);
- Assertions.assertTrue(task.scanFullTable());
+ // FULL algorithm: scan the full table and fill LINEAR-style params
(raw column reference).
+ task.getSampleParams(params, 10,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null));
Assertions.assertEquals("1", params.get("scaleFactor"));
Assertions.assertEquals("", params.get("sampleHints"));
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ params.get("rowCount2"));
Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
Assertions.assertNull(params.get("preAggHint"));
Assertions.assertEquals("COUNT(1)", params.get("rowCount"));
params.clear();
- task.getSampleParams(params, 10000);
- Assertions.assertEquals("10000", params.get("rowCount"));
+ // LINEAR algorithm with sample tablets.
+ task.getSampleParams(params, 10000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
+ Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ params.get("rowCount2"));
+ Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ params.clear();
+
+ // DUJ1 algorithm with sample tablets: rowCount2 and ndvFunction must
reference the cte1
+ // output columns (col_value/count), not the raw column name,
otherwise the DUJ1 SQL
+ // fails with "Unknown column".
+ task.getSampleParams(params, 10000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
+ Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+ Assertions.assertEquals("(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)",
+ params.get("rowCount2"));
+
Assertions.assertTrue(params.get("ndvFunction").contains("`t1`.`col_value`"),
+ "DUJ1 ndvFunction must reference cte1 output, got: " +
params.get("ndvFunction"));
params.clear();
new MockUp<OlapTable>() {
@@ -392,7 +423,9 @@ public class OlapAnalysisTaskTest {
task = new OlapAnalysisTask();
task.col = new Column("testColumn", Type.INT, false, null, null, "");
task.setTable(new OlapTable());
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("SUM(`t1`.`count`) * COUNT(`t1`.`col_value`) /
(SUM(`t1`.`count`) - SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null,
1, 0)) + SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null, 1, 0)) *
SUM(`t1`.`count`) / 1000)", params.get("ndvFunction"));
@@ -415,7 +448,9 @@ public class OlapAnalysisTaskTest {
task = new OlapAnalysisTask();
task.col = new Column("testColumn", Type.INT, false, null, null, "");
task.setTable(new OlapTable());
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("/*+PREAGGOPEN*/", params.get("preAggHint"));
params.clear();
@@ -428,7 +463,9 @@ public class OlapAnalysisTaskTest {
task = new OlapAnalysisTask();
task.col = new Column("testColumn", Type.INT, false, null, null, "");
task.setTable(new OlapTable());
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertNull(params.get("preAggHint"));
params.clear();
@@ -447,7 +484,9 @@ public class OlapAnalysisTaskTest {
task = new OlapAnalysisTask();
task.col = new Column("test", PrimitiveType.INT);
task.setTable(new OlapTable());
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
@@ -462,7 +501,9 @@ public class OlapAnalysisTaskTest {
task = new OlapAnalysisTask();
task.col = new Column("test", PrimitiveType.INT);
task.setTable(new OlapTable());
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("1000", params.get("ndvFunction"));
@@ -482,7 +523,9 @@ public class OlapAnalysisTaskTest {
task = new OlapAnalysisTask();
task.col = new Column("test", PrimitiveType.INT);
task.setTable(new OlapTable());
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("20.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("1000", params.get("ndvFunction"));
@@ -494,13 +537,64 @@ public class OlapAnalysisTaskTest {
true, null, null, null);
task.setKeyColumnSampleTooManyRows(true);
task.setTable(new OlapTable());
- task.getSampleParams(params, 2000000000);
+ task.getSampleParams(params, 2000000000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("2.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("2000000000", params.get("ndvFunction"));
Assertions.assertEquals("limit 1000000000", params.get("limit"));
}
+ @Test
+ public void testGetSampleCollectInfo() {
+ final long[] sampleRows = {100L};
+ final Pair<List<Long>, Long>[] sampleTablets = new Pair[]
{Pair.of(Lists.newArrayList(1L, 2L), 100L)};
+ final boolean[] linearTemplate = {false};
+ new MockUp<OlapAnalysisTask>() {
+ @Mock
+ protected long getSampleRows() {
+ return sampleRows[0];
+ }
+
+ @Mock
+ protected Pair<List<Long>, Long> getSampleTablets() {
+ return sampleTablets[0];
+ }
+
+ @Mock
+ protected boolean useLinearAnalyzeTemplate() {
+ return linearTemplate[0];
+ }
+ };
+
+ OlapAnalysisTask task = new OlapAnalysisTask();
+ // tableRowCount <= sample rows -> full table scan.
+ OlapAnalysisTask.SampleCollectInfo info =
task.getSampleCollectInfo(10);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.FULL, info.algorithm);
+
+ // tableRowCount > sample rows -> sample tablets, and
useLinearAnalyzeTemplate decides.
+ linearTemplate[0] = true;
+ info = task.getSampleCollectInfo(10000);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.LINEAR, info.algorithm);
+
+ // sample tablets contain too few rows -> fall back to full table scan.
+ sampleTablets[0] = Pair.of(Lists.newArrayList(1L, 2L), 10L);
+ info = task.getSampleCollectInfo(10000);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.FULL, info.algorithm);
+ // Debug point useDUJ1Template forces DUJ1 even when the row count
would normally
+ // fall back to a full table scan, so tests are not affected by BE row
count timing.
+ new MockUp<DebugPointUtil>() {
+ @Mock
+ public boolean isEnable(String debugPointName) {
+ return true;
+ }
+ };
+ sampleTablets[0] = Pair.of(Lists.newArrayList(1L, 2L), 10L);
+ info = task.getSampleCollectInfo(10000);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.DUJ1, info.algorithm);
+ }
+
@Test
public void testGetSkipPartitionId(@Mocked OlapTable tableIf) throws
AnalysisException {
// test null partition list
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
