This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7979a5a5e7a [fix](statistics) Use one algorithm decision to pick both
params and template in sample analyze (#66578)
7979a5a5e7a is described below
commit 7979a5a5e7a64dac55e544dabfe0cd6c5273bf9e
Author: yujun <[email protected]>
AuthorDate: Tue Aug 11 11:16:32 2026 +0800
[fix](statistics) Use one algorithm decision to pick both params and
template in sample analyze (#66578)
## Proposed changes
Fix sample analyze generating invalid SQL when the DUJ1 template is
forced on a small table.
`doSample()` previously filled SQL params and selected the SQL template
through two independent decisions: `getSampleParams()` decided based on
`tableRowCount`/`scanFullTable` (filling LINEAR-style params that
reference the raw column name), while the template was picked separately
via `useLinearAnalyzeTemplate()`. When `useDUJ1Template` was forced, the
DUJ1 template was chosen but the params were still filled by the
FULL-scan branch with `${colName}` references. The DUJ1 template's
`cte1` only exposes `col_value`/`count`/`column_length`, so the
generated SQL failed to bind, e.g. `Unknown column 'id' in 'table list'
in PROJECT clause`, which made `test_analyze_long_string` flaky.
This PR makes the algorithm a single decision: `getSampleCollectInfo()`
decides `AnalyzeSampleAlgorithm { FULL, LINEAR, DUJ1 }` once, and both
param filling and template selection derive from the same algorithm, so
params always match the template.
## Key changes
- Add `AnalyzeSampleAlgorithm` enum (FULL/LINEAR/DUJ1) in
`BaseAnalysisTask`.
- Rename `getAnalyzeAlgorithm` to `getSampleCollectInfo`: decide the
algorithm once and return it together with the picked sample tablets.
- Fill params (`getSampleParams`/`setSampleParamsByAlgorithm`) and pick
the SQL template (`doSample`) from the same algorithm decision.
- Remove the `scanFullTable` field, its setter/getter, and all related
checks.
## Unit test
- Updated `OlapAnalysisTaskTest` to cover FULL/LINEAR/DUJ1 param
filling, template selection driven by the algorithm, and the
sample-tablets-not-enough fallback.
---
.../apache/doris/statistics/BaseAnalysisTask.java | 15 ++
.../apache/doris/statistics/OlapAnalysisTask.java | 157 +++++++++++++--------
.../doris/statistics/OlapAnalysisTaskTest.java | 118 +++++++++++++---
3 files changed, 208 insertions(+), 82 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 1e68e01210b..6ad286562ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -65,6 +65,21 @@ public abstract class BaseAnalysisTask {
public static final long LIMIT_SIZE = 1024 * 1024 * 1024; // 1GB
public static final double LIMIT_FACTOR = 1.2;
+ /**
+ * The statistics collection algorithm chosen for a sampled analyze task.
It determines
+ * both the SQL template used to collect stats and the params rendered
into it, so callers
+ * must derive the template and the params from the same algorithm
decision.
+ */
+ public enum AnalyzeSampleAlgorithm {
+ // Full table scan without sampling, used when the table is small or
the sample
+ // tablets contain too few rows. Statistics are computed with the
LINEAR template.
+ FULL,
+ // Linear estimator, used for single unique key column / single
distribution column.
+ LINEAR,
+ // DUJ1 estimator, based on the PostgreSQL analyze algorithm.
+ DUJ1
+ }
+
/**
* Marker string embedded in {@code assert_true} inside statistics
collection SQL.
* When any row's string column length exceeds the configured limit, BE
throws an
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 45e84da9177..a7ac364c984 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -68,7 +68,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
private boolean keyColumnSampleTooManyRows = false;
private boolean partitionColumnSampleTooManyRows = false;
- private boolean scanFullTable = false;
private static final long MAXIMUM_SAMPLE_ROWS = 1_000_000_000;
public static final long NO_SKIP_TABLET_ID = -1;
@@ -123,16 +122,17 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
long tableRowCount = info.indexId == -1
? tbl.getRowCount()
: ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
- getSampleParams(params, tableRowCount);
+ SampleCollectInfo collectInfo = getSampleCollectInfo(tableRowCount);
+ getSampleParams(params, tableRowCount, collectInfo);
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql;
- if (useLinearAnalyzeTemplate()) {
- sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
- } else {
+ if (collectInfo.algorithm == AnalyzeSampleAlgorithm.DUJ1) {
sql = stringSubstitutor.replace(DUJ1_ANALYZE_TEMPLATE);
+ } else {
+ sql = stringSubstitutor.replace(LINEAR_ANALYZE_TEMPLATE);
}
- LOG.info("Analyze param: scanFullTable {}, partitionColumnTooMany {},
keyColumnTooMany {}",
- scanFullTable, partitionColumnSampleTooManyRows,
keyColumnSampleTooManyRows);
+ LOG.info("Analyze param: algorithm {}, partitionColumnTooMany {},
keyColumnTooMany {}",
+ collectInfo.algorithm, partitionColumnSampleTooManyRows,
keyColumnSampleTooManyRows);
LOG.debug(sql);
runQuery(sql);
}
@@ -234,9 +234,8 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
LOG.info("Add large tablet {} in table {} back, with rows {}",
largeTabletId, tbl.getName(), largeTabletRows);
}
- if (selectedRows < targetSampleRows) {
- scanFullTable = true;
- } else if (forPartitionColumn && selectedRows > MAXIMUM_SAMPLE_ROWS) {
+ // If sampled rows are not enough, the caller falls back to a full
table scan.
+ if (forPartitionColumn && selectedRows > MAXIMUM_SAMPLE_ROWS) {
// If the selected tablets for partition column contain too many
rows, change to linear sample.
partitionColumnSampleTooManyRows = true;
sampleTabletIds.clear();
@@ -251,12 +250,58 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
return Pair.of(sampleTabletIds, selectedRows);
}
+ /**
+ * Result of the sample analyze algorithm decision: the chosen algorithm
and, when
+ * sampling is actually performed, the picked tablets used to build the
{@code TABLET(...)}
+ * hint. Kept local to the sampling flow so the long-lived task holds no
sampling state.
+ */
+ protected static class SampleCollectInfo {
+ public final AnalyzeSampleAlgorithm algorithm;
+ public final Pair<List<Long>, Long> sampleTablets;
+
+ public SampleCollectInfo(AnalyzeSampleAlgorithm algorithm,
Pair<List<Long>, Long> sampleTablets) {
+ this.algorithm = algorithm;
+ this.sampleTablets = sampleTablets;
+ }
+ }
+
+ /**
+ * Decide which statistics algorithm to use and, when sampling is actually
performed,
+ * collect the sample tablets. Call this before
+ * {@link #getSampleParams(Map, long, SampleCollectInfo)} so the params
are always filled
+ * according to the algorithm used to pick the template.
+ */
+ protected SampleCollectInfo getSampleCollectInfo(long tableRowCount) {
+ // Debug point used by tests to force the DUJ1 algorithm. Check it
before the
+ // row-count based FULL decision so tests are not affected by BE row
count report
+ // timing (e.g. a small table whose row count is not fully reported
yet).
+ if (DebugPointUtil.isEnable("OlapAnalysisTask.useDUJ1Template")) {
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
getSampleTablets());
+ }
+ long targetSampleRows = getSampleRows();
+ // If table row count is less than the target sample row count, simple
scan the full table.
+ if (tableRowCount <= targetSampleRows) {
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null);
+ }
+ Pair<List<Long>, Long> sampleTablets = getSampleTablets();
+ if (sampleTablets.second < targetSampleRows) {
+ // Sampled tablets contain too few rows, fall back to a full table
scan.
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null);
+ }
+ if (useLinearAnalyzeTemplate()) {
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
sampleTablets);
+ }
+ return new SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
sampleTablets);
+ }
+
/**
* Get the sql params for this sample task.
* @param params Sql params to use in analyze task.
* @param tableRowCount BE reported table/index row count.
+ * @param info The analyze algorithm decision from {@link
#getSampleCollectInfo(long)}.
*/
- protected void getSampleParams(Map<String, String> params, long
tableRowCount) {
+ protected void getSampleParams(Map<String, String> params, long
tableRowCount,
+ SampleCollectInfo info) {
long targetSampleRows = getSampleRows();
params.put("rowCount", String.valueOf(tableRowCount));
params.put("type", col.getType().toString());
@@ -270,44 +315,51 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("preAggHint", "/*+PREAGGOPEN*/");
}
- // If table row count is less than the target sample row count, simple
scan the full table.
- if (tableRowCount <= targetSampleRows) {
+ // If the algorithm is FULL (table too small or sampled rows not
enough), scan the full table.
+ if (info.algorithm == AnalyzeSampleAlgorithm.FULL) {
params.put("scaleFactor", "1");
params.put("sampleHints", "");
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
// For full table scan, use COUNT(1) for table row count.
params.put("rowCount", "COUNT(1)");
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
- scanFullTable = true;
- return;
- }
- Pair<List<Long>, Long> sampleTabletsInfo = getSampleTablets();
- String tabletStr = sampleTabletsInfo.first.stream()
- .map(Object::toString)
- .collect(Collectors.joining(", "));
- String sampleHints = scanFullTable ? "" : String.format("TABLET(%s)",
tabletStr);
- params.put("sampleHints", sampleHints);
- long selectedRows = sampleTabletsInfo.second;
- long finalScanRows = selectedRows;
- double scaleFactor = scanFullTable ? 1 : (double) tableRowCount /
finalScanRows;
- params.put("scaleFactor", String.valueOf(scaleFactor));
-
- // If the tablets to be sampled are too large, use limit to control
the rows to read, and re-calculate
- // the scaleFactor.
- if (needLimit()) {
- finalScanRows = Math.min(targetSampleRows, selectedRows);
- if (col.isKey() && keyColumnSampleTooManyRows) {
- finalScanRows = MAXIMUM_SAMPLE_ROWS;
- }
- // Empty table doesn't need to limit.
- if (finalScanRows > 0) {
- scaleFactor = (double) tableRowCount / finalScanRows;
- params.put("limit", "limit " + finalScanRows);
- params.put("scaleFactor", String.valueOf(scaleFactor));
+ } else {
+ String tabletStr = info.sampleTablets.first.stream()
+ .map(Object::toString)
+ .collect(Collectors.joining(", "));
+ params.put("sampleHints", String.format("TABLET(%s)", tabletStr));
+ long selectedRows = info.sampleTablets.second;
+ long finalScanRows = selectedRows;
+ double scaleFactor = (double) tableRowCount / finalScanRows;
+ params.put("scaleFactor", String.valueOf(scaleFactor));
+
+ // If the tablets to be sampled are too large, use limit to
control the rows to read, and re-calculate
+ // the scaleFactor.
+ if (needLimit()) {
+ finalScanRows = Math.min(targetSampleRows, selectedRows);
+ if (col.isKey() && keyColumnSampleTooManyRows) {
+ finalScanRows = MAXIMUM_SAMPLE_ROWS;
+ }
+ // Empty table doesn't need to limit.
+ if (finalScanRows > 0) {
+ scaleFactor = (double) tableRowCount / finalScanRows;
+ params.put("limit", "limit " + finalScanRows);
+ params.put("scaleFactor", String.valueOf(scaleFactor));
+ }
}
}
- // Set algorithm related params.
- if (useLinearAnalyzeTemplate()) {
+ setSampleParamsByAlgorithm(params, tableRowCount, info.algorithm);
+ }
+
+ /**
+ * Set the algorithm related params. Must be called with the same
algorithm that the
+ * caller uses to pick the template, so the params always match the
template.
+ */
+ protected void setSampleParamsByAlgorithm(Map<String, String> params, long
tableRowCount,
+ AnalyzeSampleAlgorithm algorithm) {
+ if (algorithm == AnalyzeSampleAlgorithm.DUJ1) {
+ params.put("ndvFunction",
getNdvFunction(String.valueOf(tableRowCount)));
+ params.put("dataSizeFunction", getDataSizeFunction(col, true));
+ params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)");
+ } else {
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
// For single unique key, use count as ndv.
if (isSingleUniqueKey()) {
@@ -315,10 +367,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
} else {
params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
}
- } else {
- params.put("ndvFunction",
getNdvFunction(String.valueOf(tableRowCount)));
- params.put("dataSizeFunction", getDataSizeFunction(col, true));
- params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)");
}
}
@@ -504,9 +552,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
* @return Return true when need to limit.
*/
protected boolean needLimit() {
- if (scanFullTable) {
- return false;
- }
// Key column is sorted, use limit will cause the ndv not accurate
enough, so skip key columns.
if (col.isKey() && !keyColumnSampleTooManyRows) {
return false;
@@ -537,7 +582,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
if (DebugPointUtil.isEnable("OlapAnalysisTask.useDUJ1Template")) {
return false;
}
- if (partitionColumnSampleTooManyRows || scanFullTable) {
+ if (partitionColumnSampleTooManyRows) {
return true;
}
if (isSingleUniqueKey()) {
@@ -590,14 +635,4 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
public void setPartitionColumnSampleTooManyRows(boolean value) {
partitionColumnSampleTooManyRows = value;
}
-
- @VisibleForTesting
- public void setScanFullTable(boolean value) {
- scanFullTable = value;
- }
-
- @VisibleForTesting
- public boolean scanFullTable() {
- return scanFullTable;
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index ae29fc4c9ce..930fa60d36a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -38,11 +38,13 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
+import org.apache.doris.statistics.BaseAnalysisTask.AnalyzeSampleAlgorithm;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.TStorageMedium;
@@ -141,7 +143,11 @@ public class OlapAnalysisTaskTest {
OlapAnalysisTask olapAnalysisTask = Mockito.spy(new
OlapAnalysisTask());
Mockito.doReturn(new ResultRow(Lists.newArrayList("1",
"2"))).when(olapAnalysisTask).collectMinMax();
-
Mockito.doNothing().when(olapAnalysisTask).getSampleParams(ArgumentMatchers.any(),
ArgumentMatchers.anyLong());
+
Mockito.doNothing().when(olapAnalysisTask).getSampleParams(ArgumentMatchers.any(),
+ ArgumentMatchers.anyLong(), ArgumentMatchers.any());
+ Mockito.doReturn(new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)))
+
.when(olapAnalysisTask).getSampleCollectInfo(ArgumentMatchers.anyLong());
Mockito.doReturn(true).when(olapAnalysisTask).useLinearAnalyzeTemplate();
Mockito.doAnswer(inv -> {
String sql = inv.getArgument(0);
@@ -191,7 +197,10 @@ public class OlapAnalysisTaskTest {
+ "t2) SELECT * FROM cte2 CROSS JOIN cte3", sql);
return null;
}).when(olapAnalysisTask).runQuery(ArgumentMatchers.anyString());
-
Mockito.doReturn(false).when(olapAnalysisTask).useLinearAnalyzeTemplate();
+ // Second run: the DUJ1 algorithm is chosen, so the DUJ1 template must
be used.
+ Mockito.doReturn(new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)))
+
.when(olapAnalysisTask).getSampleCollectInfo(ArgumentMatchers.anyLong());
olapAnalysisTask.doSample();
}
@@ -284,11 +293,6 @@ public class OlapAnalysisTaskTest {
task.setPartitionColumnSampleTooManyRows(true);
Assertions.assertTrue(task.useLinearAnalyzeTemplate());
- task.setPartitionColumnSampleTooManyRows(false);
- task.setScanFullTable(true);
- Assertions.assertTrue(task.useLinearAnalyzeTemplate());
-
- task.setScanFullTable(false);
task.setPartitionColumnSampleTooManyRows(false);
OlapAnalysisTask spyTask = Mockito.spy(task);
Mockito.doReturn(true).when(spyTask).isSingleUniqueKey();
@@ -303,22 +307,45 @@ public class OlapAnalysisTaskTest {
Mockito.doReturn(Pair.of(Lists.newArrayList(1L, 2L),
100L)).when(task).getSampleTablets();
Mockito.doReturn(false).when(task).needLimit();
Mockito.doReturn(false).when(task).useLinearAnalyzeTemplate();
+ Mockito.doReturn(false).when(task).isSingleUniqueKey();
OlapTable mockTable = Mockito.mock(OlapTable.class);
Mockito.when(mockTable.getKeysType()).thenReturn(KeysType.DUP_KEYS);
task.col = new Column("testColumn", Type.INT, true, null, null, "");
task.setTable(mockTable);
- task.getSampleParams(params, 10);
- Assertions.assertTrue(task.scanFullTable());
+ // FULL algorithm: scan the full table and fill LINEAR-style params
(raw column reference).
+ task.getSampleParams(params, 10,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null));
Assertions.assertEquals("1", params.get("scaleFactor"));
Assertions.assertEquals("", params.get("sampleHints"));
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ params.get("rowCount2"));
Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
Assertions.assertNull(params.get("preAggHint"));
Assertions.assertEquals("COUNT(1)", params.get("rowCount"));
params.clear();
- task.getSampleParams(params, 10000);
- Assertions.assertEquals("10000", params.get("rowCount"));
+ // LINEAR algorithm with sample tablets.
+ task.getSampleParams(params, 10000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
+ Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ params.get("rowCount2"));
+ Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ params.clear();
+
+ // DUJ1 algorithm with sample tablets: rowCount2 and ndvFunction must
reference the cte1
+ // output columns (col_value/count), not the raw column name,
otherwise the DUJ1 SQL
+ // fails with "Unknown column".
+ task.getSampleParams(params, 10000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
+ Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
+ Assertions.assertEquals("(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)",
+ params.get("rowCount2"));
+
Assertions.assertTrue(params.get("ndvFunction").contains("`t1`.`col_value`"),
+ "DUJ1 ndvFunction must reference cte1 output, got: " +
params.get("ndvFunction"));
params.clear();
OlapTable mockTable2 = Mockito.mock(OlapTable.class);
@@ -330,7 +357,9 @@ public class OlapAnalysisTaskTest {
Mockito.doReturn(false).when(task).useLinearAnalyzeTemplate();
task.col = new Column("testColumn", Type.INT, false, null, null, "");
task.setTable(mockTable2);
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("SUM(`t1`.`count`) * COUNT(`t1`.`col_value`) /
(SUM(`t1`.`count`) - SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null,
1, 0)) + SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null, 1, 0)) *
SUM(`t1`.`count`) / 1000)", params.get("ndvFunction"));
@@ -349,7 +378,9 @@ public class OlapAnalysisTaskTest {
Mockito.doReturn(false).when(task).useLinearAnalyzeTemplate();
task.col = new Column("testColumn", Type.INT, false, null, null, "");
task.setTable(mockTable3);
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("/*+PREAGGOPEN*/", params.get("preAggHint"));
params.clear();
@@ -363,7 +394,9 @@ public class OlapAnalysisTaskTest {
Mockito.doReturn(false).when(task).useLinearAnalyzeTemplate();
task.col = new Column("testColumn", Type.INT, false, null, null, "");
task.setTable(mockTable4);
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertNull(params.get("preAggHint"));
params.clear();
@@ -377,7 +410,9 @@ public class OlapAnalysisTaskTest {
Mockito.when(mockTable5.getKeysType()).thenReturn(KeysType.DUP_KEYS);
task.col = new Column("test", PrimitiveType.INT);
task.setTable(mockTable5);
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
@@ -393,7 +428,9 @@ public class OlapAnalysisTaskTest {
Mockito.when(mockTable6.getKeysType()).thenReturn(KeysType.DUP_KEYS);
task.col = new Column("test", PrimitiveType.INT);
task.setTable(mockTable6);
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("1000", params.get("ndvFunction"));
@@ -409,7 +446,9 @@ public class OlapAnalysisTaskTest {
Mockito.when(mockTable7.getKeysType()).thenReturn(KeysType.DUP_KEYS);
task.col = new Column("test", PrimitiveType.INT);
task.setTable(mockTable7);
- task.getSampleParams(params, 1000);
+ task.getSampleParams(params, 1000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("20.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("1000", params.get("ndvFunction"));
@@ -428,13 +467,45 @@ public class OlapAnalysisTaskTest {
true, null, null, null);
task.setKeyColumnSampleTooManyRows(true);
task.setTable(mockTable8);
- task.getSampleParams(params, 2000000000);
+ task.getSampleParams(params, 2000000000,
+ new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("2.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("2000000000", params.get("ndvFunction"));
Assertions.assertEquals("limit 1000000000", params.get("limit"));
}
+ @Test
+ public void testGetSampleCollectInfo() {
+ // tableRowCount <= sample rows -> full table scan.
+ OlapAnalysisTask task = Mockito.spy(new OlapAnalysisTask());
+ Mockito.doReturn(100L).when(task).getSampleRows();
+ Mockito.doReturn(false).when(task).useLinearAnalyzeTemplate();
+ OlapAnalysisTask.SampleCollectInfo info =
task.getSampleCollectInfo(10);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.FULL, info.algorithm);
+
+ // tableRowCount > sample rows -> sample tablets, and
useLinearAnalyzeTemplate decides.
+ Mockito.doReturn(Pair.of(Lists.newArrayList(1L, 2L),
100L)).when(task).getSampleTablets();
+ Mockito.doReturn(true).when(task).useLinearAnalyzeTemplate();
+ info = task.getSampleCollectInfo(10000);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.LINEAR, info.algorithm);
+
+ // sample tablets contain too few rows -> fall back to full table scan.
+ Mockito.doReturn(Pair.of(Lists.newArrayList(1L, 2L),
10L)).when(task).getSampleTablets();
+ info = task.getSampleCollectInfo(10000);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.FULL, info.algorithm);
+
+ // Debug point useDUJ1Template forces DUJ1 even when the row count
would normally
+ // fall back to a full table scan, so tests are not affected by BE row
count timing.
+ try (MockedStatic<DebugPointUtil> mocked =
Mockito.mockStatic(DebugPointUtil.class)) {
+ mocked.when(() ->
DebugPointUtil.isEnable("OlapAnalysisTask.useDUJ1Template")).thenReturn(true);
+ Mockito.doReturn(Pair.of(Lists.newArrayList(1L, 2L),
10L)).when(task).getSampleTablets();
+ info = task.getSampleCollectInfo(10000);
+ Assertions.assertEquals(AnalyzeSampleAlgorithm.DUJ1,
info.algorithm);
+ }
+ }
+
@Test
public void testGetSkipPartitionId() throws AnalysisException {
OlapTable tableIf = Mockito.mock(OlapTable.class);
@@ -885,7 +956,8 @@ public class OlapAnalysisTaskTest {
OlapAnalysisTask task = Mockito.spy(new OlapAnalysisTask());
Mockito.doReturn(new ResultRow(Lists.newArrayList("1",
"2"))).when(task).collectMinMax();
- Mockito.doNothing().when(task).getSampleParams(ArgumentMatchers.any(),
ArgumentMatchers.anyLong());
+ Mockito.doNothing().when(task).getSampleParams(ArgumentMatchers.any(),
ArgumentMatchers.anyLong(),
+ ArgumentMatchers.any());
Mockito.doAnswer(invocation -> {
String sql = invocation.getArgument(0);
Assertions.assertTrue(sql.contains("as `hot_value`"), sql);
@@ -908,9 +980,13 @@ public class OlapAnalysisTaskTest {
task.db = databaseIf;
task.tableSample = new TableSample(false, 100L);
- Mockito.doReturn(true).when(task).useLinearAnalyzeTemplate();
+ Mockito.doReturn(new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)))
+ .when(task).getSampleCollectInfo(Mockito.anyLong());
task.doSample();
- Mockito.doReturn(false).when(task).useLinearAnalyzeTemplate();
+ Mockito.doReturn(new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1,
+ Pair.of(Lists.newArrayList(1L, 2L), 100L)))
+ .when(task).getSampleCollectInfo(Mockito.anyLong());
task.doSample();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
