This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 989973f8b35 [test](mtmv) Fix regression test unstable and add relevant
test log #46299 #46546 (#47112)
989973f8b35 is described below
commit 989973f8b35bf4dfc0d7962b39fe723555ce3115
Author: seawinde <[email protected]>
AuthorDate: Thu Feb 6 15:12:04 2025 +0800
[test](mtmv) Fix regression test unstable and add relevant test log #46299
#46546 (#47112)
### What problem does this PR solve?
1 test](mtmv) Fix regression test unstable and add relevant test log
https://github.com/apache/doris/pull/46299
cb6672a7
2 [test](mtmv) Fix sync mv not partition in rewrite test and some other
test problems
https://github.com/apache/doris/pull/46546
4bd106b5
3 Fix mv regression test fail occasionally and add some log
https://github.com/apache/doris/pull/47103
2e614510
---
.../mv/InitMaterializationContextHook.java | 7 +-
.../doris/nereids/stats/StatsCalculator.java | 4 +
.../java/org/apache/doris/qe/SessionVariable.java | 10 +-
regression-test/data/mv_p0/await/await.out | Bin 0 -> 3493 bytes
.../{unique => unique_rewrite}/unique_rewrite.out | Bin
.../mv/external_table/part_partition_invalid.out | Bin 2031 -> 1324 bytes
.../nereids_syntax_p0/mv/newMv/multi_slot4.out | Bin 198 -> 274 bytes
.../org/apache/doris/regression/suite/Suite.groovy | 159 +++++++++++++--------
.../suites/auth_call/test_ddl_mv_auth.groovy | 4 +-
.../suites/auth_p0/test_select_column_auth.groovy | 2 +-
.../test_mv_case/test_mv_case.groovy | 13 +-
.../create_view_nereids/create_view_use_mv.groovy | 4 +-
...test_export_table_with_materialized_view.groovy | 6 +-
.../no_await.groovy => await/await.groovy} | 61 +++++---
.../suites/mv_p0/no_await/no_await.groovy | 21 +++
.../unique_rewrite.groovy | 0
.../mv_contain_external_table.groovy | 3 +-
.../external_table/part_partition_invalid.groovy | 31 +---
.../mv/external_table/single_external_table.groovy | 6 +-
.../mv/is_in_debug_mode/is_in_debug_mode.groovy | 3 +
.../mv/nested_mtmv/nested_mtmv.groovy | 6 +-
.../mv/union_rewrite/usercase_union_rewrite.groovy | 2 +-
.../nereids_syntax_p0/mv/newMv/multi_slot4.groovy | 10 +-
.../nereids_syntax_p0/mv/ut/distinctQuery.groovy | 2 +-
.../suites/rollup_p0/test_materialized_view.groovy | 66 +++------
.../test_dup_mv_schema_change.groovy | 4 +-
.../test_uniq_mv_schema_change.groovy | 21 +--
27 files changed, 240 insertions(+), 205 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index db270390f9b..2556278c9e3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -175,10 +175,10 @@ public class InitMaterializationContextHook implements
PlannerHook {
}
for (Map.Entry<Long, MaterializedIndexMeta> entry :
olapTable.getVisibleIndexIdToMeta().entrySet()) {
long indexId = entry.getKey();
+ String indexName = olapTable.getIndexNameById(indexId);
try {
if (indexId != baseIndexId) {
MaterializedIndexMeta meta = entry.getValue();
- String indexName = olapTable.getIndexNameById(indexId);
String createMvSql;
if (meta.getDefineStmt() != null) {
// get the original create mv sql
@@ -211,8 +211,9 @@ public class InitMaterializationContextHook implements
PlannerHook {
}
}
} catch (Exception exception) {
- LOG.warn(String.format("createSyncMvContexts exception, index
id is %s, index name is %s",
- entry.getValue(), entry.getValue()), exception);
+ LOG.warn(String.format("createSyncMvContexts exception, index
id is %s, index name is %s, "
+ + "table name is %s", entry.getValue(),
indexName, olapTable.getQualifiedName()),
+ exception);
}
}
return contexts;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 0249a94d67b..77b0cdd712b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -485,8 +485,12 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
// mv is selected, return its estimated stats
Optional<Statistics> optStats =
cascadesContext.getStatementContext()
.getStatistics(((Relation) olapScan).getRelationId());
+ LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is
{}, table name is {}",
+ optStats.isPresent(), tableRowCount,
olapTable.getQualifiedName());
if (optStats.isPresent()) {
double selectedPartitionsRowCount =
getSelectedPartitionRowCount(olapScan);
+ LOG.info("computeOlapScan optStats is {},
selectedPartitionsRowCount is {}", optStats.get(),
+ selectedPartitionsRowCount);
if (selectedPartitionsRowCount == -1) {
selectedPartitionsRowCount = tableRowCount;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 9e75c15dd46..d053369a4f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1060,7 +1060,7 @@ public class SessionVariable implements Serializable,
Writable {
public int maxScanKeyNum = 48;
@VariableMgr.VarAttr(name = MAX_PUSHDOWN_CONDITIONS_PER_COLUMN)
public int maxPushdownConditionsPerColumn = 1024;
- @VariableMgr.VarAttr(name = SHOW_HIDDEN_COLUMNS, flag =
VariableMgr.SESSION_ONLY)
+ @VariableMgr.VarAttr(name = SHOW_HIDDEN_COLUMNS, flag =
VariableMgr.SESSION_ONLY, needForward = true)
public boolean showHiddenColumns = false;
@VariableMgr.VarAttr(name = ALLOW_PARTITION_COLUMN_NULLABLE, description =
{
@@ -1501,25 +1501,25 @@ public class SessionVariable implements Serializable,
Writable {
/**
* For debug purpose, don't merge unique key and agg key when reading data.
*/
- @VariableMgr.VarAttr(name = SKIP_STORAGE_ENGINE_MERGE)
+ @VariableMgr.VarAttr(name = SKIP_STORAGE_ENGINE_MERGE, needForward = true)
public boolean skipStorageEngineMerge = false;
/**
* For debug purpose, skip delete predicate when reading data.
*/
- @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE)
+ @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE, needForward = true)
public boolean skipDeletePredicate = false;
/**
* For debug purpose, skip delete sign when reading data.
*/
- @VariableMgr.VarAttr(name = SKIP_DELETE_SIGN)
+ @VariableMgr.VarAttr(name = SKIP_DELETE_SIGN, needForward = true)
public boolean skipDeleteSign = false;
/**
* For debug purpose, skip delete bitmap when reading data.
*/
- @VariableMgr.VarAttr(name = SKIP_DELETE_BITMAP)
+ @VariableMgr.VarAttr(name = SKIP_DELETE_BITMAP, needForward = true)
public boolean skipDeleteBitmap = false;
// This variable replace the original FE config
`recover_with_skip_missing_version`.
diff --git a/regression-test/data/mv_p0/await/await.out
b/regression-test/data/mv_p0/await/await.out
new file mode 100644
index 00000000000..a123f490bde
Binary files /dev/null and b/regression-test/data/mv_p0/await/await.out differ
diff --git a/regression-test/data/mv_p0/unique/unique_rewrite.out
b/regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out
similarity index 100%
rename from regression-test/data/mv_p0/unique/unique_rewrite.out
rename to regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out
diff --git
a/regression-test/data/nereids_rules_p0/mv/external_table/part_partition_invalid.out
b/regression-test/data/nereids_rules_p0/mv/external_table/part_partition_invalid.out
index d70bd0bbae1..eb3d49efc9b 100644
Binary files
a/regression-test/data/nereids_rules_p0/mv/external_table/part_partition_invalid.out
and
b/regression-test/data/nereids_rules_p0/mv/external_table/part_partition_invalid.out
differ
diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
index 5b500067986..264a653fd3a 100644
Binary files a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
and b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out differ
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 47826ca639e..3002db74128 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -763,6 +763,9 @@ class Suite implements GroovyInterceptable {
(new CreateMVAction(context, sql)).run()
}
+ // Should use create_sync_mv, this method only check the sync mv in
current db
+ // If has multi sync mv in db, may make mistake
+ @Deprecated
void createMV(String sql, String expection) {
(new CreateMVAction(context, sql, expection)).run()
}
@@ -1160,7 +1163,7 @@ class Suite implements GroovyInterceptable {
Connection getTargetConnection() {
return context.getTargetConnection(this)
}
-
+
boolean deleteFile(String filePath) {
def file = new File(filePath)
file.delete()
@@ -1461,71 +1464,101 @@ class Suite implements GroovyInterceptable {
return debugPoint
}
- void waitingMTMVTaskFinishedByMvName(String mvName) {
+ def waitingMTMVTaskFinishedByMvName = { mvName, dbName = context.dbName ->
Thread.sleep(2000);
- String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC"
+ String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}'
order by CreateTime ASC"
String status = "NULL"
List<List<Object>> result
long startTime = System.currentTimeMillis()
long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
- do {
+ List<String> toCheckTaskRow = new ArrayList<>();
+ while (timeoutTimestamp > System.currentTimeMillis() && (status ==
'PENDING' || status == 'RUNNING' || status == 'NULL')) {
result = sql(showTasks)
- logger.info("result: " + result.toString())
- if (!result.isEmpty()) {
- status = result.last().get(4)
- }
+ logger.info("current db is " + dbName + ", showTasks is " +
result.toString())
+ if (result.isEmpty()) {
+ logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is
empty")
+ Thread.sleep(1000);
+ continue;
+ }
+ toCheckTaskRow = result.last();
+ status = toCheckTaskRow.get(4)
logger.info("The state of ${showTasks} is ${status}")
Thread.sleep(1000);
- } while (timeoutTimestamp > System.currentTimeMillis() && (status ==
'PENDING' || status == 'RUNNING' || status == 'NULL'))
+ }
if (status != "SUCCESS") {
logger.info("status is not success")
}
Assert.assertEquals("SUCCESS", status)
- logger.info("waitingMTMVTaskFinished analyze mv name is " +
result.last().get(5))
- sql "analyze table ${result.last().get(6)}.${mvName} with sync;"
+ def show_tables = sql """
+ show tables from ${toCheckTaskRow.get(6)};
+ """
+ def db_id = getDbId(toCheckTaskRow.get(6))
+ def table_id = getTableId(toCheckTaskRow.get(6), mvName)
+ logger.info("waitingMTMVTaskFinished analyze mv name is " + mvName
+ + ", db name is " + toCheckTaskRow.get(6)
+ + ", show_tables are " + show_tables
+ + ", db_id is " + db_id
+ + ", table_id " + table_id)
+ sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;"
}
- void waitingMTMVTaskFinishedByMvNameAllowCancel(String mvName) {
+ def waitingMTMVTaskFinishedByMvNameAllowCancel = {mvName, dbName =
context.dbName ->
Thread.sleep(2000);
- String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC"
+ String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}'
order by CreateTime ASC"
+
String status = "NULL"
List<List<Object>> result
long startTime = System.currentTimeMillis()
long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
- do {
+ List<String> toCheckTaskRow = new ArrayList<>();
+ while (timeoutTimestamp > System.currentTimeMillis() && (status ==
'PENDING' || status == 'RUNNING' || status == 'NULL' || status == 'CANCELED'))
{
result = sql(showTasks)
- logger.info("result: " + result.toString())
- if (!result.isEmpty()) {
- status = result.last().get(4)
- }
+ logger.info("current db is " + dbName + ", showTasks result: " +
result.toString())
+ if (result.isEmpty()) {
+ logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is
empty")
+ Thread.sleep(1000);
+ continue;
+ }
+ toCheckTaskRow = result.last()
+ status = toCheckTaskRow.get(4)
logger.info("The state of ${showTasks} is ${status}")
Thread.sleep(1000);
- } while (timeoutTimestamp > System.currentTimeMillis() && (status ==
'PENDING' || status == 'RUNNING' || status == 'NULL' || status == 'CANCELED'))
+ }
if (status != "SUCCESS") {
logger.info("status is not success")
assertTrue(result.toString().contains("same table"))
}
// Need to analyze materialized view for cbo to choose the
materialized view accurately
- logger.info("waitingMTMVTaskFinished analyze mv name is " +
result.last().get(5))
- sql "analyze table ${result.last().get(6)}.${mvName} with sync;"
+ logger.info("waitingMTMVTaskFinished analyze mv name is " +
toCheckTaskRow.get(5))
+ sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;"
}
- void waitingMVTaskFinishedByMvName(String dbName, String tableName) {
+ void waitingMVTaskFinishedByMvName(String dbName, String tableName, String
indexName) {
Thread.sleep(2000)
- String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName}
where TableName='${tableName}' ORDER BY CreateTime ASC"
+ String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName}
where TableName='${tableName}' ORDER BY CreateTime DESC"
String status = "NULL"
List<List<Object>> result
long startTime = System.currentTimeMillis()
long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
- do {
+ List<String> toCheckTaskRow = new ArrayList<>();
+ while (timeoutTimestamp > System.currentTimeMillis() && (status !=
'FINISHED')) {
result = sql(showTasks)
- logger.info("result: " + result.toString())
- if (!result.isEmpty()) {
- status = result.last().get(8)
+ logger.info("crrent db is " + dbName + ", showTasks result: " +
result.toString())
+ // just consider current db
+ for (List<String> taskRow : result) {
+ if (taskRow.get(5).equals(indexName)) {
+ toCheckTaskRow = taskRow;
+ }
+ }
+ if (toCheckTaskRow.isEmpty()) {
+ logger.info("waitingMVTaskFinishedByMvName toCheckTaskRow is
empty")
+ Thread.sleep(1000);
+ continue;
}
+ status = toCheckTaskRow.get(8)
logger.info("The state of ${showTasks} is ${status}")
Thread.sleep(1000);
- } while (timeoutTimestamp > System.currentTimeMillis() && (status !=
'FINISHED'))
+ }
if (status != "FINISHED") {
logger.info("status is not success")
}
@@ -1579,8 +1612,23 @@ class Suite implements GroovyInterceptable {
}
Assert.assertEquals("SUCCESS", status)
// Need to analyze materialized view for cbo to choose the
materialized view accurately
- logger.info("waitingMTMVTaskFinished analyze mv name is " +
result.last().get(5))
+ def show_tables = sql """
+ show tables from ${result.last().get(6)};
+ """
+ def db_id = getDbId(result.last().get(6))
+ def table_id = getTableId(result.last().get(6), result.last().get(5))
+ logger.info("waitingMTMVTaskFinished analyze mv name is " +
result.last().get(5)
+ + ", db name is " + result.last().get(6)
+ + ", show_tables are " + show_tables
+ + ", db_id is " + db_id
+ + ", table_id " + table_id)
sql "analyze table ${result.last().get(6)}.${result.last().get(5)}
with sync;"
+ String db = result.last().get(6)
+ String table = result.last().get(5)
+ result = sql("show table stats ${db}.${table}")
+ logger.info("table stats: " + result.toString())
+ result = sql("show index stats ${db}.${table} ${table}")
+ logger.info("index stats: " + result.toString())
}
void waitingMTMVTaskFinishedNotNeedSuccess(String jobName) {
@@ -1604,36 +1652,6 @@ class Suite implements GroovyInterceptable {
}
}
- def getMVJobState = { tableName ->
- def jobStateResult = sql """ SHOW ALTER TABLE ROLLUP WHERE
TableName='${tableName}' ORDER BY CreateTime DESC limit 1"""
- if (jobStateResult == null || jobStateResult.isEmpty()) {
- logger.info("show alter table roll is empty" + jobStateResult)
- return "NOT_READY"
- }
- logger.info("getMVJobState jobStateResult is " +
jobStateResult.toString())
- if (!jobStateResult[0][8].equals("FINISHED")) {
- return "NOT_READY"
- }
- return "FINISHED";
- }
- def waitForRollUpJob = (tbName, timeoutMillisecond) -> {
-
- long startTime = System.currentTimeMillis()
- long timeoutTimestamp = startTime + timeoutMillisecond
-
- String result
- while (timeoutTimestamp > System.currentTimeMillis()){
- result = getMVJobState(tbName)
- if (result == "FINISHED") {
- sleep(200)
- return
- } else {
- sleep(200)
- }
- }
- Assert.assertEquals("FINISHED", result)
- }
-
void testFoldConst(String foldSql) {
String openFoldConstant = "set debug_skip_fold_constant=false";
sql(openFoldConstant)
@@ -1901,6 +1919,15 @@ class Suite implements GroovyInterceptable {
return isReady
}
+ def create_sync_mv = { db, table_name, mv_name, mv_sql ->
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name} ON ${table_name};"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ AS ${mv_sql}
+ """
+ waitingMVTaskFinishedByMvName(db, table_name, mv_name)
+ }
+
def create_async_mv = { db, mv_name, mv_sql ->
sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}"""
@@ -2333,6 +2360,20 @@ class Suite implements GroovyInterceptable {
mv_rewrite_fail(query_sql, mv_name, true)
}
+ def async_create_mv = { db, mv_sql, mv_name ->
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS ${mv_sql}
+ """
+
+ def job_name = getJobName(db, mv_name);
+ waitingMTMVTaskFinished(job_name)
+ }
+
def token = context.config.metaServiceToken
def instance_id = context.config.multiClusterInstance
def get_be_metric = { ip, port, field ->
diff --git a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
index 92ec1d9eae0..4b481060440 100644
--- a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
@@ -80,9 +80,9 @@ suite("test_ddl_mv_auth","p0,auth_call") {
connect(user, "${pwd}", context.config.jdbcUrl) {
sql """use ${dbName}"""
sql """create materialized view ${mvName} as select username from
${dbName}.${tableName};"""
- waitingMVTaskFinishedByMvName(dbName, tableName)
+ waitingMVTaskFinishedByMvName(dbName, tableName, mvName)
sql """alter table ${dbName}.${tableName} add rollup
${rollupName}(username)"""
- waitingMVTaskFinishedByMvName(dbName, tableName)
+ waitingMVTaskFinishedByMvName(dbName, tableName, rollupName)
def mv_res = sql """desc ${dbName}.${tableName} all;"""
logger.info("mv_res: " + mv_res)
diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy
b/regression-test/suites/auth_p0/test_select_column_auth.groovy
index c64c64990ca..ba1511c77c0 100644
--- a/regression-test/suites/auth_p0/test_select_column_auth.groovy
+++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy
@@ -84,7 +84,7 @@ suite("test_select_column_auth","p0,auth") {
(3, "333");
"""
sql """refresh MATERIALIZED VIEW ${dbName}.${mtmv_name} auto"""
- waitingMTMVTaskFinishedByMvName(mtmv_name)
+ waitingMTMVTaskFinishedByMvName(mtmv_name, dbName)
sql """grant select_priv on regression_test to ${user}"""
diff --git
a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
index 02a38bf181f..75b0f9f6e0a 100644
--- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
+++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
@@ -16,6 +16,7 @@
// under the License.
suite("test_mv_case") {
+
sql """drop table if exists test_table_aaa2;"""
sql """CREATE TABLE `test_table_aaa2` (
`ordernum` varchar(65533) NOT NULL ,
@@ -29,7 +30,7 @@ suite("test_mv_case") {
"replication_allocation" = "tag.location.default: 1"
);"""
sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON
test_table_aaa2;"""
- createMV("""create materialized view ods_zn_dnt_max1 as
+ create_sync_mv(context.dbName, "test_table_aaa2", "ods_zn_dnt_max1", """
select ordernum,max(dnt) as dnt from test_table_aaa2
group by ordernum
ORDER BY ordernum;""")
@@ -92,7 +93,7 @@ suite("test_mv_case") {
)
"""
sql """insert into tb1 select id,map_agg(a, b) from(select 123 id,3 a,'5'
b union all select 123 id, 6 a, '8' b) aa group by id"""
- createMV ("""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE
ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY
HASH(`info_id`) BUCKETS 2 PROPERTIES (
+ sql"""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON
SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`)
BUCKETS 2 PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
@@ -112,8 +113,9 @@ suite("test_mv_case") {
cast(a.id as bigint) info_id,
map_infos
from
- tb1 a;""")
- createMV ("""CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE
ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY
HASH(`info_id`) BUCKETS 2 PROPERTIES (
+ tb1 a;"""
+ waitingMTMVTaskFinishedByMvName("mv1")
+ sql """CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON
SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`)
BUCKETS 2 PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
@@ -132,6 +134,7 @@ suite("test_mv_case") {
info_id,
map_infos
from
- mv1 a;""")
+ mv1 a;"""
+ waitingMTMVTaskFinishedByMvName("mv2")
qt_select_mv """ select * from mv2 """
}
diff --git
a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
index 348dddb4f88..7605d4b28cb 100644
---
a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
+++
b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
@@ -48,8 +48,8 @@ suite("create_view_use_mv") {
(3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd',
'xxxxxxxxx', '2023-10-19'),
(1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17',
'a', 'b', 'yyyyyyyyy', '2023-10-17');"""
- createMV("""
- CREATE MATERIALIZED VIEW t_mv_mv AS select
+ create_sync_mv(context.dbName, "orders", "t_mv_mv", """
+ select
o_orderkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
diff --git
a/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
b/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
index adba998b16f..65a983f5cc4 100644
---
a/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
+++
b/regression-test/suites/export_p0/test_export_table_with_materialized_view.groovy
@@ -52,9 +52,11 @@ suite("test_export_table_with_materialized_view", "p0") {
)
DISTRIBUTED BY HASH(k1) BUCKETS 10 PROPERTIES("replication_num" = "1");
"""
- sql """
+
+ createMV("""
CREATE MATERIALIZED VIEW export_table_materialized_view AS SELECT k2,
sum(v5) FROM ${table_export_name} GROUP BY k2;
- """
+ """)
+
StringBuilder sb = new StringBuilder()
int i = 1
for (; i < 100; i++) {
diff --git a/regression-test/suites/mv_p0/no_await/no_await.groovy
b/regression-test/suites/mv_p0/await/await.groovy
similarity index 65%
copy from regression-test/suites/mv_p0/no_await/no_await.groovy
copy to regression-test/suites/mv_p0/await/await.groovy
index 866e4fdd5d3..92d83b982a5 100644
--- a/regression-test/suites/mv_p0/no_await/no_await.groovy
+++ b/regression-test/suites/mv_p0/await/await.groovy
@@ -17,9 +17,11 @@
import org.codehaus.groovy.runtime.IOGroovyMethods
-suite ("no_await") {
+suite ("await") {
- def tblName = "agg_have_dup_base_no_await"
+ String db = context.config.getDbNameByFile(context.file)
+
+ def tblName = "agg_have_dup_base_await"
def waitDrop = {
def try_times = 1000
def result = "null"
@@ -58,92 +60,111 @@ suite ("no_await") {
"""
sql "insert into ${tblName} select e1, -4, -4, 'd' from (select 1 k1) as t
lateral view explode_numbers(10000) tmp1 as e1;"
// do not await
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
+
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
}
diff --git a/regression-test/suites/mv_p0/no_await/no_await.groovy
b/regression-test/suites/mv_p0/no_await/no_await.groovy
index 866e4fdd5d3..b6ea39754ea 100644
--- a/regression-test/suites/mv_p0/no_await/no_await.groovy
+++ b/regression-test/suites/mv_p0/no_await/no_await.groovy
@@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite ("no_await") {
+ String db = context.config.getDbNameByFile(context.file)
+
def tblName = "agg_have_dup_base_no_await"
def waitDrop = {
def try_times = 1000
@@ -59,91 +61,110 @@ suite ("no_await") {
sql "insert into ${tblName} select e1, -4, -4, 'd' from (select 1 k1) as t
lateral view explode_numbers(10000) tmp1 as e1;"
// do not await
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
+ sql "sync;"
qt_mv "select sum(k1) from ${tblName}"
}
diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
b/regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy
similarity index 100%
rename from regression-test/suites/mv_p0/unique/unique_rewrite.groovy
rename to regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
index 42e27150c5f..2de15cb324f 100644
---
a/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy
@@ -21,6 +21,7 @@ suite("mv_contain_external_table",
"p0,external,hive,external_docker,external_do
logger.info("diable Hive test. then doesn't test mv rewrite")
return;
}
+ def suite_name = 'mv_contain_external_table'
// prepare table and data in hive
def hive_database = "test_mv_contain_external_table_rewrite_db"
def hive_table = "orders"
@@ -142,7 +143,7 @@ suite("mv_contain_external_table",
"p0,external,hive,external_docker,external_do
order_qt_query_sql """${query_sql}"""
// create mv
- def mv_name = 'mv_join'
+ def mv_name = suite_name + 'mv_join'
sql """drop materialized view if exists ${mv_name}"""
sql """
CREATE MATERIALIZED VIEW ${mv_name}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
index 43cde7c927c..3c2ad76c81a 100644
---
a/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
@@ -124,7 +124,7 @@ suite("part_partition_invalid",
"p0,external,external_docker") {
order_qt_query_sql """${query_sql}"""
// create partition mtmv, related partition is hive catalog
- def mv_name = 'mv_join'
+ def mv_name = suite_name + 'mv_join'
sql """drop materialized view if exists ${mv_name}"""
sql """
CREATE MATERIALIZED VIEW ${mv_name}
@@ -148,21 +148,6 @@ suite("part_partition_invalid",
"p0,external,external_docker") {
// data change in external table doesn't influence query rewrite,
// if want to use new data in external table should be refresh manually
sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table}
values(3, 3, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-19');"""
- mv_rewrite_success(query_sql, mv_name)
- order_qt_after_modify_data_without_refresh_catalog """ ${query_sql}"""
-
- // query invalid partition data, should hit mv, because not check now.
- mv_rewrite_fail("""
- ${query_sql} where o_orderdate = '2023-10-19';
- """, mv_name)
- order_qt_after_modify_and_without_refresh_catalog_19 """ ${query_sql}
where o_orderdate = '2023-10-19';"""
-
- // query valid partition data, should hit mv
- mv_rewrite_success("""
- ${query_sql} where o_orderdate = '2023-10-18';
- """, mv_name
- )
- order_qt_after_modify_and_without_refresh_catalog_18 """ ${query_sql}
where o_orderdate = '2023-10-18';"""
// refresh catalog cache
sql """ REFRESH CATALOG ${hive_catalog_name} PROPERTIES("invalid_cache" =
"true"); """
@@ -190,20 +175,6 @@ suite("part_partition_invalid",
"p0,external,external_docker") {
// test after hive add partition
sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table}
values(6, 7, 'ok', 29.5, 'x', 'y', 6, 'ss', '2023-10-20');"""
- mv_rewrite_success(query_sql, mv_name)
- order_qt_after_add_data_without_refresh_catalog """ ${query_sql}"""
-
- // query invalid partition data, should hit mv, because not check now.
- mv_rewrite_success(query_sql, mv_name)
-
- order_qt_after_add_and_without_refresh_catalog_19 """ ${query_sql} where
o_orderdate = '2023-10-19';"""
-
- // query valid partition data, should hit mv, because data not aware
- mv_rewrite_fail("""
- ${query_sql} where o_orderdate = '2023-10-20';
- """, mv_name)
-
- order_qt_after_add_and_without_refresh_catalog_20 """ ${query_sql} where
o_orderdate = '2023-10-20';"""
// refresh catalog cache
sql """ REFRESH CATALOG ${hive_catalog_name} PROPERTIES("invalid_cache" =
"true"); """
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
index 1600dc7cc22..d78899a12ec 100644
---
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
@@ -88,7 +88,8 @@ suite("single_external_table", "p0,external,hive") {
from ${hive_catalog_name}.${hive_database}.${hive_table};
"""
order_qt_query1_0_before "${query1_0}"
- async_mv_rewrite_success(olap_db, mv1_0, query1_0, "mv1_0")
+ // todo after add cost on external table, change to check
+ async_mv_rewrite_success_without_check_chosen(olap_db, mv1_0, query1_0,
"mv1_0")
order_qt_query1_0_after "${query1_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
@@ -105,7 +106,8 @@ suite("single_external_table", "p0,external,hive") {
where o_custkey > 2;
"""
order_qt_query1_1_before "${query1_1}"
- async_mv_rewrite_success(olap_db, mv1_1, query1_1, "mv1_1")
+ // todo after add cost on external table, change to check
+ async_mv_rewrite_success_without_check_chosen(olap_db, mv1_1, query1_1,
"mv1_1")
order_qt_query1_1_after "${query1_1}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
diff --git
a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
index ebc1921c0ab..e4c93660764 100644
---
a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
@@ -75,6 +75,9 @@ suite("is_in_debug_mode") {
sql """set skip_delete_sign = true;"""
mv_not_part_in("""select * from orders where o_orderkey > 1;""",
"basic_mv")
logger.info("skip_delete_sign session is " + sql("show variables like
'%skip_delete_sign%'"))
+
+ sql """drop materialized view if exists test_create_mv;"""
+
test {
sql """
CREATE MATERIALIZED VIEW test_create_mv
diff --git
a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
index 4870ec99e65..1972c2d505b 100644
--- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
@@ -742,10 +742,12 @@ suite("nested_mtmv") {
mv_rewrite_any_success(sql_2, [mv_1, mv_2])
compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
- mv_rewrite_any_success(sql_3, [mv_3, mv_4])
+ // level 1 maybe use mv_1 and mv_2, this also meets expectation
+ mv_rewrite_any_success(sql_3, [mv_3, mv_4, mv_1, mv_2])
compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
- mv_rewrite_any_success(sql_4, [mv_3, mv_4])
+ // level 1 maybe use mv_1 and mv_2, this also meets expectation
+ mv_rewrite_any_success(sql_4, [mv_3, mv_4, mv_1, mv_2])
compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5])
diff --git
a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
index a5bc5586e51..dbbc6f929ef 100644
---
a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
@@ -93,7 +93,7 @@ suite ("usercase_union_rewrite") {
sql """analyze table orders_user with sync;"""
sql """analyze table lineitem_user with sync;"""
- sql """alter table orders_user modify column o_comment set stats
('row_count'='4');"""
+ sql """alter table orders_user modify column o_comment set stats
('row_count'='7');"""
sql """alter table lineitem_user modify column l_comment set stats
('row_count'='3');"""
def create_mv_orders = { mv_name, mv_sql ->
diff --git
a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
index 7a49dbc4b37..32e1684fdc9 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
@@ -33,21 +33,29 @@ suite ("multi_slot4") {
"""
sql "insert into multi_slot4 select 1,1,1,'a';"
+ sql "insert into multi_slot4 select 1,1,1,'a';"
+ sql "insert into multi_slot4 select 1,1,1,'a';"
+ sql "insert into multi_slot4 select 2,2,2,'b';"
+ sql "insert into multi_slot4 select 2,2,2,'b';"
sql "insert into multi_slot4 select 2,2,2,'b';"
sql "insert into multi_slot4 select 3,-3,null,'c';"
sql "insert into multi_slot4 select 3,-3,null,'c';"
+ sql "insert into multi_slot4 select 3,-3,null,'c';"
+ sql "insert into multi_slot4 select 3,-3,null,'c';"
createMV ("create materialized view k1p2ap3ps as select
k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1;")
sleep(3000)
+ sql "insert into multi_slot4 select -4,-4,-4,'d';"
+ sql "insert into multi_slot4 select -4,-4,-4,'d';"
sql "insert into multi_slot4 select -4,-4,-4,'d';"
sql "SET experimental_enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
sql "analyze table multi_slot4 with sync;"
- sql """alter table multi_slot4 modify column k1 set stats
('row_count'='5');"""
+ sql """alter table multi_slot4 modify column k1 set stats
('row_count'='13');"""
sql """set enable_stats=false;"""
diff --git
a/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
b/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
index c95c8737967..10d86e3c5d0 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/ut/distinctQuery.groovy
@@ -48,7 +48,7 @@ suite ("distinctQuery") {
sql "analyze table distinctQuery with sync;"
- mv_rewrite_success("select distinct deptno from distinctQuery;",
"distinctQuery_mv")
+ mv_rewrite_any_success("select distinct deptno from distinctQuery;",
["distinctQuery_mv", "distinctQuery_mv2"])
mv_rewrite_success("select deptno, count(distinct empid) from
distinctQuery group by deptno;", "distinctQuery_mv2")
diff --git a/regression-test/suites/rollup_p0/test_materialized_view.groovy
b/regression-test/suites/rollup_p0/test_materialized_view.groovy
index ace5c9b740b..fda67f16b3d 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view.groovy
@@ -14,6 +14,10 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
+import org.awaitility.Awaitility
+import static java.util.concurrent.TimeUnit.SECONDS
+
suite("test_materialized_view") {
def tbName1 = "test_materialized_view1"
@@ -45,38 +49,15 @@ suite("test_materialized_view") {
)
DISTRIBUTED BY HASH(record_id) properties("replication_num" = "1");
"""
- sql "CREATE materialized VIEW amt_sum AS SELECT store_id, sum(sale_amt)
FROM ${tbName1} GROUP BY store_id;"
- int max_try_secs = 60
- while (max_try_secs--) {
- String res = getJobState(tbName1)
- if (res == "FINISHED" || res == "CANCELLED") {
- assertEquals("FINISHED", res)
- sleep(3000)
- break
- } else {
- Thread.sleep(2000)
- if (max_try_secs < 1) {
- println "test timeout," + "state:" + res
- assertEquals("FINISHED",res)
- }
- }
- }
- sql "CREATE materialized VIEW seller_id_order AS SELECT
store_id,seller_id, sale_amt FROM ${tbName2} ORDER BY store_id,seller_id;"
- max_try_secs = 60
- while (max_try_secs--) {
- String res = getJobState(tbName2)
- if (res == "FINISHED" || res == "CANCELLED") {
- assertEquals("FINISHED", res)
- sleep(3000)
- break
- } else {
- Thread.sleep(2000)
- if (max_try_secs < 1) {
- println "test timeout," + "state:" + res
- assertEquals("FINISHED",res)
- }
- }
- }
+
+ createMV("""
+ CREATE materialized VIEW amt_sum AS SELECT store_id, sum(sale_amt) FROM
${tbName1} GROUP BY store_id;
+ """)
+
+ createMV("""
+ CREATE materialized VIEW seller_id_order AS SELECT store_id,seller_id,
sale_amt FROM ${tbName2} ORDER BY store_id,seller_id;
+ """)
+
sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName1}';"
sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName2}';"
qt_sql "DESC ${tbName1} ALL;"
@@ -95,22 +76,11 @@ suite("test_materialized_view") {
qt_sql "SELECT * FROM ${tbName2} order by record_id;"
qt_sql "SELECT store_id, sum(sale_amt) FROM ${tbName2} GROUP BY store_id
order by store_id;"
- sql "CREATE materialized VIEW amt_count AS SELECT store_id,
count(sale_amt) FROM ${tbName1} GROUP BY store_id;"
- max_try_secs = 60
- while (max_try_secs--) {
- String res = getJobState(tbName1)
- if (res == "FINISHED" || res == "CANCELLED") {
- assertEquals("FINISHED", res)
- sleep(3000)
- break
- } else {
- Thread.sleep(2000)
- if (max_try_secs < 1) {
- println "test timeout," + "state:" + res
- assertEquals("FINISHED",res)
- }
- }
- }
+
+ createMV("""
+ CREATE materialized VIEW amt_count AS SELECT store_id, count(sale_amt)
FROM ${tbName1} GROUP BY store_id;
+ """)
+
sql "SELECT store_id, count(sale_amt) FROM ${tbName1} GROUP BY store_id;"
order_qt_sql "DESC ${tbName1} ALL;"
diff --git
a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
index 713c470436e..a6ad20ec623 100644
--- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
@@ -70,7 +70,7 @@ suite ("test_dup_mv_schema_change") {
"""
//add materialized view
- createMV("create materialized view mv1 as select date, user_id, city,
age from ${tableName};")
+ create_sync_mv(context.dbName, tableName, "mv1", """select date,
user_id, city, age from ${tableName}""")
// alter and test light schema change
if (!isCloudMode()) {
@@ -78,7 +78,7 @@ suite ("test_dup_mv_schema_change") {
}
//add materialized view
- createMV("create materialized view mv2 as select date, user_id, city,
age, cost from ${tableName};")
+ create_sync_mv(context.dbName, tableName, "mv2", """select date,
user_id, city, age, cost from ${tableName}""")
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02',
'2020-01-02', '2020-01-02', 1, 31, 21)
diff --git
a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
index eba6036c30a..1d8fdd4d1e7 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
@@ -21,20 +21,7 @@ import org.awaitility.Awaitility
suite ("test_uniq_mv_schema_change") {
def tableName = "schema_change_uniq_mv_regression_test"
- def getMVJobState = { tbName ->
- def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW
WHERE TableName='${tbName}' ORDER BY CreateTime DESC LIMIT 1 """
- return jobStateResult[0][8]
- }
- def waitForJob = (tbName, timeout) -> {
- Awaitility.await().atMost(timeout,
TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(()
-> {
- String result = getMVJobState(tbName)
- if (result == "FINISHED") {
- return true;
- }
- return false;
- });
- // when timeout awaitlity will raise a exception.
- }
+
try {
String backend_id;
@@ -78,8 +65,7 @@ suite ("test_uniq_mv_schema_change") {
//add materialized view
def mvName = "mv1"
- sql "create materialized view ${mvName} as select user_id, date, city,
age, sex from ${tableName};"
- waitForJob(tableName, 3000)
+ create_sync_mv(context.dbName, tableName, mvName, """select user_id, date,
city, age, sex from ${tableName}""")
// alter and test light schema change
if (!isCloudMode()) {
@@ -88,8 +74,7 @@ suite ("test_uniq_mv_schema_change") {
//add materialized view
def mvName2 = "mv2"
- sql "create materialized view ${mvName2} as select user_id, date, city,
age, sex, cost from ${tableName};"
- waitForJob(tableName, 3000)
+ create_sync_mv(context.dbName, tableName, mvName2, """select user_id,
date, city, age, sex, cost from ${tableName};""")
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02',
'2020-01-02', 1, 31, 21)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
