This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 5c1d61b13f9 [fix](external catalog) Fixed non-master FE log replay
during refresh #46369 #46410 (#46418)
5c1d61b13f9 is described below
commit 5c1d61b13f92991b501c2835c0dd33929f3cd4bf
Author: Mingyu Chen (Rayner) <morning...@163.com>
AuthorDate: Sun Jan 5 21:34:33 2025 +0800
[fix](external catalog) Fixed non-master FE log replay during refresh
#46369 #46410 (#46418)
bp #46369 #46410
---------
Co-authored-by: zy-kkk <zhongy...@gmail.com>
---
.../apache/doris/datasource/ExternalCatalog.java | 2 +-
.../apache/doris/datasource/ExternalDatabase.java | 2 +-
.../lower_case/test_conflict_name.groovy | 10 +-
.../lower_case/test_lower_case_meta_include.groovy | 10 +-
.../test_lower_case_meta_show_and_select.groovy | 28 ++++-
...ta_with_lower_table_conf_show_and_select.groovy | 77 +++++++++++-
.../lower_case/test_lower_case_mtmv.groovy | 10 +-
.../test_meta_cache_select_without_refresh.groovy | 10 +-
.../lower_case/test_meta_names_mapping.groovy | 10 +-
.../lower_case/test_timing_refresh_catalog.groovy | 135 ++++++++++++---------
.../lower_case/upgrade/load.groovy | 25 +++-
11 files changed, 244 insertions(+), 75 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 81a0eac2a68..05ad1c07cb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -723,7 +723,7 @@ public abstract class ExternalCatalog
public void replayInitCatalog(InitCatalogLog log) {
// If the remote name is missing during upgrade, all databases in the
Map will be reinitialized.
- if (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty()) {
+ if (log.getCreateCount() > 0 && (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty())) {
dbNameToId = Maps.newConcurrentMap();
idToDb = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index e7eb3949eae..cbacf563c32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -191,7 +191,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
// If the remote name is missing during upgrade, all tables in the Map
will be reinitialized.
- if (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty()) {
+ if (log.getCreateCount() > 0 && (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty())) {
tableNameToId = Maps.newConcurrentMap();
idToTbl = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
index 6187d47b645..43d6a68c1ad 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
@@ -18,12 +18,16 @@
suite("test_conflict_name", "p0,external,doris,meta_names_mapping") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_conflict_name_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_conflict_name; """
sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """
sql """create database if not exists internal.external_conflict_name; """
@@ -91,4 +95,6 @@ suite("test_conflict_name",
"p0,external,doris,meta_names_mapping") {
sql """drop database if exists internal.external_conflict_name; """
sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
index 854eb06a8e2..cd48678f25c 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_meta_include",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_include_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String mapping_db = """
{
"databases": [
@@ -155,4 +159,6 @@ suite("test_lower_case_meta_include",
"p0,external,doris,external_docker,externa
sql """drop catalog if exists test_lower_case_exclude """
sql """drop database if exists internal.external_INCLUDE; """
sql """drop database if exists internal.external_EXCLUDE; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
index d4efd141c2d..ac86623243d 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
@@ -15,15 +15,35 @@
// specific language governing permissions and limitations
// under the License.
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_without_conf_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ def wait_table_sync = { String db ->
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1,
TimeUnit.SECONDS).until{
+ try {
+ def res = sql "show tables from ${db}"
+ return res.size() > 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+ }
+
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_test_lower; """
sql """drop database if exists internal.external_test_UPPER; """
sql """create database if not exists internal.external_test_lower; """
@@ -71,6 +91,7 @@ suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker
"include_database_list" = "external_test_lower,external_test_UPPER"
)"""
+ wait_table_sync("test_cache_false_lower_false.external_test_lower")
test {
sql """show databases from test_cache_false_lower_false"""
@@ -168,6 +189,7 @@ suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker
"include_database_list" = "external_test_lower,external_test_UPPER"
)"""
+ wait_table_sync("test_cache_false_lower_true.external_test_lower")
test {
sql """show databases from test_cache_false_lower_true"""
@@ -251,4 +273,6 @@ suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker
sql """drop database if exists internal.external_test_lower; """
sql """drop database if exists internal.external_test_UPPER; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
index c3da24e1f55..0c2712e4686 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
@@ -15,17 +15,36 @@
// specific language governing permissions and limitations
// under the License.
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
suite("test_lower_case_meta_with_lower_table_conf_show_and_select",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_with_conf"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ def wait_table_sync = { String db ->
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1,
TimeUnit.SECONDS).until{
+ try {
+ def res = sql "show tables from ${db}"
+ return res.size() > 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+ }
+
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_test_lower_with_conf; """
- sql """drop database if exists internal.external_test_UPPER_with_conf; """
sql """create database if not exists
internal.external_test_lower_with_conf; """
sql """create table if not exists
internal.external_test_lower_with_conf.lower_with_conf
(id int, name varchar(20))
@@ -65,21 +84,29 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"only_test_lower_case_table_names" = "1"
)"""
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_1 "select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_2 "select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_3 "select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_4 "select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_1_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_2_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_3_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf1"
+
wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf1_4_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf"
@@ -114,21 +141,29 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"only_test_lower_case_table_names" = "2"
)"""
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_1 "select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_2 "select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_3 "select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_4 "select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_1_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_2_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_3_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf2"
+
wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf2_4_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf"
test {
@@ -260,21 +295,29 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"only_test_lower_case_table_names" = "1"
)"""
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_1 "select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_2 "select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_3 "select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_4 "select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_1_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_2_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_3_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf1"
+
wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf1_4_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf"
test {
@@ -308,21 +351,29 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"only_test_lower_case_table_names" = "2"
)"""
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_1 "select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_2 "select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_3 "select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_4 "select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_1_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_2_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_3_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf2"
+
wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf2_4_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf"
test {
@@ -398,7 +449,7 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"jdbc_url" = "${jdbcUrl}",
"driver_url" = "${driver_url}",
"driver_class" = "com.mysql.cj.jdbc.Driver",
- "use_meta_cache" = "truee",
+ "use_meta_cache" = "true",
"lower_case_meta_names" = "true",
"only_specified_database" = "true",
"include_database_list" = "external_test_lower_with_conf",
@@ -454,34 +505,42 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"only_test_lower_case_table_names" = "0"
)"""
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf0_1 "select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf0_2 "select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf0_1_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_false_with_conf0_2_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf"
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
test {
sql "select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf"
exception "Table [Lower_with_conf] does not exist in database
[external_test_lower_with_conf]."
}
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
test {
sql "select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf"
exception "Table [upper_with_conf] does not exist in database
[external_test_lower_with_conf]."
}
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
test {
sql "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf"
exception "Table [Lower_with_conf] does not exist in database
[external_test_lower_with_conf]."
}
sql "refresh catalog test_cache_false_lower_false_with_conf0"
+
wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf");
test {
sql "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf"
exception "Table [upper_with_conf] does not exist in database
[external_test_lower_with_conf]."
@@ -585,34 +644,42 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
"only_test_lower_case_table_names" = "0"
)"""
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf0_1 "select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf0_2 "select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf0_1_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
qt_sql_test_cache_false_lower_true_with_conf0_2_insert "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf"
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
test {
sql "select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf"
exception "Table [Lower_with_conf] does not exist in database
[external_test_lower_with_conf]."
}
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
test {
sql "select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf"
exception "Table [UPPER_with_conf] does not exist in database
[external_test_lower_with_conf]."
}
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
test {
sql "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf"
exception "Table [Lower_with_conf] does not exist in database
[external_test_lower_with_conf]."
}
sql "refresh catalog test_cache_false_lower_true_with_conf0"
+
wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf");
test {
sql "insert into
internal.external_test_lower_with_conf.with_conf_insert select * from
test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf"
exception "Table [UPPER_with_conf] does not exist in database
[external_test_lower_with_conf]."
@@ -699,4 +766,6 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """
sql """drop database if exists internal.external_test_lower_with_conf; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
index 40322a22afe..f72b3c9e68d 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_mtmv",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_case_mtmv_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """
sql """create database if not exists internal.EXTERNAL_LOWER_MTMV;"""
sql """create table if not exists internal.EXTERNAL_LOWER_MTMV.TABLE_TEST
@@ -61,4 +65,6 @@ suite("test_lower_case_mtmv",
"p0,external,doris,external_docker,external_docker
sql """drop catalog if exists test_lower_case_mtmv """
sql """drop database if exists internal.EXTERNAL_LOWER_MTMV """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
index 20e6b0f0032..17f37b2c132 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
@@ -18,12 +18,16 @@
suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_meta_cache_select_without_refresh_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """ drop database if exists
internal.external_lower_select_without_refresh; """
sql """create database if not exists
internal.external_lower_select_without_refresh;"""
@@ -89,4 +93,6 @@ suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_dock
sql """drop catalog if exists
test_meta_cache_lower_true_select_without_refresh """
sql """drop catalog if exists
test_meta_cache_lower_false_select_without_refresh """
sql """drop database if exists
internal.external_lower_select_without_refresh; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
index 1cf48b17c87..b58077295b3 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
@@ -18,12 +18,16 @@
suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_meta_names_mapping_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String validMetaNamesMapping = """
{
"databases": [
@@ -286,4 +290,6 @@ suite("test_meta_names_mapping",
"p0,external,doris,meta_names_mapping") {
sql """drop database if exists internal.external_meta_names_mapping; """
sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
index 19d7e2db180..242ca1b8a53 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
@@ -15,15 +15,40 @@
// specific language governing permissions and limitations
// under the License.
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_timing_refresh_catalog_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ def wait_table_sync = { String db ->
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1,
TimeUnit.SECONDS).until{
+ try {
+ def res = sql "show tables from ${db}"
+ return res.size() > 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+ }
+
+ def is_single_fe = {
+ def res = sql "show frontends";
+ return res.size() == 1;
+ }
+
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String mapping = """
{
"databases": [
@@ -49,7 +74,6 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
sql """insert into internal.external_timing_refresh_catalog.tbl values(1,
'lower')"""
sql """drop catalog if exists test_timing_refresh_catalog1 """
-
sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES (
"user" = "${jdbcUser}",
"type" = "jdbc",
@@ -64,22 +88,6 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
"include_database_list" = "external_timing_refresh_catalog"
)"""
- sql """drop catalog if exists test_timing_refresh_catalog2 """
-
- sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES (
- "user" = "${jdbcUser}",
- "type" = "jdbc",
- "password" = "${jdbcPassword}",
- "jdbc_url" = "${jdbcUrl}",
- "driver_url" = "${driver_url}",
- "driver_class" = "com.mysql.cj.jdbc.Driver",
- "use_meta_cache" = "false",
- "metadata_refresh_interval_seconds" = "1",
- "lower_case_meta_names" = "true",
- "only_specified_database" = "true",
- "include_database_list" = "external_timing_refresh_catalog"
- )"""
-
test {
def catalogName = "test_timing_refresh_catalog1"
for (int i = 0; i < 10; i++) {
@@ -90,19 +98,37 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
}
}
- test {
- def catalogName = "test_timing_refresh_catalog2"
- for (int i = 0; i < 10; i++) {
- sql """
- select * from
${catalogName}.external_timing_refresh_catalog.tbl
- """
- Thread.sleep(1000)
+ // when "use_meta_cache" = "false", the meta sync between FEs maybe delay,
+ // and it may cause the case unstable, so only run when there is single FE
+ if (is_single_fe()) {
+ sql """drop catalog if exists test_timing_refresh_catalog2 """
+ sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES (
+ "user" = "${jdbcUser}",
+ "type" = "jdbc",
+ "password" = "${jdbcPassword}",
+ "jdbc_url" = "${jdbcUrl}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "com.mysql.cj.jdbc.Driver",
+ "use_meta_cache" = "false",
+ "metadata_refresh_interval_seconds" = "1",
+ "lower_case_meta_names" = "true",
+ "only_specified_database" = "true",
+ "include_database_list" = "external_timing_refresh_catalog"
+ )"""
+
+ test {
+ def catalogName = "test_timing_refresh_catalog2"
+ for (int i = 0; i < 10; i++) {
+ sql """
+ select * from
${catalogName}.external_timing_refresh_catalog.tbl
+ """
+ Thread.sleep(1000)
+ }
}
}
// with mapping
sql """drop catalog if exists test_timing_refresh_catalog1 """
-
sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES (
"user" = "${jdbcUser}",
"type" = "jdbc",
@@ -118,23 +144,6 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
'meta_names_mapping' = '${mapping}'
)"""
- sql """drop catalog if exists test_timing_refresh_catalog2 """
-
- sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES (
- "user" = "${jdbcUser}",
- "type" = "jdbc",
- "password" = "${jdbcPassword}",
- "jdbc_url" = "${jdbcUrl}",
- "driver_url" = "${driver_url}",
- "driver_class" = "com.mysql.cj.jdbc.Driver",
- "use_meta_cache" = "false",
- "metadata_refresh_interval_seconds" = "1",
- "lower_case_meta_names" = "true",
- "only_specified_database" = "true",
- "include_database_list" = "external_timing_refresh_catalog",
- 'meta_names_mapping' = '${mapping}'
- )"""
-
test {
def catalogName = "test_timing_refresh_catalog1"
for (int i = 0; i < 10; i++) {
@@ -145,17 +154,33 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
}
}
- test {
- def catalogName = "test_timing_refresh_catalog2"
- for (int i = 0; i < 10; i++) {
- sql """
- select id_c from ${catalogName}.db.table_t
- """
- Thread.sleep(1000)
+ if (is_single_fe()) {
+ sql """drop catalog if exists test_timing_refresh_catalog2 """
+ sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES (
+ "user" = "${jdbcUser}",
+ "type" = "jdbc",
+ "password" = "${jdbcPassword}",
+ "jdbc_url" = "${jdbcUrl}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "com.mysql.cj.jdbc.Driver",
+ "use_meta_cache" = "false",
+ "metadata_refresh_interval_seconds" = "1",
+ "lower_case_meta_names" = "true",
+ "only_specified_database" = "true",
+ "include_database_list" = "external_timing_refresh_catalog",
+ 'meta_names_mapping' = '${mapping}'
+ )"""
+
+ test {
+ def catalogName = "test_timing_refresh_catalog2"
+ for (int i = 0; i < 10; i++) {
+ sql """
+ select id_c from ${catalogName}.db.table_t
+ """
+ Thread.sleep(1000)
+ }
}
}
- sql """drop catalog if exists test_timing_refresh_catalog1 """
- sql """drop catalog if exists test_timing_refresh_catalog2 """
- sql """drop database if exists internal.external_timing_refresh_catalog """
-}
\ No newline at end of file
+ try_sql """drop user ${jdbcUser}"""
+}
diff --git
a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
index 0ea89d4012d..782f7b37119 100644
--- a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
@@ -15,15 +15,35 @@
// specific language governing permissions and limitations
// under the License.
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
suite("test_upgrade_lower_case_catalog_prepare",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_upgrade_lower_case_catalog_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ def wait_table_sync = { String db ->
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1,
TimeUnit.SECONDS).until{
+ try {
+ def res = sql "show tables from ${db}"
+ return res.size() > 0;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+ }
+
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.upgrade_lower_case_catalog_lower;
"""
sql """drop database if exists internal.upgrade_lower_case_catalog_UPPER;
"""
sql """create database if not exists
internal.upgrade_lower_case_catalog_lower; """
@@ -59,6 +79,7 @@ suite("test_upgrade_lower_case_catalog_prepare",
"p0,external,doris,external_doc
"include_database_list" =
"upgrade_lower_case_catalog_lower,upgrade_lower_case_catalog_UPPER"
)"""
+
wait_table_sync("test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower")
test {
sql """show databases from test_upgrade_lower_case_catalog"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
