This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7df63e9bf44 Fix hive p2 cases. (#31541)
7df63e9bf44 is described below
commit 7df63e9bf449bcbea1533442c9e7b405c57c59fc
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Wed Feb 28 21:12:26 2024 +0800
Fix hive p2 cases. (#31541)
---
.../hive/test_hive_analyze_db.groovy | 225 +++++----------------
.../hive/test_hive_statistic_cache.groovy | 46 ++---
.../hive/test_hive_statistics_from_hms.groovy | 18 +-
3 files changed, 90 insertions(+), 199 deletions(-)
diff --git
a/regression-test/suites/external_table_p2/hive/test_hive_analyze_db.groovy
b/regression-test/suites/external_table_p2/hive/test_hive_analyze_db.groovy
index 7d6373ffdc5..5853d3e8af9 100644
--- a/regression-test/suites/external_table_p2/hive/test_hive_analyze_db.groovy
+++ b/regression-test/suites/external_table_p2/hive/test_hive_analyze_db.groovy
@@ -15,181 +15,64 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_hive_analyze_db",
"p2,external,hive,external_remote,external_remote_hive") {
- String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
- String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
- String catalog_name = "test_hive_analyze_db"
- sql """drop catalog if exists ${catalog_name};"""
- sql """
+ suite("test_hive_analyze_db",
"p2,external,hive,external_remote,external_remote_hive") {
+
+ def verify_column_stats_result = { column, result, count, ndv, nulls,
size, avg_size, min, max ->
+ def found = false;
+ for (int i = 0; i < result.size(); i++) {
+ if (result[i][0] == column) {
+ found = true;
+ assertEquals(count, result[i][2])
+ assertEquals(ndv, result[i][3])
+ assertEquals(nulls, result[i][4])
+ assertEquals(size, result[i][5])
+ assertEquals(avg_size, result[i][6])
+ assertEquals(min, result[i][7])
+ assertEquals(max, result[i][8])
+ }
+ }
+ assertTrue(found)
+ }
+
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
+ String catalog_name = "test_hive_analyze_db"
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
create catalog if not exists ${catalog_name} properties (
'type'='hms',
'hadoop.username' = 'hadoop',
'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
);
"""
- logger.info("catalog " + catalog_name + " created")
- sql """switch ${catalog_name};"""
- logger.info("switched to catalog " + catalog_name)
- sql """use statistics;"""
- sql """set query_timeout=300"""
- sql """analyze database statistics with sync"""
- def result = sql """show column stats statistics"""
- assertEquals(result.size(), 17)
-
- assertEquals(result[0][0], "lo_orderpriority")
- assertEquals(result[0][2], "100.0")
- assertEquals(result[0][3], "5.0")
- assertEquals(result[0][4], "0.0")
- assertEquals(result[0][5], "880.0")
- assertEquals(result[0][6], "8.8")
- assertEquals(result[0][7], "'1-URGENT'")
- assertEquals(result[0][8], "'5-LOW'")
-
- assertEquals(result[1][0], "lo_custkey")
- assertEquals(result[1][2], "100.0")
- assertEquals(result[1][3], "26.0")
- assertEquals(result[1][4], "0.0")
- assertEquals(result[1][5], "400.0")
- assertEquals(result[1][6], "4.0")
- assertEquals(result[1][7], "67423")
- assertEquals(result[1][8], "2735521")
-
- assertEquals(result[2][0], "lo_partkey")
- assertEquals(result[2][2], "100.0")
- assertEquals(result[2][3], "100.0")
- assertEquals(result[2][4], "0.0")
- assertEquals(result[2][5], "400.0")
- assertEquals(result[2][6], "4.0")
- assertEquals(result[2][7], "2250")
- assertEquals(result[2][8], "989601")
-
- assertEquals(result[3][0], "lo_revenue")
- assertEquals(result[3][2], "100.0")
- assertEquals(result[3][3], "100.0")
- assertEquals(result[3][4], "0.0")
- assertEquals(result[3][5], "400.0")
- assertEquals(result[3][6], "4.0")
- assertEquals(result[3][7], "101171")
- assertEquals(result[3][8], "8703450")
-
- assertEquals(result[4][0], "lo_commitdate")
- assertEquals(result[4][2], "100.0")
- assertEquals(result[4][3], "95.0")
- assertEquals(result[4][4], "0.0")
- assertEquals(result[4][5], "400.0")
- assertEquals(result[4][6], "4.0")
- assertEquals(result[4][7], "19920515")
- assertEquals(result[4][8], "19981016")
-
- assertEquals(result[5][0], "lo_quantity")
- assertEquals(result[5][2], "100.0")
- assertEquals(result[5][3], "46.0")
- assertEquals(result[5][4], "0.0")
- assertEquals(result[5][5], "400.0")
- assertEquals(result[5][6], "4.0")
- assertEquals(result[5][7], "1")
- assertEquals(result[5][8], "50")
-
- assertEquals(result[6][0], "lo_orderkey")
- assertEquals(result[6][2], "100.0")
- assertEquals(result[6][3], "26.0")
- assertEquals(result[6][4], "0.0")
- assertEquals(result[6][5], "400.0")
- assertEquals(result[6][6], "4.0")
- assertEquals(result[6][7], "1")
- assertEquals(result[6][8], "98")
-
- assertEquals(result[7][0], "lo_suppkey")
- assertEquals(result[7][2], "100.0")
- assertEquals(result[7][3], "100.0")
- assertEquals(result[7][4], "0.0")
- assertEquals(result[7][5], "400.0")
- assertEquals(result[7][6], "4.0")
- assertEquals(result[7][7], "4167")
- assertEquals(result[7][8], "195845")
-
- assertEquals(result[8][0], "lo_supplycost")
- assertEquals(result[8][2], "100.0")
- assertEquals(result[8][3], "100.0")
- assertEquals(result[8][4], "0.0")
- assertEquals(result[8][5], "400.0")
- assertEquals(result[8][6], "4.0")
- assertEquals(result[8][7], "58023")
- assertEquals(result[8][8], "121374")
-
- assertEquals(result[9][0], "lo_shipmode")
- assertEquals(result[9][2], "100.0")
- assertEquals(result[9][3], "7.0")
- assertEquals(result[9][4], "0.0")
- assertEquals(result[9][5], "421.0")
- assertEquals(result[9][6], "4.21")
- assertEquals(result[9][7], "'AIR'")
- assertEquals(result[9][8], "'TRUCK'")
-
- assertEquals(result[10][0], "lo_orderdate")
- assertEquals(result[10][2], "100.0")
- assertEquals(result[10][3], "26.0")
- assertEquals(result[10][4], "0.0")
- assertEquals(result[10][5], "400.0")
- assertEquals(result[10][6], "4.0")
- assertEquals(result[10][7], "19920221")
- assertEquals(result[10][8], "19980721")
-
- assertEquals(result[11][0], "lo_linenumber")
- assertEquals(result[11][2], "100.0")
- assertEquals(result[11][3], "7.0")
- assertEquals(result[11][4], "0.0")
- assertEquals(result[11][5], "400.0")
- assertEquals(result[11][6], "4.0")
- assertEquals(result[11][7], "1")
- assertEquals(result[11][8], "7")
-
- assertEquals(result[12][0], "lo_shippriority")
- assertEquals(result[12][2], "100.0")
- assertEquals(result[12][3], "1.0")
- assertEquals(result[12][4], "0.0")
- assertEquals(result[12][5], "400.0")
- assertEquals(result[12][6], "4.0")
- assertEquals(result[12][7], "0")
- assertEquals(result[12][8], "0")
-
- assertEquals(result[13][0], "lo_ordtotalprice")
- assertEquals(result[13][2], "100.0")
- assertEquals(result[13][3], "26.0")
- assertEquals(result[13][4], "0.0")
- assertEquals(result[13][5], "400.0")
- assertEquals(result[13][6], "4.0")
- assertEquals(result[13][7], "3428256")
- assertEquals(result[13][8], "36771805")
-
- assertEquals(result[14][0], "lo_extendedprice")
- assertEquals(result[14][2], "100.0")
- assertEquals(result[14][3], "100.0")
- assertEquals(result[14][4], "0.0")
- assertEquals(result[14][5], "400.0")
- assertEquals(result[14][6], "4.0")
- assertEquals(result[14][7], "104300")
- assertEquals(result[14][8], "9066094")
-
- assertEquals(result[15][0], "lo_tax")
- assertEquals(result[15][2], "100.0")
- assertEquals(result[15][3], "9.0")
- assertEquals(result[15][4], "0.0")
- assertEquals(result[15][5], "400.0")
- assertEquals(result[15][6], "4.0")
- assertEquals(result[15][7], "0")
- assertEquals(result[15][8], "8")
-
- assertEquals(result[16][0], "lo_discount")
- assertEquals(result[16][2], "100.0")
- assertEquals(result[16][3], "11.0")
- assertEquals(result[16][4], "0.0")
- assertEquals(result[16][5], "400.0")
- assertEquals(result[16][6], "4.0")
- assertEquals(result[16][7], "0")
- assertEquals(result[16][8], "10")
- }
-}
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """use statistics;"""
+ sql """set query_timeout=300"""
+ sql """analyze database statistics with sync"""
+ def result = sql """show column stats statistics"""
+ assertEquals(result.size(), 17)
+
+ verify_column_stats_result("lo_orderpriority", result, "100.0",
"5.0", "0.0", "880.0", "8.8", "'1-URGENT'", "'5-LOW'")
+ verify_column_stats_result("lo_custkey", result, "100.0", "26.0",
"0.0", "400.0", "4.0", "67423", "2735521")
+ verify_column_stats_result("lo_partkey", result, "100.0", "100.0",
"0.0", "400.0", "4.0", "2250", "989601")
+ verify_column_stats_result("lo_revenue", result, "100.0", "100.0",
"0.0", "400.0", "4.0", "101171", "8703450")
+ verify_column_stats_result("lo_commitdate", result, "100.0", "95.0",
"0.0", "400.0", "4.0", "19920515", "19981016")
+ verify_column_stats_result("lo_quantity", result, "100.0", "46.0",
"0.0", "400.0", "4.0", "1", "50")
+ verify_column_stats_result("lo_orderkey", result, "100.0", "26.0",
"0.0", "400.0", "4.0", "1", "98")
+ verify_column_stats_result("lo_suppkey", result, "100.0", "100.0",
"0.0", "400.0", "4.0", "4167", "195845")
+ verify_column_stats_result("lo_supplycost", result, "100.0", "100.0",
"0.0", "400.0", "4.0", "58023", "121374")
+ verify_column_stats_result("lo_shipmode", result, "100.0", "7.0",
"0.0", "421.0", "4.21", "'AIR'", "'TRUCK'")
+ verify_column_stats_result("lo_orderdate", result, "100.0", "26.0",
"0.0", "400.0", "4.0", "19920221", "19980721")
+ verify_column_stats_result("lo_linenumber", result, "100.0", "7.0",
"0.0", "400.0", "4.0", "1", "7")
+ verify_column_stats_result("lo_shippriority", result, "100.0", "1.0",
"0.0", "400.0", "4.0", "0", "0")
+ verify_column_stats_result("lo_ordtotalprice", result, "100.0",
"26.0", "0.0", "400.0", "4.0", "3428256", "36771805")
+ verify_column_stats_result("lo_extendedprice", result, "100.0",
"100.0", "0.0", "400.0", "4.0", "104300", "9066094")
+ verify_column_stats_result("lo_tax", result, "100.0", "9.0", "0.0",
"400.0", "4.0", "0", "8")
+ verify_column_stats_result("lo_discount", result, "100.0", "11.0",
"0.0", "400.0", "4.0", "0", "10")
+ }
+ }
diff --git
a/regression-test/suites/external_table_p2/hive/test_hive_statistic_cache.groovy
b/regression-test/suites/external_table_p2/hive/test_hive_statistic_cache.groovy
index 3b18f2a7254..de39d408a53 100644
---
a/regression-test/suites/external_table_p2/hive/test_hive_statistic_cache.groovy
+++
b/regression-test/suites/external_table_p2/hive/test_hive_statistic_cache.groovy
@@ -16,6 +16,21 @@
// under the License.
suite("test_hive_statistic_cache",
"p2,external,hive,external_remote,external_remote_hive") {
+
+ def wait_row_count_reported = { table, expected ->
+ for (int i = 0; i < 10; i++) {
+ result = sql """show table stats ${table}"""
+ logger.info("show table stats result: " + result)
+ assertTrue(result.size() == 1)
+ if (result[0][2] == "0") {
+ Thread.sleep(1000)
+ continue;
+ }
+ assertEquals(expected, result[0][2])
+ break;
+ }
+ }
+
String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
@@ -38,30 +53,15 @@ suite("test_hive_statistic_cache",
"p2,external,hive,external_remote,external_re
sql """desc part""";
sql """desc partsupp""";
sql """desc supplier""";
- Thread.sleep(1000);
- def result = sql """show table cached stats customer"""
- assertEquals(result[0][2], "150000000")
-
- result = sql """show table cached stats lineitem"""
- assertEquals(result[0][2], "5999989709")
-
- result = sql """show table cached stats region"""
- assertEquals(result[0][2], "5")
-
- result = sql """show table cached stats nation"""
- assertEquals(result[0][2], "25")
-
- result = sql """show table cached stats orders"""
- assertEquals(result[0][2], "1500000000")
-
- result = sql """show table cached stats part"""
- assertEquals(result[0][2], "200000000")
-
- result = sql """show table cached stats partsupp"""
- assertEquals(result[0][2], "800000000")
- result = sql """show table cached stats supplier"""
- assertEquals(result[0][2], "10000000")
+ wait_row_count_reported("customer", "150000000")
+ wait_row_count_reported("lineitem", "5999989709")
+ wait_row_count_reported("region", "5")
+ wait_row_count_reported("nation", "25")
+ wait_row_count_reported("orders", "1500000000")
+ wait_row_count_reported("part", "200000000")
+ wait_row_count_reported("partsupp", "800000000")
+ wait_row_count_reported("supplier", "10000000")
logger.info("catalog " + catalog_name + " created")
sql """switch ${catalog_name};"""
diff --git
a/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy
b/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy
index b25102f2278..c3c671bb035 100644
---
a/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy
+++
b/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy
@@ -152,7 +152,7 @@ suite("test_hive_statistics_from_hms",
"p2,external,hive,external_remote,externa
assertTrue(result[0][4] == "0.0")
assertTrue(result[0][5] == "6001215.0")
assertTrue(result[0][6] == "1.0")
-
+
result = sql """show column cached stats lineitem (l_comment)"""
assertTrue(result.size() == 1)
assertTrue(result[0][0] == "l_comment")
@@ -203,10 +203,18 @@ suite("test_hive_statistics_from_hms",
"p2,external,hive,external_remote,externa
assertTrue(result[0][4] == "0.0")
assertTrue(result[0][5] == "7.2006178E7")
assertTrue(result[0][6] == "11.998599950176756")
-
- result = sql """show table cached stats lineitem"""
- assertTrue(result.size() == 1)
- assertTrue(result[0][2] == "6001215")
+
+ for (int i = 0; i < 10; i++) {
+ result = sql """show table stats lineitem"""
+ logger.info("show table stats result: " + result)
+ assertTrue(result.size() == 1)
+ if (result[0][2] == "0") {
+ Thread.sleep(1000)
+ continue;
+ }
+ assertTrue(result[0][2] == "6001215")
+ break;
+ }
sql """drop catalog ${catalog_name}"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
