This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 4f3251e19 IMPALA-14582: Deflake test_show_create_table_with_stats
4f3251e19 is described below
commit 4f3251e19a5e71dded882be180cf71cc341eaaa2
Author: Riza Suminto <[email protected]>
AuthorDate: Mon Dec 1 17:24:13 2025 -0800
IMPALA-14582: Deflake test_show_create_table_with_stats
test_show_create_table_with_stats is flaky due to inconsistent metadata
is not handled/retried correctly in coordinator side. This patch deflake
it by retrying if InconsistentMetadataFetchException is caught.
This patch also fix some flake8 warnings in test_show_create_table.py,
including unused 'vector' parameter in several tests.
Testing:
Loop and pass test_show_create_table_with_stats 10 times.
Change-Id: I397b9502d92bfd756929be8e851661fd9246dd5e
Reviewed-on: http://gerrit.cloudera.org:8080/23728
Reviewed-by: Quanlong Huang <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
.../java/org/apache/impala/service/Frontend.java | 17 ++++++++++++
.../org/apache/impala/service/JniFrontend.java | 20 ++++----------
tests/metadata/test_show_create_table.py | 32 ++++++++++------------
3 files changed, 38 insertions(+), 31 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java
b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 1fd38f246..ad21b83f6 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -108,6 +108,7 @@ import org.apache.impala.analysis.StmtMetadataLoader;
import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache;
import org.apache.impala.analysis.TableName;
import org.apache.impala.analysis.TableRef;
+import org.apache.impala.analysis.ToSqlUtils;
import org.apache.impala.analysis.TruncateStmt;
import org.apache.impala.authentication.saml.ImpalaSamlClient;
import org.apache.impala.authorization.AuthorizationChecker;
@@ -3784,4 +3785,20 @@ public class Frontend {
}
return null;
}
+
+ public String getShowCreateTable(
+ TTableName tname, boolean withStats, int partitionLimit) throws
ImpalaException {
+ Frontend.RetryTracker retries = new Frontend.RetryTracker(
+ String.format("show create table %s.%s", tname.db_name,
tname.table_name));
+ while (true) {
+ try {
+ FeTable table = getCatalog().getTable(tname.getDb_name(),
tname.getTable_name());
+ if (withStats) {
+ return ToSqlUtils.getCreateTableWithStatsSql(table, partitionLimit);
+ } else {
+ return ToSqlUtils.getCreateTableSql(table);
+ }
+ } catch (InconsistentMetadataFetchException e) {
retries.handleRetryOrThrow(e); }
+ }
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 2cafe7c03..4627c0e8c 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -21,7 +21,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -42,7 +41,6 @@ import org.apache.impala.authorization.AuthorizationFactory;
import org.apache.impala.authorization.ImpalaInternalAdminUser;
import org.apache.impala.authorization.User;
import org.apache.impala.catalog.FeDataSource;
-import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.Function;
import org.apache.impala.common.UserCancelledException;
@@ -57,7 +55,6 @@ import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TBuildTestDescriptorTableParams;
import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TCivilTime;
-import org.apache.impala.thrift.TDatabase;
import org.apache.impala.thrift.TDescribeDbParams;
import org.apache.impala.thrift.TDescribeHistoryParams;
import org.apache.impala.thrift.TDescribeResult;
@@ -120,7 +117,6 @@ import java.io.IOException;
import java.lang.IllegalArgumentException;
import java.nio.charset.StandardCharsets;
import java.util.Calendar;
-import java.util.Collections;
import java.util.GregorianCalendar;
import java.util.Enumeration;
import java.util.List;
@@ -602,18 +598,14 @@ public class JniFrontend {
JniUtil.deserializeThrift(protocolFactory_, req, thriftParams);
Preconditions.checkState(req.isSetShow_create_table_params());
TTableName tname = req.getShow_create_table_params();
- FeTable table = frontend_.getCatalog().getTable(tname.getDb_name(),
- tname.getTable_name());
boolean withStats = req.isSetShow_create_table_with_stats()
&& req.show_create_table_with_stats;
- if (withStats) {
- // Get show_create_table_partition_limit from request, default to 1000
if not set
- int partitionLimit = req.isSetShow_create_table_partition_limit() ?
- req.getShow_create_table_partition_limit() : 1000;
- return ToSqlUtils.getCreateTableWithStatsSql(table, partitionLimit);
- } else {
- return ToSqlUtils.getCreateTableSql(table);
- }
+ // Get show_create_table_partition_limit from request, default to 1000 if
not
+ // set.
+ int partitionLimit = req.isSetShow_create_table_partition_limit() ?
+ req.getShow_create_table_partition_limit() :
+ 1000;
+ return frontend_.getShowCreateTable(tname, withStats, partitionLimit);
}
/**
diff --git a/tests/metadata/test_show_create_table.py
b/tests/metadata/test_show_create_table.py
index c4be7af2c..797d8bc96 100644
--- a/tests/metadata/test_show_create_table.py
+++ b/tests/metadata/test_show_create_table.py
@@ -78,35 +78,33 @@ class TestShowCreateTable(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_constraint(
- lambda v: v.get_value('table_format').file_format == 'text' and
- v.get_value('table_format').compression_codec == 'none')
+ lambda v: (v.get_value('table_format').file_format == 'text'
+ and v.get_value('table_format').compression_codec ==
'none'))
- def test_show_create_table(self, vector, unique_database):
- self.__run_show_create_table_test_case('QueryTest/show-create-table',
vector,
+ def test_show_create_table(self, unique_database):
+ self.__run_show_create_table_test_case('QueryTest/show-create-table',
unique_database)
@SkipIfFS.hbase
- def test_show_create_table_hbase(self, vector, unique_database):
-
self.__run_show_create_table_test_case('QueryTest/show-create-table-hbase',
vector,
+ def test_show_create_table_hbase(self, unique_database):
+ self.__run_show_create_table_test_case('QueryTest/show-create-table-hbase',
unique_database)
@SkipIfHive2.acid
- def test_show_create_table_full_acid(self, vector, unique_database):
+ def test_show_create_table_full_acid(self, unique_database):
self.__run_show_create_table_test_case('QueryTest/show-create-table-full-acid',
- vector,
unique_database)
@SkipIf.not_hdfs
- def test_show_create_table_paimon(self, vector, unique_database):
+ def test_show_create_table_paimon(self, unique_database):
self.__run_show_create_table_test_case('QueryTest/show-create-table-paimon',
- vector,
unique_database)
- def test_show_create_table_with_stats(self, vector, unique_database):
+ def test_show_create_table_with_stats(self, unique_database):
self.__run_show_create_table_with_stats_test_case(
- 'QueryTest/show-create-table-with-stats', vector, unique_database)
+ 'QueryTest/show-create-table-with-stats', unique_database)
- def __run_show_create_table_test_case(self, test_file_name, vector,
unique_db_name):
+ def __run_show_create_table_test_case(self, test_file_name, unique_db_name):
"""
Runs a show-create-table test file, containing the following sections:
@@ -169,7 +167,7 @@ class TestShowCreateTable(ImpalaTestSuite):
self.__exec(test_case.drop_table_sql)
def __run_show_create_table_with_stats_test_case(
- self, test_file_name, vector, unique_db_name):
+ self, test_file_name, unique_db_name):
sections = self.load_query_test_file(
self.get_workload(), test_file_name, self.VALID_SECTION_NAMES)
for test_section in sections:
@@ -451,8 +449,8 @@ class TestInfraCompat(ImpalaTestSuite):
def test_primary_key_parse(self, impala_testinfra_cursor,
table_primary_keys_map):
"""
Test the query generator's Impala -> Postgres data migrator's ability to
parse primary
- keys via SHOW CREATE TABLE. If this test fails, update
_fetch_primary_key_names, or fix
- the SHOW CREATE TABLE defect.
+ keys via SHOW CREATE TABLE. If this test fails, update
_fetch_primary_key_names, or
+ fix the SHOW CREATE TABLE defect.
"""
assert impala_testinfra_cursor._fetch_primary_key_names(
table_primary_keys_map['table']) ==
table_primary_keys_map['primary_keys']
@@ -510,7 +508,7 @@ class TestShowCreateTableIcebergProperties(ImpalaTestSuite):
lambda v: v.get_value('table_format').file_format == 'parquet'
and v.get_value('table_format').compression_codec == 'none')
- def test_iceberg_properties(self, vector, unique_database):
+ def test_iceberg_properties(self, unique_database):
"""
Test that the SHOW CREATE TABLE statement does not contain irrelevant
Iceberg-related
table properties.
