This is an automated email from the ASF dual-hosted git repository. boroknagyz pushed a commit to branch branch-4.4.0 in repository https://gitbox.apache.org/repos/asf/impala.git
commit 338fedb44703646664e2e22c6e2f35336924db22 Author: Michael Smith <[email protected]> AuthorDate: Mon May 6 16:10:02 2024 -0700 IMPALA-13061: Create query live as external table Impala determines whether a managed table is transactional based on the 'transactional' table property. It assumes any managed table with transactional=true returns non-null getValidWriteIds. When 'default_transactional_type=insert_only' is set at startup (via default_query_options), impala_query_live is created as a managed table with transactional=true, but SystemTables don't implement getValidWriteIds and are not meant to be transactional. DataSourceTable has a similar problem, and when a JDBC table is created setJdbcDataSourceProperties sets transactional=false. This patch uses CREATE EXTERNAL TABLE sys.impala_Query_live so that it is not created as a managed table and 'transactional' is not set. That avoids creating a SystemTable that Impala can't read (it encounters an IllegalStateException). Change-Id: Ie60a2bd03fabc63c85bcd9fa2489e9d47cd2aa65 Reviewed-on: http://gerrit.cloudera.org:8080/21401 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> (cherry picked from commit 1233ac3c579b5929866dba23debae63e5d2aae90) --- be/src/service/workload-management.cc | 5 ++++- tests/custom_cluster/test_query_live.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/be/src/service/workload-management.cc b/be/src/service/workload-management.cc index cf88365d3..15527fa97 100644 --- a/be/src/service/workload-management.cc +++ b/be/src/service/workload-management.cc @@ -122,7 +122,10 @@ static const Status SetupTable(InternalServer* server, const string& table_name, insert_query_opts[TImpalaQueryOptions::SYNC_DDL] = "true"; StringStreamPop create_table_sql; - create_table_sql << "CREATE TABLE IF NOT EXISTS " << table_name << "("; + create_table_sql << "CREATE "; + // System tables do not have anything to purge, and must not be managed tables. + if (is_system_table) create_table_sql << "EXTERNAL "; + create_table_sql << "TABLE IF NOT EXISTS " << table_name << "("; for (const auto& field : FIELD_DEFINITIONS) { create_table_sql << GetColumnName(field) << " " << field.db_column_type; diff --git a/tests/custom_cluster/test_query_live.py b/tests/custom_cluster/test_query_live.py index 58324b23b..bb6746614 100644 --- a/tests/custom_cluster/test_query_live.py +++ b/tests/custom_cluster/test_query_live.py @@ -34,10 +34,20 @@ class TestQueryLive(CustomClusterTestSuite): def setup_method(self, method): super(TestQueryLive, self).setup_method(method) create_match = self.assert_impalad_log_contains("INFO", r'\]\s+(\w+:\w+)\]\s+' - r'Analyzing query: CREATE TABLE IF NOT EXISTS sys.impala_query_live') + r'Analyzing query: CREATE EXTERNAL TABLE IF NOT EXISTS sys.impala_query_live') self.assert_impalad_log_contains("INFO", r'Query successfully unregistered: ' r'query_id={}'.format(create_match.group(1))) + def assert_describe_extended(self): + describe_ext_result = self.execute_query('describe extended sys.impala_query_live') + assert len(describe_ext_result.data) == 80 + system_table_re = re.compile(r'__IMPALA_SYSTEM_TABLE\s+true') + assert list(filter(system_table_re.search, describe_ext_result.data)) + external_re = re.compile(r'EXTERNAL\s+TRUE') + assert list(filter(external_re.search, describe_ext_result.data)) + external_table_re = re.compile(r'Table Type:\s+EXTERNAL_TABLE') + assert list(filter(external_table_re.search, describe_ext_result.data)) + def assert_impalads(self, profile, present=[0, 1, 2], absent=[]): for port_idx in present: assert ":" + str(DEFAULT_KRPC_PORT + port_idx) + ":" in profile @@ -120,9 +130,7 @@ class TestQueryLive(CustomClusterTestSuite): # describe query describe_result = self.execute_query('describe sys.impala_query_live') assert len(describe_result.data) == 49 - - describe_ext_result = self.execute_query('describe extended sys.impala_query_live') - assert len(describe_ext_result.data) == 82 + self.assert_describe_extended() # show create table show_create_tbl = self.execute_query('show create table sys.impala_query_live') @@ -161,6 +169,23 @@ class TestQueryLive(CustomClusterTestSuite): # Drop table at the end, it's only recreated on impalad startup. self.execute_query_expect_success(self.client, 'drop table sys.impala_query_live') + # Must come directly after "drop table sys.impala_query_live" + @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt " + "--cluster_id=test_query_live " + "--use_local_catalog=true", + catalogd_args="--enable_workload_mgmt " + "--catalog_topic_mode=minimal", + default_query_options=[ + ('default_transactional_type', 'insert_only')]) + def test_default_transactional(self): + """Asserts the query live table works when impala is started with + default_transactional_type=insert_only.""" + result = self.client.execute("select * from functional.alltypes", + fetch_profile_after_close=True) + assert_query('sys.impala_query_live', self.client, 'test_query_live', + result.runtime_profile) + self.assert_describe_extended() + @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt " "--cluster_id=test_query_live " "--use_local_catalog=true",
