This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit a46625a3c03a50275cdf91d3abc2f8e445913c2b Author: Jason Fehr <[email protected]> AuthorDate: Mon Nov 4 11:31:37 2024 -0800 IMPALA-12737: (addendum) Turn Off Log Buffering in Workload Management Init Tests Fixes the issue where custom cluster workload management tests do not disable glog log buffering in tests that wait for specific messages to be logged from the coordinators and catalogs. By default, logs are buffered up to 30 seconds. This buffering can cause unnecessary test slowness while the tests wait longer than needed for the expected log message to be flushed and can also cause flakiness where the tests do not find the expected log message before the timeout expires. Change-Id: I03ac0f0f00c93fe785db131278a706e3f5e975c2 Reviewed-on: http://gerrit.cloudera.org:8080/22021 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- tests/custom_cluster/test_workload_mgmt_init.py | 21 +++++++++------ .../test_workload_mgmt_sql_details.py | 30 ++++++++++++++-------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/custom_cluster/test_workload_mgmt_init.py b/tests/custom_cluster/test_workload_mgmt_init.py index 807f640a6..43f7c9239 100644 --- a/tests/custom_cluster/test_workload_mgmt_init.py +++ b/tests/custom_cluster/test_workload_mgmt_init.py @@ -58,10 +58,10 @@ class TestWorkloadManagementInitBase(CustomClusterTestSuite): function blocks until the workload management init process completes. If additional_impalad_opts is specified, that string is appended to the impala_args startup flag.""" - coord_opts = "--impalad_args=--enable_workload_mgmt " + coord_opts = "--impalad_args=--enable_workload_mgmt --logbuflevel=-1 " coord_opts += additional_impalad_opts - catalog_opts = "--catalogd_args=--enable_workload_mgmt " + catalog_opts = "--catalogd_args=--enable_workload_mgmt --logbuflevel=-1 " catalog_opts += additional_catalogd_opts if schema_version: @@ -175,7 +175,7 @@ class TestWorkloadManagementInitWait(TestWorkloadManagementInitBase): self.assert_catalogd_log_contains("INFO", r"Workload management table .*? will be " r"upgraded", expected_count=0) - @CustomClusterTestSuite.with_args(cluster_size=1, + @CustomClusterTestSuite.with_args(cluster_size=1, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --workload_mgmt_schema_version=1.0.0", catalogd_args="--enable_workload_mgmt " "--workload_mgmt_schema_version=1.0.0 " @@ -191,7 +191,8 @@ class TestWorkloadManagementInitWait(TestWorkloadManagementInitBase): self.run_test_case('QueryTest/workload-management-log-v1.0.0', vector, self.WM_DB) self.run_test_case('QueryTest/workload-management-live-v1.0.0', vector, self.WM_DB) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_args="--enable_workload_mgmt", + @CustomClusterTestSuite.with_args(cluster_size=1, disable_log_buffering=True, + impalad_args="--enable_workload_mgmt", catalogd_args="--enable_workload_mgmt " "--workload_mgmt_drop_tables=impala_query_log,impala_query_live") def test_create_on_version_1_1_0(self, vector): @@ -280,7 +281,8 @@ class TestWorkloadManagementInitWait(TestWorkloadManagementInitBase): TQueryTableColumn.AGGREGATE_COLUMNS: "NULL", TQueryTableColumn.ORDERBY_COLUMNS: "NULL"}) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_args="--enable_workload_mgmt", + @CustomClusterTestSuite.with_args(cluster_size=1, disable_log_buffering=True, + impalad_args="--enable_workload_mgmt", catalogd_args="--enable_workload_mgmt " "--query_log_table_props=\"foo=bar,foo1=bar1\" " "--workload_mgmt_drop_tables=impala_query_log,impala_query_live") @@ -291,7 +293,8 @@ class TestWorkloadManagementInitWait(TestWorkloadManagementInitBase): self.assert_table_prop(self.QUERY_TBL_LOG, "foo", "bar") self.assert_table_prop(self.QUERY_TBL_LIVE, "foo", "bar") - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_args="--enable_workload_mgmt", + @CustomClusterTestSuite.with_args(cluster_size=1, disable_log_buffering=True, + impalad_args="--enable_workload_mgmt", catalogd_args="--enable_workload_mgmt " "--workload_mgmt_drop_tables=impala_query_log,impala_query_live") def test_create_from_scratch(self): @@ -374,7 +377,8 @@ class TestWorkloadManagementInitWait(TestWorkloadManagementInitBase): sys.impala_query_live table contains an invalid value.""" self._run_invalid_table_prop_test(self.QUERY_TBL_LIVE, "wm_schema_version") - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_args="--enable_workload_mgmt", + @CustomClusterTestSuite.with_args(cluster_size=1, disable_log_buffering=True, + impalad_args="--enable_workload_mgmt", catalogd_args="--enable_workload_mgmt") def test_upgrade_to_latest_from_previous_binary(self): """Simulated an upgrade situation from workload management tables created by previous @@ -408,7 +412,8 @@ class TestWorkloadManagementInitWait(TestWorkloadManagementInitBase): "impala-server.completed-queries.written", 2, 60) assert_query(self.QUERY_TBL_LOG, client, impalad=impalad, query_id=res.query_id) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_args="--enable_workload_mgmt", + @CustomClusterTestSuite.with_args(cluster_size=1, disable_log_buffering=True, + impalad_args="--enable_workload_mgmt", catalogd_args="--enable_workload_mgmt") def test_start_at_1_0_0(self): """Tests the situation where workload management tables were created by the original diff --git a/tests/custom_cluster/test_workload_mgmt_sql_details.py b/tests/custom_cluster/test_workload_mgmt_sql_details.py index cfe7174d5..4777c5e3c 100644 --- a/tests/custom_cluster/test_workload_mgmt_sql_details.py +++ b/tests/custom_cluster/test_workload_mgmt_sql_details.py @@ -101,7 +101,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): self._assert_all(vector, query_text, expected_tables_queried, expected_select, expected_where, expected_join, expected_aggregate, expected_orderby) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_1(self, vector): @@ -133,7 +134,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): # Expected order by columns. ["customer.c_customer_id"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_3(self, vector): @@ -163,7 +165,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): # Expected order by columns. ["date_dim.d_year", "store_sales.ss_ext_sales_price", "item.i_brand_id"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_51(self, vector): @@ -195,7 +198,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): # Expected order by columns. ["web_sales.ws_item_sk", "store_sales.ss_item_sk", "date_dim.d_date"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_62(self, vector): @@ -228,7 +232,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): # Expected order by columns. ["warehouse.w_warehouse_name", "ship_mode.sm_type", "web_site.web_name"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_64(self, vector): @@ -298,7 +303,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): # Expected order by columns. ["item.i_product_name", "store.s_store_name", "store_sales.ss_wholesale_cost"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_66(self, vector): @@ -343,7 +349,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): # Expected order by columns. ["warehouse.w_warehouse_name"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_tpcds_75(self, vector): @@ -397,7 +404,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): "web_returns.wr_return_amt", "web_returns.wr_return_quantity", "web_sales.ws_ext_sales_price", "web_sales.ws_quantity"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_sql_having(self, vector): @@ -414,7 +422,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): ["store_sales.ss_item_sk", "store_sales.ss_quantity"], ["store_sales.ss_quantity"]) - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_complex_types(self, vector): @@ -430,7 +439,8 @@ class TestWorkloadManagementSQLDetails(TestQueryLogTableBase): [], "functional") - @CustomClusterTestSuite.with_args(cluster_size=1, impalad_graceful_shutdown=True, + @CustomClusterTestSuite.with_args( + cluster_size=1, impalad_graceful_shutdown=True, disable_log_buffering=True, impalad_args="--enable_workload_mgmt --query_log_write_interval_s=1", catalogd_args="--enable_workload_mgmt") def test_arithmetic(self, vector):
