This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 0767ae065a9bf3023ad0be3d98ebad0e4d0c32fa Author: Yida Wu <[email protected]> AuthorDate: Thu Oct 3 14:09:55 2024 -0700 IMPALA-12908: (Addendum) use RUNTIME_FILTER_WAIT_TIME_MS for tuple cache TPC testing When runtime filters arrive after tuple caching has occurred, they can't filter the cached results. This can lead to larger tuple caching result sets than expected, causing correctness check failures in TPC tests. While other solutions may exist, extending RUNTIME_FILTER_WAIT_TIME_MS is a simple fix by ensuring runtime filters are applied before tuple caching. Also set the query option enable_tuple_cache_verification to false by default, as the filter arrival time may affect the correctness check. To avoid flaky tests, change to use a more conservative approach and only enable the correctness check when explicitly specified by the testcase. Tests: Verified TPC tests pass correctness checks with increased runtime filter wait time. Change-Id: Ie70a87344c436ce8e2073575df5c5bf762ef562d Reviewed-on: http://gerrit.cloudera.org:8080/21898 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- common/thrift/Query.thrift | 2 +- tests/common/environ.py | 5 +++++ tests/query_test/test_tuple_cache_tpc_queries.py | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift index d0b8355a6..e21babc2c 100644 --- a/common/thrift/Query.thrift +++ b/common/thrift/Query.thrift @@ -752,7 +752,7 @@ struct TQueryOptions { 183: optional i32 long_polling_time_ms = 0; // See comment in ImpalaService.thrift - 184: optional bool enable_tuple_cache_verification = true; + 184: optional bool enable_tuple_cache_verification = false; } // Impala currently has three types of sessions: Beeswax, HiveServer2 and external diff --git a/tests/common/environ.py b/tests/common/environ.py index c1de7e892..165928157 100644 --- a/tests/common/environ.py +++ b/tests/common/environ.py @@ -109,6 +109,11 @@ IS_TUPLE_CACHE = ( and os.getenv("TUPLE_CACHE_CAPACITY", "") != "" ) +# Detect if we are testing with tuple cache correctness check enabled. +IS_TUPLE_CACHE_CORRECT_CHECK = ( + os.getenv("TUPLE_CACHE_DEBUG_DUMP_DIR", "") != "" +) + class ImpalaBuildFlavors: """ Represents the possible CMAKE_BUILD_TYPE values. These build flavors are needed diff --git a/tests/query_test/test_tuple_cache_tpc_queries.py b/tests/query_test/test_tuple_cache_tpc_queries.py index 984121cca..c1f5a9a57 100644 --- a/tests/query_test/test_tuple_cache_tpc_queries.py +++ b/tests/query_test/test_tuple_cache_tpc_queries.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function import pytest +from tests.common.environ import IS_TUPLE_CACHE_CORRECT_CHECK from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIf from tests.common.test_dimensions import create_single_exec_option_dimension @@ -29,6 +30,11 @@ MT_DOP_VALUES = [0, 4] def run_tuple_cache_test(self, vector, query, mtdop): vector.get_value('exec_option')['enable_tuple_cache'] = True + # Use a long runtime filter wait time (1 minute) to ensure filters arrive before + # generating the tuple cache for correctness check. + if IS_TUPLE_CACHE_CORRECT_CHECK: + vector.get_value('exec_option')['runtime_filter_wait_time_ms'] = 60000 + vector.get_value('exec_option')['enable_tuple_cache_verification'] = True vector.get_value('exec_option')['mt_dop'] = mtdop # Run twice to test write and read the tuple cache. self.run_test_case(query, vector)
