This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new f8e2f2c7d IMPALA-13925: Deflake TestFrontendConnectionLimit
f8e2f2c7d is described below
commit f8e2f2c7db80d244f829457773959e5466e8b93d
Author: Riza Suminto <[email protected]>
AuthorDate: Wed Apr 2 12:39:38 2025 -0700
IMPALA-13925: Deflake TestFrontendConnectionLimit
TestFrontendConnectionLimit is flaky after IMPALA-13907. The test
randomly picks one impalad and create multiple clients to connect to
that impalad. However, the test also limits --fe_service_threads=1. If
the random impalad picked is the first impalad, any new client might not
be able to connect because the default ImpalaTestSuite.hs2_client has
open session to the first impalad after running SET ALL query.
ImpalaTestSuite.beeswax_client does not seem to have this issue because
it does not run any query at connect().
This patch attempt to deflake the test by skip the creation of all
default impala clients. Added classmethod need_default_clients() that is
overrideable by test classes that wish to skip default client creation.
Filed IMPALA-13930 to improve ImpylaHS2Connection instantiation.
Testing:
Manually loop the tests for 20 times and pass all of them. Before this
patch, the test can get stuck if looped less than that.
Change-Id: I53c6760f5e3734397746b5a228345c9df38eabcb
Reviewed-on: http://gerrit.cloudera.org:8080/22724
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
tests/common/impala_test_suite.py | 62 +++++++++++++---------
.../test_frontend_connection_limit.py | 6 +++
2 files changed, 42 insertions(+), 26 deletions(-)
diff --git a/tests/common/impala_test_suite.py
b/tests/common/impala_test_suite.py
index b8648b836..0f1e31729 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -230,16 +230,19 @@ class ImpalaTestSuite(BaseTestSuite):
return pytest.config.option.default_test_protocol
@staticmethod
- def create_hive_client(port):
+ def create_hive_client(port=None):
"""
- Creates a HMS client to a external running metastore service at the
provided port
+ Creates a HMS client to a external running metastore service.
"""
+ metastore_host, metastore_port =
pytest.config.option.metastore_server.split(':')
+ if port is not None:
+ metastore_port = port
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
hive_transport = create_transport(
- host=pytest.config.option.metastore_server.split(':')[0],
- port=port,
+ host=metastore_host,
+ port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(hive_transport)
@@ -252,29 +255,30 @@ class ImpalaTestSuite(BaseTestSuite):
"""Setup section that runs before each test suite"""
cls.client = None
cls.beeswax_client = None
- cls.hive_client = None
cls.hs2_client = None
cls.hs2_http_client = None
- # Create a Hive Metastore Client (used for executing some test SETUP steps
- metastore_host, metastore_port =
pytest.config.option.metastore_server.split(':')
- trans_type = 'buffered'
- if pytest.config.option.use_kerberos:
- trans_type = 'kerberos'
- cls.hive_transport = create_transport(
- host=metastore_host,
- port=metastore_port,
- service=pytest.config.option.hive_service_name,
- transport_type=trans_type)
- protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
- cls.hive_client = ThriftHiveMetastore.Client(protocol)
- cls.hive_transport.open()
-
- cls.create_impala_clients()
+ cls.hive_client = None
+ cls.hive_transport = None
# Default query options are populated on demand.
cls.default_query_options = {}
cls.impalad_test_service = cls.create_impala_service()
+ # Override the shell history path so that commands run by any tests
+ # don't write any history into the developer's file.
+ os.environ['IMPALA_HISTFILE'] = '/dev/null'
+
+ if not cls.need_default_clients():
+ # Class setup is done.
+ return
+
+ # Default clients are requested.
+ # Create a Hive Metastore Client (used for executing some test SETUP steps
+ cls.hive_client, cls.hive_transport = ImpalaTestSuite.create_hive_client()
+
+ # Create default impala clients.
+ cls.create_impala_clients()
+
# There are multiple clients for interacting with the underlying storage
service.
#
# There are two main types of clients: filesystem-specific clients and CLI
clients.
@@ -333,10 +337,6 @@ class ImpalaTestSuite(BaseTestSuite):
elif IS_OZONE:
cls.filesystem_client = HadoopFsCommandLineClient("Ozone")
- # Override the shell history path so that commands run by any tests
- # don't write any history into the developer's file.
- os.environ['IMPALA_HISTFILE'] = '/dev/null'
-
@classmethod
def teardown_class(cls):
"""Setup section that runs after each test suite"""
@@ -345,16 +345,26 @@ class ImpalaTestSuite(BaseTestSuite):
cls.hive_transport.close()
cls.close_impala_clients()
- def setup_method(self, test_method):
+ def setup_method(self, test_method): # noqa: U100
"""Setup for all test method."""
self._reset_impala_clients()
- def teardown_method(self, test_method):
+ def teardown_method(self, test_method): # noqa: U100
"""Teardown for all test method.
Currently, it is only here as a placeholder for future use and complement
setup_method() declaration."""
pass
+ @classmethod
+ def need_default_clients(cls):
+ """This method controls whether test class need to create default clients
+ (impala, hive, hms, and filesystem).
+ If this is overridden to return False, Test method must create its own
clients
+ and none of helper method that relies on default client like
run_test_case()
+ or execute_query() will work.
+ """
+ return True
+
@classmethod
def create_impala_client(cls, host_port=None, protocol=None, is_hive=False):
"""
diff --git a/tests/custom_cluster/test_frontend_connection_limit.py
b/tests/custom_cluster/test_frontend_connection_limit.py
index 86e873a26..c6cd46c38 100644
--- a/tests/custom_cluster/test_frontend_connection_limit.py
+++ b/tests/custom_cluster/test_frontend_connection_limit.py
@@ -40,6 +40,12 @@ class TestFrontendConnectionLimit(CustomClusterTestSuite):
def add_test_dimensions(cls):
super(TestFrontendConnectionLimit, cls).add_test_dimensions()
+ @classmethod
+ def need_default_clients(cls):
+ """Must not create default clients because it will interfere with test
methods.
+ """
+ return False
+
def _connect_and_query(self, query, impalad):
with impalad.service.create_hs2_client() as client:
client.execute(query)
