This is an automated email from the ASF dual-hosted git repository.
jasonmfehr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new f5bb759c6 IMPALA-13600: Fix Flaky JWT Test
f5bb759c6 is described below
commit f5bb759c67b912de2b29e5f9084d7678d903122c
Author: jasonmfehr <[email protected]>
AuthorDate: Wed Dec 11 12:14:38 2024 -0800
IMPALA-13600: Fix Flaky JWT Test
The custom cluster tests for Impala shell JWT authentication all
contain magic numbers for the expected count of RPCs for the hs2-http
protocol. Thus, any time the rpcs are modified, these tests have the
potential to fail.
Since the JWT tests are focused on all JWT authentications either
succeeding or failing, the actual number of rpcs is not relevant. The
tests now use existing metrics to determine the expected rpc count.
Additionally, the tests use existing metrics to determine when the
assertions can run instead of relying on a sleep statement.
The modified tests passed locally and in Jenkins.
Change-Id: Icf0eebd74e1ce10ad24055b7fab4b1901ce61e03
Reviewed-on: http://gerrit.cloudera.org:8080/22201
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
tests/custom_cluster/test_shell_jwt_auth.py | 72 +++++++++++++++++------------
1 file changed, 43 insertions(+), 29 deletions(-)
diff --git a/tests/custom_cluster/test_shell_jwt_auth.py
b/tests/custom_cluster/test_shell_jwt_auth.py
index 14ba06cb6..8f6f9e81c 100644
--- a/tests/custom_cluster/test_shell_jwt_auth.py
+++ b/tests/custom_cluster/test_shell_jwt_auth.py
@@ -21,7 +21,6 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.test_dimensions import create_client_protocol_http_transport
-from time import sleep
from tests.shell.util import run_impala_shell_cmd
@@ -48,6 +47,9 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
"-jwt_token_auth=true -jwt_allow_without_tls=true "
.format(JWKS_JSON_PATH))
+ # Name of the Impala metric containing the total count of hs2-http
connections opened.
+ HS2_HTTP_CONNS =
"impala.thrift-server.hiveserver2-http-frontend.total-connections"
+
@classmethod
def get_workload(self):
return 'functional-query'
@@ -63,19 +65,25 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
impalad_args=IMPALAD_ARGS,
impala_log_dir="{jwt_auth_success}",
tmp_dir_placeholders=["jwt_auth_success"],
- disable_log_buffering=True)
+ disable_log_buffering=True,
+ cluster_size=1)
def test_jwt_auth_valid(self, vector):
"""Asserts the Impala shell can authenticate to Impala using JWT
authentication.
Also executes a query to ensure the authentication was successful."""
+ before_rpc_count = self.__get_rpc_count()
+
+ # Run a query and wait for it to complete.
args = ['--protocol', vector.get_value('protocol'), '-j', '--jwt_cmd',
'cat {0}'.format(TestImpalaShellJWTAuth.JWT_SIGNED_PATH),
'-q', 'select version()', '--auth_creds_ok_in_clear']
result = run_impala_shell_cmd(vector, args)
+ self.cluster.get_first_impalad().service.wait_for_metric_value(
+ "impala-server.backend-num-queries-executed", 1, timeout=15)
# Ensure the Impala coordinator is correctly reporting the jwt auth metrics
# must be done before the cluster shuts down since it calls to the
coordinator
- sleep(5)
- self.__assert_success_fail_metric(success_count_min=15,
success_count_max=16)
+ query_rpc_count = self.__get_rpc_count() - before_rpc_count
+ self.__assert_success_fail_metric(success_count=query_rpc_count)
# Shut down cluster to ensure logs flush to disk.
self._stop_impala_cluster()
@@ -99,10 +107,13 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
impalad_args=IMPALAD_ARGS,
impala_log_dir="{jwt_auth_fail}",
tmp_dir_placeholders=["jwt_auth_fail"],
- disable_log_buffering=True)
+ disable_log_buffering=True,
+ cluster_size=1)
def test_jwt_auth_expired(self, vector):
"""Asserts the Impala shell fails to authenticate when it presents a JWT
that has a
valid signature but is expired."""
+ before_rpc_count = self.__get_rpc_count()
+
args = ['--protocol', vector.get_value('protocol'), '-j', '--jwt_cmd',
'cat {0}'.format(TestImpalaShellJWTAuth.JWT_EXPIRED_PATH),
'-q', 'select version()', '--auth_creds_ok_in_clear']
@@ -110,8 +121,9 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
# Ensure the Impala coordinator is correctly reporting the jwt auth metrics
# must be done before the cluster shuts down since it calls to the
coordinator
- sleep(5)
- self.__assert_success_fail_metric(failure_count_min=4, failure_count_max=4)
+ self.__wait_for_rpc_count(before_rpc_count + 1)
+ query_rpc_count = self.__get_rpc_count() - before_rpc_count
+ self.__assert_success_fail_metric(fail_count=query_rpc_count)
# Shut down cluster to ensure logs flush to disk.
self._stop_impala_cluster()
@@ -139,10 +151,13 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
impalad_args=IMPALAD_ARGS,
impala_log_dir="{jwt_auth_invalid_jwk}",
tmp_dir_placeholders=["jwt_auth_invalid_jwk"],
- disable_log_buffering=True)
+ disable_log_buffering=True,
+ cluster_size=1)
def test_jwt_auth_invalid_jwk(self, vector):
"""Asserts the Impala shell fails to authenticate when it presents a JWT
that has a
valid signature but is expired."""
+ before_rpc_count = self.__get_rpc_count()
+
args = ['--protocol', vector.get_value('protocol'), '-j', '--jwt_cmd',
'cat {0}'.format(TestImpalaShellJWTAuth.JWT_INVALID_JWK),
'-q', 'select version()', '--auth_creds_ok_in_clear']
@@ -150,8 +165,9 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
# Ensure the Impala coordinator is correctly reporting the jwt auth metrics
# must be done before the cluster shuts down since it calls to the
coordinator
- sleep(5)
- self.__assert_success_fail_metric(failure_count_min=4, failure_count_max=4)
+ self.__wait_for_rpc_count(before_rpc_count + 1)
+ query_rpc_count = self.__get_rpc_count() - before_rpc_count
+ self.__assert_success_fail_metric(fail_count=query_rpc_count)
# Shut down cluster to ensure logs flush to disk.
self._stop_impala_cluster()
@@ -174,24 +190,22 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
assert "HTTP code 401: Unauthorized" in result.stderr
assert "Not connected to Impala, could not execute queries." in
result.stderr
- def __assert_success_fail_metric(self, success_count_min=0,
success_count_max=0,
- failure_count_min=0, failure_count_max=0):
+ def __assert_success_fail_metric(self, success_count=0, fail_count=0):
"""Impala emits metrics that count the number of successful and failed JWT
authentications. This function asserts the JWT auth success/fail counters
from the
- coordinator are within the specified ranges."""
- self.__assert_counter(
-
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-success",
- success_count_min, success_count_max)
- self.__assert_counter(
-
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-failure",
- failure_count_min, failure_count_max)
-
- def __assert_counter(self, counter_name, expected_count_min,
expected_count_max):
- """Asserts the value of the specifed counter metric from the coordinator
falls
- within the specified min and max (inclusive)."""
- counter_val =
self.cluster.impalads[0].service.get_metric_value(counter_name)
-
- assert counter_val >= expected_count_min and counter_val <=
expected_count_max, \
- "expected counter '{0}' to have a value between '{1}' and '{2}'
inclusive " \
- "but its value was {3}" \
- .format(counter_name, expected_count_min, expected_count_max,
counter_val)
+ coordinator match the expected values."""
+ actual = self.cluster.get_first_impalad().service.get_metric_values([
+
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-success",
+
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-failure"])
+
+ assert actual[0] == success_count, "Expected JWT auth success count to be
'{}' but " \
+ "was '{}'".format(success_count, actual[0])
+ assert actual[1] == fail_count, "Expected JWT auth failure count to be
'{}' but " \
+ "was '{}'".format(fail_count, actual[1])
+
+ def __get_rpc_count(self):
+ return
self.cluster.get_first_impalad().service.get_metric_value(self.HS2_HTTP_CONNS)
+
+ def __wait_for_rpc_count(self, expected_count):
+
self.cluster.get_first_impalad().service.wait_for_metric_value(self.HS2_HTTP_CONNS,
+ expected_count, allow_greater=True)
