This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8e350d0a8a7c95810c7704c8a5264a60daed783d Author: wzhou-code <[email protected]> AuthorDate: Fri Oct 7 14:29:19 2022 -0700 IMPALA-11304: impala-shell make the client retry attempts configurable Currently max tries for connecting to coordinator is hard coded to 4 in hs2-http mode. It's required to make the max tries when connecting to coordinator a configurable option, especially in the environment where coordinator is started slowly. This patch added support for configurable max tries in hs2-http mode using the new impala-shell config option '--connect_max_tries'. The default value of '--connect_max_tries' is set to 4. Testing: - Ran e2e shell tests. - Ran impala-shell with connect_max_tries as 100 before starting impala coordinator daemon, verified that impala-shell connects to coordinator after coordinator daemon was started. Change-Id: I5f7caeb91a69e71a38689785fb1636094295fdb1 Reviewed-on: http://gerrit.cloudera.org:8080/19105 Reviewed-by: Andrew Sherman <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- shell/impala_client.py | 6 ++++-- shell/impala_shell.py | 8 +++++++- shell/option_parser.py | 4 ++++ tests/shell/test_shell_commandline.py | 25 +++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/shell/impala_client.py b/shell/impala_client.py index da7e63581..36fa4a6b8 100755 --- a/shell/impala_client.py +++ b/shell/impala_client.py @@ -130,7 +130,8 @@ class ImpalaClient(object): kerberos_service_name="impala", use_ssl=False, ca_cert=None, user=None, ldap_password=None, use_ldap=False, client_connect_timeout_ms=60000, verbose=True, use_http_base_transport=False, http_path=None, - http_cookie_names=None, http_socket_timeout_s=None, value_converter=None): + http_cookie_names=None, http_socket_timeout_s=None, value_converter=None, + connect_max_tries=4): self.connected = False self.impalad_host = impalad[0] self.impalad_port = int(impalad[1]) @@ -145,6 +146,7 @@ class ImpalaClient(object): self.use_ldap = use_ldap self.client_connect_timeout_ms = int(client_connect_timeout_ms) self.http_socket_timeout_s = http_socket_timeout_s + self.connect_max_tries = connect_max_tries self.default_query_options = {} self.query_option_levels = {} self.fetch_size = fetch_size @@ -652,7 +654,7 @@ class ImpalaHS2Client(ImpalaClient): # Enable retries only for hs2-http protocol. if self.use_http_base_transport: # Maximum number of tries for idempotent rpcs. - self.max_tries = 4 + self.max_tries = self.connect_max_tries else: self.max_tries = 1 # Minimum sleep interval between retry attempts. diff --git a/shell/impala_shell.py b/shell/impala_shell.py index b25e47955..542e5319f 100755 --- a/shell/impala_shell.py +++ b/shell/impala_shell.py @@ -202,6 +202,7 @@ class ImpalaShell(cmd.Cmd, object): if (options.http_socket_timeout_s != 'None' and options.http_socket_timeout_s is not None): self.http_socket_timeout_s = float(options.http_socket_timeout_s) + self.connect_max_tries = options.connect_max_tries self.verbose = options.verbose self.prompt = ImpalaShell.DISCONNECTED_PROMPT self.server_version = ImpalaShell.UNKNOWN_SERVER_VERSION @@ -630,7 +631,8 @@ class ImpalaShell(cmd.Cmd, object): use_http_base_transport=True, http_path=self.http_path, http_cookie_names=self.http_cookie_names, http_socket_timeout_s=self.http_socket_timeout_s, - value_converter=value_converter) + value_converter=value_converter, + connect_max_tries=self.connect_max_tries) elif protocol == 'beeswax': return ImpalaBeeswaxClient(self.impalad, self.fetch_size, self.kerberos_host_fqdn, self.use_kerberos, self.kerberos_service_name, self.use_ssl, @@ -2127,6 +2129,10 @@ def impala_shell_main(): " expressing seconds, or None", file=sys.stderr) raise FatalShellException() + if options.connect_max_tries < 1: + print("connect_max_tries must be greater than or equal to 1", file=sys.stderr) + raise FatalShellException() + options.variables = parse_variables(options.keyval) # Override query_options from config file with those specified on the command line. diff --git a/shell/option_parser.py b/shell/option_parser.py index abfc1794c..46277be1b 100755 --- a/shell/option_parser.py +++ b/shell/option_parser.py @@ -279,6 +279,10 @@ def get_option_parser(defaults): help="Timeout in seconds after which the socket will time out" " if the associated operation cannot be completed. Set to None to" " disable any timeout. Only supported for hs2-http mode.") + parser.add_option("--connect_max_tries", type="int", + dest="connect_max_tries", default=4, + help="Maximum number of times that an idempotent RPC connection to " + "the Impala coordinator will be retried in hs2-http mode.") parser.add_option("--protocol", dest="protocol", default="hs2", help="Protocol to use for client/server connection. Valid inputs are " "['hs2', 'hs2-http', 'beeswax']. 'hs2-http' uses HTTP transport " diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py index 33b4bc2dc..0c867236e 100644 --- a/tests/shell/test_shell_commandline.py +++ b/tests/shell/test_shell_commandline.py @@ -1272,6 +1272,31 @@ class TestImpalaShell(ImpalaTestSuite): assert result.stderr == "" assert result.stdout == "0\n" + def test_connect_max_tries(self, vector): + """Test setting different connect_max_tries values.""" + if (vector.get_value('strict_hs2_protocol') + or vector.get_value('protocol') != 'hs2-http'): + pytest.skip("connect_max_tries not supported in strict hs2 mode." + " Only supported with hs2-http protocol.") + + # Test connect_max_tries=-1, expect errors + args = ['--quiet', '-B', '--query', 'select 0;'] + result = run_impala_shell_cmd(vector, args + ['--connect_max_tries=-1'], + expect_success=False) + expected_err = ("connect_max_tries must be greater than or equal to 1") + assert result.stderr.splitlines()[0] == expected_err + + # Test connect_max_tries=0, expect errors + result = run_impala_shell_cmd(vector, args + ['--connect_max_tries=0'], + expect_success=False) + expected_err = ("connect_max_tries must be greater than or equal to 1") + assert result.stderr.splitlines()[0] == expected_err + + # Test connect_max_tries>0, expect success + result = run_impala_shell_cmd(vector, args + ['--connect_max_tries=2']) + assert result.stderr == "" + assert result.stdout == "0\n" + def test_trailing_whitespace(self, vector): """Test CSV output with trailing whitespace"""
