This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit ba3518366a503efceb859102c8e6d6450083ae38 Author: Joe McDonnell <[email protected]> AuthorDate: Sun Feb 26 14:40:24 2023 -0800 IMPALA-11952 (part 4): Fix odds and ends: Octals, long, lambda, etc. There are a variety of small python 3 syntax differences: - Octal constants need to start with 0o rather than just 0 - Long constants are not supported (i.e. numbers ending with L) - Lambda syntax is slightly different - The 'ur' string mode is no longer supported Testing: - check-python-syntax.sh now passes Change-Id: Ie027a50ddf6a2a0db4b34ec9b49484ce86947f20 Reviewed-on: http://gerrit.cloudera.org:8080/19554 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Michael Smith <[email protected]> --- bin/check-rat-report.py | 2 +- docker/test-with-docker.py | 2 +- tests/comparison/data_generator_mapred_common.py | 2 +- tests/comparison/leopard/front_end.py | 2 +- tests/comparison/leopard/impala_docker_env.py | 3 +- tests/comparison/leopard/report.py | 40 ++++++++++++------------ tests/custom_cluster/test_breakpad.py | 2 +- tests/custom_cluster/test_client_ssl.py | 2 +- tests/custom_cluster/test_thrift_socket.py | 2 +- tests/util/get_parquet_metadata.py | 2 +- 10 files changed, 29 insertions(+), 30 deletions(-) diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py index 3c6870bee..0346a16ee 100755 --- a/bin/check-rat-report.py +++ b/bin/check-rat-report.py @@ -66,7 +66,7 @@ for r in resources: if not excluded: typename = r.findall('type')[0].attrib['name'] if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary'] - and clean_name[-4:] <> '.jar'): + and clean_name[-4:] != '.jar'): sys.stderr.write( "%s: %s\n" % ('UNAPPROVED' if approvals else "NO APPROVALS; " + typename, clean_name)) diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py index 47e505a7e..55ae96372 100755 --- a/docker/test-with-docker.py +++ b/docker/test-with-docker.py @@ -790,7 +790,7 @@ class TestSuiteRunner(object): # io-file-mgr-test expects a real-ish file system at /tmp; # we mount a temporary directory into the container to appease it. tmpdir = tempfile.mkdtemp(prefix=test_with_docker.name + "-" + self.name) - os.chmod(tmpdir, 01777) + os.chmod(tmpdir, 0o1777) # Container names are sometimes used as hostnames, and DNS names shouldn't # have underscores. container_name = test_with_docker.name + "-" + self.name.replace("_", "-") diff --git a/tests/comparison/data_generator_mapred_common.py b/tests/comparison/data_generator_mapred_common.py index bcf162089..df6c087b5 100644 --- a/tests/comparison/data_generator_mapred_common.py +++ b/tests/comparison/data_generator_mapred_common.py @@ -62,7 +62,7 @@ class TextTableDataGenerator(object): val_generator = col_val_generators[col_idx] for idx in xrange(val_buffer_size): val = next(val_generator) - val_buffer[idx] = "\N" if val is None else val + val_buffer[idx] = r"\N" if val is None else val for col_idx, col in enumerate(cols): if col_idx > 0: # Postgres doesn't seem to have an option to specify that the last column value diff --git a/tests/comparison/leopard/front_end.py b/tests/comparison/leopard/front_end.py index 9752f7b98..ada5bd130 100755 --- a/tests/comparison/leopard/front_end.py +++ b/tests/comparison/leopard/front_end.py @@ -309,7 +309,7 @@ def front_page(): 'index.template', assets=ASSETS, reports=sorted( - app.reports.items(), key=lambda (k, report): report.run_date, reverse=True), + app.reports.items(), key=lambda k, report: report.run_date, reverse=True), schedule_items=schedule_items) if __name__ == '__main__': diff --git a/tests/comparison/leopard/impala_docker_env.py b/tests/comparison/leopard/impala_docker_env.py index 8cb60101b..63fc33199 100755 --- a/tests/comparison/leopard/impala_docker_env.py +++ b/tests/comparison/leopard/impala_docker_env.py @@ -302,6 +302,7 @@ class ImpalaDockerEnv(object): # data as a volume to bypass AUFS. See also the README for Leopard. LOG.info('Warming testdata cluster external volume') self.start_new_container() + volume_map = None with settings( warn_only=True, host_string=self.host, @@ -325,8 +326,6 @@ class ImpalaDockerEnv(object): volume_map = { HOST_TESTDATA_EXTERNAL_VOLUME_PATH: DOCKER_TESTDATA_VOLUME_PATH, } - else: - volume_map = None self.start_new_container(volume_map=volume_map) LOG.info('Container Started') diff --git a/tests/comparison/leopard/report.py b/tests/comparison/leopard/report.py index d1bbf5cca..db463c993 100644 --- a/tests/comparison/leopard/report.py +++ b/tests/comparison/leopard/report.py @@ -49,23 +49,23 @@ class Report(object): def classify_error(self, error): d = { - ur'LINE \d+:': 'Postgres_error', - ur'Permission denied': 'permission_denied', - ur'^AnalysisException': 'AnalysisException', - ur'^Column \d+ in row \d+ does not match': 'mismatch', - ur'^Could not connect': 'could_not_connect', - ur'^IllegalStateException': 'IllegalStateException', - ur'^Invalid query handle: ': 'invalid_query_handle', - ur'^Invalid or unknown query handle: ': 'invalid_query_handle', - ur'^Known issue:': 'known_issue', - ur'^Operation is in ERROR_STATE': 'error_state', - ur'^Query timed out after \d+ seconds': 'timeout', - ur'^Row counts do not match': 'row_counts', - ur'^Too much data': 'too_much_data', - ur'^Unknown expr node type: \d+': 'unkown_node', - ur'^Year is out of valid range': 'year_range', - ur'^[A-Za-z]+ out of range': 'out_of_range', - ur'^division by zero': 'division_by_zero'} + r'LINE \d+:': 'Postgres_error', + r'Permission denied': 'permission_denied', + r'^AnalysisException': 'AnalysisException', + r'^Column \d+ in row \d+ does not match': 'mismatch', + r'^Could not connect': 'could_not_connect', + r'^IllegalStateException': 'IllegalStateException', + r'^Invalid query handle: ': 'invalid_query_handle', + r'^Invalid or unknown query handle: ': 'invalid_query_handle', + r'^Known issue:': 'known_issue', + r'^Operation is in ERROR_STATE': 'error_state', + r'^Query timed out after \d+ seconds': 'timeout', + r'^Row counts do not match': 'row_counts', + r'^Too much data': 'too_much_data', + r'^Unknown expr node type: \d+': 'unkown_node', + r'^Year is out of valid range': 'year_range', + r'^[A-Za-z]+ out of range': 'out_of_range', + r'^division by zero': 'division_by_zero'} for r in d: if re.search(r, error): @@ -94,7 +94,7 @@ class Report(object): stack = query_result['formatted_stack'] if stack: for line in stack.split('\n'): - match = re.search(ur'(impala::.*) \(', line) + match = re.search(r'(impala::.*) \(', line) if match: return match.group(1) else: @@ -106,10 +106,10 @@ class Report(object): def clean_frame(frame): #remove memory address from each frame - reg = re.match(ur'#\d+ *0x[0123456789abcdef]* in (.*)', frame) + reg = re.match(r'#\d+ *0x[0123456789abcdef]* in (.*)', frame) if reg: return reg.group(1) # this is for matching lines like "#7 SLL_Next (this=0x9046780, src=0x90467c8... - reg = re.match(ur'#\d+ *(\S.*)', frame) + reg = re.match(r'#\d+ *(\S.*)', frame) if reg: return reg.group(1) return frame diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py index aee47772c..1d17f8570 100644 --- a/tests/custom_cluster/test_breakpad.py +++ b/tests/custom_cluster/test_breakpad.py @@ -463,7 +463,7 @@ class TestLogging(TestBreakpadBase): test_error_msg = ('123456789abcde_' * 64) # 1 KB error message test_debug_actions = 'LOG_MAINTENANCE_STDERR:[email protected]@' + test_error_msg daemon = 'impalad' - os.chmod(self.tmp_dir, 0744) + os.chmod(self.tmp_dir, 0o744) expected_log_max_bytes = int(1.2 * 1024**2) # 1.2 MB self.assert_logs(daemon, 0, expected_log_max_bytes) diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py index b2a5267d3..b02a688aa 100644 --- a/tests/custom_cluster/test_client_ssl.py +++ b/tests/custom_cluster/test_client_ssl.py @@ -35,7 +35,7 @@ from tests.common.test_dimensions import create_client_protocol_dimension from tests.shell.util import run_impala_shell_cmd, run_impala_shell_cmd_no_expect, \ ImpalaShell, create_impala_shell_executable_dimension -REQUIRED_MIN_OPENSSL_VERSION = 0x10001000L +REQUIRED_MIN_OPENSSL_VERSION = 0x10001000 # Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5 # with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already if IS_REDHAT_DERIVATIVE: diff --git a/tests/custom_cluster/test_thrift_socket.py b/tests/custom_cluster/test_thrift_socket.py index 42d5fa142..c276384fd 100644 --- a/tests/custom_cluster/test_thrift_socket.py +++ b/tests/custom_cluster/test_thrift_socket.py @@ -27,7 +27,7 @@ from tests.common.test_vector import ImpalaTestVector from tests.common.test_dimensions import create_client_protocol_dimension from tests.shell.util import ImpalaShell -REQUIRED_MIN_OPENSSL_VERSION = 0x10001000L +REQUIRED_MIN_OPENSSL_VERSION = 0x10001000 # Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5 # with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already if IS_REDHAT_DERIVATIVE: diff --git a/tests/util/get_parquet_metadata.py b/tests/util/get_parquet_metadata.py index 107b33934..db4ef3f41 100644 --- a/tests/util/get_parquet_metadata.py +++ b/tests/util/get_parquet_metadata.py @@ -41,7 +41,7 @@ def julian_day_to_date(julian_day): arbitrarily and can be validated with an online converter like http://aa.usno.navy.mil/jdconverter?ID=AA&jd=2457755 """ - return date(2017, 01, 01) + timedelta(julian_day - 2457755) + return date(2017, 1, 1) + timedelta(julian_day - 2457755) def nanos_to_time(nanos):
