This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 148888e3e IMPALA-11822: Optimize the Refresh/Invalidate event
processing by skipping unnecessary events
new 566df8089 IMPALA-11959: Add Python 3 virtualenv
new 82bd087fb IMPALA-11973: Add absolute_import, division to all eligible
Python files
new eb66d00f9 IMPALA-11974: Fix lazy list operators for Python 3
compatibility
new c233634d7 IMPALA-11975: Fix Dictionary methods to work with Python 3
new aa4050b4d IMPALA-11976: Fix use of deprecated functions/fields removed
in Python 3
new 0c7c6a335 IMPALA-11977: Fix Python 3 broken imports and object model
differences
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 6 +-
bin/banned_py3k_warnings.txt | 26 ++++
bin/bootstrap_toolchain.py | 3 +
bin/check-pylint-py3k.sh | 140 +++++++++++++++++++
bin/dump_breakpad_symbols.py | 1 +
bin/generate_minidump_collection_testdata.py | 6 +-
bin/get_code_size.py | 3 +-
bin/impala-config.sh | 3 +-
testdata/bin/kill-mini-dfs.sh => bin/impala-pip3 | 4 +-
testdata/bin/kill-mini-dfs.sh => bin/impala-pylint | 4 +-
bin/{impala-py.test => impala-python3} | 6 +-
...a-python-common.sh => impala-python3-common.sh} | 6 +-
bin/init-impala-python.sh | 26 +++-
bin/inline_pom.py | 2 +-
bin/load-data.py | 6 +-
bin/parse-thrift-profile.py | 4 +-
bin/rat_exclude_files.txt | 1 +
bin/run-workload.py | 8 +-
bin/single_node_perf_run.py | 3 +-
bin/start-impala-cluster.py | 12 +-
docker/setup_build_context.py | 5 +-
infra/python/bootstrap_virtualenv.py | 152 ++++++++++++++-------
infra/python/deps/pip_download.py | 3 +-
.../python/deps/py2-requirements.txt | 26 ++--
.../python/deps/py3-requirements.txt | 23 ++--
infra/python/deps/requirements.txt | 20 +--
infra/python/deps/setuptools-requirements.txt | 2 +-
lib/python/impala_py_lib/gdb/impala-gdb.py | 6 +-
lib/python/impala_py_lib/helpers.py | 1 +
lib/python/impala_py_lib/profiles.py | 2 +-
testdata/bin/check-hbase-nodes.py | 3 +-
testdata/bin/generate-schema-statements.py | 9 +-
testdata/bin/generate-test-vectors.py | 6 +-
testdata/bin/load-tpc-kudu.py | 2 +-
testdata/bin/load_nested.py | 6 +-
testdata/bin/random_avro_schema.py | 2 +
testdata/bin/rewrite-iceberg-metadata.py | 14 +-
testdata/bin/wait-for-hiveserver2.py | 6 +-
testdata/bin/wait-for-metastore.py | 6 +-
testdata/common/cgroups.py | 3 +-
testdata/common/text_delims_table.py | 5 +-
testdata/common/widetable.py | 13 +-
tests/authorization/test_authorization.py | 1 +
tests/authorization/test_authorized_proxy.py | 1 +
tests/authorization/test_provider.py | 1 +
tests/authorization/test_ranger.py | 4 +-
tests/beeswax/impala_beeswax.py | 14 +-
tests/benchmark/plugins/clear_buffer_cache.py | 1 +
tests/benchmark/plugins/vtune_plugin.py | 4 +-
tests/benchmark/report_benchmark_results.py | 17 ++-
.../catalog_service/test_catalog_service_client.py | 1 +
tests/catalog_service/test_large_num_partitions.py | 1 +
tests/common/base_test_suite.py | 1 +
tests/common/custom_cluster_test_suite.py | 81 +++++------
tests/common/environ.py | 3 +-
tests/common/file_utils.py | 1 +
tests/common/iceberg_test_suite.py | 1 +
tests/common/impala_cluster.py | 7 +-
tests/common/impala_connection.py | 12 +-
tests/common/impala_service.py | 1 +
tests/common/impala_test_suite.py | 13 +-
tests/common/kudu_test_suite.py | 6 +-
tests/common/network.py | 1 +
tests/common/parametrize.py | 1 +
tests/common/patterns.py | 1 +
tests/common/resource_pool_config.py | 2 +-
tests/common/skip.py | 1 +
tests/common/test_dimensions.py | 4 +-
tests/common/test_result_verifier.py | 29 +++-
tests/common/test_vector.py | 5 +-
tests/comparison/cli_options.py | 1 +
tests/comparison/cluster.py | 40 +++---
tests/comparison/common.py | 13 +-
tests/comparison/compat.py | 1 +
tests/comparison/data_generator.py | 13 +-
tests/comparison/data_generator_mapred_common.py | 14 +-
tests/comparison/db_connection.py | 16 +--
tests/comparison/db_types.py | 8 +-
tests/comparison/discrepancy_searcher.py | 24 ++--
tests/comparison/funcs.py | 5 +-
tests/comparison/leopard/controller.py | 3 +-
tests/comparison/leopard/front_end.py | 5 +-
tests/comparison/leopard/impala_docker_env.py | 4 +-
tests/comparison/leopard/job.py | 4 +-
tests/comparison/leopard/report.py | 1 +
tests/comparison/leopard/schedule_item.py | 1 +
tests/comparison/model_translator.py | 1 +
tests/comparison/query.py | 17 +--
tests/comparison/query_flattener.py | 1 +
tests/comparison/query_generator.py | 30 ++--
tests/comparison/query_profile.py | 27 ++--
tests/comparison/random_val_generator.py | 1 +
tests/comparison/statement_generator.py | 4 +-
tests/comparison/tests/conftest.py | 1 +
tests/comparison/tests/fake_query.py | 1 +
.../hive/test_hive_create_agg_or_analytic_tree.py | 1 +
.../test_hive_create_relational_join_condition.py | 1 +
tests/comparison/tests/query_object_testdata.py | 1 +
tests/comparison/tests/test_cluster.py | 1 +
tests/comparison/tests/test_cursor.py | 1 +
tests/comparison/tests/test_query_generator.py | 3 +-
tests/comparison/tests/test_query_objects.py | 1 +
tests/comparison/tests/test_use_nested_with.py | 1 +
tests/comparison/util/verify-oracle-connection.py | 2 +-
tests/conftest.py | 11 +-
tests/custom_cluster/test_admission_controller.py | 53 +++----
tests/custom_cluster/test_alloc_fail.py | 1 +
tests/custom_cluster/test_always_false_filter.py | 1 +
tests/custom_cluster/test_auto_scaling.py | 2 +
.../custom_cluster/test_automatic_invalidation.py | 1 +
tests/custom_cluster/test_blacklist.py | 5 +-
.../test_blacklisted_dbs_and_tables.py | 1 +
tests/custom_cluster/test_breakpad.py | 11 +-
tests/custom_cluster/test_catalog_hms_failures.py | 2 +-
tests/custom_cluster/test_catalog_wait.py | 1 +
tests/custom_cluster/test_client_ssl.py | 2 +-
tests/custom_cluster/test_codegen_cache.py | 2 +
.../custom_cluster/test_compact_catalog_updates.py | 1 +
tests/custom_cluster/test_concurrent_ddls.py | 8 +-
.../custom_cluster/test_concurrent_kudu_create.py | 4 +-
tests/custom_cluster/test_coordinators.py | 1 +
tests/custom_cluster/test_custom_hive_configs.py | 1 +
tests/custom_cluster/test_custom_statestore.py | 4 +-
tests/custom_cluster/test_data_cache.py | 1 +
tests/custom_cluster/test_delegation.py | 1 +
.../test_disable_catalog_data_ops.py | 1 +
tests/custom_cluster/test_disable_features.py | 1 +
.../test_disk_spill_configurations.py | 1 +
tests/custom_cluster/test_events_custom_configs.py | 5 +-
.../test_exchange_deferred_batches.py | 2 +
tests/custom_cluster/test_exchange_delays.py | 1 +
tests/custom_cluster/test_exchange_eos.py | 1 +
tests/custom_cluster/test_executor_groups.py | 12 +-
.../test_frontend_connection_limit.py | 3 +-
tests/custom_cluster/test_geospatial_library.py | 2 +
.../custom_cluster/test_hbase_hms_column_order.py | 1 +
tests/custom_cluster/test_hdfs_fd_caching.py | 2 +
tests/custom_cluster/test_hdfs_timeout.py | 1 +
tests/custom_cluster/test_hedged_reads.py | 1 +
.../test_hive_parquet_codec_interop.py | 1 +
.../test_hive_parquet_timestamp_conversion.py | 1 +
.../custom_cluster/test_hive_text_codec_interop.py | 1 +
tests/custom_cluster/test_hs2.py | 1 +
tests/custom_cluster/test_hs2_fault_injection.py | 4 +-
.../test_incremental_metadata_updates.py | 2 +
tests/custom_cluster/test_insert_behaviour.py | 1 +
tests/custom_cluster/test_jvm_mem_tracking.py | 1 +
tests/custom_cluster/test_krpc_mem_usage.py | 1 +
tests/custom_cluster/test_krpc_metrics.py | 1 +
tests/custom_cluster/test_krpc_options.py | 1 +
tests/custom_cluster/test_krpc_socket.py | 1 +
tests/custom_cluster/test_kudu.py | 1 +
tests/custom_cluster/test_kudu_not_available.py | 1 +
.../test_kudu_table_create_without_hms.py | 1 +
tests/custom_cluster/test_lineage.py | 1 +
tests/custom_cluster/test_local_catalog.py | 15 +-
tests/custom_cluster/test_local_tz_conversion.py | 1 +
tests/custom_cluster/test_logging.py | 1 +
tests/custom_cluster/test_mem_reservations.py | 6 +-
.../test_metadata_no_events_processing.py | 1 +
tests/custom_cluster/test_metadata_replicas.py | 6 +-
.../test_metastore_events_cleanup.py | 1 +
tests/custom_cluster/test_metastore_service.py | 2 +
tests/custom_cluster/test_mt_dop.py | 1 +
tests/custom_cluster/test_observability.py | 1 +
.../custom_cluster/test_parquet_max_page_header.py | 10 +-
tests/custom_cluster/test_partition.py | 1 +
tests/custom_cluster/test_pause_monitor.py | 1 +
tests/custom_cluster/test_permanent_udfs.py | 1 +
tests/custom_cluster/test_preload_table_types.py | 2 +
tests/custom_cluster/test_process_failures.py | 6 +-
tests/custom_cluster/test_query_concurrency.py | 1 +
tests/custom_cluster/test_query_event_hooks.py | 1 +
tests/custom_cluster/test_query_expiration.py | 10 +-
tests/custom_cluster/test_query_retries.py | 9 +-
tests/custom_cluster/test_re2_max_mem.py | 1 +
tests/custom_cluster/test_redaction.py | 1 +
.../custom_cluster/test_reserved_words_version.py | 1 +
tests/custom_cluster/test_restart_services.py | 15 +-
tests/custom_cluster/test_result_spooling.py | 1 +
tests/custom_cluster/test_rpc_exception.py | 1 +
tests/custom_cluster/test_rpc_timeout.py | 4 +-
tests/custom_cluster/test_runtime_profile.py | 1 +
tests/custom_cluster/test_s3a_access.py | 1 +
tests/custom_cluster/test_saml2_sso.py | 33 +++--
tests/custom_cluster/test_scheduler_locality.py | 1 +
tests/custom_cluster/test_scratch_disk.py | 5 +-
tests/custom_cluster/test_seq_file_filtering.py | 1 +
tests/custom_cluster/test_services_rpc_errors.py | 1 +
tests/custom_cluster/test_session_expiration.py | 1 +
tests/custom_cluster/test_set_and_unset.py | 5 +-
tests/custom_cluster/test_shared_tzdb.py | 1 +
tests/custom_cluster/test_shell_commandline.py | 1 +
tests/custom_cluster/test_shell_interactive.py | 1 +
.../test_shell_interactive_reconnect.py | 1 +
.../test_startup_filesystem_checks.py | 1 +
tests/custom_cluster/test_stats_extrapolation.py | 1 +
.../test_thrift_debug_string_exception.py | 1 +
tests/custom_cluster/test_thrift_socket.py | 1 +
.../custom_cluster/test_topic_update_frequency.py | 3 +-
tests/custom_cluster/test_udf_concurrency.py | 11 +-
tests/custom_cluster/test_web_pages.py | 2 +-
tests/custom_cluster/test_wide_table_operations.py | 2 +
tests/data_errors/test_data_errors.py | 1 +
tests/experiments/test_targeted_perf.py | 1 +
tests/failure/test_failpoints.py | 2 +
tests/hs2/hs2_test_suite.py | 6 +-
tests/hs2/test_fetch.py | 1 +
tests/hs2/test_fetch_first.py | 14 +-
tests/hs2/test_fetch_timeout.py | 1 +
tests/hs2/test_hs2.py | 11 +-
tests/hs2/test_json_endpoints.py | 6 +-
tests/infra/test_perf_infra.py | 1 +
tests/infra/test_stress_infra.py | 1 +
tests/infra/test_utils.py | 1 +
tests/metadata/test_catalogd_debug_actions.py | 1 +
tests/metadata/test_compute_stats.py | 4 +-
tests/metadata/test_ddl.py | 26 ++--
tests/metadata/test_ddl_base.py | 1 +
tests/metadata/test_event_processing.py | 1 +
tests/metadata/test_explain.py | 1 +
tests/metadata/test_hdfs_encryption.py | 1 +
tests/metadata/test_hdfs_permissions.py | 1 +
tests/metadata/test_hidden_files.py | 1 +
tests/metadata/test_hms_integration.py | 7 +-
tests/metadata/test_last_ddl_time_update.py | 10 +-
tests/metadata/test_load.py | 8 +-
tests/metadata/test_metadata_query_statements.py | 1 +
tests/metadata/test_partition_metadata.py | 1 +
tests/metadata/test_recover_partitions.py | 16 ++-
tests/metadata/test_recursive_listing.py | 2 +
tests/metadata/test_refresh_partition.py | 1 +
tests/metadata/test_reset_metadata.py | 1 +
tests/metadata/test_reuse_partitions.py | 1 +
tests/metadata/test_set.py | 1 +
tests/metadata/test_show_create_table.py | 1 +
tests/metadata/test_stale_metadata.py | 1 +
tests/metadata/test_stats_extrapolation.py | 6 +-
tests/metadata/test_testcase_builder.py | 1 +
tests/metadata/test_views_compatibility.py | 1 +
tests/observability/test_jvm_metrics.py | 1 +
tests/observability/test_log_fragments.py | 1 +
tests/observability/test_profile_tool.py | 1 +
tests/performance/query.py | 4 +
tests/performance/query_exec_functions.py | 5 +-
tests/performance/query_executor.py | 1 +
tests/performance/scheduler.py | 8 +-
tests/performance/workload.py | 3 +-
tests/performance/workload_runner.py | 1 +
tests/query_test/test_acid.py | 1 +
tests/query_test/test_acid_row_validation.py | 1 +
tests/query_test/test_aggregation.py | 11 +-
tests/query_test/test_analytic_tpcds.py | 1 +
tests/query_test/test_async_codegen.py | 1 +
tests/query_test/test_avro_schema_resolution.py | 2 +
tests/query_test/test_beeswax.py | 1 +
tests/query_test/test_cancellation.py | 10 +-
tests/query_test/test_cast_with_format.py | 2 +
tests/query_test/test_chars.py | 1 +
tests/query_test/test_codegen.py | 1 +
tests/query_test/test_compressed_formats.py | 5 +-
tests/query_test/test_datasketches.py | 1 +
tests/query_test/test_datastream_sender.py | 1 +
tests/query_test/test_date_queries.py | 1 +
tests/query_test/test_decimal_casting.py | 16 ++-
tests/query_test/test_decimal_fuzz.py | 14 +-
tests/query_test/test_decimal_queries.py | 1 +
tests/query_test/test_delimited_text.py | 1 +
tests/query_test/test_errorlog.py | 1 +
tests/query_test/test_exprs.py | 14 +-
tests/query_test/test_fetch.py | 1 +
tests/query_test/test_geospatial_functions.py | 1 +
tests/query_test/test_hash_join_timer.py | 5 +-
tests/query_test/test_hbase_queries.py | 1 +
tests/query_test/test_hdfs_caching.py | 5 +-
tests/query_test/test_hdfs_file_mods.py | 1 +
tests/query_test/test_iceberg.py | 2 +
tests/query_test/test_insert.py | 1 +
tests/query_test/test_insert_behaviour.py | 2 +
tests/query_test/test_insert_parquet.py | 18 ++-
tests/query_test/test_insert_permutation.py | 6 +-
tests/query_test/test_invalid_test_header.py | 1 +
tests/query_test/test_io_metrics.py | 1 +
tests/query_test/test_join_queries.py | 1 +
tests/query_test/test_kudu.py | 7 +-
tests/query_test/test_lifecycle.py | 1 +
tests/query_test/test_limit.py | 2 +-
tests/query_test/test_limit_pushdown_analytic.py | 1 +
tests/query_test/test_local_fs.py | 1 +
tests/query_test/test_mem_usage_scaling.py | 3 +
tests/query_test/test_mt_dop.py | 1 +
tests/query_test/test_multiple_filesystems.py | 1 +
tests/query_test/test_nested_types.py | 1 +
tests/query_test/test_observability.py | 1 +
tests/query_test/test_orc_stats.py | 1 +
tests/query_test/test_parquet_bloom_filter.py | 2 +
.../test_parquet_late_materialization.py | 1 +
tests/query_test/test_parquet_page_index.py | 1 +
tests/query_test/test_parquet_stats.py | 1 +
tests/query_test/test_partitioning.py | 1 +
tests/query_test/test_queries.py | 1 +
tests/query_test/test_query_compilation.py | 1 +
tests/query_test/test_query_mem_limit.py | 7 +-
tests/query_test/test_query_opts.py | 1 +
tests/query_test/test_resource_limits.py | 1 +
tests/query_test/test_result_spooling.py | 1 +
tests/query_test/test_rows_availability.py | 1 +
tests/query_test/test_runtime_filters.py | 1 +
tests/query_test/test_scanners.py | 3 +-
tests/query_test/test_scanners_fuzz.py | 6 +-
tests/query_test/test_scratch_limit.py | 1 +
tests/query_test/test_sfs.py | 1 +
tests/query_test/test_sort.py | 19 ++-
tests/query_test/test_spilling.py | 1 +
tests/query_test/test_tablesample.py | 1 +
tests/query_test/test_tpcds_queries.py | 1 +
tests/query_test/test_tpch_nested_queries.py | 1 +
tests/query_test/test_tpch_queries.py | 4 +-
tests/query_test/test_udfs.py | 1 +
tests/query_test/test_utf8_strings.py | 1 +
tests/run-tests.py | 4 +-
tests/shell/test_cookie_util.py | 1 +
tests/shell/test_shell_client.py | 7 +-
tests/shell/test_shell_commandline.py | 12 +-
tests/shell/test_shell_interactive.py | 42 +++---
tests/shell/util.py | 2 +-
tests/statestore/test_statestore.py | 17 ++-
tests/stress/concurrent_select.py | 45 +++---
tests/stress/extract_min_mem.py | 4 +-
tests/stress/mem_broker.py | 1 +
tests/stress/queries.py | 6 +-
tests/stress/query_retries_stress_runner.py | 12 +-
tests/stress/query_runner.py | 14 +-
tests/stress/runtime_info.py | 12 +-
tests/stress/stress_util.py | 1 +
tests/stress/test_acid_stress.py | 28 ++--
tests/stress/test_ddl_stress.py | 6 +-
tests/stress/test_insert_stress.py | 15 +-
tests/stress/util.py | 2 +-
tests/unittests/test_command.py | 1 +
tests/unittests/test_file_parser.py | 4 +-
tests/unittests/test_result_verifier.py | 5 +-
tests/util/acid_txn.py | 2 +-
tests/util/adls_util.py | 1 +
tests/util/auto_scaler.py | 3 +-
tests/util/calculation_util.py | 6 +-
tests/util/cancel_util.py | 1 +
tests/util/cluster_controller.py | 1 +
tests/util/compute_table_stats.py | 1 +
tests/util/concurrent_workload.py | 14 +-
tests/util/event_processor_utils.py | 1 +
tests/util/failpoints_util.py | 1 +
tests/util/filesystem_base.py | 6 +-
tests/util/filesystem_utils.py | 1 +
tests/util/get_parquet_metadata.py | 5 +-
tests/util/hdfs_util.py | 11 +-
tests/util/iceberg_util.py | 1 +
tests/util/parse_util.py | 1 +
tests/util/plugin_runner.py | 1 +
tests/util/run_impyla_http_query.py | 2 +-
tests/util/shell_util.py | 1 +
tests/util/ssh_util.py | 8 +-
tests/util/test_file_parser.py | 12 +-
tests/util/thrift_util.py | 1 +
tests/util/web_pages_util.py | 1 +
tests/verifiers/mem_usage_verifier.py | 1 +
tests/verifiers/metric_verifier.py | 1 +
tests/verifiers/test_verify_metrics.py | 1 +
tests/webserver/test_web_pages.py | 1 +
369 files changed, 1409 insertions(+), 690 deletions(-)
create mode 100644 bin/banned_py3k_warnings.txt
create mode 100755 bin/check-pylint-py3k.sh
copy testdata/bin/kill-mini-dfs.sh => bin/impala-pip3 (88%)
copy testdata/bin/kill-mini-dfs.sh => bin/impala-pylint (90%)
copy bin/{impala-py.test => impala-python3} (91%)
copy bin/{impala-python-common.sh => impala-python3-common.sh} (87%)
copy be/src/exec/sequence/CMakeLists.txt =>
infra/python/deps/py2-requirements.txt (65%)
copy be/src/exec/sequence/CMakeLists.txt =>
infra/python/deps/py3-requirements.txt (70%)
