This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 7d07192e8 IMPALA-9627: Use universal_newlines for Python 3
7d07192e8 is described below
commit 7d07192e891271254898bb8ac32d67cc4028e4c6
Author: Michael Smith <[email protected]>
AuthorDate: Thu Apr 27 09:06:54 2023 -0700
IMPALA-9627: Use universal_newlines for Python 3
Fixes subprocess.check_output calls for Python 3 using
universal_newlines=True.
Change-Id: I3dae9113635cf23ae02f1f630de311e64119c456
Reviewed-on: http://gerrit.cloudera.org:8080/19812
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
bin/compare_branches.py | 16 ++++++++--------
bin/get_code_size.py | 3 ++-
bin/start-impala-cluster.py | 6 ++++--
docker/monitor.py | 5 +++--
docker/test-with-docker.py | 2 +-
tests/common/impala_cluster.py | 7 ++++---
tests/common/impala_service.py | 6 ++++--
7 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/bin/compare_branches.py b/bin/compare_branches.py
index 7ce51b3d7..5fcb3fbbe 100755
--- a/bin/compare_branches.py
+++ b/bin/compare_branches.py
@@ -139,8 +139,9 @@ def build_commit_map(branch, merge_base):
fields = ['%H', '%s', '%an', '%cd', '%b']
pretty_format = '\x1f'.join(fields) + '\x1e'
result = OrderedDict()
- for line in subprocess.check_output(["git", "log", branch, "^" + merge_base,
- "--pretty=" + pretty_format, "--color=never"]).split('\x1e'):
+ for line in subprocess.check_output(
+ ["git", "log", branch, "^" + merge_base, "--pretty=" + pretty_format,
+ "--color=never"], universal_newlines=True).split('\x1e'):
if line == "":
# if no changes are identified by the git log, we get an empty string
continue
@@ -180,9 +181,10 @@ def cherrypick(cherry_pick_hashes,
full_target_branch_name, partial_ok):
return
# Cherrypicking only makes sense if we're on the equivalent of the target
branch.
- head_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
+ head_sha = subprocess.check_output(
+ ['git', 'rev-parse', 'HEAD'], universal_newlines=True).strip()
target_branch_sha = subprocess.check_output(
- ['git', 'rev-parse', full_target_branch_name]).strip()
+ ['git', 'rev-parse', full_target_branch_name],
universal_newlines=True).strip()
if head_sha != target_branch_sha:
print("Cannot cherrypick because %s (%s) and HEAD (%s) are divergent." % (
full_target_branch_name, target_branch_sha, head_sha))
@@ -231,7 +233,7 @@ def main():
full_target_branch_name = options.target_branch
merge_base = subprocess.check_output(["git", "merge-base",
- full_source_branch_name, full_target_branch_name]).strip()
+ full_source_branch_name, full_target_branch_name],
universal_newlines=True).strip()
source_commits = build_commit_map(full_source_branch_name, merge_base)
target_commits = build_commit_map(full_target_branch_name, merge_base)
@@ -270,9 +272,7 @@ def main():
logging.debug("NOT ignoring commit {0} since not in ignored commits
({1},{2})"
.format(commit_hash, options.source_branch,
options.target_branch))
if not change_in_target and not ignore_by_config and not
ignore_by_commit_message:
- print(u'{0} {1} ({2}) - {3}'
- .format(commit_hash, msg.decode('utf8'), date, author.decode('utf8'))
- .encode('utf8'))
+ print('{0} {1} ({2}) - {3}'.format(commit_hash, msg, date, author))
cherry_pick_hashes.append(commit_hash)
jira_keys += jira_key_pat.findall(msg)
diff --git a/bin/get_code_size.py b/bin/get_code_size.py
index c69007057..dedded74d 100755
--- a/bin/get_code_size.py
+++ b/bin/get_code_size.py
@@ -30,7 +30,8 @@ from prettytable import PrettyTable
def get_bin_size_data(file):
data = ""
try:
- data = subprocess.check_output(["size", "-B", "-t", file],
stderr=subprocess.STDOUT)
+ data = subprocess.check_output(["size", "-B", "-t", file],
+ stderr=subprocess.STDOUT, universal_newlines=True)
except Exception as e:
data = e.output
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 54c47b86d..5a916dc46 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -722,7 +722,8 @@ class DockerMiniClusterOperations(object):
mount_args + mem_limit_args + [image_tag] + args)
LOG.info("Running command {0}".format(run_cmd))
check_call(run_cmd)
- port_mapping = check_output(["docker", "port", container_name])
+ port_mapping = check_output(["docker", "port", container_name],
+ universal_newlines=True)
LOG.info("Launched container {0} with port mapping:\n{1}".format(
container_name, port_mapping))
@@ -740,7 +741,8 @@ class DockerMiniClusterOperations(object):
def __get_network_info__(self):
"""Get the output of "docker network inspect" as a python data
structure."""
- output = check_output(["docker", "network", "inspect", self.network_name])
+ output = check_output(["docker", "network", "inspect", self.network_name],
+ universal_newlines=True)
# Only one network should be present in the top level array.
return json.loads(output)[0]
diff --git a/docker/monitor.py b/docker/monitor.py
index 261dfb270..f1692376d 100644
--- a/docker/monitor.py
+++ b/docker/monitor.py
@@ -70,7 +70,8 @@ def _memory():
Swap: 0 0 0
"""
- free_lines = subprocess.check_output(["free", "-b", "-w"]).split('\n')
+ free_lines = subprocess.check_output(["free", "-b", "-w"],
+ universal_newlines=True).split('\n')
free_grid = [x.split() for x in free_lines]
# Identify columns for "total" and "available"
total_idx = free_grid[0].index("total")
@@ -179,7 +180,7 @@ class ContainerMonitor(object):
# Ubuntu systems typically mount cpuacct cgroup in
/sys/fs/cgroup/cpu,cpuacct,
# but this can vary by OS distribution.
all_cgroups = subprocess.check_output(
- "findmnt -n -o TARGET -t cgroup --source cgroup".split()
+ "findmnt -n -o TARGET -t cgroup --source cgroup".split(),
universal_newlines=True
).split("\n")
cpuacct_root = [c for c in all_cgroups if "cpuacct" in c][0]
memory_root = [c for c in all_cgroups if "memory" in c][0]
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index 5d473f2a0..c7377f452 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -396,7 +396,7 @@ def _call(args, check=True):
def _check_output(*args, **kwargs):
"""Wrapper for subprocess.check_output, with logging."""
logging.info("Running: %s, %s; cmdline: %s.", args, kwargs, " ".join(*args))
- return subprocess.check_output(*args, **kwargs)
+ return subprocess.check_output(*args, universal_newlines=True, **kwargs)
def _make_dir_if_not_exist(*parts):
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index a9bc5431a..60de1f80d 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -268,7 +268,8 @@ class ImpalaCluster(object):
statestoreds = []
catalogd = None
admissiond = None
- output = check_output(["docker", "network", "inspect",
self.docker_network])
+ output = check_output(["docker", "network", "inspect",
self.docker_network],
+ universal_newlines=True)
# Only one network should be present in the top level array.
for container_id in json.loads(output)[0]["Containers"]:
container_info = get_container_info(container_id)
@@ -630,8 +631,8 @@ def run_daemon(daemon_binary, args, build_type="latest",
env_vars={}, output_fil
def get_container_info(container_id):
"""Get the output of "docker container inspect" as a python data
structure."""
- containers = json.loads(
- check_output(["docker", "container", "inspect", container_id]))
+ containers = json.loads(check_output(["docker", "container", "inspect",
container_id],
+ universal_newlines=True))
# Only one container should be present in the top level array.
assert len(containers) == 1, json.dumps(containers, indent=4)
return containers[0]
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 84e008b8a..17cb917d6 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -185,8 +185,10 @@ class BaseImpalaService(object):
# Requests a minidump for each running impalad/catalogd. The minidump will
be
# written to the processes's minidump_path. For simplicity, we leave it
there,
# as it will be preserved along with everything else in the log directory.
- impalad_pids = subprocess.check_output(["pgrep",
"impalad"]).split("\n")[:-1]
- catalogd_pids = subprocess.check_output(["pgrep", "-f",
"catalogd"]).split("\n")[:-1]
+ impalad_pids = subprocess.check_output(["pgrep", "impalad"],
+ universal_newlines=True).split("\n")[:-1]
+ catalogd_pids = subprocess.check_output(["pgrep", "-f", "catalogd"],
+ universal_newlines=True).split("\n")[:-1]
minidump_diag_string = "Dumping minidumps for impalads/catalogds...\n"
for pid in impalad_pids:
self.__request_minidump(pid)
