This is an automated email from the ASF dual-hosted git repository. csringhofer pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit f4e75510948bdb72f2d5206161fee12e5b6d0888 Author: Joe McDonnell <[email protected]> AuthorDate: Wed May 21 20:00:50 2025 -0700 IMPALA-14087: Fix shell live_progress output display issue on Python 3 When running the shell in a terminal with live_progress=true, live progress overwrites its output by using the ANSI up character to rewrite lines with updated on the query progress. On Python 3, we found that the updates to clear the live progress were overwriting the actual output in the terminal. e.g. +----------+ | count(*) | +----------+ Fetched 1 row(s) in 5.20s To avoid this, the live progress lines need to be fully flushed to stderr before starting to output the result to stdout. This adds a flush call in OverwritingStdErrOutputStream::clear() to force this. Testing: - Hand tested queries with live progress - Added test that redirects stdout and stderr to the same file and verifies that no ANSI up character comes after the query output Change-Id: Id2e21224253f76b2a04767a57b3ade49ce2c914f Reviewed-on: http://gerrit.cloudera.org:8080/22941 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Riza Suminto <[email protected]> --- shell/impala_shell/shell_output.py | 1 + tests/shell/test_shell_interactive.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/shell/impala_shell/shell_output.py b/shell/impala_shell/shell_output.py index 9101957fc..b2e638c4f 100644 --- a/shell/impala_shell/shell_output.py +++ b/shell/impala_shell/shell_output.py @@ -225,5 +225,6 @@ class OverwritingStdErrOutputStream(object): sys.stderr.write(self.UP * self.last_line_count) sys.stderr.write(self.last_clean_text) sys.stderr.write(self.UP * self.last_line_count) + sys.stderr.flush() self.last_line_count = 0 self.last_clean_text = "" diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index 822a9c675..bedc10216 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -302,6 +302,30 @@ class TestImpalaShellInteractive(ImpalaTestSuite): result = p2.get_result() assert "VIETNAM" in result.stdout + def test_live_progress_no_overlap(self, vector): + if vector.get_value('strict_hs2_protocol'): + pytest.skip("Live progress not supported in strict hs2 mode.") + local_file = NamedTemporaryFile(delete=True) + # Sending both stdout and stderr to the same file simulates a terminal + # window that is displaying both stdout and stderr. + p = ImpalaShell(vector, stdout_file=local_file, stderr_file=local_file) + p.send_cmd("select count(*) from functional.alltypes a where id = sleep(10);") + result = p.get_result() + with open(local_file.name, "r") as fi: + # ANSI Escape code for up. live_progress uses this to overwrite content. + up = "\x1b[A" + result = fi.read() + # We don't want any up characters after we get the result. Up characters + # can cause the terminal to omit/overwrite pieces of the result output. + # Find the '| count(*) |' output, which is part of the result + count_star = "| count(*) |" + count_star_offset = result.find(count_star) + # There should be no up character after that offset + remainder = result[count_star_offset + len(count_star):] + up_found = remainder.find(up) != -1 + # Use repr to keep the escape characters visible + assert not up_found, "Found up character after result: {0}".format(repr(result)) + def test_compute_stats_with_live_progress_options(self, vector, unique_database): """Test that setting LIVE_PROGRESS options won't cause COMPUTE STATS query fail""" if vector.get_value('strict_hs2_protocol'):
