This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 756d9af1c9a12d40b5f5e23fecfce373c056c693 Author: jiangwel <[email protected]> AuthorDate: Thu Nov 14 20:06:03 2024 +0900 IMPALA-13448: Log cause when failing to flush lineage events, audit events or profiles When impala fails to flush lineage events, audit events or profiles, only the log file name is logged: "Could not open log file: filename". Now we will log "Could not open log file: filename, reason". e.g: "Could not open log file: filename, cause: Permission denied" Testing: - added custom cluster tests in test_logging.py Change-Id: I5b281d807e47aad98fc256af4e0c2a9dd417c7ac Reviewed-on: http://gerrit.cloudera.org:8080/22070 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/util/simple-logger.cc | 7 ++++- tests/custom_cluster/test_logging.py | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/be/src/util/simple-logger.cc b/be/src/util/simple-logger.cc index 62ada31b1..2d6f60390 100644 --- a/be/src/util/simple-logger.cc +++ b/be/src/util/simple-logger.cc @@ -19,6 +19,7 @@ #include <mutex> #include <regex> +#include <cerrno> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> @@ -39,6 +40,7 @@ using boost::posix_time::ptime; using boost::posix_time::time_from_string; using kudu::JoinPathSegments; using namespace impala; +using std::strerror; const ptime EPOCH = time_from_string("1970-01-01 00:00:00.000"); @@ -131,7 +133,10 @@ Status SimpleLogger::FlushInternal() { log_file_.close(); } log_file_.open(log_file_name_.c_str(), std::ios_base::app | std::ios_base::out); - if (!log_file_.is_open()) return Status("Could not open log file: " + log_file_name_); + if(log_file_.fail()) { + return Status("Could not open log file: " + + log_file_name_ + ", cause: " + strerror(errno)); + } return Status::OK(); } diff --git a/tests/custom_cluster/test_logging.py b/tests/custom_cluster/test_logging.py index 861348489..09d1fc5cc 100644 --- a/tests/custom_cluster/test_logging.py +++ b/tests/custom_cluster/test_logging.py @@ -16,11 +16,15 @@ # under the License. from __future__ import absolute_import, division, print_function +import logging +import time import pytest +import os from tests.common.custom_cluster_test_suite import CustomClusterTestSuite +LOG = logging.getLogger(__name__) class TestLoggingCore(CustomClusterTestSuite): """Test existence of certain log lines under some scenario.""" @@ -63,3 +67,50 @@ class TestLoggingCore(CustomClusterTestSuite): disable_log_buffering=True) def test_max_errors_no_downgrade(self): self._test_max_errors(2, -1, False) + + +class TestLogFlushPermissionDenied(CustomClusterTestSuite): + """Test logging of failures to open log files with cause Permission denied.""" + LOG_FLUSH_FAILURES_DIR = "log_flush_failures_dir" + + @classmethod + def get_workload(cls): + return 'functional-query' + + def setup_method(self, method): + # Override parent + super(TestLogFlushPermissionDenied, self).setup_method(method) + tmp_dir = self.get_tmp_dir(self.LOG_FLUSH_FAILURES_DIR) + self.orig_permissions = os.stat(tmp_dir).st_mode + os.chmod(tmp_dir, 0) + + def teardown_method(self, method): + # Override parent + os.chmod(self.get_tmp_dir(self.LOG_FLUSH_FAILURES_DIR), self.orig_permissions) + super(TestLogFlushPermissionDenied, self).teardown_method(method) + + def __test_permission_denied(self, log_dir): + self.assert_impalad_log_contains("INFO", + r"Could not open log file: {0}.*, cause: Permission denied".format(log_dir), 2) + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args="--lineage_event_log_dir={" + LOG_FLUSH_FAILURES_DIR + "}", + tmp_dir_placeholders=[LOG_FLUSH_FAILURES_DIR]) + def test_lineage_log_failure(self): + self.__test_permission_denied(self.get_tmp_dir(self.LOG_FLUSH_FAILURES_DIR)) + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args="--audit_event_log_dir={" + LOG_FLUSH_FAILURES_DIR + "}", + tmp_dir_placeholders=[LOG_FLUSH_FAILURES_DIR]) + def test_audit_log_failure(self): + self.__test_permission_denied(self.get_tmp_dir(self.LOG_FLUSH_FAILURES_DIR)) + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args="--profile_log_dir={" + LOG_FLUSH_FAILURES_DIR + "}", + tmp_dir_placeholders=[LOG_FLUSH_FAILURES_DIR]) + def test_profiles_failure(self): + time.sleep(5) + self.__test_permission_denied(self.get_tmp_dir(self.LOG_FLUSH_FAILURES_DIR))
