This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit d91d99c08e469b4cd40d81ce1aeb8c2bec596880 Author: stiga-huang <[email protected]> AuthorDate: Fri Aug 16 10:13:19 2024 +0800 IMPALA-13303: FileSystemUtil.listFiles() should handle non-recursive case FileSystemUtil.listFiles() is used in FileMetadataLoader#loadInternal() to list the files with block locations. When table property "impala.disable.recursive.listing" is set to true, it's supposed to skip files in the sub dirs. However, for FileSystems that don't support recursive listFiles(), we always create a RecursingIterator and don't respect the 'recursive' argument. This patch fixes the issue by adding the check for the 'recursive' argument and use the non-recursive iterator when it's false. Tests - Add test in test_recursive_listing.py to reveal the issue Change-Id: Ia930e6071963d53561ce79896bff9d19720468a4 Reviewed-on: http://gerrit.cloudera.org:8080/21680 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- fe/src/main/java/org/apache/impala/common/FileSystemUtil.java | 4 +++- tests/metadata/test_recursive_listing.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java index 70d9ff59d..1d8d8c437 100644 --- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java +++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java @@ -929,9 +929,11 @@ public class FileSystemUtil { // survive from transient sub-directories. if (hasRecursiveListFiles(fs)) { baseIterator = fs.listFiles(p, recursive); - } else { + } else if (recursive) { baseIterator = new RecursingIterator<>(fs, p, debugAction, FileSystemUtil::listLocatedStatusIterator); + } else { + baseIterator = listLocatedStatusIterator(fs, p); } return new FilterIterator(p, baseIterator); } catch (FileNotFoundException e) { diff --git a/tests/metadata/test_recursive_listing.py b/tests/metadata/test_recursive_listing.py index 631f6439d..78674b6ad 100644 --- a/tests/metadata/test_recursive_listing.py +++ b/tests/metadata/test_recursive_listing.py @@ -126,6 +126,11 @@ class TestRecursiveListing(ImpalaTestSuite): self.execute_query_expect_success(self.client, "refresh {0}".format(fq_tbl_name)) assert len(self._show_files(fq_tbl_name)) == 1 assert len(self._get_rows(fq_tbl_name)) == 1 + # Also test initial metadata loading + self.execute_query_expect_success( + self.client, "invalidate metadata {0}".format(fq_tbl_name)) + assert len(self._show_files(fq_tbl_name)) == 1 + assert len(self._get_rows(fq_tbl_name)) == 1 # Re-enable. self.execute_query_expect_success(self.client, ("alter table {0} set tblproperties(" + "'impala.disable.recursive.listing'='false')").format(fq_tbl_name))
