This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 124de2a1e69 [fix](external) record not found file number (#38253)
124de2a1e69 is described below
commit 124de2a1e693dfe044262f502bfda49acd613df0
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Wed Jul 24 11:40:01 2024 +0800
[fix](external) record not found file number (#38253)
followup #37042
1. The previous PR miss one place to record "not found" file num
2. Catch "NoSuchKey" error and treat it as "not found"
---
be/src/vec/exec/format/orc/vorc_reader.cpp | 4 +++-
be/src/vec/exec/scan/vfile_scanner.cpp | 16 +++++++++++-----
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp
b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 9c016a4565e..e2ba3a57be8 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -261,7 +261,9 @@ Status OrcReader::_create_file_reader() {
if (_io_ctx && _io_ctx->should_stop && _err_msg == "stop") {
return Status::EndOfFile("stop");
}
- if (_err_msg.find("No such file or directory") != std::string::npos) {
+ // one for fs, the other is for oss.
+ if (_err_msg.find("No such file or directory") != std::string::npos ||
+ _err_msg.find("NoSuchKey") != std::string::npos) {
return Status::NotFound(_err_msg);
}
return Status::InternalError("Init OrcReader failed. reason = {}",
_err_msg);
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp
b/be/src/vec/exec/scan/vfile_scanner.cpp
index 3ef07c50b64..e066905895e 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -939,13 +939,19 @@ Status VFileScanner::_get_next_reader() {
}
COUNTER_UPDATE(_file_counter, 1);
- if (init_status.is<END_OF_FILE>() ||
init_status.is<ErrorCode::NOT_FOUND>()) {
- // The VFileScanner for external table may try to open not exist
files,
- // Because FE file cache for external table may out of date.
- // So, NOT_FOUND for VFileScanner is not a fail case.
- // Will remove this after file reader refactor.
+ // The VFileScanner for external table may try to open not exist files,
+ // Because FE file cache for external table may out of date.
+ // So, NOT_FOUND for VFileScanner is not a fail case.
+ // Will remove this after file reader refactor.
+ if (init_status.is<END_OF_FILE>()) {
COUNTER_UPDATE(_empty_file_counter, 1);
continue;
+ } else if (init_status.is<ErrorCode::NOT_FOUND>()) {
+ if (config::ignore_not_found_file_in_external_table) {
+ COUNTER_UPDATE(_not_found_file_counter, 1);
+ continue;
+ }
+ return Status::InternalError("failed to find reader, err: {}",
init_status.to_string());
} else if (!init_status.ok()) {
return Status::InternalError("failed to init reader, err: {}",
init_status.to_string());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
