This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3eb2eba42ab [enhance](S3) Print the oss request id for each error s3
request (#32499)
3eb2eba42ab is described below
commit 3eb2eba42ab9c45e5b0606af3e971cc709406bad
Author: AlexYue <yj976240...@gmail.com>
AuthorDate: Wed Mar 20 23:32:12 2024 +0800
[enhance](S3) Print the oss request id for each error s3 request (#32499)
---
be/src/io/fs/err_utils.cpp | 13 +++++++------
be/src/io/fs/s3_file_writer.cpp | 34 +++++++++++++++++++++++-----------
be/src/io/fs/s3_file_writer.h | 1 +
3 files changed, 31 insertions(+), 17 deletions(-)
diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp
index 35f4b79a304..8552c647cdd 100644
--- a/be/src/io/fs/err_utils.cpp
+++ b/be/src/io/fs/err_utils.cpp
@@ -115,16 +115,17 @@ Status s3fs_error(const Aws::S3::S3Error& err,
std::string_view msg) {
using namespace Aws::Http;
switch (err.GetResponseCode()) {
case HttpResponseCode::NOT_FOUND:
- return Status::Error<NOT_FOUND, false>("{}: {} {} type={}", msg,
err.GetExceptionName(),
- err.GetMessage(),
err.GetErrorType());
+ return Status::Error<NOT_FOUND, false>("{}: {} {} type={},
request_id={}", msg,
+ err.GetExceptionName(),
err.GetMessage(),
+ err.GetErrorType(),
err.GetRequestId());
case HttpResponseCode::FORBIDDEN:
- return Status::Error<PERMISSION_DENIED, false>("{}: {} {} type={}",
msg,
+ return Status::Error<PERMISSION_DENIED, false>("{}: {} {} type={},
request_id={}", msg,
err.GetExceptionName(),
err.GetMessage(),
- err.GetErrorType());
+ err.GetErrorType(),
err.GetRequestId());
default:
return Status::Error<ErrorCode::INTERNAL_ERROR, false>(
- "{}: {} {} code={} type={}", msg, err.GetExceptionName(),
err.GetMessage(),
- err.GetResponseCode(), err.GetErrorType());
+ "{}: {} {} code={} type={}, request_id={}", msg,
err.GetExceptionName(),
+ err.GetMessage(), err.GetResponseCode(), err.GetErrorType(),
err.GetRequestId());
}
}
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index c6f94807814..dcc11a21c64 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -181,13 +181,14 @@ Status S3FileWriter::_abort() {
outcome.GetError().GetResponseCode() ==
Aws::Http::HttpResponseCode::NOT_FOUND) {
LOG(INFO) << "Abort multipart upload successfully"
<< "bucket=" << _bucket << ", key=" << _path.native()
- << ", upload_id=" << _upload_id;
+ << ", upload_id=" << _upload_id << ", whole parts=" <<
_dump_completed_part();
_aborted = true;
return Status::OK();
}
- return s3fs_error(outcome.GetError(),
- fmt::format("failed to abort multipart upload {}
upload_id={}",
- _path.native(), _upload_id));
+ return s3fs_error(
+ outcome.GetError(),
+ fmt::format("failed to abort multipart upload {} upload_id={},
whole parts={}",
+ _path.native(), _upload_id, _dump_completed_part()));
}
Status S3FileWriter::close() {
@@ -412,8 +413,9 @@ Status S3FileWriter::_complete() {
_wait_until_finish("Complete");
DBUG_EXECUTE_IF("s3_file_writer::_complete:1", { _cur_part_num++; });
if (_failed || _completed_parts.size() != _cur_part_num) {
- _st = Status::InternalError("error status {}, complete parts {}, cur
part num {}", _st,
- _completed_parts.size(), _cur_part_num);
+ _st = Status::InternalError(
+ "error status {}, complete parts {}, cur part num {}, whole
parts {}", _st,
+ _completed_parts.size(), _cur_part_num,
_dump_completed_part());
LOG(WARNING) << _st;
return _st;
}
@@ -426,8 +428,9 @@ Status S3FileWriter::_complete() {
for (size_t i = 0; i < _completed_parts.size(); i++) {
if (_completed_parts[i]->GetPartNumber() != i + 1) [[unlikely]] {
auto st = Status::InternalError(
- "error status {}, part num not continous, expected num {},
actual num {}", _st,
- i + 1, _completed_parts[i]->GetPartNumber());
+ "error status {}, part num not continous, expected num {},
actual num {}, "
+ "whole parts {}",
+ _st, i + 1, _completed_parts[i]->GetPartNumber(),
_dump_completed_part());
LOG(WARNING) << st;
_st = st;
return st;
@@ -448,9 +451,10 @@ Status S3FileWriter::_complete() {
auto complete_outcome = _client->CompleteMultipartUpload(complete_request);
if (!complete_outcome.IsSuccess()) {
- _st = s3fs_error(complete_outcome.GetError(),
- fmt::format("failed to complete multi part upload {},
upload_id={}",
- _path.native(), _upload_id));
+ _st = s3fs_error(
+ complete_outcome.GetError(),
+ fmt::format("failed to complete multi part upload {},
upload_id={}, whole parts={}",
+ _path.native(), _upload_id,
_dump_completed_part()));
LOG(WARNING) << _st;
return _st;
}
@@ -507,4 +511,12 @@ void S3FileWriter::_put_object(UploadFileBuffer& buf) {
s3_file_created_total << 1;
}
+std::string S3FileWriter::_dump_completed_part() const {
+ std::string view;
+ for (const auto& part : _completed_parts) {
+ view.append(fmt::format("part {}, ", view, part->GetPartNumber()));
+ }
+ return view;
+}
+
} // namespace doris::io
diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h
index a841e481776..c9d7f1f479c 100644
--- a/be/src/io/fs/s3_file_writer.h
+++ b/be/src/io/fs/s3_file_writer.h
@@ -53,6 +53,7 @@ public:
private:
Status _abort();
+ [[nodiscard]] std::string _dump_completed_part() const;
void _wait_until_finish(std::string_view task_name);
Status _complete();
Status _create_multi_upload_request();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
