Gabriel39 commented on code in PR #66825:
URL: https://github.com/apache/doris/pull/66825#discussion_r3841144301
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java:
##########
@@ -175,17 +181,53 @@ public List<THivePartitionUpdate>
mergePartitions(List<THivePartitionUpdate> hiv
private void
collectUncompletedMpuPendingUploads(List<THivePartitionUpdate> hivePUs) {
for (THivePartitionUpdate pu : hivePUs) {
- if (pu.getS3MpuPendingUploads() != null) {
- for (TS3MPUPendingUpload s3MPUPendingUpload :
pu.getS3MpuPendingUploads()) {
- uncompletedMpuPendingUploads.add(
- new
UncompletedMpuPendingUpload(s3MPUPendingUpload,
pu.getLocation().getWritePath()));
+ List<TS3MPUPendingUpload> uploads = pu.getS3MpuPendingUploads();
+ if (uploads == null) {
+ continue;
+ }
+ String writePath = pu.getLocation() == null ? null :
pu.getLocation().getWritePath();
+ if (Strings.isNullOrEmpty(writePath)) {
+ // One malformed record must not prevent valid sibling uploads
from being cleaned up.
+ LOG.warn("Skipping MPU cleanup record without a write path");
+ continue;
+ }
+ for (TS3MPUPendingUpload upload : uploads) {
+ if (!isCompleteObjectStoreUpload(upload)) {
+ LOG.warn("Skipping incomplete MPU cleanup record for write
path {}", writePath);
+ continue;
}
+ uncompletedMpuPendingUploads.add(new
UncompletedMpuPendingUpload(upload, writePath));
+ }
+ }
+ }
+
+ private static boolean isCompleteObjectStoreUpload(TS3MPUPendingUpload
upload) {
+ return upload != null && !Strings.isNullOrEmpty(upload.getUploadId())
+ && !Strings.isNullOrEmpty(upload.getBucket()) &&
!Strings.isNullOrEmpty(upload.getKey());
+ }
+
+ private void validateObjectStoreCommitRecords() {
+ if (fileType != TFileType.FILE_S3) {
+ return;
+ }
+ for (THivePartitionUpdate update : hivePartitionUpdates) {
+ int fileCount = update.getFileNames() == null ? 0 :
update.getFileNames().size();
+ List<TS3MPUPendingUpload> uploads =
update.getS3MpuPendingUploads();
+ int uploadCount = uploads == null ? 0 : uploads.size();
+ boolean completeRecords = uploads != null
Review Comment:
Fixed in 643370c0f9. S3FileWriter now performs post-wait failure,
expected-count, and contiguous-part validation before transferring completion
ownership to the FE. HMSTransaction also rejects empty or gapped etag maps
before metadata commit. Added missing-tail BE and gapped-middle FE regression
coverage.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
