ChenSammi commented on code in PR #8974:
URL: https://github.com/apache/ozone/pull/8974#discussion_r2333585644
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java:
##########
@@ -628,45 +674,178 @@ private void sendDeleteKeysRequestAndClearList(String
volume, String bucket,
getOzoneManager(), omRequest, clientId,
callId.getAndIncrement());
long endTime = System.nanoTime();
LOG.debug("DeleteKeys request with {} keys cost {} ns", keyCount,
endTime - startTime);
- i += batchSize;
- startIndex += batchSize;
+ long deletedCount = keyCount;
+ long deletedSize = keysList.replicatedSizeSubList(startIndex,
endIndex)
+ .stream().mapToLong(Long::longValue).sum();
if (response != null) {
if (!response.getSuccess()) {
// log the failure and continue the iterating
- LOG.error("DeleteKeys request failed with " +
response.getMessage() +
- " for volume: {}, bucket: {}", volume, bucket);
- continue;
+ LOG.error("DeleteKeys request " + response.getStatus() + "
failed for volume: {}, bucket: {}",
+ volume, bucket);
+ if (response.getDeleteKeysResponse().hasUnDeletedKeys()) {
+ DeleteKeyArgs unDeletedKeys =
response.getDeleteKeysResponse().getUnDeletedKeys();
+ for (String key : unDeletedKeys.getKeysList()) {
+ Long size = keysList.getReplicatedSize(key);
+ if (size == null) {
+ LOG.error("Undeleted key {}/{}/{} doesn't in keyLists",
volume, bucket, key);
+ continue;
+ }
+ deletedCount -= 1;
+ deletedSize -= size;
+ }
+ }
+ for (DeleteKeyError e :
response.getDeleteKeysResponse().getErrorsList()) {
+ Long size = keysList.getReplicatedSize(e.getKey());
+ if (size == null) {
+ LOG.error("Deleted error key {}/{}/{} doesn't in
keyLists", volume, bucket, e.getKey());
+ continue;
+ }
+ deletedCount -= 1;
+ deletedSize -= size;
+ }
} else {
LOG.debug("DeleteKeys request of total {} keys, {} not
deleted", keyCount,
response.getDeleteKeysResponse().getErrorsCount());
}
}
if (dir) {
- numDirDeleted += keyCount;
- metrics.incrNumDirDeleted(keyCount);
+ numDirDeleted += deletedCount;
+ metrics.incrNumDirDeleted(deletedCount);
} else {
- numKeyDeleted += keyCount;
- metrics.incrNumKeyDeleted(keyCount);
+ numKeyDeleted += deletedCount;
+ sizeKeyDeleted += deletedSize;
+ metrics.incrNumKeyDeleted(deletedCount);
+ metrics.incrSizeKeyDeleted(deletedSize);
}
+ i += keyCount;
+ startIndex += keyCount;
} else {
batchSize /= 2;
}
}
- keysList.clear();
} catch (ServiceException e) {
LOG.error("Failed to send DeleteKeysRequest", e);
+ } finally {
+ keysList.clear();
}
}
- private void moveKeysToTrash(String volume, String bucket,
LimitedExpiredObjectList keysList) {
- for (int index = 0; index < keysList.size(); index++) {
+ private void moveKeysToTrash(OmBucketInfo bucket, LimitedExpiredObjectList
keysList) {
+ if (keysList.isEmpty()) {
+ return;
+ }
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ String trashCurrent;
+ try {
+ trashCurrent = checkAndCreateTrashDirectoryIfNeeded(bucket);
+ } catch (IOException e) {
+ keysList.clear();
+ return;
+ }
+
+ for (int i = 0; i < keysList.size(); i++) {
+ String keyName = keysList.getName(i);
+ String targetKeyName = trashCurrent + OM_KEY_PREFIX + keyName;
+ KeyArgs keyArgs = KeyArgs.newBuilder().setKeyName(keyName)
+ .setVolumeName(volumeName).setBucketName(bucketName).build();
+
+ /**
+ * Trash examples:
+ * /s3v/test/readme -> /s3v/test/.Trash/hadoop/Current/readme
+ * /s3v/test/dir1/readme -> /s3v/test/.Trash/hadoop/Current/dir1/readme
+ */
+ RenameKeyRequest renameKeyRequest = RenameKeyRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setToKeyName(targetKeyName)
+ .setUpdateID(keysList.getUpdateID(i))
+ .build();
+
+ // send request out
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .setRenameKeyRequest(renameKeyRequest)
+ .build();
try {
- ozoneTrash.moveToTrash(new Path(OM_KEY_PREFIX + volume +
OM_KEY_PREFIX + bucket + OM_KEY_PREFIX +
- keysList.getName(index)));
- } catch (IOException e) {
- // log failure and continue
- LOG.warn("Failed to move key {} to trash", keysList.getName(index),
e);
+ // perform preExecute as ratis submit do no perform preExecute
+ OMClientRequest omClientRequest =
OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager);
+ omRequest = omClientRequest.preExecute(ozoneManager);
Review Comment:
Good point.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
