zhangyue19921010 commented on a change in pull request #4078:
URL: https://github.com/apache/hudi/pull/4078#discussion_r782001297
##########
File path:
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTimelineArchiveLog.java
##########
@@ -134,12 +161,222 @@ public boolean archiveIfRequired(HoodieEngineContext
context) throws IOException
LOG.info("No Instants to archive");
}
+ if (config.getArchiveAutoMergeEnable() &&
!StorageSchemes.isAppendSupported(metaClient.getFs().getScheme())) {
+ mergeArchiveFilesIfNecessary(context);
+ }
return success;
} finally {
close();
}
}
+ /**
+ * Here Hoodie can merge the small archive files into a new larger one.
+ * Only used for filesystem which is not supported append operation.
+ * The hole merge small archive files operation has four stages:
+ * 1. Build merge plan with merge candidates/merged file name infos.
+ * 2. Do merge.
+ * 3. Delete all the candidates.
+ * 4. Delete the merge plan.
+ * @param context HoodieEngineContext
+ * @throws IOException
+ */
+ private void mergeArchiveFilesIfNecessary(HoodieEngineContext context)
throws IOException {
+ Path planPath = new Path(metaClient.getArchivePath(),
mergeArchivePlanName);
+ // Flush reminded content if existed and open a new write
+ reOpenWriter();
+ // List all archive files
+ FileStatus[] fsStatuses = metaClient.getFs().globStatus(
+ new Path(metaClient.getArchivePath() + "/.commits_.archive*"));
+ // Sort files by version suffix in reverse (implies reverse chronological
order)
+ Arrays.sort(fsStatuses, new
HoodieArchivedTimeline.ArchiveFileVersionComparator());
+
+ int archiveFilesMergeBatch = config.getArchiveFilesMergeBatchSize();
+ long smallFileLimitBytes = config.getArchiveMergeSmallFileLimitBytes();
+
+ List<FileStatus> mergeCandidate = getMergeCandidates(smallFileLimitBytes,
fsStatuses);
+
+ if (mergeCandidate.size() >= archiveFilesMergeBatch) {
+ List<String> candidateFiles = mergeCandidate.stream().map(fs ->
fs.getPath().toString()).collect(Collectors.toList());
Review comment:
Sure. https://issues.apache.org/jira/browse/HUDI-3212
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
