peterxcli commented on code in PR #8214:
URL: https://github.com/apache/ozone/pull/8214#discussion_r2032352369
##########
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java:
##########
@@ -1382,6 +1417,62 @@ public void pruneSstFiles() {
}
}
+ /**
+ * Defines the task that removes OMKeyInfo from SST files from backup
directory to
+ * save disk space.
+ */
+ public void pruneSstFileValues() {
+ if (!shouldRun()) {
+ return;
+ }
+ Path sstBackupDirPath = Paths.get(sstBackupDir);
+
+ try (Stream<Path> files = Files.list(sstBackupDirPath)
+ .filter(file -> file.endsWith(ROCKSDB_SST_SUFFIX))) {
+ ManagedEnvOptions envOptions = new ManagedEnvOptions();
+ ManagedOptions managedOptions = new ManagedOptions();
+ for (Path file : files.collect(Collectors.toList())) {
+ // Write the file.sst => file.sst.tmp
+ File sstFile = file.toFile();
+ File prunedSSTFile = Files.createFile(sstBackupDirPath
+ .resolve( sstFile.getName() + ".tmp")).toFile();
+
+ ManagedSstFileWriter sstFileWriter = new
ManagedSstFileWriter(envOptions, managedOptions);
+ sstFileWriter.open(prunedSSTFile.getAbsolutePath());
+
+ ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>(
+ managedOptions, sstFile.getAbsolutePath(), 2 * 1024 * 1024);
+ Function<ManagedRawSSTFileIterator.KeyValue, Pair<byte[], Integer>>
keyValueFuntion =
+ keyValue -> Pair.of(keyValue.getKey(), keyValue.getType());
+ ManagedRawSSTFileIterator<Pair<byte[], Integer>> itr =
sstFileReader.newIterator(keyValueFuntion, null, null);
+
+ while(itr.hasNext()) {
+ Pair<byte[], Integer> keyValue = itr.next();
+ if (keyValue.getValue() == 0) {
+ sstFileWriter.delete(keyValue.getKey());
+ } else {
+ sstFileWriter.put(keyValue.getKey(), new byte[0]);
+ }
+ }
+ sstFileWriter.finish();
+
+ // Take a write lock on the SST File
+ getSSTFileLock(sstFile.getAbsolutePath()).writeLock().lock();
+ try {
+ // Move file.sst.tmp file.sst and replace existing file atomically
Review Comment:
```suggestion
// Move file.sst.tmp to file.sst and replace existing file
atomically
```
##########
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java:
##########
@@ -1382,6 +1417,62 @@ public void pruneSstFiles() {
}
}
+ /**
+ * Defines the task that removes OMKeyInfo from SST files from backup
directory to
+ * save disk space.
+ */
+ public void pruneSstFileValues() {
+ if (!shouldRun()) {
+ return;
+ }
+ Path sstBackupDirPath = Paths.get(sstBackupDir);
+
+ try (Stream<Path> files = Files.list(sstBackupDirPath)
+ .filter(file -> file.endsWith(ROCKSDB_SST_SUFFIX))) {
+ ManagedEnvOptions envOptions = new ManagedEnvOptions();
+ ManagedOptions managedOptions = new ManagedOptions();
+ for (Path file : files.collect(Collectors.toList())) {
+ // Write the file.sst => file.sst.tmp
+ File sstFile = file.toFile();
+ File prunedSSTFile = Files.createFile(sstBackupDirPath
+ .resolve( sstFile.getName() + ".tmp")).toFile();
+
+ ManagedSstFileWriter sstFileWriter = new
ManagedSstFileWriter(envOptions, managedOptions);
+ sstFileWriter.open(prunedSSTFile.getAbsolutePath());
+
+ ManagedRawSSTFileReader sstFileReader = new ManagedRawSSTFileReader<>(
+ managedOptions, sstFile.getAbsolutePath(), 2 * 1024 * 1024);
+ Function<ManagedRawSSTFileIterator.KeyValue, Pair<byte[], Integer>>
keyValueFuntion =
+ keyValue -> Pair.of(keyValue.getKey(), keyValue.getType());
+ ManagedRawSSTFileIterator<Pair<byte[], Integer>> itr =
sstFileReader.newIterator(keyValueFuntion, null, null);
+
+ while(itr.hasNext()) {
+ Pair<byte[], Integer> keyValue = itr.next();
+ if (keyValue.getValue() == 0) {
+ sstFileWriter.delete(keyValue.getKey());
+ } else {
+ sstFileWriter.put(keyValue.getKey(), new byte[0]);
+ }
+ }
+ sstFileWriter.finish();
+
+ // Take a write lock on the SST File
+ getSSTFileLock(sstFile.getAbsolutePath()).writeLock().lock();
+ try {
+ // Move file.sst.tmp file.sst and replace existing file atomically
+ Files.move(prunedSSTFile.toPath(), file,
StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
Review Comment:
```suggestion
Files.deleteIfExists(prunedSSTFile.toPath());
throw new RuntimeException(e);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org
For additional commands, e-mail: issues-h...@ozone.apache.org
