This is an automated email from the ASF dual-hosted git repository. yuzelin pushed a commit to branch release-1.4 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 97813f617cf4f580c798c4d1f8f460846a9f97c9 Author: yuzelin <[email protected]> AuthorDate: Wed Apr 1 15:29:03 2026 +0800 [core] Fix that latestSnapshotOfUser might list snapshot directory to find the earliest (#7572) Previously, latestSnapshotOfUser called earliestSnapshotId() to determine the loop bound before iterating snapshots. During checkpoint, if the snapshot pointed to by the EARLIEST hint is concurrently expired, the hint misses and earliestSnapshotId() falls back to findByListFiles(), which lists the entire snapshot directory. This method is called during every checkpoint via prepareCommit → createWriterCleanChecker → latestCommittedIdentifier → latestSnapshotOfUserFromFilesystem. For jobs with high parallelism and a large number of snapshots (e.g. 10000+), this might cause massive listStatus requests, easily triggering QPS limits on object storage (e.g. OSS QpsLimitExceeded). The fix removes the earliestSnapshotId() call and instead iterates backward from the latest snapshot, stopping when a FileNotFoundException is encountered (indicating an expired snapshot). Other exceptions are thrown directly. --- .../org/apache/paimon/utils/SnapshotManager.java | 28 +++++----------------- .../apache/paimon/utils/SnapshotManagerTest.java | 4 ++-- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java index 414ef3f5b7..b6a818604e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java @@ -600,31 +600,14 @@ public class SnapshotManager implements Serializable { if (latestId == null) { return Optional.empty(); } - if (earliestId == null) { - earliestId = - Preconditions.checkNotNull( - earliestSnapshotId(), - "Latest snapshot id is not null, but earliest snapshot id is null. " - + "This is unexpected."); - } - for (long id = latestId; id >= earliestId; id--) { + long searchEnd = earliestId != null ? earliestId : Snapshot.FIRST_SNAPSHOT_ID; + for (long id = latestId; id >= searchEnd; id--) { Snapshot snapshot; try { - snapshot = snapshot(id); - } catch (Exception e) { - long newEarliestId = - Preconditions.checkNotNull( - earliestSnapshotId(), - "Latest snapshot id is not null, but earliest snapshot id is null. " - + "This is unexpected."); - - // this is a valid snapshot, should throw exception - if (id >= newEarliestId) { - throw e; - } - - // this is an expired snapshot + snapshot = tryGetSnapshot(id); + } catch (FileNotFoundException e) { + // this snapshot has been expired, stop searching LOG.warn( "Snapshot #" + id @@ -632,6 +615,7 @@ public class SnapshotManager implements Serializable { + user + ") is not found."); break; + // other exceptions will be thrown } if (user.equals(snapshot.commitUser())) { diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java index 19ae865692..486f0b3fe5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java @@ -334,7 +334,7 @@ public class SnapshotManagerTest { SnapshotManager snapshotManager = newSnapshotManager(localFileIO, new Path(tempDir.toString())); // create 100 snapshots using user "lastCommitUser" - for (long i = 0; i < 100; i++) { + for (long i = 1; i <= 100; i++) { Snapshot snapshot = new Snapshot( i, @@ -375,7 +375,7 @@ public class SnapshotManagerTest { Thread.sleep(100); // expire snapshot - localFileIO.deleteQuietly(snapshotManager.snapshotPath(0)); + localFileIO.deleteQuietly(snapshotManager.snapshotPath(1)); thread.join(); assertThat(exception.get()).isNull();
