yihua commented on code in PR #20080:
URL: https://github.com/apache/hudi/pull/20080#discussion_r4170710652
##########
hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java:
##########
@@ -229,6 +238,40 @@ public void testStrayFilesAreFilteredOut() throws
Exception {
"Stray files should be filtered out by listPartitions");
}
+ /**
+ * The meta folder is never a partition, so listing must neither probe it
for partition metadata
+ * nor list it as a partition, even when it holds a stray partition metafile.
+ */
+ @Test
+ public void testMetaFolderIsNotProbedForPartitionMetadata() throws Exception
{
Review Comment:
Done, the test method is package-private now.
##########
hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java:
##########
@@ -229,6 +238,40 @@ public void testStrayFilesAreFilteredOut() throws
Exception {
"Stray files should be filtered out by listPartitions");
}
+ /**
+ * The meta folder is never a partition, so listing must neither probe it
for partition metadata
+ * nor list it as a partition, even when it holds a stray partition metafile.
+ */
+ @Test
+ public void testMetaFolderIsNotProbedForPartitionMetadata() throws Exception
{
+ hoodieTestTable = hoodieTestTable.addCommit("100");
+ for (String partition : ONE_LEVEL_PARTITIONS) {
+ hoodieTestTable = hoodieTestTable.withPartitionMetaFiles(partition)
+ .withBaseFilesInPartition(partition, IntStream.range(0,
2).toArray());
+ }
+ try (OutputStream out = metaClient.getStorage().create(
+ new StoragePath(metaClient.getMetaPath(),
HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX))) {
+ out.write("stray".getBytes());
+ }
+
+ StorageConfiguration<Configuration> conf =
HadoopFSUtils.getStorageConfWithCopy(
+ metaClient.getStorageConf().unwrapAs(Configuration.class));
+ RecordingLocalFileSystem.register(conf.unwrap());
+ RecordingLocalFileSystem.reset();
+ HoodieStorage storage = HoodieStorageUtils.getStorage(basePath, conf);
+ FileSystemBackedTableMetadata fileSystemBackedTableMetadata = new
FileSystemBackedTableMetadata(
+ new HoodieLocalEngineContext(conf), metaClient.getTableConfig(),
storage, basePath);
+ List<String> partitions =
fileSystemBackedTableMetadata.getAllPartitionPaths();
+
+ Assertions.assertEquals(ONE_LEVEL_PARTITIONS,
partitions.stream().sorted().collect(Collectors.toList()));
Review Comment:
Done, `assertEquals` is statically imported.
##########
hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java:
##########
@@ -229,6 +238,40 @@ public void testStrayFilesAreFilteredOut() throws
Exception {
"Stray files should be filtered out by listPartitions");
}
+ /**
+ * The meta folder is never a partition, so listing must neither probe it
for partition metadata
+ * nor list it as a partition, even when it holds a stray partition metafile.
+ */
+ @Test
+ public void testMetaFolderIsNotProbedForPartitionMetadata() throws Exception
{
+ hoodieTestTable = hoodieTestTable.addCommit("100");
+ for (String partition : ONE_LEVEL_PARTITIONS) {
+ hoodieTestTable = hoodieTestTable.withPartitionMetaFiles(partition)
+ .withBaseFilesInPartition(partition, IntStream.range(0,
2).toArray());
+ }
+ try (OutputStream out = metaClient.getStorage().create(
+ new StoragePath(metaClient.getMetaPath(),
HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX))) {
+ out.write("stray".getBytes());
+ }
+
+ StorageConfiguration<Configuration> conf =
HadoopFSUtils.getStorageConfWithCopy(
+ metaClient.getStorageConf().unwrapAs(Configuration.class));
+ RecordingLocalFileSystem.register(conf.unwrap());
+ RecordingLocalFileSystem.reset();
+ HoodieStorage storage = HoodieStorageUtils.getStorage(basePath, conf);
+ FileSystemBackedTableMetadata fileSystemBackedTableMetadata = new
FileSystemBackedTableMetadata(
+ new HoodieLocalEngineContext(conf), metaClient.getTableConfig(),
storage, basePath);
+ List<String> partitions =
fileSystemBackedTableMetadata.getAllPartitionPaths();
+
+ Assertions.assertEquals(ONE_LEVEL_PARTITIONS,
partitions.stream().sorted().collect(Collectors.toList()));
+ Predicate<Call> partitionMetadataProbe =
Call.pathContains(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX);
+
Assertions.assertTrue(RecordingLocalFileSystem.count(partitionMetadataProbe.and(Call.underMetaFolder().negate()))
> 0,
Review Comment:
Done, `assertTrue` is statically imported.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
