This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch dedupe-parquet-filter-rebuild in repository https://gitbox.apache.org/repos/asf/hudi.git
commit bb30f901f646f0a8adf2e6d2e2a2ec9a35147ce4 Author: voon <[email protected]> AuthorDate: Mon Sep 14 12:35:14 2026 +0800 test(spark): cover every filter arm of the rebuild Exercise both the re-spelled and the absent-column arm for each leaf filter type, and the AlwaysTrue/AlwaysFalse passthrough, in rebuildFilterFromParquet. --- .../parquet/TestParquetSchemaEvolutionUtils.scala | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestParquetSchemaEvolutionUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestParquetSchemaEvolutionUtils.scala index d6d77a5c82b0..0df5f28af986 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestParquetSchemaEvolutionUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestParquetSchemaEvolutionUtils.scala @@ -28,7 +28,7 @@ import org.apache.parquet.hadoop.metadata.FileMetaData import org.apache.parquet.schema.{MessageType, Type, Types} import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName import org.apache.spark.sql.execution.datasources.parquet.VariantParquetTestFixtures.{shreddedVariant, stringKeyMap, threeLevelList, twoLevelList, unshreddedVariant} -import org.apache.spark.sql.sources.{AlwaysTrue, And, EqualTo, Filter, GreaterThan, GreaterThanOrEqual, IsNotNull, IsNull, Not, Or, StringStartsWith} +import org.apache.spark.sql.sources.{AlwaysFalse, AlwaysTrue, And, EqualNullSafe, EqualTo, Filter, GreaterThan, GreaterThanOrEqual, In, IsNotNull, IsNull, LessThan, LessThanOrEqual, Not, Or, StringContains, StringEndsWith, StringStartsWith} import org.apache.spark.sql.types.{ArrayType, BinaryType, IntegerType, MapType, MetadataBuilder, StringType, StructField, StructType} import org.junit.jupiter.api.{Assertions, Test} @@ -354,6 +354,21 @@ class TestParquetSchemaEvolutionUtils { // cannot be evaluated must not skip any of the file's row groups. Assertions.assertEquals(AlwaysTrue, rebuild(IsNotNull("added"))) + // Every leaf filter type takes both arms: re-spelled when the file holds the column under + // another name, AlwaysTrue when it does not hold it at all. + val leafFilters: Seq[String => Filter] = Seq( + EqualTo(_, "x"), EqualNullSafe(_, "x"), GreaterThan(_, "x"), GreaterThanOrEqual(_, "x"), + LessThan(_, "x"), LessThanOrEqual(_, "x"), In(_, Array[Any]("x", "y")), IsNull(_), IsNotNull(_), + StringStartsWith(_, "x"), StringEndsWith(_, "x"), StringContains(_, "x")) + leafFilters.foreach { leaf => + Assertions.assertEquals(leaf("original"), rebuild(leaf("renamed"))) + Assertions.assertEquals(AlwaysTrue, rebuild(leaf("added")), s"${leaf("added")} on an absent column") + } + + // The constant filters reference no column and pass through as they are. + Assertions.assertEquals(AlwaysTrue, rebuild(AlwaysTrue)) + Assertions.assertEquals(AlwaysFalse, rebuild(AlwaysFalse)) + // And/Or/Not rebuild their children. Assertions.assertEquals( And(EqualTo("original", "x"), AlwaysTrue),
