rangareddy commented on issue #14793:
URL: https://github.com/apache/hudi/issues/14793#issuecomment-4900139541
Hi @yihua
Partition pruning is not happening for nested partition field.
**Reproducible code:**
```python
from pyspark.sql.types import StructType, StructField, StringType, LongType
table_name = "nested_part"
base_path = f"s3://warehouse/{table_name}"
print(f"table path: {base_path}")
def check(name: str, actual: any, expected: any):
global failures
ok = actual == expected
status = "PASS" if ok else "FAIL"
if not ok:
failures += 1
print(f"[{status}] {name:<40} actual={actual} expected={expected}")
schema = StructType([
StructField("_row_key", StringType(), False),
StructField("timestamp", LongType(), False),
StructField("name", StringType(), True),
StructField("fare", StructType([
StructField("value", LongType(), True),
StructField("currency", StringType(), True)
]), True)
])
rows = [
("r1", 1, "alice", (10, "USD")),
("r2", 1, "bob", (20, "USD")),
("r3", 1, "carol", (30, "EUR"))
]
df = spark.createDataFrame(rows, schema)
# Write partitioned by the NESTED field fare.currency
df.write.format("hudi") \
.option("hoodie.table.name", table_name) \
.option("hoodie.datasource.write.recordkey.field", "_row_key") \
.option("hoodie.datasource.write.partitionpath.field", "fare.currency") \
.option("hoodie.datasource.write.precombine.field", "timestamp") \
.option("hoodie.datasource.write.keygenerator.class",
"org.apache.hudi.keygen.ComplexKeyGenerator") \
.option("hoodie.datasource.write.hive_style_partitioning", "true") \
.option("hoodie.metadata.enable", "false") \
.mode("overwrite") \
.save(base_path)
print(">>> write OK (partitioned by nested fare.currency)")
failures = 0
# Read back mapping the Try-Catch Exception block
try:
df = spark.read.format("hudi").load(base_path)
cnt = df.count()
df2 = df.selectExpr("fare.currency AS c")
cur_list = df2.distinct().collect()
# Map elements out of Row structures and sort list cleanly
currencies = sorted([
row[0] for row in cur_list
])
print(">>> read OK")
check("read row count", cnt, 3)
check("nested partition values", currencies, ["EUR", "USD"])
# ------- Partition Pruning Check Block -------
df3 = spark.read.format("hudi").load(base_path).where("fare.currency =
'USD'")
# Capture physical plan string from internal JVM execution plan
physical_plan = df3._jdf.queryExecution().executedPlan().toString()
# Verify that the pruned directory is referenced, and the unpruned
directory 'EUR' is excluded
has_pruned_partition = "fare.currency=USD" in physical_plan or
"fare.currency=27USD27" in physical_plan
has_skipped_partition = "fare.currency=EUR" in physical_plan
# Also verify that a record check matches pruned records count
pruned_cnt = df3.count()
check("partition pruning applied (USD path targeted)",
has_pruned_partition, True)
check("partition pruning applied (EUR path excluded)",
has_skipped_partition, False)
check("pruned dataset row count", pruned_cnt, 2)
df3.show()
except Exception as e:
failures += 1
msg = str(e)
if "Cannot find column" in msg and "fare.currency" in msg:
print(f">>> REPRODUCED the #14793 bug: {msg}")
else:
print(f">>> read FAILED with a different error: {e}")
# Summary printouts
if failures == 0:
print("\nRESULT: PASS — nested-field partition path works and supports
pruning in HoodieFileIndex (#14793 fixed).")
else:
print(f"\nRESULT: {failures} CHECK(S) FAILED.")
```
**Output:**
```
table path: s3://warehouse/nested_part
>>> write OK (partitioned by nested fare.currency)
>>> read OK
[PASS] read row count actual=3 expected=3
[PASS] nested partition values actual=['EUR', 'USD']
expected=['EUR', 'USD']
[FAIL] partition pruning applied (USD path targeted) actual=False
expected=True
[PASS] partition pruning applied (EUR path excluded) actual=False
expected=False
[PASS] pruned dataset row count actual=2 expected=2
+-------------------+--------------------+------------------+----------------------+--------------------+--------+---------+-----+---------+--------+
|_hoodie_commit_time|_hoodie_commit_seqno|_hoodie_record_key|_hoodie_partition_path|
_hoodie_file_name|_row_key|timestamp| name| fare|currency|
+-------------------+--------------------+------------------+----------------------+--------------------+--------+---------+-----+---------+--------+
| 20260707042842194|20260707042842194...| r1|
fare.currency=USD|59f191d4-91d2-4d7...| r1| 1|alice|{10, USD}|
USD|
| 20260707042842194|20260707042842194...| r2|
fare.currency=USD|59f191d4-91d2-4d7...| r2| 1| bob|{20, USD}|
USD|
+-------------------+--------------------+------------------+----------------------+--------------------+--------+---------+-----+---------+--------+
RESULT: 1 CHECK(S) FAILED.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
