alamb commented on code in PR #15769:
URL: https://github.com/apache/datafusion/pull/15769#discussion_r2074120924
##########
datafusion/core/tests/sql/path_partition.rs:
##########
@@ -57,55 +53,6 @@ use object_store::{
use object_store::{Attributes, MultipartUpload, PutMultipartOpts, PutPayload};
use url::Url;
-#[tokio::test]
-async fn parquet_partition_pruning_filter() -> Result<()> {
Review Comment:
can you remind me again why this test was removed ?
##########
datafusion/datasource-parquet/src/mod.rs:
##########
@@ -244,7 +242,7 @@ impl ParquetExecBuilder {
inner: DataSourceExec::new(Arc::new(base_config.clone())),
base_config,
predicate,
- pruning_predicate: parquet.pruning_predicate,
+ pruning_predicate: None, // for backwards compat since
`ParquetExec` is only for backwards compat anyway
Review Comment:
I filed a PR to track this
- https://github.com/apache/datafusion/issues/15950
##########
datafusion/datasource-parquet/src/source.rs:
##########
@@ -589,4 +559,49 @@ impl FileSource for ParquetSource {
}
}
}
+
+ fn try_pushdown_filters(
+ &self,
+ fd: FilterDescription,
+ config: &datafusion_common::config::ConfigOptions,
+ ) -> datafusion_common::Result<FilterPushdownResult<Arc<dyn FileSource>>> {
+ let Some(file_schema) = self.file_schema.clone() else {
Review Comment:
Let's file that follow on ticket
##########
datafusion/sqllogictest/test_files/push_down_filter.slt:
##########
@@ -18,7 +18,7 @@
# Test push down filter
statement ok
-set datafusion.explain.logical_plan_only = true;
+set datafusion.explain.physical_plan_only = true;
Review Comment:
💯
##########
docs/source/library-user-guide/upgrading.md:
##########
@@ -46,6 +46,16 @@ schema. To upgrade structs which implement `PhysicalExpr`
you need to implement
the `return_field` function. There are numerous examples in the `physical-expr`
crate.
+### `FileFormat::supports_filters_pushdown` replaced with
`FileSource::try_pushdown_filters`
Review Comment:
👍
##########
datafusion/sqllogictest/test_files/push_down_filter.slt:
##########
@@ -179,9 +190,9 @@ LOCATION
'test_files/scratch/parquet/test_filter_with_limit/';
query TT
explain select * from test_filter_with_limit where value = 2 limit 1;
----
-logical_plan
-01)Limit: skip=0, fetch=1
-02)--TableScan: test_filter_with_limit projection=[part_key, value],
full_filters=[test_filter_with_limit.value = Int32(2)], fetch=1
+physical_plan
+01)CoalescePartitionsExec: fetch=1
+02)--DataSourceExec: file_groups={3 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_filter_with_limit/part-0.parquet],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_filter_with_limit/part-1.parquet],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_filter_with_limit/part-2.parquet]]},
projection=[part_key, value], limit=1, file_type=parquet, predicate=value@1 =
2, pruning_predicate=value_null_count@2 != row_count@3 AND value_min@0 <= 2 AND
2 <= value_max@1, required_guarantees=[value in (2)]
Review Comment:
it is good to see this the predicate pushed down
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
