alamb commented on code in PR #12135:
URL: https://github.com/apache/datafusion/pull/12135#discussion_r1747606086
##########
datafusion/core/src/datasource/listing/table.rs:
##########
@@ -826,27 +829,37 @@ impl TableProvider for ListingTable {
&self,
filters: &[&Expr],
) -> Result<Vec<TableProviderFilterPushDown>> {
- Ok(filters
+ filters
.iter()
.map(|filter| {
if expr_applicable_for_cols(
&self
.options
.table_partition_cols
.iter()
- .map(|x| x.0.as_str())
+ .map(|col| col.0.as_str())
.collect::<Vec<_>>(),
filter,
) {
// if filter can be handled by partition pruning, it is
exact
- TableProviderFilterPushDown::Exact
- } else {
- // otherwise, we still might be able to handle the filter
with file
- // level mechanisms such as Parquet row group pruning.
- TableProviderFilterPushDown::Inexact
+ return Ok(TableProviderFilterPushDown::Exact);
+ }
+
+ // if we can't push it down completely with only the
filename-based/path-based
Review Comment:
this looks really nice now
##########
datafusion/core/src/datasource/file_format/mod.rs:
##########
@@ -138,6 +139,33 @@ pub trait FileFormat: Send + Sync + fmt::Debug {
) -> Result<Arc<dyn ExecutionPlan>> {
not_impl_err!("Writer not implemented for this format")
}
+
+ /// Check if the specified file format has support for pushing down the
provided filters within
Review Comment:
👍 very nice
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
