xudong963 commented on code in PR #18868:
URL: https://github.com/apache/datafusion/pull/18868#discussion_r2568668461
##########
datafusion/datasource-parquet/src/row_group_filter.rs:
##########
@@ -70,6 +78,49 @@ impl RowGroupAccessPlanFilter {
self.access_plan
}
+ /// Returns the is_fully_matched vector
+ pub fn is_fully_matched(&self) -> &Vec<bool> {
+ &self.is_fully_matched
+ }
+
+ /// Prunes the access plan based on the limit and fully contained row
groups.
Review Comment:
done in
https://github.com/apache/datafusion/pull/18868/commits/fdef90399c5c768d4ad00e75f7701677eb6c0f2f
##########
datafusion/datasource-parquet/src/row_group_filter.rs:
##########
@@ -135,13 +186,56 @@ impl RowGroupAccessPlanFilter {
// try to prune the row groups in a single call
match predicate.prune(&pruning_stats) {
Ok(values) => {
- // values[i] is false means the predicate could not be true
for row group i
+ let mut fully_contained_candidates_original_idx: Vec<usize> =
Vec::new();
for (idx, &value) in
row_group_indexes.iter().zip(values.iter()) {
if !value {
self.access_plan.skip(*idx);
metrics.row_groups_pruned_statistics.add_pruned(1);
} else {
metrics.row_groups_pruned_statistics.add_matched(1);
+ fully_contained_candidates_original_idx.push(*idx);
+ }
+ }
+
+ // Note: this part of code shouldn't be expensive with a
limited number of row groups
Review Comment:
done
https://github.com/apache/datafusion/pull/18868/commits/fdef90399c5c768d4ad00e75f7701677eb6c0f2f
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
