adriangb commented on code in PR #25760: URL: https://github.com/apache/datafusion/pull/25760#discussion_r4137251458
########## datafusion/physical-expr/src/filter.rs: ########## @@ -0,0 +1,367 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! [`PhysicalFilter`]: a filter that an operator applies to its rows. +//! +//! Many operators (`FilterExec`, file scans, joins) apply a boolean +//! [`PhysicalExpr`] as a filter. A [`PhysicalFilter`] holds that filter as an +//! ordered list of [`FilterConjunct`]s. Each conjunct carries the properties +//! that a consumer needs to decide how to apply it (for example, whether it +//! is required for correctness). +//! +//! The expression of each conjunct must return a `Boolean` value. A row +//! passes a conjunct only when the value is `true`: `false` and `NULL` +//! remove the row. The type system does not enforce this rule. The code that +//! makes a [`FilterConjunct`] must make sure that the expression is boolean. +//! +//! A [`PhysicalFilter`] is *not* a [`PhysicalExpr`]. Use +//! [`PhysicalFilter::to_expr`] to get one expression (the `AND` of all +//! conjuncts) for code that only accepts expressions. + +use std::fmt; +use std::sync::Arc; + +use datafusion_common::Result; + +use crate::PhysicalExpr; +use crate::utils::{conjunction, conjunction_opt, split_conjunction}; + +/// One conjunct of a [`PhysicalFilter`]. +/// +/// A row passes the filter only if it passes all *required* conjuncts. A +/// consumer can skip an *optional* conjunct (for example, a dynamic filter +/// from a hash join) without an effect on the result, because another +/// operator removes the same rows again. +/// +/// The expression must return a `Boolean` value. A row passes the conjunct +/// only when the value is `true` (`NULL` removes the row, as `false` does). +/// The constructors do not check the type. See the [module docs](self). +#[derive(Debug, Clone)] +pub struct FilterConjunct { + expr: Arc<dyn PhysicalExpr>, + optional: bool, +} + +impl FilterConjunct { + /// A conjunct that the consumer must apply. + pub fn required(expr: Arc<dyn PhysicalExpr>) -> Self { + Self { + expr, + optional: false, + } + } + + /// A conjunct that the consumer can skip without an effect on the result. + pub fn optional(expr: Arc<dyn PhysicalExpr>) -> Self { + Self { + expr, + optional: true, + } + } + + /// The expression of this conjunct. It returns a `Boolean` value. + pub fn expr(&self) -> &Arc<dyn PhysicalExpr> { + &self.expr + } + + /// Consume this conjunct and return its expression. + pub fn into_expr(self) -> Arc<dyn PhysicalExpr> { + self.expr + } + + /// `true` if a consumer can skip this conjunct. + pub fn is_optional(&self) -> bool { + self.optional + } + + /// Replace the expression (for example, after a column remap) and keep + /// all other properties. + pub fn with_expr(self, expr: Arc<dyn PhysicalExpr>) -> Self { + Self { expr, ..self } + } +} + +impl From<Arc<dyn PhysicalExpr>> for FilterConjunct { + fn from(expr: Arc<dyn PhysicalExpr>) -> Self { + Self::required(expr) + } +} + +impl fmt::Display for FilterConjunct { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.expr) + } +} + +/// A filter: an ordered list of [`FilterConjunct`]s. A row passes the filter +/// when it passes all conjuncts. +/// +/// The order is the order in which the filter was built. Consumers can +/// change the evaluation order. +/// +/// An empty filter lets all rows pass. +#[derive(Debug, Clone, Default)] +pub struct PhysicalFilter { + conjuncts: Vec<FilterConjunct>, +} + +impl PhysicalFilter { + /// Create a filter from a list of conjuncts. + pub fn new(conjuncts: impl IntoIterator<Item = FilterConjunct>) -> Self { + Self { + conjuncts: conjuncts.into_iter().collect(), + } + } + + /// Adapter: a filter with `expr` as its single *required* conjunct. + /// + /// The expression is not split, so [`Self::to_expr`] returns `expr` + /// unchanged. Use [`Self::split`] to get one conjunct for each term of + /// the root `AND` chain. + pub fn from_expr(expr: Arc<dyn PhysicalExpr>) -> Self { + Self::new([FilterConjunct::required(expr)]) + } + + /// A filter with one *required* conjunct for each term of the root `AND` + /// chain of `expr`. + pub fn split(expr: &Arc<dyn PhysicalExpr>) -> Self { + Self::new( + split_conjunction(expr) + .into_iter() + .map(|e| FilterConjunct::required(Arc::clone(e))), + ) + } + + /// All conjuncts, in order. + pub fn conjuncts(&self) -> &[FilterConjunct] { + &self.conjuncts + } + + /// Consume the filter and return its conjuncts. + pub fn into_conjuncts(self) -> Vec<FilterConjunct> { + self.conjuncts + } + + /// `true` if the filter has no conjuncts (all rows pass). + pub fn is_empty(&self) -> bool { + self.conjuncts.is_empty() + } + + /// The required conjuncts, in order. + pub fn required(&self) -> impl Iterator<Item = &FilterConjunct> { + self.conjuncts.iter().filter(|c| !c.is_optional()) + } + + /// The optional conjuncts, in order. + pub fn optional(&self) -> impl Iterator<Item = &FilterConjunct> { + self.conjuncts.iter().filter(|c| c.is_optional()) + } + + /// Add conjuncts at the end. + pub fn extend(&mut self, conjuncts: impl IntoIterator<Item = FilterConjunct>) { Review Comment: ```suggestion pub fn extend(&mut self, conjuncts: impl IntoIterator<Item = impl Into<FilterConjunct>>) { ``` ? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
