alamb commented on code in PR #14653:
URL: https://github.com/apache/datafusion/pull/14653#discussion_r1970165386
##########
datafusion/core/src/dataframe/mod.rs:
##########
@@ -183,6 +183,8 @@ pub struct DataFrame {
// Box the (large) SessionState to reduce the size of DataFrame on the
stack
session_state: Box<SessionState>,
plan: LogicalPlan,
+ // whether we can skip validation for projection ops
Review Comment:
Could you add some additional comments here about what circumstances permit
validation to be skipped?
##########
datafusion/core/benches/dataframe.rs:
##########
@@ -68,8 +67,7 @@ fn run(column_count: u32, ctx: Arc<SessionContext>) {
}
fn criterion_benchmark(c: &mut Criterion) {
- // 500 takes far too long right now
- for column_count in [10, 100, 200 /* 500 */] {
+ for column_count in [10, 100, 200, 500] {
Review Comment:
🎉
##########
datafusion/expr/src/logical_plan/builder.rs:
##########
@@ -1612,13 +1621,33 @@ pub fn union_by_name(
pub fn project(
plan: LogicalPlan,
expr: impl IntoIterator<Item = impl Into<Expr>>,
+) -> Result<LogicalPlan> {
+ project_with_validation(plan, expr.into_iter().map(|e| (e, true)))
+}
+
+/// Create Projection. Similar to project except that the expressions
+/// passed in have a flag to indicate if that expression requires
+/// validation (normalize & columnize) (true) or not (false)
+/// # Errors
+/// This function errors under any of the following conditions:
+/// * Two or more expressions have the same name
+/// * An invalid expression is used (e.g. a `sort` expression)
+pub fn project_with_validation(
Review Comment:
does this need to be `pub`? It seems like it is an implementation detail of
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
