MrGranday commented on code in PR #17314:
URL: https://github.com/apache/datafusion/pull/17314#discussion_r2303036694
##########
datafusion/core/src/dataframe/mod.rs:
##########
@@ -232,19 +242,31 @@ pub struct DataFrame {
projection_requires_validation: bool,
}
-impl DataFrame {
- /// Create a new `DataFrame ` based on an existing `LogicalPlan`
- ///
- /// This is a low-level method and is not typically used by end users. See
- /// [`SessionContext::read_csv`] and other methods for creating a
- /// `DataFrame` from an existing datasource.
- pub fn new(session_state: SessionState, plan: LogicalPlan) -> Self {
- Self {
- session_state: Box::new(session_state),
- plan,
- projection_requires_validation: true,
- }
+pub async fn cache(self) -> Result<DataFrame> {
+ if self.session_state.config.local_cache {
+ // Eager caching (current behavior)
+ let context =
SessionContext::new_with_state((*self.session_state).clone());
+ let plan = self.clone().create_physical_plan().await?;
+ let schema = plan.schema();
+
+ // collect_partitioned now only needs the plan
+ let partitions = collect_partitioned(plan).await?;
+
+ let mem_table = MemTable::try_new(schema, partitions)?;
+ context.read_table(Arc::new(mem_table))
+ } else {
+ // Lazy caching: return LogicalPlan::Cache
Review Comment:
Thanks I’ll use `self.into_parts()` to split the state and plan instead of
manual cloning.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
