alexanderbianchi commented on code in PR #67:
URL: https://github.com/apache/datafusion-iceberg/pull/67#discussion_r4586965472


##########
crates/datafusion/src/read/scan.rs:
##########
@@ -98,20 +117,134 @@ impl IcebergDataSource {
 /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`].
 ///
 /// A table provider passes the table's schema and selects columns with
-/// [`Self::with_projection`]. To rebuild a source, such as in another process,
+/// [`Self::with_projection`]. Call [`Self::build`] for lazy discovery or
+/// [`Self::build_eager`] to return a source of already-assigned files.
+///
+/// To rebuild a source, such as in another process,
 /// pass its [`IcebergDataSource::schema`], which is already projected, and no
-/// projection.
-#[derive(Debug, Clone)]
+/// projection. For already-planned files, reconstruct an eager source with
+/// [`Self::from_planned_files`] instead of discovering files again.
+#[derive(Clone)]
 pub struct IcebergDataSourceBuilder {
-    table: Table,
+    input: ScanInput,
     schema: ArrowSchemaRef,
     projection: Option<Vec<usize>>,
     snapshot_id: Option<i64>,
     predicate: Option<Predicate>,
     fetch: Option<usize>,
+    reader: Option<ArrowReader>,
+}
+
+#[derive(Clone)]
+enum ScanInput {
+    Table(Table),
+    Planned {
+        groups: Vec<Vec<FileScanTask>>,
+        file_io: FileIO,
+    },
+}
+
+impl std::fmt::Debug for IcebergDataSourceBuilder {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IcebergDataSourceBuilder")
+            .field("schema", &self.schema)
+            .field("projection", &self.projection)
+            .field("snapshot_id", &self.snapshot_id)
+            .field("predicate", &self.predicate)
+            .field("fetch", &self.fetch)
+            .finish_non_exhaustive()
+    }
 }
 
 impl IcebergDataSourceBuilder {
+    /// Build from already-planned files without a table, catalog or discovery 
I/O.
+    /// Each group is one local output partition; empty groups and supplied 
order
+    /// are preserved. `schema` must already be projected to match the tasks.
+    /// `reader` must use the supplied FileIO and process-local 
runtime/settings.
+    /// Reader clones share their delete loader; their settings are not 
overwritten.
+    ///
+    /// Finish with [`Self::build_planned`], which reserves descriptor memory 
in the
+    /// supplied query context. Discovery-only projection, snapshot and 
predicate
+    /// settings are rejected rather than silently ignored. Statistics are 
unknown.
+    /// Use [`EagerIcebergDataSource::with_planned_files`] to replace the 
assignment
+    /// of an existing source instead.
+    ///
+    /// ```
+    /// use datafusion::arrow::datatypes::SchemaRef;
+    /// use datafusion::common::Result;
+    /// use datafusion::execution::TaskContext;
+    /// use iceberg::arrow::ArrowReader;
+    /// use iceberg::io::FileIO;
+    /// use iceberg::scan::FileScanTask;
+    /// use iceberg_datafusion::read::{EagerIcebergDataSource, 
IcebergDataSourceBuilder};
+    ///
+    /// fn restore(
+    ///     schema: SchemaRef,
+    ///     groups: Vec<Vec<FileScanTask>>,
+    ///     file_io: FileIO,
+    ///     local_reader: ArrowReader,
+    ///     fetch: Option<usize>,
+    ///     context: &TaskContext,
+    /// ) -> Result<EagerIcebergDataSource> {
+    ///     IcebergDataSourceBuilder::from_planned_files(schema, groups, 
file_io, local_reader)
+    ///         .with_fetch(fetch)
+    ///         .build_planned(context)
+    /// }
+    /// ```
+    pub fn from_planned_files(

Review Comment:
    These APIs aren’t test-only—DFD is the primary use case. Workers receive 
already-planned file tasks and need to reconstruct a data source without 
loading a table or rediscovering files. Comet and other systems also supply 
their own Iceberg tasks. This is about composability and reuse. I’m happy to 
improve the builder’s shape, but the reconstruction capability serves real 
consumers, not just tests.



##########
crates/datafusion/src/read/scan.rs:
##########
@@ -98,20 +117,134 @@ impl IcebergDataSource {
 /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`].
 ///
 /// A table provider passes the table's schema and selects columns with
-/// [`Self::with_projection`]. To rebuild a source, such as in another process,
+/// [`Self::with_projection`]. Call [`Self::build`] for lazy discovery or
+/// [`Self::build_eager`] to return a source of already-assigned files.
+///
+/// To rebuild a source, such as in another process,
 /// pass its [`IcebergDataSource::schema`], which is already projected, and no
-/// projection.
-#[derive(Debug, Clone)]
+/// projection. For already-planned files, reconstruct an eager source with
+/// [`Self::from_planned_files`] instead of discovering files again.
+#[derive(Clone)]
 pub struct IcebergDataSourceBuilder {
-    table: Table,
+    input: ScanInput,
     schema: ArrowSchemaRef,
     projection: Option<Vec<usize>>,
     snapshot_id: Option<i64>,
     predicate: Option<Predicate>,
     fetch: Option<usize>,
+    reader: Option<ArrowReader>,
+}
+
+#[derive(Clone)]
+enum ScanInput {
+    Table(Table),
+    Planned {
+        groups: Vec<Vec<FileScanTask>>,
+        file_io: FileIO,
+    },
+}
+
+impl std::fmt::Debug for IcebergDataSourceBuilder {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IcebergDataSourceBuilder")
+            .field("schema", &self.schema)
+            .field("projection", &self.projection)
+            .field("snapshot_id", &self.snapshot_id)
+            .field("predicate", &self.predicate)
+            .field("fetch", &self.fetch)
+            .finish_non_exhaustive()
+    }
 }
 
 impl IcebergDataSourceBuilder {
+    /// Build from already-planned files without a table, catalog or discovery 
I/O.
+    /// Each group is one local output partition; empty groups and supplied 
order
+    /// are preserved. `schema` must already be projected to match the tasks.
+    /// `reader` must use the supplied FileIO and process-local 
runtime/settings.
+    /// Reader clones share their delete loader; their settings are not 
overwritten.
+    ///
+    /// Finish with [`Self::build_planned`], which reserves descriptor memory 
in the
+    /// supplied query context. Discovery-only projection, snapshot and 
predicate
+    /// settings are rejected rather than silently ignored. Statistics are 
unknown.
+    /// Use [`EagerIcebergDataSource::with_planned_files`] to replace the 
assignment
+    /// of an existing source instead.
+    ///
+    /// ```
+    /// use datafusion::arrow::datatypes::SchemaRef;
+    /// use datafusion::common::Result;
+    /// use datafusion::execution::TaskContext;
+    /// use iceberg::arrow::ArrowReader;
+    /// use iceberg::io::FileIO;
+    /// use iceberg::scan::FileScanTask;
+    /// use iceberg_datafusion::read::{EagerIcebergDataSource, 
IcebergDataSourceBuilder};
+    ///
+    /// fn restore(
+    ///     schema: SchemaRef,
+    ///     groups: Vec<Vec<FileScanTask>>,
+    ///     file_io: FileIO,
+    ///     local_reader: ArrowReader,
+    ///     fetch: Option<usize>,
+    ///     context: &TaskContext,
+    /// ) -> Result<EagerIcebergDataSource> {
+    ///     IcebergDataSourceBuilder::from_planned_files(schema, groups, 
file_io, local_reader)
+    ///         .with_fetch(fetch)
+    ///         .build_planned(context)
+    /// }
+    /// ```
+    pub fn from_planned_files(

Review Comment:
    These APIs aren’t test-only—DFD is the one use case. Workers receive 
already-planned file tasks and need to reconstruct a data source without 
loading a table or rediscovering files. Comet and other systems also supply 
their own Iceberg tasks. This is about composability and reuse. I’m happy to 
improve the builder’s shape, but the reconstruction capability serves real 
consumers, not just tests.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to