adriangb commented on code in PR #15865:
URL: https://github.com/apache/datafusion/pull/15865#discussion_r2082446025
##########
datafusion/datasource/src/schema_adapter.rs:
##########
@@ -334,4 +340,126 @@ impl SchemaMapper for SchemaMapping {
let record_batch = RecordBatch::try_new_with_options(schema, cols,
&options)?;
Ok(record_batch)
}
+
+ /// Adapts file-level column `Statistics` to match the `table_schema`
+ fn map_column_statistics(
+ &self,
+ file_col_statistics: &[ColumnStatistics],
+ ) -> datafusion_common::Result<Vec<ColumnStatistics>> {
+ let mut table_col_statistics = vec![];
+
+ // Map the statistics for each field in the file schema to the
corresponding field in the
+ // table schema, if a field is not present in the file schema, we need
to fill it with `ColumnStatistics::new_unknown`
+ for (_, file_col_idx) in self
+ .projected_table_schema
+ .fields()
+ .iter()
+ .zip(&self.field_mappings)
+ {
+ if let Some(file_col_idx) = file_col_idx {
+ table_col_statistics.push(
+ file_col_statistics
+ .get(*file_col_idx)
+ .cloned()
+ .unwrap_or_default(),
+ );
+ } else {
+ table_col_statistics.push(ColumnStatistics::new_unknown());
+ }
+ }
Review Comment:
What I mean is that I think we have enough information to make the stats
`null_count: Precision::Exact(row_count)` and absent for the rest
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
