asolimando commented on code in PR #24520:
URL: https://github.com/apache/datafusion/pull/24520#discussion_r3832008623
##########
datafusion/physical-plan/src/filter.rs:
##########
@@ -406,6 +411,10 @@ impl FilterExec {
}
};
+ let num_rows = match (match_limit, num_rows.get_value()) {
Review Comment:
I think we should rescale accordingly `byte_size` and `total_byte_size`
after limiting `num_rows`
##########
datafusion/datasource/src/file_scan_config/mod.rs:
##########
@@ -2133,6 +2161,117 @@ mod tests {
}
// sets default for configs that play no role in projections
+ fn config_with_constraints(
+ table_schema: TableSchema,
+ statistics: Statistics,
+ constraints: Vec<Constraint>,
+ projection: Option<Vec<usize>>,
+ ) -> FileScanConfig {
+ FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(MockSource::new(table_schema)),
+ )
+ .with_statistics(statistics)
+ .with_constraints(Constraints::new_unverified(constraints))
+ .with_projection_indices(projection)
+ .unwrap()
+ .build()
+ }
+
+ /// A key column has one distinct value per row, which no file format
records.
+ #[test]
+ fn key_columns_report_a_distinct_count() {
+ let file_schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("part", DataType::Int32, false),
+ Field::new("code", DataType::Int32, true),
+ ]));
+ let table_schema =
TableSchema::builder(Arc::clone(&file_schema)).build();
+ let mut statistics = Statistics::new_unknown(&file_schema);
+ statistics.num_rows = Precision::Exact(100);
+ // A unique column may repeat NULL, which is not a distinct value.
+ statistics.column_statistics[2].null_count = Precision::Exact(10);
+
+ let stats = |constraints| {
+ config_with_constraints(
+ table_schema.clone(),
+ statistics.clone(),
+ constraints,
+ None,
+ )
+ .statistics()
+ };
+
+ // A primary key cannot be null, so the count is as exact as the row
count.
+ let primary_key = stats(vec![Constraint::PrimaryKey(vec![0])]);
+ assert_eq!(
+ primary_key.column_statistics[0].distinct_count,
+ Precision::Exact(100)
+ );
+ assert_eq!(
+ primary_key.column_statistics[1].distinct_count,
+ Precision::Absent
+ );
+
+ // The nulls a unique column may repeat are known here, so this is
exact too.
+ let unique = stats(vec![Constraint::Unique(vec![2])]);
+ assert_eq!(
+ unique.column_statistics[2].distinct_count,
+ Precision::Exact(90)
+ );
+
+ // With an unknown null count it is not.
Review Comment:
Nit: I'd repeat "not exact." as it's not evident it reads after the previous
comment
##########
datafusion/datasource/src/file_scan_config/mod.rs:
##########
@@ -568,6 +570,32 @@ impl FileScanConfigBuilder {
}
}
+/// Records that a key column holds one distinct value per row, which no file
format
+/// stores. Single-column keys only: a composite key says nothing about its
columns.
+fn add_key_distinct_counts(constraints: &Constraints, statistics: &mut
Statistics) {
Review Comment:
Minor: I guess this would be applicable as-is to more table providers, it's
fine to do as follow-up but maybe we could move the function somewhere else
higher up in `datasource`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
