compiler-segfault commented on code in PR #27283:
URL: https://github.com/apache/datafusion/pull/27283#discussion_r4537996943
##########
datafusion/core/tests/parquet/constant_columns.rs:
##########
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Tests that statistics-derived "constant columns" are not substituted when
+//! the statistics cannot prove that the column is constant.
+//!
+//! Parquet writers exclude `NaN` from floating point `min`/`max` statistics
+//! (the format requires it), so a column holding `1.0`, `NaN` and `1.0` has
+//! `min == max == 1.0` and `null_count == 0`. Treating that as a constant
+//! column silently rewrites the `NaN` row.
+
+use std::sync::Arc;
+
+use arrow::array::{Array, Float64Array};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::record_batch::RecordBatch;
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use parquet::arrow::ArrowWriter;
+use parquet::file::properties::WriterProperties;
+use tempfile::NamedTempFile;
+
+/// Write a single `Float64` column named `f` with statistics enabled (the
+/// default), which is what makes the column eligible for statistics-based
+/// constant substitution.
+fn write_float_parquet(values: Vec<f64>) -> NamedTempFile {
+ let schema = Arc::new(Schema::new(vec![Field::new("f", DataType::Float64,
true)]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Float64Array::from(values))],
+ )
+ .unwrap();
+
+ let file = tempfile::Builder::new()
+ .suffix(".parquet")
+ .tempfile()
+ .expect("tempfile creation");
+ let props = WriterProperties::builder().build();
+ let mut writer =
+ ArrowWriter::try_new(file.reopen().unwrap(), schema,
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ file
+}
+
+async fn setup(values: Vec<f64>) -> (SessionContext, NamedTempFile) {
+ let file = write_float_parquet(values);
+ let ctx = SessionContext::new();
+ ctx.register_parquet(
+ "t",
+ file.path().to_str().unwrap(),
+ ParquetReadOptions::default(),
+ )
+ .await
+ .unwrap();
+ (ctx, file)
+}
+
+#[tokio::test]
+async fn nan_rows_are_not_replaced_by_the_statistics_constant() {
Review Comment:
Thanks, moved it to a sqllogictest:
datafusion/sqllogictest/test_files/parquet_constant_columns.slt (5f0bfe29), and
rebased the branch on current main.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
