Rich-T-kid commented on code in PR #27500:
URL: https://github.com/apache/datafusion/pull/27500#discussion_r4620151407


##########
datafusion/datasource-parquet/src/metadata.rs:
##########
@@ -447,27 +502,18 @@ impl<'a> DFParquetMetadata<'a> {
             })
             .unwrap_or(schema);
 
-        let schema = if self.enable_rle_to_dictionary {
+        let schema = if self.enable_rle_to_dictionary
+            || self.dictionary_read_max_distinct_values.is_some()
+        {
             let schema_descr = file_metadata.schema_descr();
-            // Top-level columns that have a dictionary page in at least one 
row group.
-            let dict_cols: HashSet<String> = metadata
-                .row_groups()
-                .iter()
-                .flat_map(|rg| {
-                    rg.columns()
-                        .iter()
-                        .enumerate()
-                        .filter_map(|(col_idx, col)| {
-                            col.dictionary_page_offset()?;
-                            let col_desc = schema_descr.column(col_idx);
-                            let parts = col_desc.path().parts();
-                            // Skip nested columns: their leaf name doesn't 
match the
-                            // Arrow top-level field name.
-                            (parts.len() == 1).then(|| parts[0].clone())
-                        })
-                })
-                .collect();
-            if dict_cols.is_empty() {
+            let dict_cols = dictionary_promotion_candidates(
+                &metadata,
+                schema_descr,
+                self.dictionary_read_max_distinct_values,
+            );
+            if self.dictionary_read_max_distinct_values.is_some() {
+                set_dictionary_columns(&schema, &dict_cols).unwrap_or(schema)

Review Comment:
   Fixed in f323f0265. Per-file inference now only promotes eligible plain 
fields, so dictionaries declared by the embedded Arrow schema are preserved. 
The cross-file pass separately demotes fields that are dictionary in one file 
and plain in another. I also added the suggested regression case.



##########
datafusion/datasource-parquet/src/source.rs:
##########
@@ -650,7 +650,12 @@ impl FileSource for ParquetSource {
             enable_rle_to_dictionary: self
                 .table_parquet_options
                 .global
-                .enable_rle_to_dictionary,
+                .enable_rle_to_dictionary

Review Comment:
   Done in f323f0265. I added a crate-private dictionary_reads_enabled helper 
and use it for per-file inference, cross-file normalization, and reader 
construction. I also removed the stale comment.



##########
datafusion/sqllogictest/test_files/parquet_rle_to_dictionary.slt:
##########
@@ -100,6 +101,57 @@ Dictionary(Int32, Utf8) Int64
 statement ok
 DROP TABLE products_dict;
 
+statement ok
+set datafusion.execution.parquet.enable_rle_to_dictionary = false;
+
+statement ok
+set datafusion.execution.parquet.dictionary_read_max_distinct_values = 6;
+
+# Without NDV statistics, the row count is the conservative estimate.
+statement ok
+CREATE EXTERNAL TABLE threshold_no_ndv STORED AS PARQUET LOCATION 
'test_files/scratch/parquet_rle_to_dictionary/products.parquet';
+
+query TT
+SELECT DISTINCT arrow_typeof(product_category), arrow_typeof(status) FROM 
threshold_no_ndv;
+----
+Dictionary(Int32, Utf8) Dictionary(Int32, Utf8)
+
+statement ok
+set datafusion.execution.parquet.dictionary_read_max_distinct_values = 2;
+
+statement ok
+set datafusion.execution.parquet.write_row_group_number_distinct_values = true;
+
+# The exact-cap column stays dictionary encoded across files. mixed_card is
+# demoted because one file's row-group NDV sum exceeds the cap.
+query I
+COPY (SELECT column1 AS exact_cap, column2 AS mixed_card FROM (VALUES 
('x','a'),('x','b'))) TO 
'test_files/scratch/parquet_rle_to_dictionary/threshold/low.parquet' STORED AS 
PARQUET OPTIONS ('format.dictionary_enabled' true, 'format.max_row_group_size' 
2);
+----
+2
+
+query I
+COPY (SELECT column1 AS exact_cap, column2 AS mixed_card FROM (VALUES 
('x','a'),('x','b'),('x','c'))) TO 
'test_files/scratch/parquet_rle_to_dictionary/threshold/high.parquet' STORED AS 
PARQUET OPTIONS ('format.dictionary_enabled' true, 'format.max_row_group_size' 
2);
+----
+3
+
+# The cap remains authoritative when the legacy flag is also enabled.
+statement ok
+set datafusion.execution.parquet.enable_rle_to_dictionary = true;
+
+statement ok
+CREATE EXTERNAL TABLE thresholded STORED AS PARQUET LOCATION 
'test_files/scratch/parquet_rle_to_dictionary/threshold/';
+
+query TT
+SELECT DISTINCT arrow_typeof(exact_cap), arrow_typeof(mixed_card) FROM 
thresholded;
+----
+Dictionary(Int32, Utf8) Utf8View

Review Comment:
   Added this case in f323f0265. It writes a plain-encoded two-row column under 
cap 2 and verifies schema inference keeps it as Utf8View, so removing the 
dictionary-page check now fails the test.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to