jayzhan211 commented on code in PR #27500:
URL: https://github.com/apache/datafusion/pull/27500#discussion_r4615531658


##########
datafusion/datasource-parquet/src/source.rs:
##########
@@ -650,7 +650,12 @@ impl FileSource for ParquetSource {
             enable_rle_to_dictionary: self
                 .table_parquet_options
                 .global
-                .enable_rle_to_dictionary,
+                .enable_rle_to_dictionary

Review Comment:
   Optional cleanup: `enable_rle_to_dictionary || 
dictionary_read_max_distinct_values.is_some()` is now written in three places 
(here, `file_format.rs:466` and `metadata.rs:505`). If this copy is dropped, 
every test still passes. The cap-only mode would then read plain and cast to 
the dictionary instead of decoding straight into it. A single crate-private 
helper keeps the three from drifting apart. It also makes the comment at 
`file_format.rs:486` unnecessary, since that comment describes an OR the line 
below it doesn't do:
   
   ```rs
   pub(crate) fn dictionary_reads_enabled(options: &ParquetOptions) -> bool {
       options.enable_rle_to_dictionary
           || options.dictionary_read_max_distinct_values.is_some()
   }
   ```



##########
datafusion/sqllogictest/test_files/parquet_rle_to_dictionary.slt:
##########
@@ -100,6 +101,57 @@ Dictionary(Int32, Utf8) Int64
 statement ok
 DROP TABLE products_dict;
 
+statement ok
+set datafusion.execution.parquet.enable_rle_to_dictionary = false;
+
+statement ok
+set datafusion.execution.parquet.dictionary_read_max_distinct_values = 6;
+
+# Without NDV statistics, the row count is the conservative estimate.
+statement ok
+CREATE EXTERNAL TABLE threshold_no_ndv STORED AS PARQUET LOCATION 
'test_files/scratch/parquet_rle_to_dictionary/products.parquet';
+
+query TT
+SELECT DISTINCT arrow_typeof(product_category), arrow_typeof(status) FROM 
threshold_no_ndv;
+----
+Dictionary(Int32, Utf8) Dictionary(Int32, Utf8)
+
+statement ok
+set datafusion.execution.parquet.dictionary_read_max_distinct_values = 2;
+
+statement ok
+set datafusion.execution.parquet.write_row_group_number_distinct_values = true;
+
+# The exact-cap column stays dictionary encoded across files. mixed_card is
+# demoted because one file's row-group NDV sum exceeds the cap.
+query I
+COPY (SELECT column1 AS exact_cap, column2 AS mixed_card FROM (VALUES 
('x','a'),('x','b'))) TO 
'test_files/scratch/parquet_rle_to_dictionary/threshold/low.parquet' STORED AS 
PARQUET OPTIONS ('format.dictionary_enabled' true, 'format.max_row_group_size' 
2);
+----
+2
+
+query I
+COPY (SELECT column1 AS exact_cap, column2 AS mixed_card FROM (VALUES 
('x','a'),('x','b'),('x','c'))) TO 
'test_files/scratch/parquet_rle_to_dictionary/threshold/high.parquet' STORED AS 
PARQUET OPTIONS ('format.dictionary_enabled' true, 'format.max_row_group_size' 
2);
+----
+3
+
+# The cap remains authoritative when the legacy flag is also enabled.
+statement ok
+set datafusion.execution.parquet.enable_rle_to_dictionary = true;
+
+statement ok
+CREATE EXTERNAL TABLE thresholded STORED AS PARQUET LOCATION 
'test_files/scratch/parquet_rle_to_dictionary/threshold/';
+
+query TT
+SELECT DISTINCT arrow_typeof(exact_cap), arrow_typeof(mixed_card) FROM 
thresholded;
+----
+Dictionary(Int32, Utf8) Utf8View

Review Comment:
   No test covers the requirement that every nonempty row group has a 
dictionary page. If I delete the `row_count > 0 && 
dictionary_page_offset().is_none()` check, this slt file and the crate's unit 
tests still pass. A plain-encoded column whose row count is within the cap is 
the only input that separates the two. It fails on this head with the check 
removed (`Dictionary(Int32, Utf8)`), and passes as is:
   
   ```
   # A column without dictionary pages stays plain even when its row count is
   # within the cap.
   query I
   COPY (SELECT column1 AS no_dict_page FROM (VALUES ('x'),('x'))) TO 
'test_files/scratch/parquet_rle_to_dictionary/threshold_no_dict_page/p.parquet' 
STORED AS PARQUET OPTIONS ('format.dictionary_enabled' false);
   ----
   2
   
   statement ok
   CREATE EXTERNAL TABLE threshold_no_dict_page STORED AS PARQUET LOCATION 
'test_files/scratch/parquet_rle_to_dictionary/threshold_no_dict_page/';
   
   query T
   SELECT DISTINCT arrow_typeof(no_dict_page) FROM threshold_no_dict_page;
   ----
   Utf8View
   ```



##########
datafusion/datasource-parquet/src/metadata.rs:
##########
@@ -447,27 +502,18 @@ impl<'a> DFParquetMetadata<'a> {
             })
             .unwrap_or(schema);
 
-        let schema = if self.enable_rle_to_dictionary {
+        let schema = if self.enable_rle_to_dictionary
+            || self.dictionary_read_max_distinct_values.is_some()
+        {
             let schema_descr = file_metadata.schema_descr();
-            // Top-level columns that have a dictionary page in at least one 
row group.
-            let dict_cols: HashSet<String> = metadata
-                .row_groups()
-                .iter()
-                .flat_map(|rg| {
-                    rg.columns()
-                        .iter()
-                        .enumerate()
-                        .filter_map(|(col_idx, col)| {
-                            col.dictionary_page_offset()?;
-                            let col_desc = schema_descr.column(col_idx);
-                            let parts = col_desc.path().parts();
-                            // Skip nested columns: their leaf name doesn't 
match the
-                            // Arrow top-level field name.
-                            (parts.len() == 1).then(|| parts[0].clone())
-                        })
-                })
-                .collect();
-            if dict_cols.is_empty() {
+            let dict_cols = dictionary_promotion_candidates(
+                &metadata,
+                schema_descr,
+                self.dictionary_read_max_distinct_values,
+            );
+            if self.dictionary_read_max_distinct_values.is_some() {
+                set_dictionary_columns(&schema, &dict_cols).unwrap_or(schema)

Review Comment:
   Setting the cap demotes columns that the file's own Arrow schema declares as 
`Dictionary`. A file written from `arrow_cast(c, 'Dictionary(Int32, Utf8)')` 
reads as `Dictionary(Int32, Utf8)` without the cap, but as `Utf8View` with 
`dictionary_read_max_distinct_values = 1`. With default writer settings there 
is no NDV, so the row-count estimate demotes almost every such column. Turning 
on an option meant to add dictionary reads shouldn't take one away. Making the 
per-file step promote-only fixes this and lets the legacy path share the helper 
(net −21 lines). The cross-file step still demotes a column that is plain in 
another file:
   
   ```diff
   -            if self.dictionary_read_max_distinct_values.is_some() {
   -                set_dictionary_columns(&schema, 
&dict_cols).unwrap_or(schema)
   -            } else if dict_cols.is_empty() {
   -                schema
   -            } else {
   -                let promoted: Vec<_> = schema
   -                // ... 30 more lines of the legacy promotion
   -            }
   +            promote_dictionary_columns(&schema, 
&dict_cols).unwrap_or(schema)
   ```
   
   Test (fails on this head with `Utf8View`, passes with the change; place it 
after the `thresholded` query, where the cap is 2 and NDV writing is on):
   
   ```
   # A dictionary type from the file's Arrow schema is kept above the cap.
   query I
   COPY (SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS declared_dict 
FROM (VALUES ('a'),('b'),('c'))) TO 
'test_files/scratch/parquet_rle_to_dictionary/threshold_declared/d.parquet' 
STORED AS PARQUET;
   ----
   3
   
   statement ok
   CREATE EXTERNAL TABLE threshold_declared STORED AS PARQUET LOCATION 
'test_files/scratch/parquet_rle_to_dictionary/threshold_declared/';
   
   query T
   SELECT DISTINCT arrow_typeof(declared_dict) FROM threshold_declared;
   ----
   Dictionary(Int32, Utf8)
   ```
   
   <details>
   <summary>Full change for `schema_coercion.rs`, `file_format.rs`, 
`metadata.rs` (clippy clean; unit tests and `parquet_rle_to_dictionary.slt` 
pass)</summary>
   
   ```diff
   -/// Make the listed string/binary fields dictionaries and demote all other
   -/// string/binary dictionaries to their value type.
   -pub(crate) fn set_dictionary_columns(
   +/// Read the listed top-level string/binary fields as `Dictionary(Int32, 
_)`.
   +pub(crate) fn promote_dictionary_columns(
        schema: &Schema,
        dictionary_columns: &HashSet<String>,
    ) -> Option<Schema> {
        let fields = coerce_fields(schema.fields(), |_, field| {
   -        let listed = dictionary_columns.contains(field.name());
   -        let new_type = match (listed, field.data_type()) {
   -            // ...
   -        };
   -        new_type.map(|ty| field_with_new_type(field, ty))
   +        if !dictionary_columns.contains(field.name()) {
   +            return None;
   +        }
   +        let value_type = field.data_type();
   +        matches!(
   +            value_type,
   +            DataType::Utf8
   +                | DataType::LargeUtf8
   +                | DataType::Binary
   +                | DataType::LargeBinary
   +        )
   +        .then(|| {
   +            let dict = DataType::Dictionary(
   +                Box::new(DataType::Int32),
   +                Box::new(value_type.clone()),
   +            );
   +            field_with_new_type(field, dict)
   +        })
   +    })?;
   +    Some(Schema::new_with_metadata(fields, schema.metadata().clone()))
   +}
   +
   +/// Read top-level string/binary dictionaries named in `plain_columns` as 
their
   +/// value type.
   +pub(crate) fn demote_dictionary_columns(
   +    schema: &Schema,
   +    plain_columns: &HashSet<String>,
   +) -> Option<Schema> {
   +    let fields = coerce_fields(schema.fields(), |_, field| match 
field.data_type() {
   +        DataType::Dictionary(_, value_type)
   +            if plain_columns.contains(field.name())
   +                && (value_type.is_string() || value_type.is_binary()) =>
   +        {
   +            Some(field_with_new_type(field, value_type.as_ref().clone()))
   +        }
   +        _ => None,
        })?;
        Some(Schema::new_with_metadata(fields, schema.metadata().clone()))
    }
   ```
   
   `file_format.rs`:
   
   ```diff
   -                let (mut dictionary_columns, plain_columns) = schemas
   +                let (dictionary_columns, plain_columns) = schemas
    // ...
   -                dictionary_columns.retain(|name| 
!plain_columns.contains(name));
   +                let mixed: HashSet<String> = dictionary_columns
   +                    .intersection(&plain_columns)
   +                    .cloned()
   +                    .collect();
                    schemas = schemas
                        .into_iter()
                        .map(|schema| {
   -                        crate::schema_coercion::set_dictionary_columns(
   -                            &schema,
   -                            &dictionary_columns,
   -                        )
   -                        .unwrap_or(schema)
   +                        
crate::schema_coercion::demote_dictionary_columns(&schema, &mixed)
   +                            .unwrap_or(schema)
                        })
   ```
   
   `metadata.rs` also drops the now-unused `Field` import.
   </details>



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to