alamb commented on code in PR #25640:
URL: https://github.com/apache/datafusion/pull/25640#discussion_r4157860517
##########
datafusion-cli/src/functions.rs:
##########
@@ -463,6 +467,184 @@ impl TableFunctionImpl for ParquetMetadataFunc {
}
}
+/// PARQUET_FILE_METADATA table function
+#[derive(Debug)]
+pub struct ParquetFileMetadataFunc {}
+
+impl TableFunctionImpl for ParquetFileMetadataFunc {
+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn
TableProvider>> {
+ let filename = parquet_file_path("parquet_file_metadata",
args.exprs())?;
+
+ let mut reader = ParquetMetaDataReader::new();
+ reader.try_parse(&File::open(filename)?)?;
+ let footer_length = reader.metadata_size().map(|size| size as i64);
+ let metadata = reader.finish()?;
+ let file_metadata = metadata.file_metadata();
+
+ let key_values = file_metadata.key_value_metadata();
+ let mut key_value_metadata =
+ MapBuilder::new(None, StringBuilder::new(), StringBuilder::new());
+ for KeyValue { key, value } in key_values.into_iter().flatten() {
+ key_value_metadata.keys().append_value(key);
+ key_value_metadata.values().append_option(value.as_ref());
+ }
+ key_value_metadata.append(key_values.is_some())?;
+ let key_value_metadata = key_value_metadata.finish();
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("filename", DataType::Utf8, true),
+ Field::new("created_by", DataType::Utf8, true),
+ Field::new("version", DataType::Int64, true),
+ Field::new("num_rows", DataType::Int64, true),
+ Field::new("num_row_groups", DataType::Int64, true),
+ Field::new(
+ "key_value_metadata",
+ key_value_metadata.data_type().clone(),
+ true,
+ ),
+ Field::new("footer_length", DataType::Int64, true),
+ ]));
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(StringArray::from(vec![filename])),
+ Arc::new(StringArray::from(vec![file_metadata.created_by()])),
+
Arc::new(Int64Array::from(vec![i64::from(file_metadata.version())])),
+ Arc::new(Int64Array::from(vec![file_metadata.num_rows()])),
+ Arc::new(Int64Array::from(vec![metadata.num_row_groups() as
i64])),
+ Arc::new(key_value_metadata),
+ Arc::new(Int64Array::from(vec![footer_length])),
+ ],
+ )?;
+
+ Ok(Arc::new(ParquetMetadataTable { schema, batch }))
+ }
+}
+
+/// Formats the min and max of page `idx` in `index` the same way as
+/// [`convert_parquet_statistics`]
+fn convert_page_min_max(
+ index: &ColumnIndexMetaData,
+ idx: usize,
+ converted_type: ConvertedType,
+) -> (Option<String>, Option<String>) {
+ fn primitive<T: ToString>(
+ index: &PrimitiveColumnIndex<T>,
+ idx: usize,
+ ) -> (Option<String>, Option<String>) {
+ (
+ index.min_value(idx).map(T::to_string),
+ index.max_value(idx).map(T::to_string),
+ )
+ }
+ let bytes = |val: &[u8]| match std::str::from_utf8(val) {
+ Ok(s) if converted_type == ConvertedType::UTF8 => s.to_string(),
+ _ => format!("{val:?}"),
+ };
+
+ match index {
Review Comment:
As a follow on, we could also potentially use this:
https://docs.rs/parquet/latest/parquet/arrow/arrow_reader/statistics/struct.StatisticsConverter.html#method.row_group_mins
To convert the min/max values and then call the arrow cast kernel to turn
them into strings
There is similar code for data page mins here:
https://docs.rs/parquet/latest/parquet/arrow/arrow_reader/statistics/struct.StatisticsConverter.html#method.data_page_mins
That would handle things like min/max dates better (I think this is just
going to show the raw values rather than formatted as a date)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
