mbutrovich commented on code in PR #26407:
URL: https://github.com/apache/datafusion/pull/26407#discussion_r4427828850
##########
datafusion/datasource-parquet/src/file_format.rs:
##########
@@ -521,22 +549,51 @@ impl FileFormat for ParquetFormat {
.await?;
let file_metadata_cache =
state.runtime_env().cache_manager.get_file_metadata_cache();
- let metadata = DFParquetMetadata::new(store, object)
- .with_metadata_size_hint(self.metadata_size_hint())
- .with_decryption_properties(file_decryption_properties)
- .with_file_metadata_cache(Some(file_metadata_cache))
- .fetch_metadata()
- .await?;
- let statistics = DFParquetMetadata::statistics_from_parquet_metadata(
- &metadata,
- &table_schema,
- )?;
- let ordering =
- crate::metadata::ordering_from_parquet_metadata(&metadata,
&table_schema)?;
- Ok(
- datafusion_datasource::file_format::FileMeta::new(statistics)
- .with_ordering(ordering),
- )
+ let metadata_size_hint = self.metadata_size_hint();
+
+ // Decoding the footer and converting its statistics is CPU bound and
+ // grows with the number of columns. Callers such as `ListingTable` run
+ // this for many files concurrently from a single planning task, so run
+ // it in its own task to spread that work over the runtime's threads
+ // instead of decoding every file on the thread polling the planner.
+ // As in `spawn_buffered`, only spawn from a multi-thread tokio
runtime.
+ match tokio::runtime::Handle::try_current() {
Review Comment:
The suggested change looks right to me. @jayzhan211 or @alamb are you able
to apply it directly if @cetra3 doesn't circle back to the PR?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
