tustvold commented on code in PR #13690:
URL: https://github.com/apache/datafusion/pull/13690#discussion_r1875038450
##########
datafusion/execution/src/runtime_env.rs:
##########
@@ -155,6 +161,79 @@ impl RuntimeEnv {
.get_store(url.as_ref())
.map_err(DataFusionError::from)
}
+
+ /// Return the current DedicatedExecutor
+ pub fn dedicated_executor(&self) -> Option<&DedicatedExecutor> {
+ self.dedicated_executor.as_ref()
+ }
+
+ /// Run an async future that will do IO operations on the IO thread pool
+ /// if there is a [`DedicatedExecutor`] registered
+ ///
+ /// If no DedicatedExecutor is registered, runs the operation on the
current
+ /// thread pool
+ ///
+ /// See [`DedicatedExecutor`] for more details
+ pub async fn spawn_io<Fut>(&self, fut: Fut) -> Fut::Output
+ where
+ Fut: Future + Send + 'static,
+ Fut::Output: Send,
+ {
+ if self.dedicated_executor().is_some() {
+ println!("Running IO on dedicated executor");
+ // TODO it is strange that the io thread is tied directly to a
thread
+ // local rather than bound to an instance
+ DedicatedExecutor::spawn_io(fut).await
+ } else {
+ // otherwise run on the current runtime
+ println!("Running IO on current runtime");
+ fut.await
+ }
+ }
+
+ /// Run an async future that will do CPU operations on the CPU task pool
+ /// if there is a [`DedicatedExecutor`] registered
+ ///
+ /// If no DedicatedExecutor is registered, runs the operation on the
current
+ /// thread pool
+ ///
+ /// See [`DedicatedExecutor`] for more details
+ pub async fn spawn_cpu<Fut>(&self, fut: Fut) -> Result<Fut::Output>
Review Comment:
Actually I think this might actually be a better way to approach this
problem, whilst CPU-bound tasks are a lot more common in DF, they're also a lot
easier to work with... I wrote the idea up here :thinking:
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
