diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 325ae91d27bbf..8be23f927edba 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -70,6 +70,7 @@ use datafusion_functions_aggregate::expr_fn::{ use async_trait::async_trait; use datafusion_catalog::Session; use datafusion_expr::extension_types::DFArrayFormatterFactory; +use futures::future::BoxFuture; /// Contains options that control how data is /// written out from a DataFrame @@ -2716,7 +2717,36 @@ impl TableProvider for DataFrameTableProvider { self.table_type } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } +} + +impl DataFrameTableProvider { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } + + async fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rs index 50ee96da3dff0..f670c9cc444aa 100644 --- a/datafusion/core/src/datasource/dynamic_file.rs +++ b/datafusion/core/src/datasource/dynamic_file.rs @@ -31,6 +31,7 @@ use datafusion_common::plan_datafusion_err; use datafusion_session::SessionStore; use async_trait::async_trait; +use futures::future::BoxFuture; /// [DynamicListTableFactory] is a factory that can create a [ListingTable] from the given url. #[derive(Default, Debug)] @@ -53,7 +54,28 @@ impl DynamicListTableFactory { #[async_trait] impl UrlTableFactory for DynamicListTableFactory { - async fn try_new(&self, url: &str) -> Result>> { + fn try_new<'life0, 'life1, 'async_trait>( + &'life0 self, + url: &'life1 str, + ) -> BoxFuture<'async_trait, Result>>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.try_new_boxed(url) + } +} + +impl DynamicListTableFactory { + fn try_new_boxed<'a>( + &'a self, + url: &'a str, + ) -> BoxFuture<'a, Result>>> { + Box::pin(self.try_new_inner(url)) + } + + async fn try_new_inner(&self, url: &str) -> Result>> { let Ok(table_url) = ListingTableUrl::parse(url) else { return Ok(None); }; diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 8ef780ea1e972..4babf4a421f4e 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -17,6 +17,7 @@ //! User facing options for the file formats readers +use std::future::ready; use std::sync::Arc; #[cfg(feature = "avro")] @@ -44,6 +45,7 @@ use datafusion_common::{ use async_trait::async_trait; use datafusion_datasource_json::file_format::JsonFormat; use datafusion_expr::SortExpr; +use futures::future::BoxFuture; /// Options that control the reading of CSV files. /// @@ -602,26 +604,37 @@ pub trait ReadOptions<'a> { } /// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function. - async fn _get_resolved_schema( + fn _get_resolved_schema<'life0, 'async_trait>( &'a self, - config: &SessionConfig, + config: &'life0 SessionConfig, state: SessionState, table_path: ListingTableUrl, schema: Option<&'a Schema>, - ) -> Result + ) -> BoxFuture<'async_trait, Result> where 'a: 'async_trait, + 'life0: 'async_trait, + Self: 'async_trait, { if let Some(s) = schema { - return Ok(Arc::new(s.to_owned())); + return Box::pin(ready(Ok(Arc::new(s.to_owned())))); } - self.to_listing_options(config, state.default_table_options()) - .infer_schema(&state, &table_path) - .await + let listing_options = + self.to_listing_options(config, state.default_table_options()); + infer_schema_boxed(listing_options, state, table_path) } } +/// Infers a schema from `table_path`, boxed for [`ReadOptions::_get_resolved_schema`]. +fn infer_schema_boxed( + listing_options: ListingOptions, + state: SessionState, + table_path: ListingTableUrl, +) -> BoxFuture<'static, Result> { + Box::pin(async move { listing_options.infer_schema(&state, &table_path).await }) +} + #[async_trait] impl ReadOptions<'_> for CsvReadOptions<'_> { fn to_listing_options( @@ -649,14 +662,18 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -696,14 +713,18 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -730,14 +751,18 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -760,14 +785,18 @@ impl ReadOptions<'_> for AvroReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -789,14 +818,18 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 56a5d5779596c..7b435822f275f 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -21,6 +21,7 @@ use datafusion_catalog_listing::{ListingOptions, ListingTableConfig}; use datafusion_common::{config_datafusion_err, internal_datafusion_err}; use datafusion_session::Session; use futures::StreamExt; +use futures::future::BoxFuture; use std::collections::HashMap; /// Extension trait for [`ListingTableConfig`] that supports inferring schemas @@ -47,17 +48,50 @@ pub trait ListingTableConfigExt { #[async_trait] impl ListingTableConfigExt for ListingTableConfig { - async fn infer_options( + fn infer_options<'life0, 'async_trait>( self, - state: &dyn Session, - ) -> datafusion_common::Result { - let store = if let Some(url) = self.table_paths.first() { + state: &'life0 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + infer_options_boxed(self, state) + } + + fn infer<'life0, 'async_trait>( + self, + state: &'life0 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + infer_boxed(self, state) + } +} + +/// Body of [`ListingTableConfigExt::infer`]. +fn infer_boxed<'a>( + config: ListingTableConfig, + state: &'a dyn Session, +) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(async move { config.infer_options(state).await?.infer_schema(state).await }) +} + +/// Body of [`ListingTableConfigExt::infer_options`]. +fn infer_options_boxed<'a>( + config: ListingTableConfig, + state: &'a dyn Session, +) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(async move { + let store = if let Some(url) = config.table_paths.first() { state.runtime_env().object_store(url)? } else { - return Ok(self); + return Ok(config); }; - let file = self + let file = config .table_paths .first() .unwrap() @@ -95,12 +129,8 @@ impl ListingTableConfigExt for ListingTableConfig { let listing_options = ListingOptions::new(file_format).with_file_extension(listing_file_extension); - Ok(self.with_listing_options(listing_options)) - } - - async fn infer(self, state: &dyn Session) -> datafusion_common::Result { - self.infer_options(state).await?.infer_schema(state).await - } + Ok(config.with_listing_options(listing_options)) + }) } #[cfg(test)] diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs index 68f6743189447..116b8985b4115 100644 --- a/datafusion/core/src/datasource/listing_table_factory.rs +++ b/datafusion/core/src/datasource/listing_table_factory.rs @@ -36,6 +36,7 @@ use datafusion_expr::CreateExternalTable; use async_trait::async_trait; use datafusion_catalog::Session; +use futures::future::BoxFuture; /// A `TableProviderFactory` capable of creating new `ListingTable`s #[derive(Debug, Default)] @@ -50,7 +51,31 @@ impl ListingTableFactory { #[async_trait] impl TableProviderFactory for ListingTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl ListingTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_inner(state, cmd)) + } + + async fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rs index 9d4b67632a019..84bb338835c86 100644 --- a/datafusion/core/src/datasource/provider.rs +++ b/datafusion/core/src/datasource/provider.rs @@ -23,6 +23,7 @@ use async_trait::async_trait; use datafusion_catalog::Session; use datafusion_expr::CreateExternalTable; pub use datafusion_expr::{TableProviderFilterPushDown, TableType}; +use futures::future::BoxFuture; use crate::catalog::{TableProvider, TableProviderFactory}; use crate::datasource::listing_table_factory::ListingTableFactory; @@ -48,7 +49,31 @@ impl DefaultTableFactory { #[async_trait] impl TableProviderFactory for DefaultTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl DefaultTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_inner(state, cmd)) + } + + async fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index bfd38faacf816..99b16857bb1bc 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -85,6 +85,7 @@ use datafusion_sql::{ use async_trait::async_trait; use chrono::{DateTime, Utc}; +use futures::future::BoxFuture; use itertools::Itertools; use log::{debug, info}; use object_store::ObjectStore; @@ -291,11 +292,16 @@ impl Session for SessionState { SessionState::statistics_registry(self) } - async fn create_physical_plan( - &self, - logical_plan: &LogicalPlan, - ) -> datafusion_common::Result> { - self.create_physical_plan(logical_plan).await + fn create_physical_plan<'life0, 'life1, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan) } fn create_physical_expr( @@ -351,6 +357,22 @@ impl Session for SessionState { } } +impl SessionState { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan)) + } + + async fn create_physical_plan_inner( + &self, + logical_plan: &LogicalPlan, + ) -> datafusion_common::Result> { + self.create_physical_plan(logical_plan).await + } +} + impl SessionState { pub(crate) fn resolve_table_ref( &self, @@ -2336,7 +2358,31 @@ struct DefaultQueryPlanner {} #[async_trait] impl QueryPlanner for DefaultQueryPlanner { /// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution - async fn create_physical_plan( + fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + session_state: &'life2 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan, session_state) + } +} + +impl DefaultQueryPlanner { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + session_state: &'a dyn Session, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan, session_state)) + } + + async fn create_physical_plan_inner( &self, logical_plan: &LogicalPlan, session_state: &dyn Session, diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 3c1e7b50780a5..6e68ed9ce69b3 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -112,6 +112,7 @@ use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Sessio use async_trait::async_trait; use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper}; +use futures::future::BoxFuture; use futures::{StreamExt, TryStreamExt}; use indexmap::IndexSet; use itertools::{Itertools, multiunzip}; @@ -153,22 +154,18 @@ pub struct DefaultPhysicalPlanner { #[async_trait] impl PhysicalPlanner for DefaultPhysicalPlanner { /// Create a physical plan from a logical plan - async fn create_physical_plan( - &self, - logical_plan: &LogicalPlan, - session_state: &dyn Session, - ) -> Result> { - if let Some(plan) = self - .handle_explain_or_analyze(logical_plan, session_state) - .await? - { - return Ok(plan); - } - let plan = self - .create_initial_plan(logical_plan, session_state) - .await?; - - self.optimize_physical_plan(plan, session_state, |_, _| {}) + fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + session_state: &'life2 dyn Session, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan, session_state) } /// Create a physical expression from a logical expression @@ -193,6 +190,34 @@ impl PhysicalPlanner for DefaultPhysicalPlanner { } } +impl DefaultPhysicalPlanner { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + session_state: &'a dyn Session, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan, session_state)) + } + + async fn create_physical_plan_inner( + &self, + logical_plan: &LogicalPlan, + session_state: &dyn Session, + ) -> Result> { + if let Some(plan) = self + .handle_explain_or_analyze(logical_plan, session_state) + .await? + { + return Ok(plan); + } + let plan = self + .create_initial_plan(logical_plan, session_state) + .await?; + + self.optimize_physical_plan(plan, session_state, |_, _| {}) + } +} + #[derive(Debug)] struct ExecutionPlanChild { /// Index needed to order children of parent to ensure consistency with original @@ -329,7 +354,7 @@ impl DefaultPhysicalPlanner { &'a self, logical_plan: &'a LogicalPlan, session_state: &'a dyn Session, - ) -> futures::future::BoxFuture<'a, Result>> { + ) -> BoxFuture<'a, Result>> { Box::pin(async move { // When `enable_physical_uncorrelated_scalar_subquery` is disabled, the // `ScalarSubqueryToJoin` optimizer rule rewrites all uncorrelated diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs index d70c0d186d007..036988c8fcc76 100644 --- a/datafusion/core/src/test_util/mod.rs +++ b/datafusion/core/src/test_util/mod.rs @@ -26,6 +26,7 @@ use futures::Stream; use std::collections::HashMap; use std::fmt::Formatter; use std::fs::File; +use std::future::ready; use std::io::Write; use std::path::Path; use std::sync::Arc; @@ -53,6 +54,7 @@ use datafusion_expr::{ use std::pin::Pin; use async_trait::async_trait; +use futures::future::BoxFuture; use tempfile::TempDir; // backwards compatibility @@ -182,9 +184,33 @@ pub struct TestTableFactory {} #[async_trait] impl TableProviderFactory for TestTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + session: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(session, cmd) + } +} + +impl TestTableFactory { + fn create_boxed<'a>( + &'a self, + session: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.create_inner(session, cmd))) + } + + fn create_inner( &self, - _: &dyn Session, + _session: &dyn Session, cmd: &CreateExternalTable, ) -> Result> { let Some(location) = cmd.locations.first() else {