From 9b394461a905f62dee55e540a7c03eb8f329fce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 17:06:40 +0200 Subject: [PATCH 1/2] perf(core): cut `datafusion` core compile time ~4.5x Third and largest instance of the problem from #24325 and #24326. 76% of this crate's compile time is the trait solver (`evaluate_obligation`), 75.0s of it proving `Send`/`Sync`. `#[async_trait]` gives each `async fn` a `where 'life0: 'async_trait, ..` clause. rustc only serves auto-trait obligations from its global evaluation cache when the `ParamEnv` is empty, so the `Send`/`Sync` proof for everything the returned future captures is redone per method. In this crate the captured sets include `SessionState`, `&LogicalPlan` and `ListingTableConfig`, each of which reaches a large fraction of the logical-plan type graph. Grouping the goals by the `Self` type in their `ParamEnv` shows how concentrated it is -- 14 impls, top 10 = 78%: 7.73s ParquetReadOptions 5.18s DynamicListTableFactory 7.54s JsonReadOptions 4.56s ListingTableFactory 7.07s DefaultPhysicalPlanner 4.47s TestTableFactory 6.59s CsvReadOptions 4.30s ListingTableConfig 5.68s DataFrameTableProvider 4.28s DefaultQueryPlanner 5.37s (trait default bodies) 4.15s DefaultTableFactory 4.11s SessionState 3.72s ArrowReadOptions For contrast, in the same compile 31,818 goals with an empty `ParamEnv` cost 0.19s in total -- 6us each, against ~1.3ms for the same kind of goal under `async_trait`'s bounds. Each of those methods is now the hand-written desugaring of `async fn`, which only forwards; the coroutine is built in a shim with no where-clauses so its proofs land in the global cache. The `ReadOptions` family collapses to a single proof: all five impls already delegated to the `_get_resolved_schema` default body, which now hands the coroutine to a free `infer_schema_boxed`. Bodies are moved verbatim; ones with no `.await` become plain fns wrapped in `ready(..)`. Note `DefaultPhysicalPlanner::create_initial_plan` already used exactly this shape (`-> BoxFuture<'a, _>` plus `Box::pin(async move ..)`), there for recursion rather than for compile time. Interleaved A/B of `cargo rustc -p datafusion --lib`: base: 74.4s 69.9s fix: 16.6s 15.5s A third pair ran under heavy load from a concurrent build (base 219.8s, fix 32.5s) and is excluded; its ratio was consistent. Co-Authored-By: Claude Opus 5 --- datafusion/core/src/dataframe/mod.rs | 32 ++++- .../core/src/datasource/dynamic_file.rs | 24 +++- .../src/datasource/file_format/options.rs | 111 +++++++++++++----- .../core/src/datasource/listing/table.rs | 43 +++++-- .../src/datasource/listing_table_factory.rs | 27 ++++- datafusion/core/src/datasource/provider.rs | 27 ++++- .../core/src/execution/session_state.rs | 58 ++++++++- datafusion/core/src/physical_planner.rs | 59 +++++++--- datafusion/core/src/test_util/mod.rs | 30 ++++- 9 files changed, 338 insertions(+), 73 deletions(-) diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 325ae91d27bbf..8be23f927edba 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -70,6 +70,7 @@ use datafusion_functions_aggregate::expr_fn::{ use async_trait::async_trait; use datafusion_catalog::Session; use datafusion_expr::extension_types::DFArrayFormatterFactory; +use futures::future::BoxFuture; /// Contains options that control how data is /// written out from a DataFrame @@ -2716,7 +2717,36 @@ impl TableProvider for DataFrameTableProvider { self.table_type } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } +} + +impl DataFrameTableProvider { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } + + async fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rs index 50ee96da3dff0..f670c9cc444aa 100644 --- a/datafusion/core/src/datasource/dynamic_file.rs +++ b/datafusion/core/src/datasource/dynamic_file.rs @@ -31,6 +31,7 @@ use datafusion_common::plan_datafusion_err; use datafusion_session::SessionStore; use async_trait::async_trait; +use futures::future::BoxFuture; /// [DynamicListTableFactory] is a factory that can create a [ListingTable] from the given url. #[derive(Default, Debug)] @@ -53,7 +54,28 @@ impl DynamicListTableFactory { #[async_trait] impl UrlTableFactory for DynamicListTableFactory { - async fn try_new(&self, url: &str) -> Result>> { + fn try_new<'life0, 'life1, 'async_trait>( + &'life0 self, + url: &'life1 str, + ) -> BoxFuture<'async_trait, Result>>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.try_new_boxed(url) + } +} + +impl DynamicListTableFactory { + fn try_new_boxed<'a>( + &'a self, + url: &'a str, + ) -> BoxFuture<'a, Result>>> { + Box::pin(self.try_new_inner(url)) + } + + async fn try_new_inner(&self, url: &str) -> Result>> { let Ok(table_url) = ListingTableUrl::parse(url) else { return Ok(None); }; diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 8ef780ea1e972..8137a7ae8dbe5 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -17,6 +17,7 @@ //! User facing options for the file formats readers +use std::future::ready; use std::sync::Arc; #[cfg(feature = "avro")] @@ -44,6 +45,7 @@ use datafusion_common::{ use async_trait::async_trait; use datafusion_datasource_json::file_format::JsonFormat; use datafusion_expr::SortExpr; +use futures::future::BoxFuture; /// Options that control the reading of CSV files. /// @@ -602,26 +604,46 @@ pub trait ReadOptions<'a> { } /// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function. - async fn _get_resolved_schema( + // Compile-time optimization: written as the desugaring of `async fn` so the + // coroutine is built by `infer_schema_boxed`, whose `ParamEnv` is empty. + // Under `#[async_trait]`'s `'a: 'async_trait` bounds rustc cannot cache the + // future's `Send`/`Sync` proofs globally, and re-proves `SessionState`'s + // whole type graph for every impl of this trait. + fn _get_resolved_schema<'life0, 'async_trait>( &'a self, - config: &SessionConfig, + config: &'life0 SessionConfig, state: SessionState, table_path: ListingTableUrl, schema: Option<&'a Schema>, - ) -> Result + ) -> BoxFuture<'async_trait, Result> where 'a: 'async_trait, + 'life0: 'async_trait, + Self: Sync + 'async_trait, { if let Some(s) = schema { - return Ok(Arc::new(s.to_owned())); + return Box::pin(ready(Ok(Arc::new(s.to_owned())))); } - self.to_listing_options(config, state.default_table_options()) - .infer_schema(&state, &table_path) - .await + let listing_options = + self.to_listing_options(config, state.default_table_options()); + infer_schema_boxed(listing_options, state, table_path) } } +/// Infers a schema from `table_path`, boxed for [`ReadOptions::_get_resolved_schema`]. +/// +/// Free function on purpose: it has no where-clauses, so the returned future's +/// auto-trait obligations are proved in an empty `ParamEnv` and land in rustc's +/// global evaluation cache, shared by every `ReadOptions` impl. +fn infer_schema_boxed( + listing_options: ListingOptions, + state: SessionState, + table_path: ListingTableUrl, +) -> BoxFuture<'static, Result> { + Box::pin(async move { listing_options.infer_schema(&state, &table_path).await }) +} + #[async_trait] impl ReadOptions<'_> for CsvReadOptions<'_> { fn to_listing_options( @@ -649,14 +671,19 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -696,14 +723,19 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -730,14 +762,19 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -760,14 +797,19 @@ impl ReadOptions<'_> for AvroReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -789,14 +831,19 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 56a5d5779596c..652201d05563b 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -21,6 +21,7 @@ use datafusion_catalog_listing::{ListingOptions, ListingTableConfig}; use datafusion_common::{config_datafusion_err, internal_datafusion_err}; use datafusion_session::Session; use futures::StreamExt; +use futures::future::BoxFuture; use std::collections::HashMap; /// Extension trait for [`ListingTableConfig`] that supports inferring schemas @@ -47,17 +48,39 @@ pub trait ListingTableConfigExt { #[async_trait] impl ListingTableConfigExt for ListingTableConfig { - async fn infer_options( + // Compile-time optimization; the coroutine is built by `infer_options_boxed`, + // whose `ParamEnv` is empty, so rustc caches the future's `Send`/`Sync` proofs + // globally instead of re-proving `ListingTableConfig`'s type graph here. + fn infer_options<'life0, 'async_trait>( self, - state: &dyn Session, - ) -> datafusion_common::Result { - let store = if let Some(url) = self.table_paths.first() { + state: &'life0 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + infer_options_boxed(self, state) + } + + async fn infer(self, state: &dyn Session) -> datafusion_common::Result { + self.infer_options(state).await?.infer_schema(state).await + } +} + +/// Body of [`ListingTableConfigExt::infer_options`], boxed in a function with no +/// where-clauses so its auto-trait obligations are proved in an empty `ParamEnv`. +fn infer_options_boxed<'a>( + config: ListingTableConfig, + state: &'a dyn Session, +) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(async move { + let store = if let Some(url) = config.table_paths.first() { state.runtime_env().object_store(url)? } else { - return Ok(self); + return Ok(config); }; - let file = self + let file = config .table_paths .first() .unwrap() @@ -95,12 +118,8 @@ impl ListingTableConfigExt for ListingTableConfig { let listing_options = ListingOptions::new(file_format).with_file_extension(listing_file_extension); - Ok(self.with_listing_options(listing_options)) - } - - async fn infer(self, state: &dyn Session) -> datafusion_common::Result { - self.infer_options(state).await?.infer_schema(state).await - } + Ok(config.with_listing_options(listing_options)) + }) } #[cfg(test)] diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs index 68f6743189447..116b8985b4115 100644 --- a/datafusion/core/src/datasource/listing_table_factory.rs +++ b/datafusion/core/src/datasource/listing_table_factory.rs @@ -36,6 +36,7 @@ use datafusion_expr::CreateExternalTable; use async_trait::async_trait; use datafusion_catalog::Session; +use futures::future::BoxFuture; /// A `TableProviderFactory` capable of creating new `ListingTable`s #[derive(Debug, Default)] @@ -50,7 +51,31 @@ impl ListingTableFactory { #[async_trait] impl TableProviderFactory for ListingTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl ListingTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_inner(state, cmd)) + } + + async fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rs index 9d4b67632a019..84bb338835c86 100644 --- a/datafusion/core/src/datasource/provider.rs +++ b/datafusion/core/src/datasource/provider.rs @@ -23,6 +23,7 @@ use async_trait::async_trait; use datafusion_catalog::Session; use datafusion_expr::CreateExternalTable; pub use datafusion_expr::{TableProviderFilterPushDown, TableType}; +use futures::future::BoxFuture; use crate::catalog::{TableProvider, TableProviderFactory}; use crate::datasource::listing_table_factory::ListingTableFactory; @@ -48,7 +49,31 @@ impl DefaultTableFactory { #[async_trait] impl TableProviderFactory for DefaultTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl DefaultTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_inner(state, cmd)) + } + + async fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index bfd38faacf816..99b16857bb1bc 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -85,6 +85,7 @@ use datafusion_sql::{ use async_trait::async_trait; use chrono::{DateTime, Utc}; +use futures::future::BoxFuture; use itertools::Itertools; use log::{debug, info}; use object_store::ObjectStore; @@ -291,11 +292,16 @@ impl Session for SessionState { SessionState::statistics_registry(self) } - async fn create_physical_plan( - &self, - logical_plan: &LogicalPlan, - ) -> datafusion_common::Result> { - self.create_physical_plan(logical_plan).await + fn create_physical_plan<'life0, 'life1, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan) } fn create_physical_expr( @@ -351,6 +357,22 @@ impl Session for SessionState { } } +impl SessionState { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan)) + } + + async fn create_physical_plan_inner( + &self, + logical_plan: &LogicalPlan, + ) -> datafusion_common::Result> { + self.create_physical_plan(logical_plan).await + } +} + impl SessionState { pub(crate) fn resolve_table_ref( &self, @@ -2336,7 +2358,31 @@ struct DefaultQueryPlanner {} #[async_trait] impl QueryPlanner for DefaultQueryPlanner { /// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution - async fn create_physical_plan( + fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + session_state: &'life2 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan, session_state) + } +} + +impl DefaultQueryPlanner { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + session_state: &'a dyn Session, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan, session_state)) + } + + async fn create_physical_plan_inner( &self, logical_plan: &LogicalPlan, session_state: &dyn Session, diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 3c1e7b50780a5..6e68ed9ce69b3 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -112,6 +112,7 @@ use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Sessio use async_trait::async_trait; use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper}; +use futures::future::BoxFuture; use futures::{StreamExt, TryStreamExt}; use indexmap::IndexSet; use itertools::{Itertools, multiunzip}; @@ -153,22 +154,18 @@ pub struct DefaultPhysicalPlanner { #[async_trait] impl PhysicalPlanner for DefaultPhysicalPlanner { /// Create a physical plan from a logical plan - async fn create_physical_plan( - &self, - logical_plan: &LogicalPlan, - session_state: &dyn Session, - ) -> Result> { - if let Some(plan) = self - .handle_explain_or_analyze(logical_plan, session_state) - .await? - { - return Ok(plan); - } - let plan = self - .create_initial_plan(logical_plan, session_state) - .await?; - - self.optimize_physical_plan(plan, session_state, |_, _| {}) + fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + session_state: &'life2 dyn Session, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan, session_state) } /// Create a physical expression from a logical expression @@ -193,6 +190,34 @@ impl PhysicalPlanner for DefaultPhysicalPlanner { } } +impl DefaultPhysicalPlanner { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + session_state: &'a dyn Session, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan, session_state)) + } + + async fn create_physical_plan_inner( + &self, + logical_plan: &LogicalPlan, + session_state: &dyn Session, + ) -> Result> { + if let Some(plan) = self + .handle_explain_or_analyze(logical_plan, session_state) + .await? + { + return Ok(plan); + } + let plan = self + .create_initial_plan(logical_plan, session_state) + .await?; + + self.optimize_physical_plan(plan, session_state, |_, _| {}) + } +} + #[derive(Debug)] struct ExecutionPlanChild { /// Index needed to order children of parent to ensure consistency with original @@ -329,7 +354,7 @@ impl DefaultPhysicalPlanner { &'a self, logical_plan: &'a LogicalPlan, session_state: &'a dyn Session, - ) -> futures::future::BoxFuture<'a, Result>> { + ) -> BoxFuture<'a, Result>> { Box::pin(async move { // When `enable_physical_uncorrelated_scalar_subquery` is disabled, the // `ScalarSubqueryToJoin` optimizer rule rewrites all uncorrelated diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs index d70c0d186d007..036988c8fcc76 100644 --- a/datafusion/core/src/test_util/mod.rs +++ b/datafusion/core/src/test_util/mod.rs @@ -26,6 +26,7 @@ use futures::Stream; use std::collections::HashMap; use std::fmt::Formatter; use std::fs::File; +use std::future::ready; use std::io::Write; use std::path::Path; use std::sync::Arc; @@ -53,6 +54,7 @@ use datafusion_expr::{ use std::pin::Pin; use async_trait::async_trait; +use futures::future::BoxFuture; use tempfile::TempDir; // backwards compatibility @@ -182,9 +184,33 @@ pub struct TestTableFactory {} #[async_trait] impl TableProviderFactory for TestTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + session: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(session, cmd) + } +} + +impl TestTableFactory { + fn create_boxed<'a>( + &'a self, + session: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.create_inner(session, cmd))) + } + + fn create_inner( &self, - _: &dyn Session, + _session: &dyn Session, cmd: &CreateExternalTable, ) -> Result> { let Some(location) = cmd.locations.first() else { From 1afc9ecc3689baf2729fbcf783c37e92d9bb57b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 18:25:32 +0200 Subject: [PATCH 2/2] perf(core): drop the last two sources of repeated auto-trait proving Follow-up within the same crate, found by re-profiling after the previous commit. `evaluate_obligation` was still 11.15s of core's 31.3s, and grouping those goals by the `Self` type in their `ParamEnv` pointed at exactly two things: 4.37s ListingTableConfig 2.43s JsonReadOptions 2.22s ParquetReadOptions 2.14s CsvReadOptions 0.23s (empty ParamEnv, 33,391 goals -- 7us each) 1. `ListingTableConfigExt::infer` was still an `async fn`, so it captured `self: ListingTableConfig` and paid for a walk of its type graph. It now uses the same shim as `infer_options` next to it. 2. `ReadOptions::_get_resolved_schema` still carried `Self: Sync`, which `#[async_trait]` needed when the body was a coroutine capturing `&self`. After the previous commit it is not a coroutine and does not capture `&self`, so the bound is dead weight -- and it forced every caller to prove its own type `Sync` structurally (through arrow's `DataType`/`Schema`) in a non-empty `ParamEnv`. `ArrowReadOptions` was already cheap because it has fewer fields; Csv/Json/Parquet were not. Note this relaxes a bound on a public trait method. Nothing in tree overrides `_get_resolved_schema` (all five impls only implement `get_resolved_schema`), and the underscore prefix marks it as an internal helper, but an external override written with `#[async_trait]` would generate `Self: Sync` and no longer match. `cargo rustc -p datafusion --lib` with `-Ztime-passes`: 88.9s base 18.6s after the previous commit 8.2s after this one `evaluate_obligation` goes 75.0s -> 11.15s -> **234ms** over 34,724 goals, i.e. 6.7us per goal, the same rate as goals with an empty `ParamEnv`. What remains in this crate is LLVM: 7.6s emitting objects, 4.2s in LLVM passes. Co-Authored-By: Claude Opus 5 --- .../src/datasource/file_format/options.rs | 16 +----------- .../core/src/datasource/listing/table.rs | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 8137a7ae8dbe5..4babf4a421f4e 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -604,11 +604,6 @@ pub trait ReadOptions<'a> { } /// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function. - // Compile-time optimization: written as the desugaring of `async fn` so the - // coroutine is built by `infer_schema_boxed`, whose `ParamEnv` is empty. - // Under `#[async_trait]`'s `'a: 'async_trait` bounds rustc cannot cache the - // future's `Send`/`Sync` proofs globally, and re-proves `SessionState`'s - // whole type graph for every impl of this trait. fn _get_resolved_schema<'life0, 'async_trait>( &'a self, config: &'life0 SessionConfig, @@ -619,7 +614,7 @@ pub trait ReadOptions<'a> { where 'a: 'async_trait, 'life0: 'async_trait, - Self: Sync + 'async_trait, + Self: 'async_trait, { if let Some(s) = schema { return Box::pin(ready(Ok(Arc::new(s.to_owned())))); @@ -632,10 +627,6 @@ pub trait ReadOptions<'a> { } /// Infers a schema from `table_path`, boxed for [`ReadOptions::_get_resolved_schema`]. -/// -/// Free function on purpose: it has no where-clauses, so the returned future's -/// auto-trait obligations are proved in an empty `ParamEnv` and land in rustc's -/// global evaluation cache, shared by every `ReadOptions` impl. fn infer_schema_boxed( listing_options: ListingOptions, state: SessionState, @@ -671,7 +662,6 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -723,7 +713,6 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -762,7 +751,6 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -797,7 +785,6 @@ impl ReadOptions<'_> for AvroReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -831,7 +818,6 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - // Compile-time optimization; see `ReadOptions::_get_resolved_schema`. fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 652201d05563b..7b435822f275f 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -48,9 +48,6 @@ pub trait ListingTableConfigExt { #[async_trait] impl ListingTableConfigExt for ListingTableConfig { - // Compile-time optimization; the coroutine is built by `infer_options_boxed`, - // whose `ParamEnv` is empty, so rustc caches the future's `Send`/`Sync` proofs - // globally instead of re-proving `ListingTableConfig`'s type graph here. fn infer_options<'life0, 'async_trait>( self, state: &'life0 dyn Session, @@ -62,13 +59,27 @@ impl ListingTableConfigExt for ListingTableConfig { infer_options_boxed(self, state) } - async fn infer(self, state: &dyn Session) -> datafusion_common::Result { - self.infer_options(state).await?.infer_schema(state).await + fn infer<'life0, 'async_trait>( + self, + state: &'life0 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + infer_boxed(self, state) } } -/// Body of [`ListingTableConfigExt::infer_options`], boxed in a function with no -/// where-clauses so its auto-trait obligations are proved in an empty `ParamEnv`. +/// Body of [`ListingTableConfigExt::infer`]. +fn infer_boxed<'a>( + config: ListingTableConfig, + state: &'a dyn Session, +) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(async move { config.infer_options(state).await?.infer_schema(state).await }) +} + +/// Body of [`ListingTableConfigExt::infer_options`]. fn infer_options_boxed<'a>( config: ListingTableConfig, state: &'a dyn Session,