diff --git a/ballista/rust/core/src/serde/physical_plan/mod.rs b/ballista/rust/core/src/serde/physical_plan/mod.rs index 0361df6776077..ba2fd3cefa316 100644 --- a/ballista/rust/core/src/serde/physical_plan/mod.rs +++ b/ballista/rust/core/src/serde/physical_plan/mod.rs @@ -53,7 +53,7 @@ use datafusion::physical_plan::hash_join::{HashJoinExec, PartitionMode}; use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; -use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::sorts::sort2::SortExec2; use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::windows::{create_window_expr, WindowAggExec}; use datafusion::physical_plan::{ @@ -522,7 +522,7 @@ impl AsExecutionPlan for PhysicalPlanNode { } }) .collect::, _>>()?; - Ok(Arc::new(SortExec::try_new(exprs, input)?)) + Ok(Arc::new(SortExec2::try_new(exprs, input)?)) } PhysicalPlanType::Unresolved(unresolved_shuffle) => { let schema = Arc::new(convert_required!(unresolved_shuffle.schema)?); @@ -849,7 +849,7 @@ impl AsExecutionPlan for PhysicalPlanNode { }, ))), }) - } else if let Some(exec) = plan.downcast_ref::() { + } else if let Some(exec) = plan.downcast_ref::() { let input = protobuf::PhysicalPlanNode::try_from_physical_plan( exec.input().to_owned(), extension_codec, @@ -1032,7 +1032,7 @@ mod roundtrip_tests { hash_aggregate::{AggregateMode, HashAggregateExec}, hash_join::{HashJoinExec, PartitionMode}, limit::{GlobalLimitExec, LocalLimitExec}, - sorts::sort::SortExec, + sorts::sort2::SortExec2, AggregateExpr, ExecutionPlan, Partitioning, PhysicalExpr, Statistics, }, prelude::SessionContext, @@ -1193,7 +1193,7 @@ mod roundtrip_tests { }, }, ]; - roundtrip_test(Arc::new(SortExec::try_new( + roundtrip_test(Arc::new(SortExec2::try_new( sort_exprs, Arc::new(EmptyExec::new(false, schema)), )?)) diff --git a/ballista/rust/core/src/utils.rs b/ballista/rust/core/src/utils.rs index 6670ab5cedd83..1e66728f4b37c 100644 --- a/ballista/rust/core/src/utils.rs +++ b/ballista/rust/core/src/utils.rs @@ -53,7 +53,7 @@ use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::hash_aggregate::HashAggregateExec; use datafusion::physical_plan::hash_join::HashJoinExec; use datafusion::physical_plan::projection::ProjectionExec; -use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::sorts::sort2::SortExec2; use datafusion::physical_plan::{metrics, ExecutionPlan, RecordBatchStream}; use futures::{Stream, StreamExt}; @@ -153,8 +153,8 @@ fn build_exec_plan_diagram( ) -> Result { let operator_str = if plan.as_any().downcast_ref::().is_some() { "HashAggregateExec" - } else if plan.as_any().downcast_ref::().is_some() { - "SortExec" + } else if plan.as_any().downcast_ref::().is_some() { + "SortExec2" } else if plan.as_any().downcast_ref::().is_some() { "ProjectionExec" } else if plan.as_any().downcast_ref::().is_some() { diff --git a/ballista/rust/scheduler/src/planner.rs b/ballista/rust/scheduler/src/planner.rs index d7ee22bbdf353..c3d884b5acc11 100644 --- a/ballista/rust/scheduler/src/planner.rs +++ b/ballista/rust/scheduler/src/planner.rs @@ -274,7 +274,7 @@ mod test { use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; use datafusion::physical_plan::hash_aggregate::{AggregateMode, HashAggregateExec}; use datafusion::physical_plan::hash_join::HashJoinExec; - use datafusion::physical_plan::sorts::sort::SortExec; + use datafusion::physical_plan::sorts::sort2::SortExec2; use datafusion::physical_plan::{ coalesce_partitions::CoalescePartitionsExec, projection::ProjectionExec, }; @@ -361,7 +361,7 @@ mod test { // verify stage 2 let stage2 = stages[2].children()[0].clone(); - let sort = downcast_exec!(stage2, SortExec); + let sort = downcast_exec!(stage2, SortExec2); let coalesce_partitions = sort.children()[0].clone(); let coalesce_partitions = downcast_exec!(coalesce_partitions, CoalescePartitionsExec); diff --git a/benchmarks/queries/q1.sql b/benchmarks/queries/q1.sql index a0fcf159e2094..9af11d3c99faa 100644 --- a/benchmarks/queries/q1.sql +++ b/benchmarks/queries/q1.sql @@ -1,21 +1,12 @@ select l_returnflag, l_linestatus, - sum(l_quantity) as sum_qty, - sum(l_extendedprice) as sum_base_price, - sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, - sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, - avg(l_quantity) as avg_qty, - avg(l_extendedprice) as avg_price, - avg(l_discount) as avg_disc, - count(*) as count_order + l_quantity, + l_extendedprice, + l_discount, + l_tax from lineitem -where - l_shipdate <= date '1998-09-02' -group by - l_returnflag, - l_linestatus order by - l_returnflag, - l_linestatus; \ No newline at end of file + l_extendedprice, + l_discount; \ No newline at end of file diff --git a/datafusion/core/src/physical_optimizer/repartition.rs b/datafusion/core/src/physical_optimizer/repartition.rs index d98fa4162fa27..1656a51ec828b 100644 --- a/datafusion/core/src/physical_optimizer/repartition.rs +++ b/datafusion/core/src/physical_optimizer/repartition.rs @@ -245,7 +245,7 @@ mod tests { use crate::physical_plan::hash_aggregate::{AggregateMode, HashAggregateExec}; use crate::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use crate::physical_plan::projection::ProjectionExec; - use crate::physical_plan::sorts::sort::SortExec; + use crate::physical_plan::sorts::sort2::SortExec2; use crate::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use crate::physical_plan::union::UnionExec; use crate::physical_plan::{displayable, Statistics}; @@ -290,7 +290,7 @@ mod tests { expr: col("c1", &schema()).unwrap(), options: SortOptions::default(), }]; - Arc::new(SortExec::try_new(sort_exprs, input).unwrap()) + Arc::new(SortExec2::try_new(sort_exprs, input).unwrap()) } fn projection_exec(input: Arc) -> Arc { @@ -413,7 +413,7 @@ mod tests { "GlobalLimitExec: limit=100", "LocalLimitExec: limit=100", // data is sorted so can't repartition here - "SortExec: [c1@0 ASC]", + "SortExec2: [c1@0 ASC]", "ParquetExec: limit=None, partitions=[x], projection=[c1]", ]; @@ -431,7 +431,7 @@ mod tests { "FilterExec: c1@0", // data is sorted so can't repartition here even though // filter would benefit from parallelism, the answers might be wrong - "SortExec: [c1@0 ASC]", + "SortExec2: [c1@0 ASC]", "ParquetExec: limit=None, partitions=[x], projection=[c1]", ]; @@ -519,7 +519,7 @@ mod tests { let expected = &[ "SortPreservingMergeExec: [c1@0 ASC]", // Expect repartition on the input to the sort (as it can benefit from additional parallelism) - "SortExec: [c1@0 ASC]", + "SortExec2: [c1@0 ASC]", "ProjectionExec: expr=[c1@0 as c1]", "RepartitionExec: partitioning=RoundRobinBatch(10)", "ParquetExec: limit=None, partitions=[x], projection=[c1]", @@ -536,7 +536,7 @@ mod tests { let expected = &[ "SortPreservingMergeExec: [c1@0 ASC]", // Expect repartition on the input to the sort (as it can benefit from additional parallelism) - "SortExec: [c1@0 ASC]", + "SortExec2: [c1@0 ASC]", "FilterExec: c1@0", "RepartitionExec: partitioning=RoundRobinBatch(10)", "ParquetExec: limit=None, partitions=[x], projection=[c1]", @@ -555,7 +555,7 @@ mod tests { let expected = &[ "SortPreservingMergeExec: [c1@0 ASC]", // Expect repartition on the input to the sort (as it can benefit from additional parallelism) - "SortExec: [c1@0 ASC]", + "SortExec2: [c1@0 ASC]", "ProjectionExec: expr=[c1@0 as c1]", "FilterExec: c1@0", // repartition is lowest down diff --git a/datafusion/core/src/physical_plan/planner.rs b/datafusion/core/src/physical_plan/planner.rs index 7504dd4385e43..1c08b17014f27 100644 --- a/datafusion/core/src/physical_plan/planner.rs +++ b/datafusion/core/src/physical_plan/planner.rs @@ -45,7 +45,7 @@ use crate::physical_plan::hash_join::HashJoinExec; use crate::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use crate::physical_plan::projection::ProjectionExec; use crate::physical_plan::repartition::RepartitionExec; -use crate::physical_plan::sorts::sort::SortExec; +use crate::physical_plan::sorts::sort2::SortExec2; use crate::physical_plan::udf; use crate::physical_plan::windows::WindowAggExec; use crate::physical_plan::{join_utils, Partitioning}; @@ -457,9 +457,9 @@ impl DefaultPhysicalPlanner { }) .collect::>>()?; Arc::new(if can_repartition { - SortExec::new_with_partitioning(sort_keys, input_exec, true) + SortExec2::new_with_partitioning(sort_keys, input_exec, true) } else { - SortExec::try_new(sort_keys, input_exec)? + SortExec2::try_new(sort_keys, input_exec)? }) }; @@ -704,7 +704,7 @@ impl DefaultPhysicalPlanner { )), }) .collect::>>()?; - Ok(Arc::new(SortExec::try_new(sort_expr, physical_input)?) ) + Ok(Arc::new(SortExec2::try_new(sort_expr, physical_input)?) ) } LogicalPlan::Join(Join { left, diff --git a/datafusion/core/src/physical_plan/sorts/mod.rs b/datafusion/core/src/physical_plan/sorts/mod.rs index 818546f316fc4..6d5f648855027 100644 --- a/datafusion/core/src/physical_plan/sorts/mod.rs +++ b/datafusion/core/src/physical_plan/sorts/mod.rs @@ -37,6 +37,7 @@ use std::sync::Arc; use std::task::{Context, Poll}; pub mod sort; +pub mod sort2; pub mod sort_preserving_merge; /// A `SortKeyCursor` is created from a `RecordBatch`, and a set of @@ -128,6 +129,14 @@ impl SortKeyCursor { ))); } + if self.is_finished() && other.is_finished() { + return Ok(Ordering::Equal); + } else if self.is_finished() { + return Ok(Ordering::Greater); + } else if other.is_finished() { + return Ok(Ordering::Less); + } + let zipped: Vec<((&ArrayRef, &ArrayRef), &SortOptions)> = self .sort_columns .iter() diff --git a/datafusion/core/src/physical_plan/sorts/sort2.rs b/datafusion/core/src/physical_plan/sorts/sort2.rs new file mode 100644 index 0000000000000..a9819de01870a --- /dev/null +++ b/datafusion/core/src/physical_plan/sorts/sort2.rs @@ -0,0 +1,981 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Sort that deals with an arbitrary size of the input. +//! It will do in-memory sorting if it has enough memory budget +//! but spills to disk if needed. + +use crate::error::{DataFusionError, Result}; +use crate::execution::context::TaskContext; +use crate::execution::memory_manager::{ + human_readable_size, ConsumerType, MemoryConsumer, MemoryConsumerId, MemoryManager, +}; +use crate::execution::runtime_env::RuntimeEnv; +use crate::physical_plan::common::{batch_byte_size, IPCWriter, SizedRecordBatchStream}; +use crate::physical_plan::expressions::PhysicalSortExpr; +use crate::physical_plan::metrics::{ + BaselineMetrics, CompositeMetricsSet, MemTrackingMetrics, MetricsSet, +}; +use crate::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeStream; +use crate::physical_plan::sorts::SortedStream; +use crate::physical_plan::stream::RecordBatchReceiverStream; +use crate::physical_plan::{ + DisplayFormatType, Distribution, EmptyRecordBatchStream, ExecutionPlan, Partitioning, + SendableRecordBatchStream, Statistics, +}; +use crate::prelude::SessionConfig; +use arrow::array::ArrayRef; +pub use arrow::compute::SortOptions; +use arrow::compute::{lexsort_to_indices, take, SortColumn, TakeOptions}; +use arrow::datatypes::SchemaRef; +use arrow::error::Result as ArrowResult; +use arrow::ipc::reader::FileReader; +use arrow::record_batch::RecordBatch; +use async_trait::async_trait; +use futures::lock::Mutex; +use futures::StreamExt; +use log::{debug, error}; +use std::any::Any; +use std::fmt; +use std::fmt::{Debug, Formatter}; +use std::fs::File; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tempfile::NamedTempFile; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::task; + +/// Sort arbitrary size of data to get a total order (may spill several times during sorting based on free memory available). +/// +/// The basic architecture of the algorithm: +/// 1. get a non-empty new batch from input +/// 2. check with the memory manager if we could buffer the batch in memory +/// 2.1 if memory sufficient, then buffer batch in memory, go to 1. +/// 2.2 if the memory threshold is reached, sort all buffered batches and spill to file. +/// buffer the batch in memory, go to 1. +/// 3. when input is exhausted, merge all in memory batches and spills to get a total order. +struct ExternalSorter2 { + id: MemoryConsumerId, + schema: SchemaRef, + in_mem_batches: Mutex>, + spills: Mutex>, + /// Sort expressions + expr: Vec, + session_config: Arc, + runtime: Arc, + metrics_set: CompositeMetricsSet, + metrics: BaselineMetrics, +} + +impl ExternalSorter2 { + pub fn new( + partition_id: usize, + schema: SchemaRef, + expr: Vec, + metrics_set: CompositeMetricsSet, + session_config: Arc, + runtime: Arc, + ) -> Self { + let metrics = metrics_set.new_intermediate_baseline(partition_id); + Self { + id: MemoryConsumerId::new(partition_id), + schema, + in_mem_batches: Mutex::new(vec![]), + spills: Mutex::new(vec![]), + expr, + session_config, + runtime, + metrics_set, + metrics, + } + } + + async fn insert_batch( + &self, + input: RecordBatch, + tracking_metrics: &MemTrackingMetrics, + ) -> Result<()> { + if input.num_rows() > 0 { + let size = batch_byte_size(&input); + self.try_grow(size).await?; + self.metrics.mem_used().add(size); + let mut in_mem_batches = self.in_mem_batches.lock().await; + // NB timer records time taken on drop, so there are no + // calls to `timer.done()` below. + let _timer = tracking_metrics.elapsed_compute().timer(); + let partial = sort_batch(input, self.schema.clone(), &self.expr)?; + in_mem_batches.push(partial); + } + Ok(()) + } + + async fn spilled_before(&self) -> bool { + let spills = self.spills.lock().await; + !spills.is_empty() + } + + /// MergeSort in mem batches as well as spills into total order with `SortPreservingMergeStream`. + async fn sort(&self) -> Result { + let partition = self.partition_id(); + let batch_size = self.session_config.batch_size; + let mut in_mem_batches = self.in_mem_batches.lock().await; + + if self.spilled_before().await { + let mut streams: Vec = vec![]; + if in_mem_batches.len() > 0 { + let in_mem_stream = in_mem_partial_sort( + &mut *in_mem_batches, + self.schema.clone(), + &self.expr, + batch_size, + &self.metrics_set, + partition, + self.runtime.clone(), + false, + )?; + let prev_used = self.metrics.mem_used().set(0); + streams.push(SortedStream::new(in_mem_stream, prev_used)); + } + + let mut spills = self.spills.lock().await; + + for spill in spills.drain(..) { + let stream = read_spill_as_stream(spill, self.schema.clone())?; + streams.push(SortedStream::new(stream, 0)); + } + let tracking_metrics = self + .metrics_set + .new_final_tracking(partition, self.runtime.clone()); + Ok(Box::pin(SortPreservingMergeStream::new_from_streams( + streams, + self.schema.clone(), + &self.expr, + tracking_metrics, + self.session_config.batch_size, + ))) + } else if in_mem_batches.len() > 0 { + let result = in_mem_partial_sort( + &mut *in_mem_batches, + self.schema.clone(), + &self.expr, + batch_size, + &self.metrics_set, + partition, + self.runtime.clone(), + true, + ); + // Report to the memory manager we are no longer using memory + self.metrics.mem_used().set(0); + result + } else { + Ok(Box::pin(EmptyRecordBatchStream::new(self.schema.clone()))) + } + } + + fn used(&self) -> usize { + self.metrics.mem_used().value() + } + + fn spilled_bytes(&self) -> usize { + self.metrics.spilled_bytes().value() + } + + fn spill_count(&self) -> usize { + self.metrics.spill_count().value() + } +} + +impl Debug for ExternalSorter2 { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.debug_struct("ExternalSorter2") + .field("id", &self.id()) + .field("memory_used", &self.used()) + .field("spilled_bytes", &self.spilled_bytes()) + .field("spill_count", &self.spill_count()) + .finish() + } +} + +impl Drop for ExternalSorter2 { + fn drop(&mut self) { + self.runtime.drop_consumer(self.id(), self.used()); + } +} + +#[async_trait] +impl MemoryConsumer for ExternalSorter2 { + fn name(&self) -> String { + "ExternalSorter2".to_owned() + } + + fn id(&self) -> &MemoryConsumerId { + &self.id + } + + fn memory_manager(&self) -> Arc { + self.runtime.memory_manager.clone() + } + + fn type_(&self) -> &ConsumerType { + &ConsumerType::Requesting + } + + async fn spill(&self) -> Result { + debug!( + "{}[{}] spilling sort data of {} to disk while inserting ({} time(s) so far)", + self.name(), + self.id(), + self.used(), + self.spill_count() + ); + + let partition = self.partition_id(); + let mut in_mem_batches = self.in_mem_batches.lock().await; + // we could always get a chance to free some memory as long as we are holding some + if in_mem_batches.len() == 0 { + return Ok(0); + } + + let spillfile = self.runtime.disk_manager.create_tmp_file()?; + let stream = in_mem_partial_sort( + &mut *in_mem_batches, + self.schema.clone(), + &*self.expr, + self.session_config.batch_size, + &self.metrics_set, + partition, + self.runtime.clone(), + false, + ); + + spill_partial_sorted_stream(&mut stream?, spillfile.path(), self.schema.clone()) + .await?; + let mut spills = self.spills.lock().await; + let used = self.metrics.mem_used().set(0); + self.metrics.record_spill(used); + spills.push(spillfile); + Ok(used) + } + + fn mem_used(&self) -> usize { + self.metrics.mem_used().value() + } +} + +/// consume the non-empty `sorted_bathes` and do in_mem_sort +#[allow(clippy::too_many_arguments)] +fn in_mem_partial_sort( + buffered_batches: &mut Vec, + schema: SchemaRef, + expressions: &[PhysicalSortExpr], + batch_size: usize, + metrics_set: &CompositeMetricsSet, + partition: usize, + runtime: Arc, + final_metric: bool, +) -> Result { + let get_metrics = |final_metric| -> MemTrackingMetrics { + if final_metric { + metrics_set.new_final_tracking(partition, runtime.clone()) + } else { + metrics_set.new_intermediate_tracking(partition, runtime.clone()) + } + }; + + assert_ne!(buffered_batches.len(), 0); + if buffered_batches.len() == 1 { + let tracking_metrics = get_metrics(final_metric); + let result = buffered_batches.pop(); + Ok(Box::pin(SizedRecordBatchStream::new( + schema, + vec![Arc::new(result.unwrap())], + tracking_metrics, + ))) + } else { + let sorted_streams = buffered_batches + .drain(..) + .map(|batch| { + let tracking_metrics = get_metrics(false); + SortedStream::new( + Box::pin(SizedRecordBatchStream::new( + schema.clone(), + vec![Arc::new(batch)], + tracking_metrics, + )), + 0, // do not track mem since been tracked by inner stream + ) + }) + .collect::>(); + + let tracking_metrics = get_metrics(final_metric); + Ok(Box::pin(SortPreservingMergeStream::new_from_streams( + sorted_streams, + schema, + expressions, + tracking_metrics, + batch_size, + ))) + } +} + +async fn spill_partial_sorted_stream( + in_mem_stream: &mut SendableRecordBatchStream, + path: &Path, + schema: SchemaRef, +) -> Result<()> { + let (sender, receiver) = tokio::sync::mpsc::channel(2); + let path: PathBuf = path.into(); + let handle = task::spawn_blocking(move || write_sorted(receiver, path, schema)); + while let Some(item) = in_mem_stream.next().await { + sender.send(item).await.ok(); + } + drop(sender); + match handle.await { + Ok(r) => r, + Err(e) => Err(DataFusionError::Execution(format!( + "Error occurred while spilling {}", + e + ))), + } +} + +fn read_spill_as_stream( + path: NamedTempFile, + schema: SchemaRef, +) -> Result { + let (sender, receiver): ( + Sender>, + Receiver>, + ) = tokio::sync::mpsc::channel(2); + let join_handle = task::spawn_blocking(move || { + if let Err(e) = read_spill(sender, path.path()) { + error!("Failure while reading spill file: {:?}. Error: {}", path, e); + } + }); + Ok(RecordBatchReceiverStream::create( + &schema, + receiver, + join_handle, + )) +} + +fn write_sorted( + mut receiver: Receiver>, + path: PathBuf, + schema: SchemaRef, +) -> Result<()> { + let mut writer = IPCWriter::new(path.as_ref(), schema.as_ref())?; + while let Some(batch) = receiver.blocking_recv() { + writer.write(&batch?)?; + } + writer.finish()?; + debug!( + "Spilled {} batches of total {} rows to disk, memory released {}", + writer.num_batches, + writer.num_rows, + human_readable_size(writer.num_bytes as usize), + ); + Ok(()) +} + +fn read_spill(sender: Sender>, path: &Path) -> Result<()> { + let file = BufReader::new(File::open(&path)?); + let reader = FileReader::try_new(file, None)?; + for batch in reader { + sender + .blocking_send(batch) + .map_err(|e| DataFusionError::Execution(format!("{}", e)))?; + } + Ok(()) +} + +/// External Sort execution plan +#[derive(Debug)] +pub struct SortExec2 { + /// Input schema + input: Arc, + /// Sort expressions + expr: Vec, + /// Containing all metrics set created during sort + metrics_set: CompositeMetricsSet, + /// Preserve partitions of input plan + preserve_partitioning: bool, +} + +impl SortExec2 { + /// Create a new sort execution plan + pub fn try_new( + expr: Vec, + input: Arc, + ) -> Result { + Ok(Self::new_with_partitioning(expr, input, false)) + } + + /// Create a new sort execution plan with the option to preserve + /// the partitioning of the input plan + pub fn new_with_partitioning( + expr: Vec, + input: Arc, + preserve_partitioning: bool, + ) -> Self { + Self { + expr, + input, + metrics_set: CompositeMetricsSet::new(), + preserve_partitioning, + } + } + + /// Input schema + pub fn input(&self) -> &Arc { + &self.input + } + + /// Sort expressions + pub fn expr(&self) -> &[PhysicalSortExpr] { + &self.expr + } +} + +#[async_trait] +impl ExecutionPlan for SortExec2 { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + self.input.schema() + } + + /// Get the output partitioning of this plan + fn output_partitioning(&self) -> Partitioning { + if self.preserve_partitioning { + self.input.output_partitioning() + } else { + Partitioning::UnknownPartitioning(1) + } + } + + fn required_child_distribution(&self) -> Distribution { + if self.preserve_partitioning { + Distribution::UnspecifiedDistribution + } else { + Distribution::SinglePartition + } + } + + fn children(&self) -> Vec> { + vec![self.input.clone()] + } + + fn relies_on_input_order(&self) -> bool { + // this operator resorts everything + false + } + + fn benefits_from_input_partitioning(&self) -> bool { + false + } + + fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { + Some(&self.expr) + } + + fn with_new_children( + &self, + children: Vec>, + ) -> Result> { + match children.len() { + 1 => Ok(Arc::new(SortExec2::try_new( + self.expr.clone(), + children[0].clone(), + )?)), + _ => Err(DataFusionError::Internal( + "SortExec2 wrong number of children".to_string(), + )), + } + } + + async fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + if !self.preserve_partitioning { + if 0 != partition { + return Err(DataFusionError::Internal(format!( + "SortExec2 invalid partition {}", + partition + ))); + } + + // sort needs to operate on a single partition currently + if 1 != self.input.output_partitioning().partition_count() { + return Err(DataFusionError::Internal( + "SortExec2 requires a single input partition".to_owned(), + )); + } + } + + let input = self.input.execute(partition, context.clone()).await?; + + do_sort( + input, + partition, + self.expr.clone(), + self.metrics_set.clone(), + context, + ) + .await + } + + fn metrics(&self) -> Option { + Some(self.metrics_set.aggregate_all()) + } + + fn fmt_as( + &self, + t: DisplayFormatType, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + match t { + DisplayFormatType::Default => { + let expr: Vec = self.expr.iter().map(|e| e.to_string()).collect(); + write!(f, "SortExec2: [{}]", expr.join(",")) + } + } + } + + fn statistics(&self) -> Statistics { + self.input.statistics() + } +} + +fn sort_batch( + batch: RecordBatch, + schema: SchemaRef, + expr: &[PhysicalSortExpr], +) -> ArrowResult { + // TODO: pushup the limit expression to sort + let indices = lexsort_to_indices( + &expr + .iter() + .map(|e| e.evaluate_to_sort_column(&batch)) + .collect::>>()?, + None, + )?; + + // reorder all rows based on sorted indices + RecordBatch::try_new( + schema, + batch + .columns() + .iter() + .map(|column| { + take( + column.as_ref(), + &indices, + // disable bound check overhead since indices are already generated from + // the same record batch + Some(TakeOptions { + check_bounds: false, + }), + ) + }) + .collect::>>()?, + ) +} + +async fn do_sort( + mut input: SendableRecordBatchStream, + partition_id: usize, + expr: Vec, + metrics_set: CompositeMetricsSet, + context: Arc, +) -> Result { + let schema = input.schema(); + let tracking_metrics = + metrics_set.new_intermediate_tracking(partition_id, context.runtime_env()); + let sorter = ExternalSorter2::new( + partition_id, + schema.clone(), + expr, + metrics_set, + Arc::new(context.session_config()), + context.runtime_env(), + ); + context.runtime_env().register_requester(sorter.id()); + while let Some(batch) = input.next().await { + let batch = batch?; + sorter.insert_batch(batch, &tracking_metrics).await?; + } + sorter.sort().await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; + use crate::execution::context::SessionConfig; + use crate::execution::runtime_env::RuntimeConfig; + use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; + use crate::physical_plan::expressions::col; + use crate::physical_plan::memory::MemoryExec; + use crate::physical_plan::{ + collect, + file_format::{CsvExec, FileScanConfig}, + }; + use crate::prelude::SessionContext; + use crate::test; + use crate::test::assert_is_pending; + use crate::test::exec::{assert_strong_count_converges_to_zero, BlockingExec}; + use crate::test_util; + use arrow::array::*; + use arrow::compute::SortOptions; + use arrow::datatypes::*; + use futures::FutureExt; + use std::collections::{BTreeMap, HashMap}; + + #[tokio::test] + async fn test_in_mem_sort() -> Result<()> { + let session_ctx = SessionContext::new(); + let task_ctx = session_ctx.task_ctx(); + let schema = test_util::aggr_test_schema(); + let partitions = 4; + let (_, files) = + test::create_partitioned_csv("aggregate_test_100.csv", partitions)?; + + let csv = CsvExec::new( + FileScanConfig { + object_store: Arc::new(LocalFileSystem {}), + file_schema: Arc::clone(&schema), + file_groups: files, + statistics: Statistics::default(), + projection: None, + limit: None, + table_partition_cols: vec![], + }, + true, + b',', + ); + + let sort_exec = Arc::new(SortExec2::try_new( + vec![ + // c1 string column + PhysicalSortExpr { + expr: col("c1", &schema)?, + options: SortOptions::default(), + }, + // c2 uin32 column + PhysicalSortExpr { + expr: col("c2", &schema)?, + options: SortOptions::default(), + }, + // c7 uin8 column + PhysicalSortExpr { + expr: col("c7", &schema)?, + options: SortOptions::default(), + }, + ], + Arc::new(CoalescePartitionsExec::new(Arc::new(csv))), + )?); + + let result = collect(sort_exec, task_ctx).await?; + + assert_eq!(result.len(), 1); + + let columns = result[0].columns(); + + let c1 = as_string_array(&columns[0]); + assert_eq!(c1.value(0), "a"); + assert_eq!(c1.value(c1.len() - 1), "e"); + + let c2 = as_primitive_array::(&columns[1]); + assert_eq!(c2.value(0), 1); + assert_eq!(c2.value(c2.len() - 1), 5,); + + let c7 = as_primitive_array::(&columns[6]); + assert_eq!(c7.value(0), 15); + assert_eq!(c7.value(c7.len() - 1), 254,); + + Ok(()) + } + + #[tokio::test] + async fn test_sort_spill() -> Result<()> { + // trigger spill there will be 4 batches with 5.5KB for each + let config = RuntimeConfig::new().with_memory_limit(12288, 1.0); + let runtime = Arc::new(RuntimeEnv::new(config)?); + let session_ctx = SessionContext::with_config_rt(SessionConfig::new(), runtime); + + let schema = test_util::aggr_test_schema(); + let partitions = 4; + let (_, files) = + test::create_partitioned_csv("aggregate_test_100.csv", partitions)?; + + let csv = CsvExec::new( + FileScanConfig { + object_store: Arc::new(LocalFileSystem {}), + file_schema: Arc::clone(&schema), + file_groups: files, + statistics: Statistics::default(), + projection: None, + limit: None, + table_partition_cols: vec![], + }, + true, + b',', + ); + + let sort_exec = Arc::new(SortExec2::try_new( + vec![ + // c1 string column + PhysicalSortExpr { + expr: col("c1", &schema)?, + options: SortOptions::default(), + }, + // c2 uin32 column + PhysicalSortExpr { + expr: col("c2", &schema)?, + options: SortOptions::default(), + }, + // c7 uin8 column + PhysicalSortExpr { + expr: col("c7", &schema)?, + options: SortOptions::default(), + }, + ], + Arc::new(CoalescePartitionsExec::new(Arc::new(csv))), + )?); + + let task_ctx = session_ctx.task_ctx(); + let result = collect(sort_exec.clone(), task_ctx).await?; + + assert_eq!(result.len(), 1); + + // Now, validate metrics + let metrics = sort_exec.metrics().unwrap(); + + assert_eq!(metrics.output_rows().unwrap(), 100); + assert!(metrics.elapsed_compute().unwrap() > 0); + assert!(metrics.spill_count().unwrap() > 0); + assert!(metrics.spilled_bytes().unwrap() > 0); + + let columns = result[0].columns(); + + let c1 = as_string_array(&columns[0]); + assert_eq!(c1.value(0), "a"); + assert_eq!(c1.value(c1.len() - 1), "e"); + + let c2 = as_primitive_array::(&columns[1]); + assert_eq!(c2.value(0), 1); + assert_eq!(c2.value(c2.len() - 1), 5,); + + let c7 = as_primitive_array::(&columns[6]); + assert_eq!(c7.value(0), 15); + assert_eq!(c7.value(c7.len() - 1), 254,); + + Ok(()) + } + + #[tokio::test] + async fn test_sort_metadata() -> Result<()> { + let session_ctx = SessionContext::new(); + let task_ctx = session_ctx.task_ctx(); + let field_metadata: BTreeMap = + vec![("foo".to_string(), "bar".to_string())] + .into_iter() + .collect(); + let schema_metadata: HashMap = + vec![("baz".to_string(), "barf".to_string())] + .into_iter() + .collect(); + + let mut field = Field::new("field_name", DataType::UInt64, true); + field.set_metadata(Some(field_metadata.clone())); + let schema = Schema::new_with_metadata(vec![field], schema_metadata.clone()); + let schema = Arc::new(schema); + + let data: ArrayRef = + Arc::new(vec![3, 2, 1].into_iter().map(Some).collect::()); + + let batch = RecordBatch::try_new(schema.clone(), vec![data]).unwrap(); + let input = + Arc::new(MemoryExec::try_new(&[vec![batch]], schema.clone(), None).unwrap()); + + let sort_exec = Arc::new(SortExec2::try_new( + vec![PhysicalSortExpr { + expr: col("field_name", &schema)?, + options: SortOptions::default(), + }], + input, + )?); + + let result: Vec = collect(sort_exec, task_ctx).await?; + + let expected_data: ArrayRef = + Arc::new(vec![1, 2, 3].into_iter().map(Some).collect::()); + let expected_batch = + RecordBatch::try_new(schema.clone(), vec![expected_data]).unwrap(); + + // Data is correct + assert_eq!(&vec![expected_batch], &result); + + // explicitlty ensure the metadata is present + assert_eq!( + result[0].schema().fields()[0].metadata(), + &Some(field_metadata) + ); + assert_eq!(result[0].schema().metadata(), &schema_metadata); + + Ok(()) + } + + #[tokio::test] + async fn test_lex_sort_by_float() -> Result<()> { + let session_ctx = SessionContext::new(); + let task_ctx = session_ctx.task_ctx(); + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Float32, true), + Field::new("b", DataType::Float64, true), + ])); + + // define data. + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Float32Array::from(vec![ + Some(f32::NAN), + None, + None, + Some(f32::NAN), + Some(1.0_f32), + Some(1.0_f32), + Some(2.0_f32), + Some(3.0_f32), + ])), + Arc::new(Float64Array::from(vec![ + Some(200.0_f64), + Some(20.0_f64), + Some(10.0_f64), + Some(100.0_f64), + Some(f64::NAN), + None, + None, + Some(f64::NAN), + ])), + ], + )?; + + let sort_exec = Arc::new(SortExec2::try_new( + vec![ + PhysicalSortExpr { + expr: col("a", &schema)?, + options: SortOptions { + descending: true, + nulls_first: true, + }, + }, + PhysicalSortExpr { + expr: col("b", &schema)?, + options: SortOptions { + descending: false, + nulls_first: false, + }, + }, + ], + Arc::new(MemoryExec::try_new(&[vec![batch]], schema, None)?), + )?); + + assert_eq!(DataType::Float32, *sort_exec.schema().field(0).data_type()); + assert_eq!(DataType::Float64, *sort_exec.schema().field(1).data_type()); + + let result: Vec = collect(sort_exec.clone(), task_ctx).await?; + let metrics = sort_exec.metrics().unwrap(); + assert!(metrics.elapsed_compute().unwrap() > 0); + assert_eq!(metrics.output_rows().unwrap(), 8); + assert_eq!(result.len(), 1); + + let columns = result[0].columns(); + + assert_eq!(DataType::Float32, *columns[0].data_type()); + assert_eq!(DataType::Float64, *columns[1].data_type()); + + let a = as_primitive_array::(&columns[0]); + let b = as_primitive_array::(&columns[1]); + + // convert result to strings to allow comparing to expected result containing NaN + let result: Vec<(Option, Option)> = (0..result[0].num_rows()) + .map(|i| { + let aval = if a.is_valid(i) { + Some(a.value(i).to_string()) + } else { + None + }; + let bval = if b.is_valid(i) { + Some(b.value(i).to_string()) + } else { + None + }; + (aval, bval) + }) + .collect(); + + let expected: Vec<(Option, Option)> = vec![ + (None, Some("10".to_owned())), + (None, Some("20".to_owned())), + (Some("NaN".to_owned()), Some("100".to_owned())), + (Some("NaN".to_owned()), Some("200".to_owned())), + (Some("3".to_owned()), Some("NaN".to_owned())), + (Some("2".to_owned()), None), + (Some("1".to_owned()), Some("NaN".to_owned())), + (Some("1".to_owned()), None), + ]; + + assert_eq!(expected, result); + + Ok(()) + } + + #[tokio::test] + async fn test_drop_cancel() -> Result<()> { + let session_ctx = SessionContext::new(); + let task_ctx = session_ctx.task_ctx(); + let schema = + Arc::new(Schema::new(vec![Field::new("a", DataType::Float32, true)])); + + let blocking_exec = Arc::new(BlockingExec::new(Arc::clone(&schema), 1)); + let refs = blocking_exec.refs(); + let sort_exec = Arc::new(SortExec2::try_new( + vec![PhysicalSortExpr { + expr: col("a", &schema)?, + options: SortOptions::default(), + }], + blocking_exec, + )?); + + let fut = collect(sort_exec, task_ctx); + let mut fut = fut.boxed(); + + assert_is_pending(&mut fut); + drop(fut); + assert_strong_count_converges_to_zero(refs).await; + + Ok(()) + } +} diff --git a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs index 3fb480de21bde..2946753fca4a9 100644 --- a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs +++ b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs @@ -553,45 +553,54 @@ impl SortPreservingMergeStream { let elapsed_compute = self.tracking_metrics.elapsed_compute().clone(); let _timer = elapsed_compute.timer(); - match self.min_heap.pop() { - Some(mut cursor) => { - let stream_idx = cursor.stream_idx; - let batch_idx = self.batches[stream_idx].len() - 1; - let row_idx = cursor.advance(); - - let mut cursor_finished = false; - // insert the cursor back to min_heap if the record batch is not exhausted - if !cursor.is_finished() { - self.min_heap.push(cursor); - } else { - cursor_finished = true; - self.cursor_finished[stream_idx] = true; - } + let in_progress_empty = self.in_progress.is_empty(); + let mut cursor = self.min_heap.peek_mut().unwrap(); + let cursor_finished = cursor.is_finished(); + + if cursor_finished { + std::mem::drop(cursor); + return if in_progress_empty { + Poll::Ready(None) + } else { + Poll::Ready(Some(self.build_record_batch())) + }; + } - self.in_progress.push(RowIndex { - stream_idx, - batch_idx, - row_idx, - }); + let stream_idx = cursor.stream_idx; + let row_idx = cursor.advance(); + + let mut cursor_finished = false; + // insert the cursor back to min_heap if the record batch is not exhausted + if cursor.is_finished() { + std::mem::drop(cursor); + cursor_finished = true; + self.cursor_finished[stream_idx] = true; + } else { + std::mem::drop(cursor); + } - if self.in_progress.len() == self.batch_size { - return Poll::Ready(Some(self.build_record_batch())); - } + let batch_idx = self.batches[stream_idx].len() - 1; - // If removed the last row from the cursor, need to fetch a new record - // batch if possible, before looping round again - if cursor_finished { - match futures::ready!(self.maybe_poll_stream(cx, stream_idx)) { - Ok(_) => {} - Err(e) => { - self.aborted = true; - return Poll::Ready(Some(Err(e))); - } - } + self.in_progress.push(RowIndex { + stream_idx, + batch_idx, + row_idx, + }); + + if self.in_progress.len() == self.batch_size { + return Poll::Ready(Some(self.build_record_batch())); + } + + // If removed the last row from the cursor, need to fetch a new record + // batch if possible, before looping round again + if cursor_finished { + match futures::ready!(self.maybe_poll_stream(cx, stream_idx)) { + Ok(_) => {} + Err(e) => { + self.aborted = true; + return Poll::Ready(Some(Err(e))); } } - None if self.in_progress.is_empty() => return Poll::Ready(None), - None => return Poll::Ready(Some(self.build_record_batch())), } } } @@ -617,7 +626,7 @@ mod tests { use crate::physical_plan::expressions::col; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::physical_plan::memory::MemoryExec; - use crate::physical_plan::sorts::sort::SortExec; + use crate::physical_plan::sorts::sort2::SortExec2; use crate::physical_plan::{collect, common}; use crate::test::{self, assert_is_pending}; use crate::{assert_batches_eq, test_util}; @@ -881,7 +890,7 @@ mod tests { context: Arc, ) -> RecordBatch { let sort_exec = - Arc::new(SortExec::new_with_partitioning(sort.clone(), input, true)); + Arc::new(SortExec2::new_with_partitioning(sort.clone(), input, true)); sorted_merge(sort_exec, sort, context).await } @@ -891,7 +900,7 @@ mod tests { context: Arc, ) -> RecordBatch { let merge = Arc::new(CoalescePartitionsExec::new(src)); - let sort_exec = Arc::new(SortExec::try_new(sort, merge).unwrap()); + let sort_exec = Arc::new(SortExec2::try_new(sort, merge).unwrap()); let mut result = collect(sort_exec, context).await.unwrap(); assert_eq!(result.len(), 1); result.remove(0) diff --git a/datafusion/core/tests/order_spill_fuzz.rs b/datafusion/core/tests/order_spill_fuzz.rs index c052382d5eac7..dd363ed6138c9 100644 --- a/datafusion/core/tests/order_spill_fuzz.rs +++ b/datafusion/core/tests/order_spill_fuzz.rs @@ -27,6 +27,7 @@ use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; use datafusion::physical_plan::expressions::{col, PhysicalSortExpr}; use datafusion::physical_plan::memory::MemoryExec; use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::sorts::sort2::SortExec2; use datafusion::physical_plan::{collect, ExecutionPlan}; use datafusion::prelude::{SessionConfig, SessionContext}; use fuzz_utils::{add_empty_batches, batches_to_vec, partitions_to_sorted_vec}; @@ -36,12 +37,22 @@ use std::sync::Arc; #[tokio::test] async fn test_sort_1k_mem() { - run_sort(1024, vec![(5, false), (2000, true), (1000000, true)]).await + run_sort(1024, vec![(5, false), (2000, true), (1000000, true)]).await; +} + +#[tokio::test] +async fn test_sort_1k_mem_2() { + run_sort2(1024, vec![(5, false), (2000, true), (1000000, true)]).await; } #[tokio::test] async fn test_sort_100k_mem() { - run_sort(102400, vec![(5, false), (2000, false), (1000000, true)]).await + run_sort(102400, vec![(5, false), (2000, false), (1000000, true)]).await; +} + +#[tokio::test] +async fn test_sort_100k_mem_2() { + run_sort2(102400, vec![(5, false), (2000, false), (1000000, true)]).await; } #[tokio::test] @@ -50,7 +61,16 @@ async fn test_sort_unlimited_mem() { usize::MAX, vec![(5, false), (2000, false), (1000000, false)], ) - .await + .await; +} + +#[tokio::test] +async fn test_sort_unlimited_mem2() { + run_sort2( + usize::MAX, + vec![(5, false), (2000, false), (1000000, false)], + ) + .await; } /// Sort the input using SortExec and ensure the results are correct according to `Vec::sort` @@ -97,6 +117,50 @@ async fn run_sort(pool_size: usize, size_spill: Vec<(usize, bool)>) { } } +/// Sort the input using SortExec and ensure the results are correct according to `Vec::sort` +async fn run_sort2(pool_size: usize, size_spill: Vec<(usize, bool)>) { + for (size, spill) in size_spill { + let input = vec![make_staggered_batches(size)]; + let first_batch = input + .iter() + .flat_map(|p| p.iter()) + .next() + .expect("at least one batch"); + let schema = first_batch.schema(); + + let sort = vec![PhysicalSortExpr { + expr: col("x", &schema).unwrap(), + options: SortOptions { + descending: false, + nulls_first: true, + }, + }]; + + let exec = MemoryExec::try_new(&input, schema, None).unwrap(); + let sort = Arc::new(SortExec2::try_new(sort, Arc::new(exec)).unwrap()); + + let runtime_config = RuntimeConfig::new().with_memory_manager( + MemoryManagerConfig::try_new_limit(pool_size, 1.0).unwrap(), + ); + let runtime = Arc::new(RuntimeEnv::new(runtime_config).unwrap()); + let session_ctx = SessionContext::with_config_rt(SessionConfig::new(), runtime); + + let task_ctx = session_ctx.task_ctx(); + let collected = collect(sort.clone(), task_ctx).await.unwrap(); + + let expected = partitions_to_sorted_vec(&input); + let actual = batches_to_vec(&collected); + + if spill { + assert_ne!(sort.metrics().unwrap().spill_count().unwrap(), 0); + } else { + assert_eq!(sort.metrics().unwrap().spill_count().unwrap(), 0); + } + + assert_eq!(expected, actual, "failure in @ pool_size {}", pool_size); + } +} + /// Return randomly sized record batches in a field named 'x' of type `Int32` /// with randomized i32 content fn make_staggered_batches(len: usize) -> Vec { diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs index ef6ea52dd7740..fe1094542beff 100644 --- a/datafusion/core/tests/sql/explain_analyze.rs +++ b/datafusion/core/tests/sql/explain_analyze.rs @@ -60,7 +60,7 @@ async fn explain_analyze_baseline_metrics() { ); assert_metrics!( &formatted, - "SortExec: [c1@0 ASC NULLS LAST]", + "SortExec2: [c1@0 ASC NULLS LAST]", "metrics=[output_rows=5, elapsed_compute=" ); assert_metrics!( @@ -108,7 +108,7 @@ async fn explain_analyze_baseline_metrics() { use datafusion::physical_plan; use datafusion::physical_plan::sorts; - plan.as_any().downcast_ref::().is_some() + plan.as_any().downcast_ref::().is_some() || plan.as_any().downcast_ref::().is_some() // CoalescePartitionsExec doesn't do any work so is not included || plan.as_any().downcast_ref::().is_some() @@ -648,7 +648,7 @@ async fn test_physical_plan_display_indent() { let physical_plan = ctx.create_physical_plan(&plan).await.unwrap(); let expected = vec![ "GlobalLimitExec: limit=10", - " SortExec: [the_min@2 DESC]", + " SortExec2: [the_min@2 DESC]", " CoalescePartitionsExec", " ProjectionExec: expr=[c1@0 as c1, MAX(aggregate_test_100.c12)@1 as MAX(aggregate_test_100.c12), MIN(aggregate_test_100.c12)@2 as the_min]", " HashAggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[MAX(aggregate_test_100.c12), MIN(aggregate_test_100.c12)]",