Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions datafusion/core/tests/memory_limit/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -921,15 +921,19 @@ async fn test_external_sort_zero_merge_reservation() {

let ctx = SessionContext::new_with_config_rt(config, runtime);

let query = "select * from generate_series(1,10000000) as t1(v1) order by v1 desc;";
// The input and its sorted copy exceed the 10 MiB pool.
let query = "select * from generate_series(1,1048576) as t1(v1) order by v1 desc;";
let df = ctx.sql(query).await.unwrap();

let physical_plan = df.create_physical_plan().await.unwrap();
let task_ctx = Arc::new(TaskContext::from(&ctx.state()));
let stream = physical_plan.execute(0, task_ctx).unwrap();

// Ensures execution succeed
let _result = collect(stream).await;
let result = collect(stream).await.unwrap();
assert_eq!(
result.iter().map(RecordBatch::num_rows).sum::<usize>(),
1_048_576
);

// Ensures the query spilled during execution
let spill_count = plan_spill_count(physical_plan.as_ref());
Expand Down
12 changes: 9 additions & 3 deletions datafusion/core/tests/sql/runtime_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ use crate::helper::plan_metrics::plan_spill_count;

#[tokio::test]
async fn test_memory_limit_with_spill() {
let ctx = SessionContext::new();
let ctx =
SessionContext::new_with_config(SessionConfig::new().with_target_partitions(1));

ctx.sql("SET datafusion.runtime.memory_limit = '1M'")
.await
Expand All @@ -38,14 +39,19 @@ async fn test_memory_limit_with_spill() {
.await
.unwrap();

let query = "select * from generate_series(1,10000000) as t1(v1) order by v1 desc;";
// Two MiB of input exceeds the one MiB pool without a large fixture.
let query = "select * from generate_series(1,262144) as t1(v1) order by v1 desc;";
let df = ctx.sql(query).await.unwrap();

let plan = df.create_physical_plan().await.unwrap();
let task_ctx = Arc::new(TaskContext::from(&ctx.state()));
let stream = plan.execute(0, task_ctx).unwrap();

let _results = collect(stream).await;
let results = collect(stream).await.unwrap();
assert_eq!(
results.iter().map(|batch| batch.num_rows()).sum::<usize>(),
262_144
);
let spill_count = plan_spill_count(plan.as_ref());
assert!(spill_count > 0, "Expected spills but none occurred");
}
Expand Down
Loading