diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index d78f3559bd8..f3912024370 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -236,6 +236,7 @@ depends on, on the driver and executor classpath, for example | Option | Default | Description | |----------------------------|---------|-------------| | `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. | +| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing output. This does not cap total native memory or I/O read-ahead. | | `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. | | `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. | | `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. | diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java index 8658cf5198b..22c090c6419 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java @@ -238,6 +238,7 @@ public Scan scan(ScanOptions options) { byte[] selectionRoaringBitmap = selectionRoaringBitmap(options); long limit = options.limit().orElse(0L); boolean ordered = options.ordered(); + int splitConcurrency = options.splitConcurrency().orElse(0); long scanPtr = dev.vortex.jni.NativeScan.create( pointer, @@ -249,7 +250,8 @@ public Scan scan(ScanOptions options) { selectionRoaringBitmap, selectionMode.code(), limit, - ordered); + ordered, + splitConcurrency); return Scan.fromPointer(session, scanPtr); } diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java index 25f03e76f7f..43e713581ad 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java @@ -9,6 +9,7 @@ import java.io.UncheckedIOException; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.OptionalLong; import org.immutables.value.Value; import org.roaringbitmap.longlong.Roaring64NavigableMap; @@ -59,6 +60,25 @@ default boolean ordered() { return false; } + /** + * The most splits of each partition the native scan decodes at once, in all, and the most of its batches converted + * to Arrow ahead of the caller. Absent keeps the native default, which scales with the machine's cores. + * + *

Each split being decoded holds the decoded chunks of every column it overlaps, so a scan running alongside + * others, such as one Spark task of several in an executor, can lower this to reduce memory held by concurrent + * decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split + * while preparing output. This does not cap total native memory or I/O read-ahead. + */ + OptionalInt splitConcurrency(); + + @Value.Check + default void validateSplitConcurrency() { + if (splitConcurrency().isPresent() && splitConcurrency().getAsInt() <= 0) { + throw new IllegalArgumentException("split concurrency must be positive, got " + + splitConcurrency().getAsInt()); + } + } + @Value.Check default void validateSelectionPayload() { boolean hasIndices = selectionIndices().isPresent(); diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java index ff80b30caaa..ff4ad73a317 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java @@ -27,6 +27,7 @@ private NativeScan() {} * {@code selectionRoaringBitmap}) * @param limit max rows to return, or {@code 0} for "no limit" * @param ordered true to preserve row order across partitions + * @param splitConcurrency the most splits of each partition read at once, or {@code 0} for the native default */ public static native long create( long dataSourcePointer, @@ -38,7 +39,8 @@ public static native long create( byte[] selectionRoaringBitmap, byte selectionInclude, long limit, - boolean ordered); + boolean ordered, + int splitConcurrency); /** Free a scan pointer. */ public static native void free(long pointer); diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java b/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java index c72e67f4051..cb0c6ff5b8a 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java @@ -12,6 +12,7 @@ import dev.vortex.api.ScanOptions.SelectionMode; import java.io.ByteArrayInputStream; import java.io.DataInputStream; +import java.util.OptionalInt; import java.util.OptionalLong; import org.junit.jupiter.api.Test; import org.roaringbitmap.longlong.Roaring64NavigableMap; @@ -39,6 +40,21 @@ public void defaultsReadEveryRowAndColumn() { assertTrue(options.selectionRoaringBitmap().isEmpty()); assertEquals(SelectionMode.INCLUDE_ALL, options.selectionMode()); assertFalse(options.ordered()); + assertEquals(OptionalInt.empty(), options.splitConcurrency()); + } + + @Test + public void splitConcurrencyMustBePositive() { + for (int value : new int[] {0, -1}) { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> ScanOptions.builder().splitConcurrency(value).build(), + () -> "no exception for " + value); + assertEquals("split concurrency must be positive, got " + value, exception.getMessage()); + } + assertEquals( + OptionalInt.of(2), + ScanOptions.builder().splitConcurrency(2).build().splitConcurrency()); } @Test diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java index 2b270c3e165..f4adefed093 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java @@ -184,6 +184,18 @@ public void testProjectedScanWithFilter() throws Exception { assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people); } + @Test + public void testScanWithSplitConcurrencyReadsEveryRow() throws Exception { + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Session session = Session.create(); + DataSource ds = DataSource.open(session, writePath); + + ScanOptions options = + ScanOptions.builder().splitConcurrency(1).ordered(true).build(); + List people = readAll(ds, options, allocator, TestMinimal::readFullBatch); + assertEquals(MINIMAL_DATA, people); + } + @Test public void testSelectionIncludesRows() throws Exception { BufferAllocator allocator = ArrowAllocation.rootAllocator(); diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java index 5bc35a81808..9f1ad11c2f8 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java @@ -30,31 +30,47 @@ public final class VortexIo implements Serializable { private static final int DEFAULT_READ_CONCURRENCY = 0; + /** + * Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the + * decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default + * keeps several splits in flight per core. Lowering this reduces memory held by concurrent split decoding and Arrow + * conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing + * output. This does not cap total native memory or I/O read-ahead. Zero keeps the native default. + */ + public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency"; + + private static final int DEFAULT_SPLIT_CONCURRENCY = 0; + private final SerializableHadoopConf conf; private final int readConcurrency; + private final int splitConcurrency; /** Captures {@code hadoopConf} and the read settings found in the format options. */ public static VortexIo create(VortexOptions options, Configuration hadoopConf) { Objects.requireNonNull(options, "options"); Objects.requireNonNull(hadoopConf, "hadoopConf"); - return new VortexIo(new SerializableHadoopConf(hadoopConf), parseReadConcurrency(options)); + return new VortexIo( + new SerializableHadoopConf(hadoopConf), + nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY), + nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY)); } /** Hadoop I/O over a default configuration, for callers that have no Spark session to draw one from. */ public static VortexIo defaults() { - return new VortexIo(new SerializableHadoopConf(new Configuration()), DEFAULT_READ_CONCURRENCY); + return new VortexIo( + new SerializableHadoopConf(new Configuration()), DEFAULT_READ_CONCURRENCY, DEFAULT_SPLIT_CONCURRENCY); } - private VortexIo(SerializableHadoopConf conf, int readConcurrency) { + private VortexIo(SerializableHadoopConf conf, int readConcurrency, int splitConcurrency) { this.conf = conf; this.readConcurrency = readConcurrency; + this.splitConcurrency = splitConcurrency; } - private static int parseReadConcurrency(VortexOptions options) { - int parsed = options.getInt(READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY); + private static int nonNegative(VortexOptions options, String key, int defaultValue) { + int parsed = options.getInt(key, defaultValue); if (parsed < 0) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "%s must be >= 0, got %d", READ_CONCURRENCY_OPTION, parsed)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "%s must be >= 0, got %d", key, parsed)); } return parsed; } @@ -68,6 +84,11 @@ public int readConcurrency() { return readConcurrency; } + /** The most splits of a file the native scan reads at once; zero keeps the native default. */ + public int splitConcurrency() { + return splitConcurrency; + } + /** * Opens a byte source for {@code file}, stating it only if the listing that produced it did not report a size. The * caller owns the result and must close it once the scan built on it is done. diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java index 44de7a423ed..1d8c93e0562 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java @@ -115,6 +115,9 @@ public VortexPartitionReader( projectedSchema, arrowFields, caseSensitive, file.toPath().toString()); var options = ScanOptions.builder(); + if (io.splitConcurrency() > 0) { + options.splitConcurrency(io.splitConcurrency()); + } // Always project, an empty read schema included: a query that needs only partition columns or only a // row count must not pull every data column off storage. Struct columns that Spark pruned read only the // nested fields the query uses. diff --git a/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java b/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java index 6d528f49f1f..05d4518d985 100644 --- a/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java +++ b/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import dev.vortex.spark.io.VortexIo; import java.nio.file.Path; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -43,7 +44,9 @@ public void setUp() { @AfterEach public void resetConf() { - for (String option : new String[] {READ_CONCURRENCY, VortexSparkSession.PROVIDER_OPTION}) { + for (String option : + new String[] {READ_CONCURRENCY, VortexIo.SPLIT_CONCURRENCY_OPTION, VortexSparkSession.PROVIDER_OPTION + }) { spark.conf().unset(option); spark.conf().unset(DATA_SOURCE_PREFIX + option); } @@ -84,6 +87,19 @@ void aSessionSettingAppliesToV1Reads() { assertSessionSettingAppliesToReads(write("v1"), "FileScan vortex", READ_CONCURRENCY); } + @Test + void aSplitConcurrencyReadsEveryRowThroughV1AndV2() { + String path = write("split_concurrency"); + for (String useV1 : new String[] {"", "vortex"}) { + spark.conf().set(USE_V1, useV1); + Dataset read = spark.read() + .format("vortex") + .option(VortexIo.SPLIT_CONCURRENCY_OPTION, "1") + .load(path); + assertEquals(45L, read.selectExpr("sum(id)").first().getLong(0), "useV1List=" + useV1); + } + } + @Test void aSessionSettingAppliesToWrites() { assertSessionSettingAppliesToWrites("written", VortexSparkSession.PROVIDER_OPTION); diff --git a/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java b/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java index a7800f25786..019f811fca0 100644 --- a/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java +++ b/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java @@ -18,10 +18,30 @@ final class VortexIoTest { @Test - void readConcurrencyDefaultsToNativeDefault() { + void concurrenciesDefaultToNativeDefaults() { VortexIo io = VortexIo.create(VortexOptions.empty(), new Configuration()); assertEquals(0, io.readConcurrency()); + assertEquals(0, io.splitConcurrency()); + } + + @Test + void splitConcurrencyIsParsedAndValidated() { + Configuration conf = new Configuration(); + + assertEquals( + 2, + VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "2"), conf) + .splitConcurrency()); + assertEquals( + 2, + VortexIo.create(options("VORTEX.SPLITCONCURRENCY", "2"), conf).splitConcurrency()); + assertThrows( + IllegalArgumentException.class, + () -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "-1"), conf)); + assertThrows( + IllegalArgumentException.class, + () -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "many"), conf)); } @Test @@ -54,11 +74,14 @@ void readConcurrencyIsFoundUnderAnyKeyCase() { void settingsAndConfigurationSurviveSerialization() throws IOException, ClassNotFoundException { Configuration conf = new Configuration(); conf.set("fs.s3a.endpoint", "https://storage.example"); - VortexIo io = VortexIo.create(options(VortexIo.READ_CONCURRENCY_OPTION, "4"), conf); + VortexIo io = VortexIo.create( + VortexOptions.of(Map.of(VortexIo.READ_CONCURRENCY_OPTION, "4", VortexIo.SPLIT_CONCURRENCY_OPTION, "3")), + conf); VortexIo shipped = roundTrip(io); assertEquals(4, shipped.readConcurrency()); + assertEquals(3, shipped.splitConcurrency()); assertEquals("https://storage.example", shipped.hadoopConf().get("fs.s3a.endpoint")); } diff --git a/vortex-ffi/src/scan.rs b/vortex-ffi/src/scan.rs index b58caddfb69..e14301c3475 100644 --- a/vortex-ffi/src/scan.rs +++ b/vortex-ffi/src/scan.rs @@ -208,6 +208,7 @@ fn scan_request(opts: *const vx_scan_options) -> VortexResult { limit, partition_selection: Selection::All, partition_range: None, + split_concurrency: None, }) } diff --git a/vortex-jni/src/scan.rs b/vortex-jni/src/scan.rs index 125558256f9..030013185b3 100644 --- a/vortex-jni/src/scan.rs +++ b/vortex-jni/src/scan.rs @@ -11,6 +11,7 @@ //! `FFI_ArrowArrayStream` that Java imports via Arrow's C Data Interface. use std::io::Cursor; +use std::num::NonZeroUsize; use std::ops::Range; use std::ptr; use std::sync::Arc; @@ -21,22 +22,26 @@ use arrow_array::ffi::FFI_ArrowSchema; use arrow_array::ffi_stream::FFI_ArrowArrayStream; use arrow_schema::ArrowError; use arrow_schema::Field; +use futures::Stream; use futures::StreamExt; use jni::EnvUnowned; use jni::objects::JByteArray; use jni::objects::JClass; use jni::objects::JLongArray; use jni::sys::jboolean; +use jni::sys::jint; use jni::sys::jlong; use vortex::array::VortexSessionExecute; use vortex::array::stream::SendableArrayStream; use vortex::buffer::Buffer; use vortex::error::VortexResult; use vortex::error::vortex_bail; +use vortex::error::vortex_err; use vortex::expr::Expression; use vortex::expr::root; use vortex::expr::stats::Precision; use vortex::io::runtime::BlockingRuntime; +use vortex::io::runtime::current::CurrentThreadRuntime; use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; use vortex::scan::DataSourceScan; use vortex::scan::PartitionRef; @@ -55,17 +60,21 @@ use crate::session::session_ref; /// Opaque scan handle. Holds a three-state machine: either the scan is pending (not yet /// realized as a stream), actively streaming partitions, or finished. +/// +/// The scan's `split_concurrency`, if set, also bounds how many of each partition's arrays are +/// converted to Arrow ahead of Java, so the conversion stays within the scan's bound. #[allow(dead_code)] pub(crate) enum NativeScan { - Pending(Box), - Started(PartitionStream), + Pending(Box, Option), + Started(PartitionStream, Option), Finished, } -/// Opaque partition handle with the same three-state machine shape. +/// Opaque partition handle with the same three-state machine shape, carrying its scan's +/// `split_concurrency`. #[allow(dead_code)] pub(crate) enum NativePartition { - Pending(PartitionRef), + Pending(PartitionRef, Option), Started(SendableArrayStream), Finished, } @@ -81,6 +90,7 @@ fn build_scan_request( selection_include: u8, limit: jlong, ordered: jboolean, + split_concurrency: jint, ) -> VortexResult { let projection = if projection_ptr == 0 { root() @@ -113,6 +123,10 @@ fn build_scan_request( }); let limit = (limit > 0).then_some(limit as u64); + let split_concurrency = + NonZeroUsize::new(usize::try_from(split_concurrency).map_err(|_| { + vortex_err!("split concurrency must not be negative, got {split_concurrency}") + })?); Ok(ScanRequest { projection, @@ -123,6 +137,7 @@ fn build_scan_request( limit, partition_selection: Selection::All, partition_range: None, + split_concurrency, }) } @@ -149,6 +164,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_create( selection_include: jni::sys::jbyte, limit: jlong, ordered: jboolean, + split_concurrency: jint, ) -> jlong { try_or_throw(&mut env, |env| { let ds = unsafe { NativeDataSource::from_ptr(data_source_ptr) }; @@ -185,10 +201,12 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_create( selection_include as u8, limit, ordered, + split_concurrency, )?; + let split_concurrency = request.split_concurrency; let scan = RUNTIME.block_on(async { ds.inner().scan(request).await })?; - Ok(Box::into_raw(Box::new(NativeScan::Pending(scan))) as jlong) + Ok(Box::into_raw(Box::new(NativeScan::Pending(scan, split_concurrency))) as jlong) }) } @@ -219,7 +237,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_arrowSchema( } let session = unsafe { session_ref(session_ptr) }; let scan = unsafe { &*(pointer as *const NativeScan) }; - let NativeScan::Pending(scan) = scan else { + let NativeScan::Pending(scan, _) = scan else { throw_runtime!("schema unavailable: scan already started"); }; let arrow_schema = session.arrow().to_arrow_schema(scan.dtype())?; @@ -241,7 +259,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_partitionCount( ) { try_or_throw(&mut env, |env| { let scan = unsafe { &*(pointer as *const NativeScan) }; - let NativeScan::Pending(scan) = scan else { + let NativeScan::Pending(scan, _) = scan else { throw_runtime!("partition count unavailable: scan already started"); }; let (rows, cardinality) = match scan.partition_count() { @@ -271,17 +289,20 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_nextPartition( // only rewrite it on the happy path where we want to keep streaming. ptr::write(slot, NativeScan::Finished); - let mut stream = match owned { - NativeScan::Pending(scan) => scan.partitions(), - NativeScan::Started(stream) => stream, + let (mut stream, split_concurrency) = match owned { + NativeScan::Pending(scan, split_concurrency) => (scan.partitions(), split_concurrency), + NativeScan::Started(stream, split_concurrency) => (stream, split_concurrency), NativeScan::Finished => return Ok(0), }; match RUNTIME.block_on(stream.next()) { Some(partition) => { let partition = partition?; - let handle = Box::into_raw(Box::new(NativePartition::Pending(partition))) as jlong; - ptr::write(slot, NativeScan::Started(stream)); + let handle = Box::into_raw(Box::new(NativePartition::Pending( + partition, + split_concurrency, + ))) as jlong; + ptr::write(slot, NativeScan::Started(stream, split_concurrency)); Ok(handle) } None => Ok(0), @@ -311,7 +332,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_rowCount( ) { try_or_throw(&mut env, |env| { let partition = unsafe { &*(pointer as *const NativePartition) }; - let NativePartition::Pending(partition) = partition else { + let NativePartition::Pending(partition, _) = partition else { throw_runtime!("row count unavailable: partition already started"); }; let (rows, cardinality) = match partition.row_count() { @@ -346,10 +367,13 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( throw_runtime!("null arrow stream address"); } - let partition = match partition { - NativePartition::Pending(p) => p, + let (partition, split_concurrency) = match partition { + NativePartition::Pending(p, split_concurrency) => (p, split_concurrency), _ => throw_runtime!("partition already consumed"), }; + let lookahead = split_concurrency + .map_or(usize::MAX, NonZeroUsize::get) + .min(POOL.worker_count().max(1)); let array_stream = partition.execute()?; let dtype = array_stream.dtype().clone(); @@ -358,28 +382,27 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( let schema = Arc::new(session.arrow().to_arrow_schema(&dtype)?); let target = Arc::new(Field::new_struct("", schema.fields().clone(), false)); - let iter = RUNTIME - .block_on_stream_thread_safe(|handle| { - array_stream - .map(move |chunk| { - let session = session.clone(); - let target = Arc::clone(&target); - handle.spawn(async move { - let chunk = chunk?; - let mut ctx = session.create_execution_ctx(); - let arrow = session.arrow().execute_arrow( - chunk, - Some(target.as_ref()), - &mut ctx, - )?; - rebase_offsets(RecordBatch::from(arrow.as_struct().clone())) - }) - }) - .buffered(POOL.worker_count().max(1)) + let handle = RUNTIME.handle(); + let batches = array_stream + .map(move |chunk| { + let session = session.clone(); + let target = Arc::clone(&target); + handle.spawn(async move { + let chunk = chunk?; + let mut ctx = session.create_execution_ctx(); + let arrow = + session + .arrow() + .execute_arrow(chunk, Some(target.as_ref()), &mut ctx)?; + rebase_offsets(RecordBatch::from(arrow.as_struct().clone())) + }) }) - .map(|result: VortexResult| { + .buffered(lookahead); + let iter = scan_iterator(&RUNTIME, split_concurrency, batches).map( + |result: VortexResult| { result.map_err(|e| ArrowError::ExternalError(Box::new(e))) - }); + }, + ); let reader = RecordBatchIteratorAdapter::new(iter, schema); let arrow_stream = FFI_ArrowArrayStream::new(Box::new(reader)); @@ -389,3 +412,69 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( Ok(()) }); } + +fn scan_iterator( + runtime: &CurrentThreadRuntime, + split_concurrency: Option, + stream: impl Stream + Send + 'static, +) -> Box + Send> { + if split_concurrency.is_some() { + // The thread-safe bridge queues another batch per core, independently of the stream's + // conversion lookahead. Drive capped scans directly so that queue cannot bypass the cap. + Box::new(runtime.block_on_stream(stream)) + } else { + Box::new(runtime.block_on_stream_thread_safe(|_| stream)) + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroUsize; + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + use futures::StreamExt; + use futures::stream; + use rstest::rstest; + use vortex::io::runtime::BlockingRuntime; + use vortex::io::runtime::current::CurrentThreadRuntime; + + use super::scan_iterator; + + #[rstest] + #[case(1)] + #[case(2)] + fn split_concurrency_bounds_completed_batches_ahead_of_the_consumer(#[case] max: usize) { + let runtime = CurrentThreadRuntime::new(); + let pool = runtime.new_pool(); + pool.set_workers(2); + let completed = Arc::new(AtomicUsize::new(0)); + let counter = Arc::clone(&completed); + let handle = runtime.handle(); + let batches = stream::iter(0..16) + .map(move |value| { + let counter = Arc::clone(&counter); + handle.spawn(async move { + counter.fetch_add(1, Ordering::SeqCst); + value + }) + }) + .buffered(max); + let mut iter = scan_iterator(&runtime, NonZeroUsize::new(max), batches); + assert_eq!(iter.next(), Some(0)); + + // Let background work finish while the consumer keeps its first batch open. + runtime.block_on(async { + for _ in 0..64 { + runtime.handle().spawn(async {}).await; + } + }); + assert!( + completed.load(Ordering::SeqCst) <= max + 1, + "completed batches escaped the conversion lookahead bound" + ); + + assert_eq!(iter.collect::>(), (1..16).collect::>()); + } +} diff --git a/vortex-layout/src/scan/layout.rs b/vortex-layout/src/scan/layout.rs index ae670958ff4..62b1805aacf 100644 --- a/vortex-layout/src/scan/layout.rs +++ b/vortex-layout/src/scan/layout.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::any::Any; +use std::num::NonZeroUsize; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; @@ -182,6 +183,7 @@ impl DataSource for LayoutReaderDataSource { row_limit, selection: scan_request.selection, ordered: scan_request.ordered, + split_concurrency: scan_request.split_concurrency, metrics_registry: self.metrics_registry.clone(), next_row: row_range.start, end_row: row_range.end, @@ -207,6 +209,7 @@ struct LayoutReaderScan { /// A row budget shared by every split of an unordered limited scan; see [`RowLimit`]. row_limit: Option, ordered: bool, + split_concurrency: Option, selection: Selection, metrics_registry: Option>, next_row: u64, @@ -276,6 +279,7 @@ impl Stream for LayoutReaderScan { limit: split_limit, row_limit: this.row_limit.clone(), ordered: this.ordered, + split_concurrency: this.split_concurrency, row_range, selection: this.selection.clone(), metrics_registry: this.metrics_registry.clone(), @@ -309,6 +313,7 @@ struct LayoutReaderSplit { /// A row budget shared with the scan's other splits (unordered limited scans only). row_limit: Option, ordered: bool, + split_concurrency: Option, row_range: Range, selection: Selection, metrics_registry: Option>, @@ -347,7 +352,8 @@ impl Partition for LayoutReaderSplit { .with_projection(self.projection) .with_some_filter(self.filter) .with_some_metrics_registry(self.metrics_registry) - .with_ordered(self.ordered); + .with_ordered(self.ordered) + .with_some_max_splits_in_flight(self.split_concurrency); let builder = match self.row_limit { Some(row_limit) => builder.with_shared_limit(row_limit), None => builder.with_some_limit(self.limit), diff --git a/vortex-layout/src/scan/multi.rs b/vortex-layout/src/scan/multi.rs index c4a0d56fe3b..1f1ee275688 100644 --- a/vortex-layout/src/scan/multi.rs +++ b/vortex-layout/src/scan/multi.rs @@ -26,6 +26,7 @@ use std::any::Any; use std::collections::VecDeque; +use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; @@ -341,6 +342,7 @@ struct BoundScanRequest { partition_range: Option>, ordered: bool, limit: Option, + split_concurrency: Option, } impl BoundScanRequest { @@ -354,6 +356,7 @@ impl BoundScanRequest { partition_range, ordered, limit, + split_concurrency, } = request; Ok(Self { @@ -368,6 +371,7 @@ impl BoundScanRequest { partition_range, ordered, limit, + split_concurrency, }) } } @@ -599,7 +603,8 @@ impl Partition for MultiLayoutPartition { .with_selection(request.selection) .with_projection(request.projection) .with_some_filter(filter) - .with_ordered(request.ordered); + .with_ordered(request.ordered) + .with_some_max_splits_in_flight(request.split_concurrency); builder = match self.row_limit { Some(row_limit) => builder.with_shared_limit(row_limit), None => builder.with_some_limit(request.limit), @@ -637,6 +642,7 @@ mod tests { use vortex_scan::ScanRequest; use super::*; + use crate::scan::test::InFlight; use crate::scan::test::TestLayoutReader; use crate::scan::test::collect_scan_values; use crate::scan::test::new_session; @@ -734,4 +740,62 @@ mod tests { assert_eq!(values.len(), 3); Ok(()) } + + /// Scans 64 one-row splits, returning the rows and the most splits read at once. + fn scan_with_split_concurrency( + split_concurrency: Option, + ) -> VortexResult<(Vec, usize)> { + let runtime = SingleThreadRuntime::default(); + let session = session_with_handle(runtime.handle()); + let in_flight = Arc::new(InFlight::default()); + let reader: LayoutReaderRef = Arc::new( + TestLayoutReader::new(64) + .with_split_size(1) + .with_in_flight(Arc::clone(&in_flight)), + ); + let source = MultiLayoutDataSource::new_with_first(reader, vec![], vec![], &session); + + let scan = runtime.block_on(source.scan(ScanRequest { + ordered: true, + split_concurrency, + ..Default::default() + }))?; + let mut values = Vec::new(); + for partition in runtime.block_on(scan.partitions().try_collect::>())? { + values.extend(collect_scan_values( + runtime.block_on_stream(partition.execute()?), + )?); + } + Ok((values, in_flight.peak())) + } + + /// A per-worker concurrency set after the bound leaves the bound in place. + #[test] + fn a_later_per_worker_concurrency_keeps_the_bound() { + let reader: LayoutReaderRef = Arc::new(TestLayoutReader::new(1)); + let builder = ScanBuilder::new(new_session(), reader) + .with_some_max_splits_in_flight(NonZeroUsize::new(2)) + .with_concurrency(8); + + assert_eq!(builder.concurrency(), 8); + assert_eq!(builder.max_splits_in_flight(), NonZeroUsize::new(2)); + } + + /// The scan reads at most `split_concurrency` splits at once, however many cores there are. + #[rstest] + #[case(1)] + #[case(2)] + fn split_concurrency_bounds_the_splits_read_at_once(#[case] max: usize) -> VortexResult<()> { + let (values, peak) = scan_with_split_concurrency(NonZeroUsize::new(max))?; + let (default_values, default_peak) = scan_with_split_concurrency(None)?; + + assert_eq!(values, (0..64).collect::>()); + assert_eq!(values, default_values); + assert_eq!(peak, max, "splits read at once"); + assert!( + default_peak > max, + "the default read {default_peak} at once" + ); + Ok(()) + } } diff --git a/vortex-layout/src/scan/repeated_scan.rs b/vortex-layout/src/scan/repeated_scan.rs index 4200d00614a..7866713a5c5 100644 --- a/vortex-layout/src/scan/repeated_scan.rs +++ b/vortex-layout/src/scan/repeated_scan.rs @@ -3,6 +3,7 @@ use std::cmp; use std::iter; +use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; use std::task::Poll; @@ -61,8 +62,8 @@ pub struct RepeatedScan { selection: Selection, /// The natural splits of the file. splits: Splits, - /// The number of splits to make progress on concurrently **per-thread**. - concurrency: usize, + /// How many splits to make progress on at once. + concurrency: SplitConcurrency, /// Function to apply to each [`ArrayRef`] within the spawned split tasks. map_fn: MapFn, /// Maximal number of rows to read (after filtering). @@ -136,7 +137,7 @@ impl RepeatedScan { row_range: Option>, selection: Selection, splits: Splits, - concurrency: usize, + concurrency: SplitConcurrency, map_fn: MapFn, limit: Option, dtype: DType, @@ -240,7 +241,7 @@ impl RepeatedScan { &self, row_range: Option>, ) -> VortexResult>> { - let concurrency = self.concurrency * get_available_parallelism().unwrap_or(1); + let concurrency = self.concurrency.splits_in_flight(); let handle = self.session.handle(); let ordered = self.ordered; let ctx = self.task_context(); @@ -383,6 +384,23 @@ fn stop_on_error( .boxed() } +/// How many splits a scan makes progress on at once: `per_worker` for each worker thread, as +/// [`get_available_parallelism`] counts them, and at most `max` in all when that is set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SplitConcurrency { + pub(crate) per_worker: usize, + pub(crate) max: Option, +} + +impl SplitConcurrency { + fn splits_in_flight(self) -> usize { + let scaled = self + .per_worker + .saturating_mul(get_available_parallelism().unwrap_or(1)); + self.max.map_or(scaled, |max| scaled.min(max.get())) + } +} + /// Run up to `concurrency` futures of `tasks` at once, in order when `ordered` is set. fn buffer( tasks: impl Stream + Send + 'static> + Send + 'static, @@ -407,6 +425,7 @@ fn intersect_ranges(left: Option<&Range>, right: Option>) -> Opt #[cfg(test)] mod tests { + use std::num::NonZeroUsize; use std::sync::Arc; use std::task::Poll; @@ -419,10 +438,20 @@ mod tests { use vortex_error::VortexResult; use vortex_error::vortex_err; + use super::SplitConcurrency; use super::limit_rows; use super::stop_on_error; use crate::scan::limit::RowLimit; + #[test] + fn split_cap_handles_oversized_per_worker_concurrency() { + let concurrency = SplitConcurrency { + per_worker: usize::MAX, + max: NonZeroUsize::new(1), + }; + assert_eq!(concurrency.splits_in_flight(), 1); + } + #[rstest] #[case::whole_array(4)] #[case::trimmed_array(2)] diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index 21dc22c04ca..d8c11e5541d 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::num::NonZeroUsize; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; @@ -39,6 +40,7 @@ use crate::layouts::row_idx::RowIdxLayoutReader; use crate::scan::limit::RowLimit; use crate::scan::limit::ScanLimit; use crate::scan::repeated_scan::RepeatedScan; +use crate::scan::repeated_scan::SplitConcurrency; use crate::scan::split_by::SplitBy; use crate::scan::splits::Splits; use crate::scan::splits::attempt_split_ranges; @@ -71,7 +73,7 @@ pub struct ScanBuilder { /// uses them instead of walking the layout. natural_splits: Option>, /// The number of splits to make progress on concurrently **per-thread**. - concurrency: usize, + concurrency: SplitConcurrency, /// Function to apply to each [`ArrayRef`] within the spawned split tasks. map_fn: Arc VortexResult + Send + Sync>, metrics_registry: Option>, @@ -98,7 +100,10 @@ impl ScanBuilder { natural_splits: None, // We default to four tasks per worker thread, which allows for some I/O lookahead // without too much impact on work-stealing. - concurrency: 4, + concurrency: SplitConcurrency { + per_worker: 4, + max: None, + }, map_fn: Arc::new(Ok), metrics_registry: None, limit: None, @@ -222,14 +227,31 @@ impl ScanBuilder { /// Returns the per-worker row-split concurrency. pub fn concurrency(&self) -> usize { - self.concurrency + self.concurrency.per_worker + } + + /// Returns the most row splits in progress at once, in all, if a bound is set. + pub fn max_splits_in_flight(&self) -> Option { + self.concurrency.max } /// The number of row splits to make progress on concurrently per-thread, must /// be greater than 0. pub fn with_concurrency(mut self, concurrency: usize) -> Self { assert!(concurrency > 0); - self.concurrency = concurrency; + self.concurrency.per_worker = concurrency; + self + } + + /// Bound the row splits in progress at once to `max` in all, whatever the per-worker + /// concurrency and the machine's core count. `None` removes the bound. A filtered, limited + /// scan may hold one more trimmed array per split while it is mapped. + /// + /// Each split in progress holds the decoded chunks of every column it overlaps, so a scan + /// that runs alongside others, such as one task of several on a machine, can set this to + /// reduce memory held by concurrent decoding. This does not cap total memory or I/O read-ahead. + pub fn with_some_max_splits_in_flight(mut self, max: Option) -> Self { + self.concurrency.max = max; self } diff --git a/vortex-layout/src/scan/test.rs b/vortex-layout/src/scan/test.rs index 7d030f2c987..53bca5c5b5b 100644 --- a/vortex-layout/src/scan/test.rs +++ b/vortex-layout/src/scan/test.rs @@ -4,6 +4,8 @@ use std::ops::Range; use std::sync::Arc; use std::sync::LazyLock; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use std::task::Poll; use parking_lot::Mutex; @@ -66,6 +68,20 @@ pub struct TestLayoutReader { fail_projection: bool, projection_masks: Option>>>, filter_exprs: Option>>>, + in_flight: Option>, +} + +/// Counts projections running at once, and the most that ever were. +#[derive(Debug, Default)] +pub struct InFlight { + now: AtomicUsize, + peak: AtomicUsize, +} + +impl InFlight { + pub fn peak(&self) -> usize { + self.peak.load(Ordering::SeqCst) + } } impl TestLayoutReader { @@ -82,6 +98,7 @@ impl TestLayoutReader { fail_projection: false, projection_masks: None, filter_exprs: None, + in_flight: None, } } @@ -111,6 +128,13 @@ impl TestLayoutReader { self } + /// Records into `in_flight` how many projections are running at once. Each projection yields + /// once before finishing, so others that the scan has started get to run alongside it. + pub fn with_in_flight(mut self, in_flight: Arc) -> Self { + self.in_flight = Some(in_flight); + self + } + pub fn with_filter_exprs(mut self, filter_exprs: Arc>>) -> Self { self.filter_exprs = Some(filter_exprs); self @@ -213,8 +237,15 @@ impl LayoutReader for TestLayoutReader { let base = self.base; let projection_masks = self.projection_masks.clone(); let fail = self.fail_projection || (self.fail_first_projection && row_range.start == 0); + let in_flight = self.in_flight.clone(); Ok(Box::pin(async move { + if let Some(in_flight) = &in_flight { + let now = in_flight.now.fetch_add(1, Ordering::SeqCst) + 1; + in_flight.peak.fetch_max(now, Ordering::SeqCst); + tokio::task::yield_now().await; + in_flight.now.fetch_sub(1, Ordering::SeqCst); + } let mask = mask.await?; if let Some(projection_masks) = projection_masks { projection_masks.lock().push(mask.true_count()); diff --git a/vortex-scan/src/lib.rs b/vortex-scan/src/lib.rs index 755ec472c6c..a0454c6c1a6 100644 --- a/vortex-scan/src/lib.rs +++ b/vortex-scan/src/lib.rs @@ -28,6 +28,7 @@ pub mod selection; pub mod strict_sorted_buffer; use std::any::Any; +use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; @@ -170,6 +171,13 @@ pub struct ScanRequest { /// order is completion order, so each partition applies the limit locally and the caller must /// trim the concatenated result. pub limit: Option, + /// The most splits of each partition to decode at once, in all. + /// + /// `None` keeps the reader's default, which scales with the machine's cores. Each split being + /// decoded holds the decoded chunks of every column it overlaps, so a scan that runs alongside + /// others, such as one task of several on a machine, can set this to bound its memory. It does + /// not bound I/O: segment reads for later splits may still be coalesced ahead of them. + pub split_concurrency: Option, } impl Default for ScanRequest { @@ -183,6 +191,7 @@ impl Default for ScanRequest { ordered: false, limit: None, partition_range: None, + split_concurrency: None, } } }