From a7b6e71ad6fa61272b05a345a10e3d09bed7ea04 Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Mon, 2 Nov 2026 12:59:29 +0000 Subject: [PATCH 1/3] feat(scan): let a scan request bound the splits it decodes at once ScanRequest gains split_concurrency: the most splits of each partition decoded at once, in all. Both layout data sources pass it to ScanBuilder::with_some_max_splits_in_flight, a cap kept apart from the per-worker concurrency. JNI takes it as ScanOptions.splitConcurrency, and also bounds the Arrow conversion it runs ahead of Java by it; Spark takes it as the vortex.splitConcurrency read option. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christopher Wirt --- docs/user-guide/spark.md | 1 + .../main/java/dev/vortex/api/DataSource.java | 4 +- .../main/java/dev/vortex/api/ScanOptions.java | 19 ++++++ .../main/java/dev/vortex/jni/NativeScan.java | 4 +- .../java/dev/vortex/api/ScanOptionsTest.java | 16 +++++ .../test/java/dev/vortex/api/TestMinimal.java | 12 ++++ .../java/dev/vortex/spark/io/VortexIo.java | 34 ++++++++-- .../spark/read/VortexPartitionReader.java | 3 + .../spark/VortexSessionOptionsTest.java | 18 ++++- .../dev/vortex/spark/io/VortexIoTest.java | 27 +++++++- vortex-ffi/src/scan.rs | 1 + vortex-jni/src/scan.rs | 54 ++++++++++----- vortex-layout/src/scan/layout.rs | 8 ++- vortex-layout/src/scan/multi.rs | 66 ++++++++++++++++++- vortex-layout/src/scan/repeated_scan.rs | 24 +++++-- vortex-layout/src/scan/scan_builder.rs | 30 +++++++-- vortex-layout/src/scan/test.rs | 31 +++++++++ vortex-scan/src/lib.rs | 9 +++ 18 files changed, 323 insertions(+), 38 deletions(-) diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index d78f3559bd8..8b4bce44ad6 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -236,6 +236,7 @@ depends on, on the driver and executor classpath, for example | Option | Default | Description | |----------------------------|---------|-------------| | `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. | +| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Each split holds the decoded chunks of every column it reads, and every task in an executor runs its own scan, so lowering this bounds a task's native memory. | | `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. | | `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. | | `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. | diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java index 8658cf5198b..22c090c6419 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java @@ -238,6 +238,7 @@ public Scan scan(ScanOptions options) { byte[] selectionRoaringBitmap = selectionRoaringBitmap(options); long limit = options.limit().orElse(0L); boolean ordered = options.ordered(); + int splitConcurrency = options.splitConcurrency().orElse(0); long scanPtr = dev.vortex.jni.NativeScan.create( pointer, @@ -249,7 +250,8 @@ public Scan scan(ScanOptions options) { selectionRoaringBitmap, selectionMode.code(), limit, - ordered); + ordered, + splitConcurrency); return Scan.fromPointer(session, scanPtr); } diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java index 25f03e76f7f..865884647ed 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java @@ -9,6 +9,7 @@ import java.io.UncheckedIOException; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.OptionalLong; import org.immutables.value.Value; import org.roaringbitmap.longlong.Roaring64NavigableMap; @@ -59,6 +60,24 @@ default boolean ordered() { return false; } + /** + * The most splits of each partition the native scan decodes at once, in all, and the most of its batches converted + * to Arrow ahead of the caller. Absent keeps the native default, which scales with the machine's cores. + * + *

Each split being decoded holds the decoded chunks of every column it overlaps, so a scan running alongside + * others, such as one Spark task of several in an executor, can set this to bound its native memory. It does not + * bound I/O: segment reads for later splits may still be coalesced ahead of them. + */ + OptionalInt splitConcurrency(); + + @Value.Check + default void validateSplitConcurrency() { + if (splitConcurrency().isPresent() && splitConcurrency().getAsInt() <= 0) { + throw new IllegalArgumentException("split concurrency must be positive, got " + + splitConcurrency().getAsInt()); + } + } + @Value.Check default void validateSelectionPayload() { boolean hasIndices = selectionIndices().isPresent(); diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java index ff80b30caaa..ff4ad73a317 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java @@ -27,6 +27,7 @@ private NativeScan() {} * {@code selectionRoaringBitmap}) * @param limit max rows to return, or {@code 0} for "no limit" * @param ordered true to preserve row order across partitions + * @param splitConcurrency the most splits of each partition read at once, or {@code 0} for the native default */ public static native long create( long dataSourcePointer, @@ -38,7 +39,8 @@ public static native long create( byte[] selectionRoaringBitmap, byte selectionInclude, long limit, - boolean ordered); + boolean ordered, + int splitConcurrency); /** Free a scan pointer. */ public static native void free(long pointer); diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java b/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java index c72e67f4051..cb0c6ff5b8a 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java @@ -12,6 +12,7 @@ import dev.vortex.api.ScanOptions.SelectionMode; import java.io.ByteArrayInputStream; import java.io.DataInputStream; +import java.util.OptionalInt; import java.util.OptionalLong; import org.junit.jupiter.api.Test; import org.roaringbitmap.longlong.Roaring64NavigableMap; @@ -39,6 +40,21 @@ public void defaultsReadEveryRowAndColumn() { assertTrue(options.selectionRoaringBitmap().isEmpty()); assertEquals(SelectionMode.INCLUDE_ALL, options.selectionMode()); assertFalse(options.ordered()); + assertEquals(OptionalInt.empty(), options.splitConcurrency()); + } + + @Test + public void splitConcurrencyMustBePositive() { + for (int value : new int[] {0, -1}) { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> ScanOptions.builder().splitConcurrency(value).build(), + () -> "no exception for " + value); + assertEquals("split concurrency must be positive, got " + value, exception.getMessage()); + } + assertEquals( + OptionalInt.of(2), + ScanOptions.builder().splitConcurrency(2).build().splitConcurrency()); } @Test diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java index 2b270c3e165..f4adefed093 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java @@ -184,6 +184,18 @@ public void testProjectedScanWithFilter() throws Exception { assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people); } + @Test + public void testScanWithSplitConcurrencyReadsEveryRow() throws Exception { + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Session session = Session.create(); + DataSource ds = DataSource.open(session, writePath); + + ScanOptions options = + ScanOptions.builder().splitConcurrency(1).ordered(true).build(); + List people = readAll(ds, options, allocator, TestMinimal::readFullBatch); + assertEquals(MINIMAL_DATA, people); + } + @Test public void testSelectionIncludesRows() throws Exception { BufferAllocator allocator = ArrowAllocation.rootAllocator(); diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java index 5bc35a81808..db60286364d 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java @@ -30,31 +30,46 @@ public final class VortexIo implements Serializable { private static final int DEFAULT_READ_CONCURRENCY = 0; + /** + * Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the + * decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default + * keeps several splits in flight per core, so setting this bounds a task's native memory. Zero keeps the native + * default. + */ + public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency"; + + private static final int DEFAULT_SPLIT_CONCURRENCY = 0; + private final SerializableHadoopConf conf; private final int readConcurrency; + private final int splitConcurrency; /** Captures {@code hadoopConf} and the read settings found in the format options. */ public static VortexIo create(VortexOptions options, Configuration hadoopConf) { Objects.requireNonNull(options, "options"); Objects.requireNonNull(hadoopConf, "hadoopConf"); - return new VortexIo(new SerializableHadoopConf(hadoopConf), parseReadConcurrency(options)); + return new VortexIo( + new SerializableHadoopConf(hadoopConf), + nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY), + nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY)); } /** Hadoop I/O over a default configuration, for callers that have no Spark session to draw one from. */ public static VortexIo defaults() { - return new VortexIo(new SerializableHadoopConf(new Configuration()), DEFAULT_READ_CONCURRENCY); + return new VortexIo( + new SerializableHadoopConf(new Configuration()), DEFAULT_READ_CONCURRENCY, DEFAULT_SPLIT_CONCURRENCY); } - private VortexIo(SerializableHadoopConf conf, int readConcurrency) { + private VortexIo(SerializableHadoopConf conf, int readConcurrency, int splitConcurrency) { this.conf = conf; this.readConcurrency = readConcurrency; + this.splitConcurrency = splitConcurrency; } - private static int parseReadConcurrency(VortexOptions options) { - int parsed = options.getInt(READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY); + private static int nonNegative(VortexOptions options, String key, int defaultValue) { + int parsed = options.getInt(key, defaultValue); if (parsed < 0) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "%s must be >= 0, got %d", READ_CONCURRENCY_OPTION, parsed)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "%s must be >= 0, got %d", key, parsed)); } return parsed; } @@ -68,6 +83,11 @@ public int readConcurrency() { return readConcurrency; } + /** The most splits of a file the native scan reads at once; zero keeps the native default. */ + public int splitConcurrency() { + return splitConcurrency; + } + /** * Opens a byte source for {@code file}, stating it only if the listing that produced it did not report a size. The * caller owns the result and must close it once the scan built on it is done. diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java index 44de7a423ed..1d8c93e0562 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java @@ -115,6 +115,9 @@ public VortexPartitionReader( projectedSchema, arrowFields, caseSensitive, file.toPath().toString()); var options = ScanOptions.builder(); + if (io.splitConcurrency() > 0) { + options.splitConcurrency(io.splitConcurrency()); + } // Always project, an empty read schema included: a query that needs only partition columns or only a // row count must not pull every data column off storage. Struct columns that Spark pruned read only the // nested fields the query uses. diff --git a/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java b/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java index 6d528f49f1f..05d4518d985 100644 --- a/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java +++ b/java/vortex-spark/common/src/test/java/dev/vortex/spark/VortexSessionOptionsTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import dev.vortex.spark.io.VortexIo; import java.nio.file.Path; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -43,7 +44,9 @@ public void setUp() { @AfterEach public void resetConf() { - for (String option : new String[] {READ_CONCURRENCY, VortexSparkSession.PROVIDER_OPTION}) { + for (String option : + new String[] {READ_CONCURRENCY, VortexIo.SPLIT_CONCURRENCY_OPTION, VortexSparkSession.PROVIDER_OPTION + }) { spark.conf().unset(option); spark.conf().unset(DATA_SOURCE_PREFIX + option); } @@ -84,6 +87,19 @@ void aSessionSettingAppliesToV1Reads() { assertSessionSettingAppliesToReads(write("v1"), "FileScan vortex", READ_CONCURRENCY); } + @Test + void aSplitConcurrencyReadsEveryRowThroughV1AndV2() { + String path = write("split_concurrency"); + for (String useV1 : new String[] {"", "vortex"}) { + spark.conf().set(USE_V1, useV1); + Dataset read = spark.read() + .format("vortex") + .option(VortexIo.SPLIT_CONCURRENCY_OPTION, "1") + .load(path); + assertEquals(45L, read.selectExpr("sum(id)").first().getLong(0), "useV1List=" + useV1); + } + } + @Test void aSessionSettingAppliesToWrites() { assertSessionSettingAppliesToWrites("written", VortexSparkSession.PROVIDER_OPTION); diff --git a/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java b/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java index a7800f25786..019f811fca0 100644 --- a/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java +++ b/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/VortexIoTest.java @@ -18,10 +18,30 @@ final class VortexIoTest { @Test - void readConcurrencyDefaultsToNativeDefault() { + void concurrenciesDefaultToNativeDefaults() { VortexIo io = VortexIo.create(VortexOptions.empty(), new Configuration()); assertEquals(0, io.readConcurrency()); + assertEquals(0, io.splitConcurrency()); + } + + @Test + void splitConcurrencyIsParsedAndValidated() { + Configuration conf = new Configuration(); + + assertEquals( + 2, + VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "2"), conf) + .splitConcurrency()); + assertEquals( + 2, + VortexIo.create(options("VORTEX.SPLITCONCURRENCY", "2"), conf).splitConcurrency()); + assertThrows( + IllegalArgumentException.class, + () -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "-1"), conf)); + assertThrows( + IllegalArgumentException.class, + () -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "many"), conf)); } @Test @@ -54,11 +74,14 @@ void readConcurrencyIsFoundUnderAnyKeyCase() { void settingsAndConfigurationSurviveSerialization() throws IOException, ClassNotFoundException { Configuration conf = new Configuration(); conf.set("fs.s3a.endpoint", "https://storage.example"); - VortexIo io = VortexIo.create(options(VortexIo.READ_CONCURRENCY_OPTION, "4"), conf); + VortexIo io = VortexIo.create( + VortexOptions.of(Map.of(VortexIo.READ_CONCURRENCY_OPTION, "4", VortexIo.SPLIT_CONCURRENCY_OPTION, "3")), + conf); VortexIo shipped = roundTrip(io); assertEquals(4, shipped.readConcurrency()); + assertEquals(3, shipped.splitConcurrency()); assertEquals("https://storage.example", shipped.hadoopConf().get("fs.s3a.endpoint")); } diff --git a/vortex-ffi/src/scan.rs b/vortex-ffi/src/scan.rs index b58caddfb69..e14301c3475 100644 --- a/vortex-ffi/src/scan.rs +++ b/vortex-ffi/src/scan.rs @@ -208,6 +208,7 @@ fn scan_request(opts: *const vx_scan_options) -> VortexResult { limit, partition_selection: Selection::All, partition_range: None, + split_concurrency: None, }) } diff --git a/vortex-jni/src/scan.rs b/vortex-jni/src/scan.rs index 125558256f9..6878af0f7d2 100644 --- a/vortex-jni/src/scan.rs +++ b/vortex-jni/src/scan.rs @@ -11,6 +11,7 @@ //! `FFI_ArrowArrayStream` that Java imports via Arrow's C Data Interface. use std::io::Cursor; +use std::num::NonZeroUsize; use std::ops::Range; use std::ptr; use std::sync::Arc; @@ -27,12 +28,14 @@ use jni::objects::JByteArray; use jni::objects::JClass; use jni::objects::JLongArray; use jni::sys::jboolean; +use jni::sys::jint; use jni::sys::jlong; use vortex::array::VortexSessionExecute; use vortex::array::stream::SendableArrayStream; use vortex::buffer::Buffer; use vortex::error::VortexResult; use vortex::error::vortex_bail; +use vortex::error::vortex_err; use vortex::expr::Expression; use vortex::expr::root; use vortex::expr::stats::Precision; @@ -55,17 +58,21 @@ use crate::session::session_ref; /// Opaque scan handle. Holds a three-state machine: either the scan is pending (not yet /// realized as a stream), actively streaming partitions, or finished. +/// +/// The scan's `split_concurrency`, if set, also bounds how many of each partition's arrays are +/// converted to Arrow ahead of Java, so the conversion stays within the scan's bound. #[allow(dead_code)] pub(crate) enum NativeScan { - Pending(Box), - Started(PartitionStream), + Pending(Box, Option), + Started(PartitionStream, Option), Finished, } -/// Opaque partition handle with the same three-state machine shape. +/// Opaque partition handle with the same three-state machine shape, carrying its scan's +/// `split_concurrency`. #[allow(dead_code)] pub(crate) enum NativePartition { - Pending(PartitionRef), + Pending(PartitionRef, Option), Started(SendableArrayStream), Finished, } @@ -81,6 +88,7 @@ fn build_scan_request( selection_include: u8, limit: jlong, ordered: jboolean, + split_concurrency: jint, ) -> VortexResult { let projection = if projection_ptr == 0 { root() @@ -113,6 +121,10 @@ fn build_scan_request( }); let limit = (limit > 0).then_some(limit as u64); + let split_concurrency = + NonZeroUsize::new(usize::try_from(split_concurrency).map_err(|_| { + vortex_err!("split concurrency must not be negative, got {split_concurrency}") + })?); Ok(ScanRequest { projection, @@ -123,6 +135,7 @@ fn build_scan_request( limit, partition_selection: Selection::All, partition_range: None, + split_concurrency, }) } @@ -149,6 +162,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_create( selection_include: jni::sys::jbyte, limit: jlong, ordered: jboolean, + split_concurrency: jint, ) -> jlong { try_or_throw(&mut env, |env| { let ds = unsafe { NativeDataSource::from_ptr(data_source_ptr) }; @@ -185,10 +199,12 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_create( selection_include as u8, limit, ordered, + split_concurrency, )?; + let split_concurrency = request.split_concurrency; let scan = RUNTIME.block_on(async { ds.inner().scan(request).await })?; - Ok(Box::into_raw(Box::new(NativeScan::Pending(scan))) as jlong) + Ok(Box::into_raw(Box::new(NativeScan::Pending(scan, split_concurrency))) as jlong) }) } @@ -219,7 +235,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_arrowSchema( } let session = unsafe { session_ref(session_ptr) }; let scan = unsafe { &*(pointer as *const NativeScan) }; - let NativeScan::Pending(scan) = scan else { + let NativeScan::Pending(scan, _) = scan else { throw_runtime!("schema unavailable: scan already started"); }; let arrow_schema = session.arrow().to_arrow_schema(scan.dtype())?; @@ -241,7 +257,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_partitionCount( ) { try_or_throw(&mut env, |env| { let scan = unsafe { &*(pointer as *const NativeScan) }; - let NativeScan::Pending(scan) = scan else { + let NativeScan::Pending(scan, _) = scan else { throw_runtime!("partition count unavailable: scan already started"); }; let (rows, cardinality) = match scan.partition_count() { @@ -271,17 +287,20 @@ pub extern "system" fn Java_dev_vortex_jni_NativeScan_nextPartition( // only rewrite it on the happy path where we want to keep streaming. ptr::write(slot, NativeScan::Finished); - let mut stream = match owned { - NativeScan::Pending(scan) => scan.partitions(), - NativeScan::Started(stream) => stream, + let (mut stream, split_concurrency) = match owned { + NativeScan::Pending(scan, split_concurrency) => (scan.partitions(), split_concurrency), + NativeScan::Started(stream, split_concurrency) => (stream, split_concurrency), NativeScan::Finished => return Ok(0), }; match RUNTIME.block_on(stream.next()) { Some(partition) => { let partition = partition?; - let handle = Box::into_raw(Box::new(NativePartition::Pending(partition))) as jlong; - ptr::write(slot, NativeScan::Started(stream)); + let handle = Box::into_raw(Box::new(NativePartition::Pending( + partition, + split_concurrency, + ))) as jlong; + ptr::write(slot, NativeScan::Started(stream, split_concurrency)); Ok(handle) } None => Ok(0), @@ -311,7 +330,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_rowCount( ) { try_or_throw(&mut env, |env| { let partition = unsafe { &*(pointer as *const NativePartition) }; - let NativePartition::Pending(partition) = partition else { + let NativePartition::Pending(partition, _) = partition else { throw_runtime!("row count unavailable: partition already started"); }; let (rows, cardinality) = match partition.row_count() { @@ -346,10 +365,13 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( throw_runtime!("null arrow stream address"); } - let partition = match partition { - NativePartition::Pending(p) => p, + let (partition, split_concurrency) = match partition { + NativePartition::Pending(p, split_concurrency) => (p, split_concurrency), _ => throw_runtime!("partition already consumed"), }; + let lookahead = split_concurrency + .map_or(usize::MAX, NonZeroUsize::get) + .min(POOL.worker_count().max(1)); let array_stream = partition.execute()?; let dtype = array_stream.dtype().clone(); @@ -375,7 +397,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( rebase_offsets(RecordBatch::from(arrow.as_struct().clone())) }) }) - .buffered(POOL.worker_count().max(1)) + .buffered(lookahead) }) .map(|result: VortexResult| { result.map_err(|e| ArrowError::ExternalError(Box::new(e))) diff --git a/vortex-layout/src/scan/layout.rs b/vortex-layout/src/scan/layout.rs index ae670958ff4..62b1805aacf 100644 --- a/vortex-layout/src/scan/layout.rs +++ b/vortex-layout/src/scan/layout.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::any::Any; +use std::num::NonZeroUsize; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; @@ -182,6 +183,7 @@ impl DataSource for LayoutReaderDataSource { row_limit, selection: scan_request.selection, ordered: scan_request.ordered, + split_concurrency: scan_request.split_concurrency, metrics_registry: self.metrics_registry.clone(), next_row: row_range.start, end_row: row_range.end, @@ -207,6 +209,7 @@ struct LayoutReaderScan { /// A row budget shared by every split of an unordered limited scan; see [`RowLimit`]. row_limit: Option, ordered: bool, + split_concurrency: Option, selection: Selection, metrics_registry: Option>, next_row: u64, @@ -276,6 +279,7 @@ impl Stream for LayoutReaderScan { limit: split_limit, row_limit: this.row_limit.clone(), ordered: this.ordered, + split_concurrency: this.split_concurrency, row_range, selection: this.selection.clone(), metrics_registry: this.metrics_registry.clone(), @@ -309,6 +313,7 @@ struct LayoutReaderSplit { /// A row budget shared with the scan's other splits (unordered limited scans only). row_limit: Option, ordered: bool, + split_concurrency: Option, row_range: Range, selection: Selection, metrics_registry: Option>, @@ -347,7 +352,8 @@ impl Partition for LayoutReaderSplit { .with_projection(self.projection) .with_some_filter(self.filter) .with_some_metrics_registry(self.metrics_registry) - .with_ordered(self.ordered); + .with_ordered(self.ordered) + .with_some_max_splits_in_flight(self.split_concurrency); let builder = match self.row_limit { Some(row_limit) => builder.with_shared_limit(row_limit), None => builder.with_some_limit(self.limit), diff --git a/vortex-layout/src/scan/multi.rs b/vortex-layout/src/scan/multi.rs index c4a0d56fe3b..1f1ee275688 100644 --- a/vortex-layout/src/scan/multi.rs +++ b/vortex-layout/src/scan/multi.rs @@ -26,6 +26,7 @@ use std::any::Any; use std::collections::VecDeque; +use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; @@ -341,6 +342,7 @@ struct BoundScanRequest { partition_range: Option>, ordered: bool, limit: Option, + split_concurrency: Option, } impl BoundScanRequest { @@ -354,6 +356,7 @@ impl BoundScanRequest { partition_range, ordered, limit, + split_concurrency, } = request; Ok(Self { @@ -368,6 +371,7 @@ impl BoundScanRequest { partition_range, ordered, limit, + split_concurrency, }) } } @@ -599,7 +603,8 @@ impl Partition for MultiLayoutPartition { .with_selection(request.selection) .with_projection(request.projection) .with_some_filter(filter) - .with_ordered(request.ordered); + .with_ordered(request.ordered) + .with_some_max_splits_in_flight(request.split_concurrency); builder = match self.row_limit { Some(row_limit) => builder.with_shared_limit(row_limit), None => builder.with_some_limit(request.limit), @@ -637,6 +642,7 @@ mod tests { use vortex_scan::ScanRequest; use super::*; + use crate::scan::test::InFlight; use crate::scan::test::TestLayoutReader; use crate::scan::test::collect_scan_values; use crate::scan::test::new_session; @@ -734,4 +740,62 @@ mod tests { assert_eq!(values.len(), 3); Ok(()) } + + /// Scans 64 one-row splits, returning the rows and the most splits read at once. + fn scan_with_split_concurrency( + split_concurrency: Option, + ) -> VortexResult<(Vec, usize)> { + let runtime = SingleThreadRuntime::default(); + let session = session_with_handle(runtime.handle()); + let in_flight = Arc::new(InFlight::default()); + let reader: LayoutReaderRef = Arc::new( + TestLayoutReader::new(64) + .with_split_size(1) + .with_in_flight(Arc::clone(&in_flight)), + ); + let source = MultiLayoutDataSource::new_with_first(reader, vec![], vec![], &session); + + let scan = runtime.block_on(source.scan(ScanRequest { + ordered: true, + split_concurrency, + ..Default::default() + }))?; + let mut values = Vec::new(); + for partition in runtime.block_on(scan.partitions().try_collect::>())? { + values.extend(collect_scan_values( + runtime.block_on_stream(partition.execute()?), + )?); + } + Ok((values, in_flight.peak())) + } + + /// A per-worker concurrency set after the bound leaves the bound in place. + #[test] + fn a_later_per_worker_concurrency_keeps_the_bound() { + let reader: LayoutReaderRef = Arc::new(TestLayoutReader::new(1)); + let builder = ScanBuilder::new(new_session(), reader) + .with_some_max_splits_in_flight(NonZeroUsize::new(2)) + .with_concurrency(8); + + assert_eq!(builder.concurrency(), 8); + assert_eq!(builder.max_splits_in_flight(), NonZeroUsize::new(2)); + } + + /// The scan reads at most `split_concurrency` splits at once, however many cores there are. + #[rstest] + #[case(1)] + #[case(2)] + fn split_concurrency_bounds_the_splits_read_at_once(#[case] max: usize) -> VortexResult<()> { + let (values, peak) = scan_with_split_concurrency(NonZeroUsize::new(max))?; + let (default_values, default_peak) = scan_with_split_concurrency(None)?; + + assert_eq!(values, (0..64).collect::>()); + assert_eq!(values, default_values); + assert_eq!(peak, max, "splits read at once"); + assert!( + default_peak > max, + "the default read {default_peak} at once" + ); + Ok(()) + } } diff --git a/vortex-layout/src/scan/repeated_scan.rs b/vortex-layout/src/scan/repeated_scan.rs index 4200d00614a..91448f6988b 100644 --- a/vortex-layout/src/scan/repeated_scan.rs +++ b/vortex-layout/src/scan/repeated_scan.rs @@ -3,6 +3,7 @@ use std::cmp; use std::iter; +use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; use std::task::Poll; @@ -61,8 +62,8 @@ pub struct RepeatedScan { selection: Selection, /// The natural splits of the file. splits: Splits, - /// The number of splits to make progress on concurrently **per-thread**. - concurrency: usize, + /// How many splits to make progress on at once. + concurrency: SplitConcurrency, /// Function to apply to each [`ArrayRef`] within the spawned split tasks. map_fn: MapFn, /// Maximal number of rows to read (after filtering). @@ -136,7 +137,7 @@ impl RepeatedScan { row_range: Option>, selection: Selection, splits: Splits, - concurrency: usize, + concurrency: SplitConcurrency, map_fn: MapFn, limit: Option, dtype: DType, @@ -240,7 +241,7 @@ impl RepeatedScan { &self, row_range: Option>, ) -> VortexResult>> { - let concurrency = self.concurrency * get_available_parallelism().unwrap_or(1); + let concurrency = self.concurrency.splits_in_flight(); let handle = self.session.handle(); let ordered = self.ordered; let ctx = self.task_context(); @@ -383,6 +384,21 @@ fn stop_on_error( .boxed() } +/// How many splits a scan makes progress on at once: `per_worker` for each worker thread, as +/// [`get_available_parallelism`] counts them, and at most `max` in all when that is set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SplitConcurrency { + pub(crate) per_worker: usize, + pub(crate) max: Option, +} + +impl SplitConcurrency { + fn splits_in_flight(self) -> usize { + let scaled = self.per_worker * get_available_parallelism().unwrap_or(1); + self.max.map_or(scaled, |max| scaled.min(max.get())) + } +} + /// Run up to `concurrency` futures of `tasks` at once, in order when `ordered` is set. fn buffer( tasks: impl Stream + Send + 'static> + Send + 'static, diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index 21dc22c04ca..08a5700d6e7 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::num::NonZeroUsize; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; @@ -39,6 +40,7 @@ use crate::layouts::row_idx::RowIdxLayoutReader; use crate::scan::limit::RowLimit; use crate::scan::limit::ScanLimit; use crate::scan::repeated_scan::RepeatedScan; +use crate::scan::repeated_scan::SplitConcurrency; use crate::scan::split_by::SplitBy; use crate::scan::splits::Splits; use crate::scan::splits::attempt_split_ranges; @@ -71,7 +73,7 @@ pub struct ScanBuilder { /// uses them instead of walking the layout. natural_splits: Option>, /// The number of splits to make progress on concurrently **per-thread**. - concurrency: usize, + concurrency: SplitConcurrency, /// Function to apply to each [`ArrayRef`] within the spawned split tasks. map_fn: Arc VortexResult + Send + Sync>, metrics_registry: Option>, @@ -98,7 +100,10 @@ impl ScanBuilder { natural_splits: None, // We default to four tasks per worker thread, which allows for some I/O lookahead // without too much impact on work-stealing. - concurrency: 4, + concurrency: SplitConcurrency { + per_worker: 4, + max: None, + }, map_fn: Arc::new(Ok), metrics_registry: None, limit: None, @@ -222,14 +227,31 @@ impl ScanBuilder { /// Returns the per-worker row-split concurrency. pub fn concurrency(&self) -> usize { - self.concurrency + self.concurrency.per_worker + } + + /// Returns the most row splits in progress at once, in all, if a bound is set. + pub fn max_splits_in_flight(&self) -> Option { + self.concurrency.max } /// The number of row splits to make progress on concurrently per-thread, must /// be greater than 0. pub fn with_concurrency(mut self, concurrency: usize) -> Self { assert!(concurrency > 0); - self.concurrency = concurrency; + self.concurrency.per_worker = concurrency; + self + } + + /// Bound the row splits in progress at once to `max` in all, whatever the per-worker + /// concurrency and the machine's core count. `None` removes the bound. A filtered, limited + /// scan may hold one more trimmed array per split while it is mapped. + /// + /// Each split in progress holds the decoded chunks of every column it overlaps, so a scan + /// that runs alongside others, such as one task of several on a machine, can set this to + /// bound its memory. + pub fn with_some_max_splits_in_flight(mut self, max: Option) -> Self { + self.concurrency.max = max; self } diff --git a/vortex-layout/src/scan/test.rs b/vortex-layout/src/scan/test.rs index 7d030f2c987..53bca5c5b5b 100644 --- a/vortex-layout/src/scan/test.rs +++ b/vortex-layout/src/scan/test.rs @@ -4,6 +4,8 @@ use std::ops::Range; use std::sync::Arc; use std::sync::LazyLock; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use std::task::Poll; use parking_lot::Mutex; @@ -66,6 +68,20 @@ pub struct TestLayoutReader { fail_projection: bool, projection_masks: Option>>>, filter_exprs: Option>>>, + in_flight: Option>, +} + +/// Counts projections running at once, and the most that ever were. +#[derive(Debug, Default)] +pub struct InFlight { + now: AtomicUsize, + peak: AtomicUsize, +} + +impl InFlight { + pub fn peak(&self) -> usize { + self.peak.load(Ordering::SeqCst) + } } impl TestLayoutReader { @@ -82,6 +98,7 @@ impl TestLayoutReader { fail_projection: false, projection_masks: None, filter_exprs: None, + in_flight: None, } } @@ -111,6 +128,13 @@ impl TestLayoutReader { self } + /// Records into `in_flight` how many projections are running at once. Each projection yields + /// once before finishing, so others that the scan has started get to run alongside it. + pub fn with_in_flight(mut self, in_flight: Arc) -> Self { + self.in_flight = Some(in_flight); + self + } + pub fn with_filter_exprs(mut self, filter_exprs: Arc>>) -> Self { self.filter_exprs = Some(filter_exprs); self @@ -213,8 +237,15 @@ impl LayoutReader for TestLayoutReader { let base = self.base; let projection_masks = self.projection_masks.clone(); let fail = self.fail_projection || (self.fail_first_projection && row_range.start == 0); + let in_flight = self.in_flight.clone(); Ok(Box::pin(async move { + if let Some(in_flight) = &in_flight { + let now = in_flight.now.fetch_add(1, Ordering::SeqCst) + 1; + in_flight.peak.fetch_max(now, Ordering::SeqCst); + tokio::task::yield_now().await; + in_flight.now.fetch_sub(1, Ordering::SeqCst); + } let mask = mask.await?; if let Some(projection_masks) = projection_masks { projection_masks.lock().push(mask.true_count()); diff --git a/vortex-scan/src/lib.rs b/vortex-scan/src/lib.rs index 755ec472c6c..a0454c6c1a6 100644 --- a/vortex-scan/src/lib.rs +++ b/vortex-scan/src/lib.rs @@ -28,6 +28,7 @@ pub mod selection; pub mod strict_sorted_buffer; use std::any::Any; +use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; @@ -170,6 +171,13 @@ pub struct ScanRequest { /// order is completion order, so each partition applies the limit locally and the caller must /// trim the concatenated result. pub limit: Option, + /// The most splits of each partition to decode at once, in all. + /// + /// `None` keeps the reader's default, which scales with the machine's cores. Each split being + /// decoded holds the decoded chunks of every column it overlaps, so a scan that runs alongside + /// others, such as one task of several on a machine, can set this to bound its memory. It does + /// not bound I/O: segment reads for later splits may still be coalesced ahead of them. + pub split_concurrency: Option, } impl Default for ScanRequest { @@ -183,6 +191,7 @@ impl Default for ScanRequest { ordered: false, limit: None, partition_range: None, + split_concurrency: None, } } } From 31ae92c169c1c0774a02556c39154f773840e936 Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Mon, 2 Nov 2026 20:40:43 +0000 Subject: [PATCH 2/3] fix(scan): keep capped JNI scans within Arrow lookahead Drive capped Arrow streams directly instead of feeding the runtime result queue, which can retain one extra completed batch per core. Preserve the existing queue for uncapped scans. Add paused-consumer regressions with two background workers. Saturate per-worker concurrency scaling before applying the absolute cap and add an overflow regression. Clarify that split concurrency reduces decoded memory rather than bounding total native memory or I/O read-ahead. Signed-off-by: Christopher Wirt --- docs/user-guide/spark.md | 2 +- .../java/dev/vortex/spark/io/VortexIo.java | 4 +- vortex-jni/src/scan.rs | 107 ++++++++++++++---- vortex-layout/src/scan/repeated_scan.rs | 15 ++- vortex-layout/src/scan/scan_builder.rs | 2 +- 5 files changed, 105 insertions(+), 25 deletions(-) diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index 8b4bce44ad6..db8245364f5 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -236,7 +236,7 @@ depends on, on the driver and executor classpath, for example | Option | Default | Description | |----------------------------|---------|-------------| | `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. | -| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Each split holds the decoded chunks of every column it reads, and every task in an executor runs its own scan, so lowering this bounds a task's native memory. | +| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. It does not cap total native memory or I/O read-ahead. | | `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. | | `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. | | `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. | diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java index db60286364d..44fca9d8b13 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java @@ -33,8 +33,8 @@ public final class VortexIo implements Serializable { /** * Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the * decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default - * keeps several splits in flight per core, so setting this bounds a task's native memory. Zero keeps the native - * default. + * keeps several splits in flight per core. Lowering this reduces memory held by concurrent split decoding and Arrow + * conversion; it does not cap total native memory or I/O read-ahead. Zero keeps the native default. */ public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency"; diff --git a/vortex-jni/src/scan.rs b/vortex-jni/src/scan.rs index 6878af0f7d2..030013185b3 100644 --- a/vortex-jni/src/scan.rs +++ b/vortex-jni/src/scan.rs @@ -22,6 +22,7 @@ use arrow_array::ffi::FFI_ArrowSchema; use arrow_array::ffi_stream::FFI_ArrowArrayStream; use arrow_schema::ArrowError; use arrow_schema::Field; +use futures::Stream; use futures::StreamExt; use jni::EnvUnowned; use jni::objects::JByteArray; @@ -40,6 +41,7 @@ use vortex::expr::Expression; use vortex::expr::root; use vortex::expr::stats::Precision; use vortex::io::runtime::BlockingRuntime; +use vortex::io::runtime::current::CurrentThreadRuntime; use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; use vortex::scan::DataSourceScan; use vortex::scan::PartitionRef; @@ -380,28 +382,27 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( let schema = Arc::new(session.arrow().to_arrow_schema(&dtype)?); let target = Arc::new(Field::new_struct("", schema.fields().clone(), false)); - let iter = RUNTIME - .block_on_stream_thread_safe(|handle| { - array_stream - .map(move |chunk| { - let session = session.clone(); - let target = Arc::clone(&target); - handle.spawn(async move { - let chunk = chunk?; - let mut ctx = session.create_execution_ctx(); - let arrow = session.arrow().execute_arrow( - chunk, - Some(target.as_ref()), - &mut ctx, - )?; - rebase_offsets(RecordBatch::from(arrow.as_struct().clone())) - }) - }) - .buffered(lookahead) + let handle = RUNTIME.handle(); + let batches = array_stream + .map(move |chunk| { + let session = session.clone(); + let target = Arc::clone(&target); + handle.spawn(async move { + let chunk = chunk?; + let mut ctx = session.create_execution_ctx(); + let arrow = + session + .arrow() + .execute_arrow(chunk, Some(target.as_ref()), &mut ctx)?; + rebase_offsets(RecordBatch::from(arrow.as_struct().clone())) + }) }) - .map(|result: VortexResult| { + .buffered(lookahead); + let iter = scan_iterator(&RUNTIME, split_concurrency, batches).map( + |result: VortexResult| { result.map_err(|e| ArrowError::ExternalError(Box::new(e))) - }); + }, + ); let reader = RecordBatchIteratorAdapter::new(iter, schema); let arrow_stream = FFI_ArrowArrayStream::new(Box::new(reader)); @@ -411,3 +412,69 @@ pub extern "system" fn Java_dev_vortex_jni_NativePartition_scanArrow( Ok(()) }); } + +fn scan_iterator( + runtime: &CurrentThreadRuntime, + split_concurrency: Option, + stream: impl Stream + Send + 'static, +) -> Box + Send> { + if split_concurrency.is_some() { + // The thread-safe bridge queues another batch per core, independently of the stream's + // conversion lookahead. Drive capped scans directly so that queue cannot bypass the cap. + Box::new(runtime.block_on_stream(stream)) + } else { + Box::new(runtime.block_on_stream_thread_safe(|_| stream)) + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroUsize; + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + use futures::StreamExt; + use futures::stream; + use rstest::rstest; + use vortex::io::runtime::BlockingRuntime; + use vortex::io::runtime::current::CurrentThreadRuntime; + + use super::scan_iterator; + + #[rstest] + #[case(1)] + #[case(2)] + fn split_concurrency_bounds_completed_batches_ahead_of_the_consumer(#[case] max: usize) { + let runtime = CurrentThreadRuntime::new(); + let pool = runtime.new_pool(); + pool.set_workers(2); + let completed = Arc::new(AtomicUsize::new(0)); + let counter = Arc::clone(&completed); + let handle = runtime.handle(); + let batches = stream::iter(0..16) + .map(move |value| { + let counter = Arc::clone(&counter); + handle.spawn(async move { + counter.fetch_add(1, Ordering::SeqCst); + value + }) + }) + .buffered(max); + let mut iter = scan_iterator(&runtime, NonZeroUsize::new(max), batches); + assert_eq!(iter.next(), Some(0)); + + // Let background work finish while the consumer keeps its first batch open. + runtime.block_on(async { + for _ in 0..64 { + runtime.handle().spawn(async {}).await; + } + }); + assert!( + completed.load(Ordering::SeqCst) <= max + 1, + "completed batches escaped the conversion lookahead bound" + ); + + assert_eq!(iter.collect::>(), (1..16).collect::>()); + } +} diff --git a/vortex-layout/src/scan/repeated_scan.rs b/vortex-layout/src/scan/repeated_scan.rs index 91448f6988b..7866713a5c5 100644 --- a/vortex-layout/src/scan/repeated_scan.rs +++ b/vortex-layout/src/scan/repeated_scan.rs @@ -394,7 +394,9 @@ pub(crate) struct SplitConcurrency { impl SplitConcurrency { fn splits_in_flight(self) -> usize { - let scaled = self.per_worker * get_available_parallelism().unwrap_or(1); + let scaled = self + .per_worker + .saturating_mul(get_available_parallelism().unwrap_or(1)); self.max.map_or(scaled, |max| scaled.min(max.get())) } } @@ -423,6 +425,7 @@ fn intersect_ranges(left: Option<&Range>, right: Option>) -> Opt #[cfg(test)] mod tests { + use std::num::NonZeroUsize; use std::sync::Arc; use std::task::Poll; @@ -435,10 +438,20 @@ mod tests { use vortex_error::VortexResult; use vortex_error::vortex_err; + use super::SplitConcurrency; use super::limit_rows; use super::stop_on_error; use crate::scan::limit::RowLimit; + #[test] + fn split_cap_handles_oversized_per_worker_concurrency() { + let concurrency = SplitConcurrency { + per_worker: usize::MAX, + max: NonZeroUsize::new(1), + }; + assert_eq!(concurrency.splits_in_flight(), 1); + } + #[rstest] #[case::whole_array(4)] #[case::trimmed_array(2)] diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index 08a5700d6e7..d8c11e5541d 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -249,7 +249,7 @@ impl ScanBuilder { /// /// Each split in progress holds the decoded chunks of every column it overlaps, so a scan /// that runs alongside others, such as one task of several on a machine, can set this to - /// bound its memory. + /// reduce memory held by concurrent decoding. This does not cap total memory or I/O read-ahead. pub fn with_some_max_splits_in_flight(mut self, max: Option) -> Self { self.concurrency.max = max; self From 11e695f928e6ea53e5d9dcc51bd34f1a2c47caa3 Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Mon, 2 Nov 2026 21:03:33 +0000 Subject: [PATCH 3/3] docs(scan): clarify split concurrency memory limits Signed-off-by: Christopher Wirt --- docs/user-guide/spark.md | 2 +- .../vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java | 5 +++-- .../common/src/main/java/dev/vortex/spark/io/VortexIo.java | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index db8245364f5..f3912024370 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -236,7 +236,7 @@ depends on, on the driver and executor classpath, for example | Option | Default | Description | |----------------------------|---------|-------------| | `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. | -| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. It does not cap total native memory or I/O read-ahead. | +| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing output. This does not cap total native memory or I/O read-ahead. | | `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. | | `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. | | `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. | diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java index 865884647ed..43e713581ad 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java @@ -65,8 +65,9 @@ default boolean ordered() { * to Arrow ahead of the caller. Absent keeps the native default, which scales with the machine's cores. * *

Each split being decoded holds the decoded chunks of every column it overlaps, so a scan running alongside - * others, such as one Spark task of several in an executor, can set this to bound its native memory. It does not - * bound I/O: segment reads for later splits may still be coalesced ahead of them. + * others, such as one Spark task of several in an executor, can lower this to reduce memory held by concurrent + * decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split + * while preparing output. This does not cap total native memory or I/O read-ahead. */ OptionalInt splitConcurrency(); diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java index 44fca9d8b13..9f1ad11c2f8 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/VortexIo.java @@ -34,7 +34,8 @@ public final class VortexIo implements Serializable { * Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the * decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default * keeps several splits in flight per core. Lowering this reduces memory held by concurrent split decoding and Arrow - * conversion; it does not cap total native memory or I/O read-ahead. Zero keeps the native default. + * conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing + * output. This does not cap total native memory or I/O read-ahead. Zero keeps the native default. */ public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency";