diff --git a/docs/user-guide/spark.md b/docs/user-guide/spark.md index d78f3559bd8..f3912024370 100644 --- a/docs/user-guide/spark.md +++ b/docs/user-guide/spark.md @@ -236,6 +236,7 @@ depends on, on the driver and executor classpath, for example | Option | Default | Description | |----------------------------|---------|-------------| | `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. | +| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing output. This does not cap total native memory or I/O read-ahead. | | `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. | | `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. | | `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. | diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java index 8658cf5198b..22c090c6419 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java @@ -238,6 +238,7 @@ public Scan scan(ScanOptions options) { byte[] selectionRoaringBitmap = selectionRoaringBitmap(options); long limit = options.limit().orElse(0L); boolean ordered = options.ordered(); + int splitConcurrency = options.splitConcurrency().orElse(0); long scanPtr = dev.vortex.jni.NativeScan.create( pointer, @@ -249,7 +250,8 @@ public Scan scan(ScanOptions options) { selectionRoaringBitmap, selectionMode.code(), limit, - ordered); + ordered, + splitConcurrency); return Scan.fromPointer(session, scanPtr); } diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java index 25f03e76f7f..43e713581ad 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java @@ -9,6 +9,7 @@ import java.io.UncheckedIOException; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.OptionalLong; import org.immutables.value.Value; import org.roaringbitmap.longlong.Roaring64NavigableMap; @@ -59,6 +60,25 @@ default boolean ordered() { return false; } + /** + * The most splits of each partition the native scan decodes at once, in all, and the most of its batches converted + * to Arrow ahead of the caller. Absent keeps the native default, which scales with the machine's cores. + * + *
Each split being decoded holds the decoded chunks of every column it overlaps, so a scan running alongside
+ * others, such as one Spark task of several in an executor, can lower this to reduce memory held by concurrent
+ * decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split
+ * while preparing output. This does not cap total native memory or I/O read-ahead.
+ */
+ OptionalInt splitConcurrency();
+
+ @Value.Check
+ default void validateSplitConcurrency() {
+ if (splitConcurrency().isPresent() && splitConcurrency().getAsInt() <= 0) {
+ throw new IllegalArgumentException("split concurrency must be positive, got "
+ + splitConcurrency().getAsInt());
+ }
+ }
+
@Value.Check
default void validateSelectionPayload() {
boolean hasIndices = selectionIndices().isPresent();
diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java
index ff80b30caaa..ff4ad73a317 100644
--- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java
+++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java
@@ -27,6 +27,7 @@ private NativeScan() {}
* {@code selectionRoaringBitmap})
* @param limit max rows to return, or {@code 0} for "no limit"
* @param ordered true to preserve row order across partitions
+ * @param splitConcurrency the most splits of each partition read at once, or {@code 0} for the native default
*/
public static native long create(
long dataSourcePointer,
@@ -38,7 +39,8 @@ public static native long create(
byte[] selectionRoaringBitmap,
byte selectionInclude,
long limit,
- boolean ordered);
+ boolean ordered,
+ int splitConcurrency);
/** Free a scan pointer. */
public static native void free(long pointer);
diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java b/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java
index c72e67f4051..cb0c6ff5b8a 100644
--- a/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java
+++ b/java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java
@@ -12,6 +12,7 @@
import dev.vortex.api.ScanOptions.SelectionMode;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
+import java.util.OptionalInt;
import java.util.OptionalLong;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
@@ -39,6 +40,21 @@ public void defaultsReadEveryRowAndColumn() {
assertTrue(options.selectionRoaringBitmap().isEmpty());
assertEquals(SelectionMode.INCLUDE_ALL, options.selectionMode());
assertFalse(options.ordered());
+ assertEquals(OptionalInt.empty(), options.splitConcurrency());
+ }
+
+ @Test
+ public void splitConcurrencyMustBePositive() {
+ for (int value : new int[] {0, -1}) {
+ IllegalArgumentException exception = assertThrows(
+ IllegalArgumentException.class,
+ () -> ScanOptions.builder().splitConcurrency(value).build(),
+ () -> "no exception for " + value);
+ assertEquals("split concurrency must be positive, got " + value, exception.getMessage());
+ }
+ assertEquals(
+ OptionalInt.of(2),
+ ScanOptions.builder().splitConcurrency(2).build().splitConcurrency());
}
@Test
diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java
index 2b270c3e165..f4adefed093 100644
--- a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java
+++ b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java
@@ -184,6 +184,18 @@ public void testProjectedScanWithFilter() throws Exception {
assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people);
}
+ @Test
+ public void testScanWithSplitConcurrencyReadsEveryRow() throws Exception {
+ BufferAllocator allocator = ArrowAllocation.rootAllocator();
+ Session session = Session.create();
+ DataSource ds = DataSource.open(session, writePath);
+
+ ScanOptions options =
+ ScanOptions.builder().splitConcurrency(1).ordered(true).build();
+ List