Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/user-guide/spark.md
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ depends on, on the driver and executor classpath, for example
| Option | Default | Description |
|----------------------------|---------|-------------|
| `vortex.readConcurrency` | `0` | Maximum Hadoop read upcalls per file; `0` uses the native default. |
| `vortex.splitConcurrency` | `0` | Most row splits of a file decoded at once by one task; `0` uses the native default of four per core. Lowering this reduces memory held by concurrent split decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing output. This does not cap total native memory or I/O read-ahead. |
| `vortex.workerThreads` | `4` | Background threads driving Vortex futures. JVM-wide, and set once per JVM by the first read or write. |
| `vortex.session.provider` | — | Fully-qualified name of a `VortexSessionProvider` supplying a custom session on the driver and on every executor. |
| `vortex.aggregatePushdown` | `true` | Answer `COUNT(*)` from file footers. |
Expand Down
4 changes: 3 additions & 1 deletion java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@ public Scan scan(ScanOptions options) {
byte[] selectionRoaringBitmap = selectionRoaringBitmap(options);
long limit = options.limit().orElse(0L);
boolean ordered = options.ordered();
int splitConcurrency = options.splitConcurrency().orElse(0);

long scanPtr = dev.vortex.jni.NativeScan.create(
pointer,
Expand All @@ -249,7 +250,8 @@ public Scan scan(ScanOptions options) {
selectionRoaringBitmap,
selectionMode.code(),
limit,
ordered);
ordered,
splitConcurrency);
return Scan.fromPointer(session, scanPtr);
}

Expand Down
20 changes: 20 additions & 0 deletions java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.io.UncheckedIOException;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.OptionalLong;
import org.immutables.value.Value;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
Expand Down Expand Up @@ -59,6 +60,25 @@ default boolean ordered() {
return false;
}

/**
* The most splits of each partition the native scan decodes at once, in all, and the most of its batches converted
* to Arrow ahead of the caller. Absent keeps the native default, which scales with the machine's cores.
*
* <p>Each split being decoded holds the decoded chunks of every column it overlaps, so a scan running alongside
* others, such as one Spark task of several in an executor, can lower this to reduce memory held by concurrent
* decoding and Arrow conversion. A scan with both a filter and a limit may also retain one trimmed array per split
* while preparing output. This does not cap total native memory or I/O read-ahead.
*/
OptionalInt splitConcurrency();

@Value.Check
default void validateSplitConcurrency() {
if (splitConcurrency().isPresent() && splitConcurrency().getAsInt() <= 0) {
throw new IllegalArgumentException("split concurrency must be positive, got "
+ splitConcurrency().getAsInt());
}
}

@Value.Check
default void validateSelectionPayload() {
boolean hasIndices = selectionIndices().isPresent();
Expand Down
4 changes: 3 additions & 1 deletion java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ private NativeScan() {}
* {@code selectionRoaringBitmap})
* @param limit max rows to return, or {@code 0} for "no limit"
* @param ordered true to preserve row order across partitions
* @param splitConcurrency the most splits of each partition read at once, or {@code 0} for the native default
*/
public static native long create(
long dataSourcePointer,
Expand All @@ -38,7 +39,8 @@ public static native long create(
byte[] selectionRoaringBitmap,
byte selectionInclude,
long limit,
boolean ordered);
boolean ordered,
int splitConcurrency);

/** Free a scan pointer. */
public static native void free(long pointer);
Expand Down
16 changes: 16 additions & 0 deletions java/vortex-jni/src/test/java/dev/vortex/api/ScanOptionsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import dev.vortex.api.ScanOptions.SelectionMode;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.util.OptionalInt;
import java.util.OptionalLong;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
Expand Down Expand Up @@ -39,6 +40,21 @@ public void defaultsReadEveryRowAndColumn() {
assertTrue(options.selectionRoaringBitmap().isEmpty());
assertEquals(SelectionMode.INCLUDE_ALL, options.selectionMode());
assertFalse(options.ordered());
assertEquals(OptionalInt.empty(), options.splitConcurrency());
}

@Test
public void splitConcurrencyMustBePositive() {
for (int value : new int[] {0, -1}) {
IllegalArgumentException exception = assertThrows(
IllegalArgumentException.class,
() -> ScanOptions.builder().splitConcurrency(value).build(),
() -> "no exception for " + value);
assertEquals("split concurrency must be positive, got " + value, exception.getMessage());
}
assertEquals(
OptionalInt.of(2),
ScanOptions.builder().splitConcurrency(2).build().splitConcurrency());
}

@Test
Expand Down
12 changes: 12 additions & 0 deletions java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,18 @@ public void testProjectedScanWithFilter() throws Exception {
assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people);
}

@Test
public void testScanWithSplitConcurrencyReadsEveryRow() throws Exception {
BufferAllocator allocator = ArrowAllocation.rootAllocator();
Session session = Session.create();
DataSource ds = DataSource.open(session, writePath);

ScanOptions options =
ScanOptions.builder().splitConcurrency(1).ordered(true).build();
List<Person> people = readAll(ds, options, allocator, TestMinimal::readFullBatch);
assertEquals(MINIMAL_DATA, people);
}

@Test
public void testSelectionIncludesRows() throws Exception {
BufferAllocator allocator = ArrowAllocation.rootAllocator();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,31 +30,47 @@ public final class VortexIo implements Serializable {

private static final int DEFAULT_READ_CONCURRENCY = 0;

/**
* Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the
* decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default
* keeps several splits in flight per core. Lowering this reduces memory held by concurrent split decoding and Arrow
* conversion. A scan with both a filter and a limit may also retain one trimmed array per split while preparing
* output. This does not cap total native memory or I/O read-ahead. Zero keeps the native default.
*/
public static final String SPLIT_CONCURRENCY_OPTION = "vortex.splitConcurrency";

private static final int DEFAULT_SPLIT_CONCURRENCY = 0;

private final SerializableHadoopConf conf;
private final int readConcurrency;
private final int splitConcurrency;

/** Captures {@code hadoopConf} and the read settings found in the format options. */
public static VortexIo create(VortexOptions options, Configuration hadoopConf) {
Objects.requireNonNull(options, "options");
Objects.requireNonNull(hadoopConf, "hadoopConf");
return new VortexIo(new SerializableHadoopConf(hadoopConf), parseReadConcurrency(options));
return new VortexIo(
new SerializableHadoopConf(hadoopConf),
nonNegative(options, READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY),
nonNegative(options, SPLIT_CONCURRENCY_OPTION, DEFAULT_SPLIT_CONCURRENCY));
}

/** Hadoop I/O over a default configuration, for callers that have no Spark session to draw one from. */
public static VortexIo defaults() {
return new VortexIo(new SerializableHadoopConf(new Configuration()), DEFAULT_READ_CONCURRENCY);
return new VortexIo(
new SerializableHadoopConf(new Configuration()), DEFAULT_READ_CONCURRENCY, DEFAULT_SPLIT_CONCURRENCY);
}

private VortexIo(SerializableHadoopConf conf, int readConcurrency) {
private VortexIo(SerializableHadoopConf conf, int readConcurrency, int splitConcurrency) {
this.conf = conf;
this.readConcurrency = readConcurrency;
this.splitConcurrency = splitConcurrency;
}

private static int parseReadConcurrency(VortexOptions options) {
int parsed = options.getInt(READ_CONCURRENCY_OPTION, DEFAULT_READ_CONCURRENCY);
private static int nonNegative(VortexOptions options, String key, int defaultValue) {
int parsed = options.getInt(key, defaultValue);
if (parsed < 0) {
throw new IllegalArgumentException(
String.format(Locale.ROOT, "%s must be >= 0, got %d", READ_CONCURRENCY_OPTION, parsed));
throw new IllegalArgumentException(String.format(Locale.ROOT, "%s must be >= 0, got %d", key, parsed));
}
return parsed;
}
Expand All @@ -68,6 +84,11 @@ public int readConcurrency() {
return readConcurrency;
}

/** The most splits of a file the native scan reads at once; zero keeps the native default. */
public int splitConcurrency() {
return splitConcurrency;
}

/**
* Opens a byte source for {@code file}, stating it only if the listing that produced it did not report a size. The
* caller owns the result and must close it once the scan built on it is done.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ public VortexPartitionReader(
projectedSchema, arrowFields, caseSensitive, file.toPath().toString());

var options = ScanOptions.builder();
if (io.splitConcurrency() > 0) {
options.splitConcurrency(io.splitConcurrency());
}
// Always project, an empty read schema included: a query that needs only partition columns or only a
// row count must not pull every data column off storage. Struct columns that Spark pruned read only the
// nested fields the query uses.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import dev.vortex.spark.io.VortexIo;
import java.nio.file.Path;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
Expand Down Expand Up @@ -43,7 +44,9 @@ public void setUp() {

@AfterEach
public void resetConf() {
for (String option : new String[] {READ_CONCURRENCY, VortexSparkSession.PROVIDER_OPTION}) {
for (String option :
new String[] {READ_CONCURRENCY, VortexIo.SPLIT_CONCURRENCY_OPTION, VortexSparkSession.PROVIDER_OPTION
}) {
spark.conf().unset(option);
spark.conf().unset(DATA_SOURCE_PREFIX + option);
}
Expand Down Expand Up @@ -84,6 +87,19 @@ void aSessionSettingAppliesToV1Reads() {
assertSessionSettingAppliesToReads(write("v1"), "FileScan vortex", READ_CONCURRENCY);
}

@Test
void aSplitConcurrencyReadsEveryRowThroughV1AndV2() {
String path = write("split_concurrency");
for (String useV1 : new String[] {"", "vortex"}) {
spark.conf().set(USE_V1, useV1);
Dataset<Row> read = spark.read()
.format("vortex")
.option(VortexIo.SPLIT_CONCURRENCY_OPTION, "1")
.load(path);
assertEquals(45L, read.selectExpr("sum(id)").first().getLong(0), "useV1List=" + useV1);
}
}

@Test
void aSessionSettingAppliesToWrites() {
assertSessionSettingAppliesToWrites("written", VortexSparkSession.PROVIDER_OPTION);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,30 @@

final class VortexIoTest {
@Test
void readConcurrencyDefaultsToNativeDefault() {
void concurrenciesDefaultToNativeDefaults() {
VortexIo io = VortexIo.create(VortexOptions.empty(), new Configuration());

assertEquals(0, io.readConcurrency());
assertEquals(0, io.splitConcurrency());
}

@Test
void splitConcurrencyIsParsedAndValidated() {
Configuration conf = new Configuration();

assertEquals(
2,
VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "2"), conf)
.splitConcurrency());
assertEquals(
2,
VortexIo.create(options("VORTEX.SPLITCONCURRENCY", "2"), conf).splitConcurrency());
assertThrows(
IllegalArgumentException.class,
() -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "-1"), conf));
assertThrows(
IllegalArgumentException.class,
() -> VortexIo.create(options(VortexIo.SPLIT_CONCURRENCY_OPTION, "many"), conf));
}

@Test
Expand Down Expand Up @@ -54,11 +74,14 @@ void readConcurrencyIsFoundUnderAnyKeyCase() {
void settingsAndConfigurationSurviveSerialization() throws IOException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set("fs.s3a.endpoint", "https://storage.example");
VortexIo io = VortexIo.create(options(VortexIo.READ_CONCURRENCY_OPTION, "4"), conf);
VortexIo io = VortexIo.create(
VortexOptions.of(Map.of(VortexIo.READ_CONCURRENCY_OPTION, "4", VortexIo.SPLIT_CONCURRENCY_OPTION, "3")),
conf);

VortexIo shipped = roundTrip(io);

assertEquals(4, shipped.readConcurrency());
assertEquals(3, shipped.splitConcurrency());
assertEquals("https://storage.example", shipped.hadoopConf().get("fs.s3a.endpoint"));
}

Expand Down
1 change: 1 addition & 0 deletions vortex-ffi/src/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ fn scan_request(opts: *const vx_scan_options) -> VortexResult<ScanRequest> {
limit,
partition_selection: Selection::All,
partition_range: None,
split_concurrency: None,
})
}

Expand Down
Loading
Loading