diff --git a/Cargo.lock b/Cargo.lock
index 0a025bc047c..f80c03f000b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12164,6 +12164,7 @@ name = "vortex-jni"
version = "0.1.0"
dependencies = [
"arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
"arrow-data 59.3.2",
"arrow-schema 59.3.2",
"async-fs",
diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java b/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java
index 7649c0f2707..1266bbe1cf5 100644
--- a/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java
+++ b/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java
@@ -6,6 +6,7 @@
import com.google.common.base.Preconditions;
import dev.vortex.VortexCleaner;
import dev.vortex.io.NativeWritable;
+import dev.vortex.jni.NativeBuffer;
import dev.vortex.jni.NativeWriter;
import java.io.Closeable;
import java.io.IOException;
@@ -13,9 +14,11 @@
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.arrow.c.ArrowArray;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.Schema;
/**
@@ -24,9 +27,10 @@
*
Open one with {@link #builder(Session, String, Schema, BufferAllocator)} to write to a URI, or
* {@link #builder(Session, NativeWritable, Schema, BufferAllocator)} to write into a caller-provided byte sink.
*
- *
Batches are accepted via the Arrow C Data Interface: callers export an Arrow record batch to an {@code ArrowArray}
- * / {@code ArrowSchema} pair and pass the addresses to {@link #writeBatch(long, long)}. The writer accepts up to four
- * in-flight batches on the session's runtime thread before back-pressuring the caller.
+ *
Use {@link #writeBatch(VectorSchemaRoot, BufferAllocator)} for zero-copy Java batches with independent buffer
+ * lifetimes. Batches are also accepted via the Arrow C Data Interface: callers export an Arrow record batch to an
+ * {@code ArrowArray} / {@code ArrowSchema} pair and pass the addresses to {@link #writeBatch(long, long)}. The writer
+ * accepts up to four in-flight batches on the session's runtime thread before back-pressuring the caller.
*
*
Call {@link #close()} to flush remaining batches and finalize the file. If the writer becomes unreachable without
* an explicit {@code close()}, {@link VortexCleaner} will flush and release native resources as a backstop — but
@@ -34,12 +38,14 @@
*/
public final class VortexWriter implements AutoCloseable {
private final long pointer;
+ private final Schema arrowSchema;
private final AtomicBoolean closed = new AtomicBoolean(false);
private volatile VortexWriteSummary summary;
- private VortexWriter(long pointer) {
+ private VortexWriter(long pointer, Schema arrowSchema) {
Preconditions.checkArgument(pointer != 0, "invalid writer pointer");
this.pointer = pointer;
+ this.arrowSchema = arrowSchema;
AtomicBoolean closedRef = this.closed;
VortexCleaner.register(this, () -> {
if (closedRef.compareAndSet(false, true)) {
@@ -170,7 +176,7 @@ public VortexWriter build() throws IOException {
if (pointer <= 0) {
throw new IOException("failed to create writer for " + destination() + " (ptr=" + pointer + ")");
}
- return new VortexWriter(pointer);
+ return new VortexWriter(pointer, arrowSchema);
}
}
@@ -183,8 +189,38 @@ private void checkOptionsApply() {
}
}
- /** Write a batch directly from Arrow C Data Interface addresses. */
- public void writeBatch(long arrowArrayAddr, long arrowSchemaAddr) throws IOException {
+ /**
+ * Write a Java Arrow batch without copying its data. Native buffers retain their allocations independently, so a
+ * retained column does not pin the entire batch. The caller may close or clear the root after this method returns,
+ * but must not overwrite its retained buffers.
+ *
+ * @param root batch to write; dictionary-encoded fields require the address-based API
+ * @param allocator allocator for temporary C Data Interface metadata
+ */
+ public synchronized void writeBatch(VectorSchemaRoot root, BufferAllocator allocator) throws IOException {
+ Preconditions.checkState(!closed.get(), "writer already closed");
+ if (!arrowSchema.equals(root.getSchema())) {
+ throw new IOException("batch schema does not match writer schema");
+ }
+ try (ArrowArray array = ArrowArray.allocateNew(allocator)) {
+ try {
+ Data.exportVectorSchemaRoot(allocator, root, null, array);
+ if (!NativeWriter.writeBatchBuffers(pointer, array.memoryAddress(), NativeBuffer.collect(root))) {
+ throw new IOException("failed to write batch");
+ }
+ } finally {
+ array.release();
+ }
+ } catch (RuntimeException e) {
+ throw new IOException("failed to write batch", e);
+ }
+ }
+
+ /**
+ * Write a batch directly from Arrow C Data Interface addresses. These buffers share the producer's batch release
+ * callback; use {@link #writeBatch(VectorSchemaRoot, BufferAllocator)} for independently retained Java allocations.
+ */
+ public synchronized void writeBatch(long arrowArrayAddr, long arrowSchemaAddr) throws IOException {
Preconditions.checkState(!closed.get(), "writer already closed");
final boolean ok;
try {
diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeBuffer.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeBuffer.java
new file mode 100644
index 00000000000..d06973eac61
--- /dev/null
+++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeBuffer.java
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+package dev.vortex.jni;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.ReferenceManager;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+/** Per-allocation ownership used by the native zero-copy importer. */
+public final class NativeBuffer {
+ private final ReferenceManager manager;
+ private final long address;
+ private final long capacity;
+
+ private NativeBuffer(ArrowBuf buffer) {
+ manager = buffer.getReferenceManager();
+ address = buffer.memoryAddress();
+ capacity = buffer.capacity();
+ }
+
+ /** Capture buffer ownership metadata while the root remains alive. */
+ public static NativeBuffer[] collect(VectorSchemaRoot root) {
+ List buffers = new ArrayList<>();
+ for (FieldVector vector : root.getFieldVectors()) {
+ collect(vector, buffers);
+ }
+ return buffers.toArray(new NativeBuffer[0]);
+ }
+
+ private static void collect(FieldVector vector, List buffers) {
+ for (ArrowBuf buffer : vector.getFieldBuffers()) {
+ if (buffer.capacity() != 0) {
+ buffers.add(new NativeBuffer(buffer));
+ }
+ }
+ for (FieldVector child : vector.getChildrenFromFields()) {
+ collect(child, buffers);
+ }
+ }
+
+ /** Buffer address, called by JNI. */
+ public long address() {
+ return address;
+ }
+
+ /** Buffer capacity, called by JNI. */
+ public long capacity() {
+ return capacity;
+ }
+
+ /** Acquire an independent native reference, called by JNI. */
+ public void retain() {
+ manager.retain();
+ }
+
+ /** Release the native reference, possibly from a writer worker thread. */
+ public void release() {
+ manager.release();
+ }
+}
diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java
index 44bbd3b5323..141edf557ca 100644
--- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java
+++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java
@@ -45,6 +45,12 @@ public static native long createStream(
*/
public static native boolean writeBatch(long writerPointer, long arrowArrayAddress, long arrowSchemaAddress);
+ /**
+ * Import a Java batch with independently retained buffer allocations. The exported array must match the schema
+ * passed to {@link #create}; the caller must serialize writes and close.
+ */
+ public static native boolean writeBatchBuffers(long writerPointer, long arrowArrayAddress, NativeBuffer[] buffers);
+
/** Number of bytes successfully written to the underlying sink so far. */
public static native long bytesWritten(long writerPointer);
diff --git a/java/vortex-jni/src/test/java/dev/vortex/jni/IndependentBuffersTest.java b/java/vortex-jni/src/test/java/dev/vortex/jni/IndependentBuffersTest.java
new file mode 100644
index 00000000000..2a8d91d751f
--- /dev/null
+++ b/java/vortex-jni/src/test/java/dev/vortex/jni/IndependentBuffersTest.java
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+package dev.vortex.jni;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import dev.vortex.api.DataSource;
+import dev.vortex.api.Scan;
+import dev.vortex.api.ScanOptions;
+import dev.vortex.api.Session;
+import dev.vortex.api.VortexWriter;
+import dev.vortex.io.NativeWritable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.apache.arrow.c.ArrowArray;
+import org.apache.arrow.c.ArrowSchema;
+import org.apache.arrow.c.Data;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ViewVarCharVector;
+import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public final class IndependentBuffersTest {
+ @TempDir
+ Path directory;
+
+ private static Schema schema() {
+ return new Schema(List.of(new Field(
+ "nested",
+ FieldType.nullable(ArrowType.Struct.INSTANCE),
+ List.of(Field.nullable("value", new ArrowType.Utf8())))));
+ }
+
+ private static void fill(VectorSchemaRoot root, int rows) {
+ StructVector nested = (StructVector) root.getVector(0);
+ VarCharVector values = nested.getChild("value", VarCharVector.class);
+ nested.setInitialCapacity(rows);
+ nested.allocateNew();
+ byte[] value = new byte[512];
+ Arrays.fill(value, (byte) 97);
+ for (int row = 0; row < rows; row++) {
+ value[0] = (byte) (row % 26 + 97);
+ values.setSafe(row, value);
+ if (row % 2 == 0) {
+ nested.setIndexDefined(row);
+ } else {
+ nested.setNull(row);
+ }
+ }
+ root.setRowCount(rows);
+ }
+
+ @Test
+ public void releasesLargeColumnsWhileSmallBitmapsRemain() throws Exception {
+ Schema schema = schema();
+ int batches = 16;
+ int rows = 8192;
+ try (BufferAllocator allocator = new RootAllocator();
+ BufferAllocator batchesAllocator = allocator.newChildAllocator("batches", 0, Long.MAX_VALUE);
+ VortexWriter writer = VortexWriter.builder(
+ Session.create(),
+ directory.resolve("memory.vortex").toString(),
+ schema,
+ allocator)
+ .build()) {
+ for (int batch = 0; batch < batches; batch++) {
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, batchesAllocator)) {
+ fill(root, rows);
+ writer.writeBatch(root, allocator);
+ }
+ }
+ long allValueBytes = (long) batches * rows * 512;
+ // Arrow export can transfer accounting to its allocator; measure the common parent.
+ long held = allocator.getAllocatedMemory();
+ assertTrue(held < allValueBytes * 3 / 4, "unrelated allocations must be released; held " + held);
+ assertEquals((long) batches * rows, writer.finish().rowCount());
+ assertEquals(0, batchesAllocator.getAllocatedMemory());
+ assertEquals(0, allocator.getAllocatedMemory());
+ }
+ }
+
+ @Test
+ public void failedImportReleasesPartialRetains() throws Exception {
+ Schema schema = schema();
+ try (BufferAllocator allocator = new RootAllocator();
+ BufferAllocator batches = allocator.newChildAllocator("batch", 0, Long.MAX_VALUE);
+ VectorSchemaRoot root = VectorSchemaRoot.create(schema, batches);
+ ArrowSchema ffiSchema = ArrowSchema.allocateNew(allocator);
+ ArrowArray array = ArrowArray.allocateNew(allocator)) {
+ fill(root, 32);
+ Data.exportSchema(allocator, schema, null, ffiSchema);
+ long writer = NativeWriter.create(
+ Session.create().nativePointer(),
+ directory.resolve("failure.vortex").toString(),
+ ffiSchema.memoryAddress(),
+ Map.of(),
+ Map.of());
+ try {
+ Data.exportVectorSchemaRoot(allocator, root, null, array);
+ NativeBuffer[] owners = NativeBuffer.collect(root);
+ assertThrows(
+ RuntimeException.class,
+ () -> NativeWriter.writeBatchBuffers(
+ writer, array.memoryAddress(), new NativeBuffer[] {owners[0]}));
+ root.clear();
+ assertEquals(0, batches.getAllocatedMemory());
+ } finally {
+ NativeWriter.close(writer);
+ array.release();
+ ffiSchema.release();
+ }
+ }
+ }
+
+ @Test
+ public void sinkFailureReleasesBuffers() throws Exception {
+ try (BufferAllocator allocator = new RootAllocator()) {
+ NativeWritable sink = new NativeWritable() {
+ @Override
+ public void write(byte[] buffer, int offset, int length) throws IOException {
+ throw new IOException("injected write failure");
+ }
+
+ @Override
+ public void flush() {}
+
+ @Override
+ public void close() {}
+ };
+ VortexWriter writer = VortexWriter.builder(Session.create(), sink, schema(), allocator)
+ .build();
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(schema(), allocator)) {
+ fill(root, 32);
+ writer.writeBatch(root, allocator);
+ }
+ IOException error = assertThrows(IOException.class, writer::finish);
+ assertTrue(error.getCause().getMessage().contains("injected write failure"), error.toString());
+ assertEquals(0, allocator.getAllocatedMemory());
+ }
+ }
+
+ @Test
+ public void emptyAndViewBatchesRoundTrip() throws Exception {
+ Schema schema = new Schema(List.of(Field.nullable("text", new ArrowType.Utf8View())));
+ Session session = Session.create();
+ String path = directory.resolve("views.vortex").toString();
+ List expected = Arrays.asList("short", null, "a long out-of-line string value", "");
+ try (BufferAllocator allocator = new RootAllocator()) {
+ try (VortexWriter writer = VortexWriter.builder(session, path, schema, allocator)
+ .build();
+ VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+ writer.writeBatch(root, allocator);
+ ViewVarCharVector values = (ViewVarCharVector) root.getVector(0);
+ values.allocateNew();
+ for (int row = 0; row < expected.size(); row++) {
+ if (expected.get(row) == null) {
+ // Arrow 19 leaves null view slots uninitialized. The existing Vortex
+ // view concatenation path reads them, so initialize the physical slot.
+ values.setSafe(row, new byte[0]);
+ values.setNull(row);
+ } else {
+ values.setSafe(row, expected.get(row).getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ root.setRowCount(expected.size());
+ writer.writeBatch(root, allocator);
+ root.clear();
+ }
+ DataSource source = DataSource.open(session, path);
+ Scan scan = source.scan(ScanOptions.of());
+ List actual = new ArrayList<>();
+ while (scan.hasNext()) {
+ try (ArrowReader reader = scan.next().scanArrow(allocator)) {
+ while (reader.loadNextBatch()) {
+ VectorSchemaRoot root = reader.getVectorSchemaRoot();
+ for (int row = 0; row < root.getRowCount(); row++) {
+ Object value = root.getVector(0).getObject(row);
+ actual.add(value == null ? null : value.toString());
+ }
+ }
+ }
+ }
+ assertEquals(expected, actual);
+ }
+ }
+
+ @Test
+ public void unalignedInputKeepsArrowAlignmentCopies() throws Exception {
+ Schema schema = new Schema(List.of(Field.nullable("number", new ArrowType.Int(32, true))));
+ Session session = Session.create();
+ String path = directory.resolve("unaligned.vortex").toString();
+ try (BufferAllocator allocator = new RootAllocator()) {
+ try (VortexWriter writer = VortexWriter.builder(session, path, schema, allocator)
+ .build();
+ VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
+ ArrowBuf validity = allocator.buffer(1);
+ ArrowBuf data = allocator.buffer(13)) {
+ validity.setByte(0, 7);
+ data.setInt(1, 11);
+ data.setInt(5, 22);
+ data.setInt(9, 33);
+ IntVector vector = (IntVector) root.getVector(0);
+ vector.loadFieldBuffers(new ArrowFieldNode(3, 0), List.of(validity, data.slice(1, 12)));
+ root.setRowCount(3);
+ assertEquals(1, vector.getDataBuffer().memoryAddress() % Integer.BYTES);
+ writer.writeBatch(root, allocator);
+ root.clear();
+ }
+ Scan scan = DataSource.open(session, path).scan(ScanOptions.of());
+ List actual = new ArrayList<>();
+ while (scan.hasNext()) {
+ try (ArrowReader reader = scan.next().scanArrow(allocator)) {
+ while (reader.loadNextBatch()) {
+ IntVector vector =
+ (IntVector) reader.getVectorSchemaRoot().getVector(0);
+ for (int row = 0; row < vector.getValueCount(); row++) {
+ actual.add(vector.get(row));
+ }
+ }
+ }
+ }
+ assertEquals(List.of(11, 22, 33), actual);
+ }
+ }
+
+ @Test
+ public void mismatchedSchemaIsRejectedBeforeImport() throws Exception {
+ Schema different = new Schema(List.of(Field.nullable("number", new ArrowType.Int(32, true))));
+ try (BufferAllocator allocator = new RootAllocator();
+ VortexWriter writer = VortexWriter.builder(
+ Session.create(),
+ directory.resolve("schema.vortex").toString(),
+ schema(),
+ allocator)
+ .build();
+ VectorSchemaRoot root = VectorSchemaRoot.create(different, allocator)) {
+ IOException error = assertThrows(IOException.class, () -> writer.writeBatch(root, allocator));
+ assertTrue(error.getMessage().contains("schema"));
+ try (VectorSchemaRoot valid = VectorSchemaRoot.create(schema(), allocator)) {
+ fill(valid, 32);
+ writer.writeBatch(valid, allocator);
+ }
+ assertEquals(32, writer.finish().rowCount());
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void writeAndFinishSerializeNativeLifetime(boolean independent) throws Exception {
+ var executor = Executors.newFixedThreadPool(2);
+ try (BufferAllocator allocator = new RootAllocator()) {
+ for (int iteration = 0; iteration < 16; iteration++) {
+ try (VortexWriter writer = VortexWriter.builder(
+ Session.create(),
+ directory
+ .resolve("race-" + iteration + ".vortex")
+ .toString(),
+ schema(),
+ allocator)
+ .build();
+ VectorSchemaRoot root = VectorSchemaRoot.create(schema(), allocator);
+ ArrowArray array = ArrowArray.allocateNew(allocator);
+ ArrowSchema ffi = ArrowSchema.allocateNew(allocator)) {
+ fill(root, 32);
+ Data.exportVectorSchemaRoot(allocator, root, null, array, ffi);
+ CyclicBarrier start = new CyclicBarrier(2);
+ try {
+ var write = executor.submit(() -> {
+ start.await();
+ try {
+ if (independent) {
+ writer.writeBatch(root, allocator);
+ } else {
+ writer.writeBatch(array.memoryAddress(), ffi.memoryAddress());
+ }
+ return true;
+ } catch (IllegalStateException closed) {
+ assertTrue(closed.getMessage().contains("closed"));
+ return false;
+ }
+ });
+ var finish = executor.submit(() -> {
+ start.await();
+ return writer.finish();
+ });
+ boolean accepted = write.get(10, TimeUnit.SECONDS);
+ assertEquals(
+ accepted ? 32 : 0,
+ finish.get(10, TimeUnit.SECONDS).rowCount());
+ } finally {
+ array.release();
+ ffi.release();
+ }
+ }
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+}
diff --git a/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java b/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java
index ba41c1b8528..9fcec319641 100644
--- a/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java
+++ b/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java
@@ -53,6 +53,7 @@
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
public final class JNIWriterTest {
private static final String ARROW_EXTENSION_NAME = "ARROW:extension:name";
@@ -275,8 +276,9 @@ public void testCreateWriterCreatesParentDirectories() throws IOException {
source.arrowSchema(allocator));
}
- @Test
- public void testWriteBatch() throws IOException {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testWriteBatch(boolean independentBuffers) throws IOException {
Path outputPath = tempDir.resolve("test_ffi.vortex");
String writePath = outputPath.toAbsolutePath().toUri().toString();
@@ -304,11 +306,16 @@ public void testWriteBatch() throws IOException {
root.setRowCount(3);
- try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator);
- ArrowSchema arrowSchemaFfi = ArrowSchema.allocateNew(allocator)) {
- Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchemaFfi);
- writer.writeBatch(arrowArray.memoryAddress(), arrowSchemaFfi.memoryAddress());
+ if (independentBuffers) {
+ writer.writeBatch(root, allocator);
+ } else {
+ try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator);
+ ArrowSchema arrowSchemaFfi = ArrowSchema.allocateNew(allocator)) {
+ Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchemaFfi);
+ writer.writeBatch(arrowArray.memoryAddress(), arrowSchemaFfi.memoryAddress());
+ }
}
+ root.clear();
bytesWhileOpen = writer.bytesWritten();
summary = writer.finish();
assertEquals(summary.fileSize(), writer.bytesWritten());
@@ -353,8 +360,9 @@ public void testWriteBatch() throws IOException {
assertEquals(List.of(30, 25, 40), ages);
}
- @Test
- public void testMapRoundTrip() throws IOException {
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testMapRoundTrip(boolean independentBuffers) throws IOException {
Path outputPath = tempDir.resolve("test_map.vortex");
String writePath = outputPath.toAbsolutePath().toUri().toString();
@@ -366,10 +374,19 @@ public void testMapRoundTrip() throws IOException {
.build();
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
populateMapRoot(root);
- try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator);
- ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator)) {
- Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchema);
- writer.writeBatch(arrowArray.memoryAddress(), arrowSchema.memoryAddress());
+ if (independentBuffers) {
+ writer.writeBatch(root, allocator);
+ } else {
+ try (ArrowArray array = ArrowArray.allocateNew(allocator);
+ ArrowSchema ffi = ArrowSchema.allocateNew(allocator)) {
+ try {
+ Data.exportVectorSchemaRoot(allocator, root, null, array, ffi);
+ writer.writeBatch(array.memoryAddress(), ffi.memoryAddress());
+ } finally {
+ array.release();
+ ffi.release();
+ }
+ }
}
}
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java
index 953ff55c5b1..02afa8dde61 100644
--- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java
@@ -5,9 +5,6 @@
import dev.vortex.api.Session;
import dev.vortex.api.VortexWriter;
-import dev.vortex.relocated.org.apache.arrow.c.ArrowArray;
-import dev.vortex.relocated.org.apache.arrow.c.ArrowSchema;
-import dev.vortex.relocated.org.apache.arrow.c.Data;
import dev.vortex.relocated.org.apache.arrow.memory.BufferAllocator;
import dev.vortex.relocated.org.apache.arrow.memory.RootAllocator;
import dev.vortex.relocated.org.apache.arrow.vector.FieldVector;
@@ -47,7 +44,6 @@ public final class VortexDataWriter implements DataWriter, AutoClos
private VortexWriter vortexWriter;
private BufferAllocator allocator;
private VectorSchemaRoot vectorSchemaRoot;
- private ArrowSchema exportedSchema;
private ArrowFieldWriter>[] fieldWriters;
private int batchRowCount = 0;
private long recordCount = 0;
@@ -106,9 +102,6 @@ public final class VortexDataWriter implements DataWriter, AutoClos
for (int i = 0; i < fields.length; i++) {
fieldWriters[i] = ArrowFieldWriter.create(vectorSchemaRoot.getVector(i), fields[i].dataType());
}
- // The native writer only reads the schema struct, so one export serves every batch.
- this.exportedSchema = ArrowSchema.allocateNew(allocator);
- Data.exportSchema(allocator, arrowSchema, null, exportedSchema);
logger.debug("Initialized VortexDataWriter for {}", filePath);
@@ -166,10 +159,7 @@ private void flushBatch() throws IOException {
}
vectorSchemaRoot.setRowCount(batchRowCount);
- try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator)) {
- Data.exportVectorSchemaRoot(allocator, vectorSchemaRoot, null, arrowArray);
- vortexWriter.writeBatch(arrowArray.memoryAddress(), exportedSchema.memoryAddress());
- }
+ vortexWriter.writeBatch(vectorSchemaRoot, allocator);
vectorSchemaRoot.clear();
for (ArrowFieldWriter> writer : fieldWriters) {
@@ -310,13 +300,6 @@ private void closeArrowResources() {
vectorSchemaRoot.close();
vectorSchemaRoot = null;
}
- if (exportedSchema != null) {
- // close() frees only the struct; the children the export allocated are freed by the release
- // callback, which the native side never calls because it only borrows the schema.
- exportedSchema.release();
- exportedSchema.close();
- exportedSchema = null;
- }
}
/**
diff --git a/vortex-jni/Cargo.toml b/vortex-jni/Cargo.toml
index 41da023ff0b..aad214616c8 100644
--- a/vortex-jni/Cargo.toml
+++ b/vortex-jni/Cargo.toml
@@ -16,6 +16,7 @@ categories = { workspace = true }
[dependencies]
arrow-array = { workspace = true, features = ["ffi"] }
+arrow-buffer = { workspace = true }
arrow-data = { workspace = true }
arrow-schema = { workspace = true }
async-fs = { workspace = true }
diff --git a/vortex-jni/src/buffer/mod.rs b/vortex-jni/src/buffer/mod.rs
new file mode 100644
index 00000000000..d2abf2187ab
--- /dev/null
+++ b/vortex-jni/src/buffer/mod.rs
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Replace the C Data Interface batch owner with independent Java allocation owners.
+
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use std::ptr::NonNull;
+use std::sync::Arc;
+
+use arrow_array::ffi::FFI_ArrowArray;
+use arrow_buffer::BooleanBuffer;
+use arrow_buffer::Buffer;
+use arrow_buffer::NullBuffer;
+use arrow_data::ArrayData;
+use jni::Env;
+use jni::JavaVM;
+use jni::objects::JObject;
+use jni::objects::JObjectArray;
+use jni::refs::Global;
+use jni::refs::IntoAuto;
+use vortex::error::vortex_err;
+
+use crate::errors::JNIError;
+use crate::io::with_jvm;
+
+struct JavaBuffer {
+ vm: JavaVM,
+ object: Global>,
+ capacity: usize,
+}
+
+impl Drop for JavaBuffer {
+ fn drop(&mut self) {
+ if let Err(error) = with_jvm(&self.vm, |env| {
+ env.call_method(
+ &self.object,
+ jni::jni_str!("release"),
+ jni::jni_sig!("()V"),
+ &[],
+ )?;
+ Ok(())
+ }) {
+ tracing::error!(%error, "failed to release Java Arrow buffer");
+ }
+ }
+}
+
+/// Snapshot producer addresses before Arrow can replace unaligned buffers with owned copies.
+pub(crate) fn exported_buffer_addresses(array: &FFI_ArrowArray) -> BTreeSet {
+ fn collect(array: &FFI_ArrowArray, addresses: &mut BTreeSet) {
+ addresses.extend((0..array.num_buffers()).map(|index| array.buffer(index) as usize));
+ for index in 0..array.num_children() {
+ collect(array.child(index), addresses);
+ }
+ if let Some(dictionary) = array.dictionary() {
+ collect(dictionary, addresses);
+ }
+ }
+ let mut addresses = BTreeSet::new();
+ collect(array, &mut addresses);
+ addresses
+}
+
+/// The caller must keep the exported root alive until every independent retain is acquired.
+pub(crate) fn import_buffers(
+ env: &mut Env,
+ data: ArrayData,
+ exported_addresses: &BTreeSet,
+ buffers: JObjectArray,
+) -> Result {
+ let mut owners = BTreeMap::new();
+ for index in 0..buffers.len(env)? {
+ // Wide schemas can contain thousands of buffers; do not accumulate JNI local references.
+ let object = buffers.get_element(env, index)?.auto();
+ let address = env
+ .call_method(&object, jni::jni_str!("address"), jni::jni_sig!("()J"), &[])?
+ .j()?;
+ let address =
+ usize::try_from(address).map_err(|_| vortex_err!("invalid Java buffer address"))?;
+ let capacity = env
+ .call_method(
+ &object,
+ jni::jni_str!("capacity"),
+ jni::jni_sig!("()J"),
+ &[],
+ )?
+ .j()?;
+ let capacity =
+ usize::try_from(capacity).map_err(|_| vortex_err!("invalid Java buffer capacity"))?;
+ let global = env.new_global_ref(&object)?;
+ let vm = env.get_java_vm()?;
+ env.call_method(&object, jni::jni_str!("retain"), jni::jni_sig!("()V"), &[])?;
+ let owner = Arc::new(JavaBuffer {
+ vm,
+ object: global,
+ capacity,
+ });
+ // Multiple slices can start at the same address. Keep the largest available extent.
+ let entry = owners.entry(address).or_insert_with(|| Arc::clone(&owner));
+ if entry.capacity < capacity {
+ *entry = owner;
+ }
+ }
+ replace_owners(data, &mut |buffer| {
+ if buffer.is_empty() {
+ return Ok(Buffer::from_slice_ref([] as [u8; 0]));
+ }
+ let address = buffer.as_ptr() as usize;
+ // Arrow's alignment copies already own their allocations. Keep only buffers whose
+ // addresses differ from the producer's, so a missing Java owner still fails import.
+ if !exported_addresses.contains(&address) {
+ return Ok(buffer.clone());
+ }
+ let (_, owner) = owners
+ .range(..=address)
+ .rev()
+ .find(|(start, owner)| {
+ let offset = address - **start;
+ offset <= owner.capacity && buffer.len() <= owner.capacity - offset
+ })
+ .ok_or_else(|| vortex_err!("imported buffer has no Java owner covering its range"))?;
+ // SAFETY: the Java reference manager was retained above and owns this exact memory
+ // range. The Arc releases that reference only after all Rust slices are dropped.
+ Ok(unsafe {
+ Buffer::from_custom_allocation(
+ NonNull::new_unchecked(buffer.as_ptr().cast_mut()),
+ buffer.len(),
+ Arc::::clone(owner),
+ )
+ })
+ })
+}
+
+fn replace_owners(
+ data: ArrayData,
+ replace: &mut impl FnMut(&Buffer) -> Result,
+) -> Result {
+ let buffers = data
+ .buffers()
+ .iter()
+ .map(&mut *replace)
+ .collect::, _>>()?;
+ let nulls = data
+ .nulls()
+ .map(|nulls| {
+ let buffer = BooleanBuffer::new(replace(nulls.buffer())?, nulls.offset(), nulls.len());
+ // SAFETY: only the owner changed; the bitmap and its null count are unchanged.
+ Ok::<_, JNIError>(unsafe { NullBuffer::new_unchecked(buffer, nulls.null_count()) })
+ })
+ .transpose()?;
+ let children = data
+ .child_data()
+ .iter()
+ .cloned()
+ .map(|child| replace_owners(child, replace))
+ .collect::, _>>()?;
+ // SAFETY: all buffers still describe exactly the imported data, with unchanged addresses,
+ // lengths, offsets and child layouts. Only their allocation owners have changed.
+ Ok(unsafe {
+ data.into_builder()
+ .buffers(buffers)
+ .nulls(nulls)
+ .child_data(children)
+ .build_unchecked()
+ })
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/vortex-jni/src/buffer/tests.rs b/vortex-jni/src/buffer/tests.rs
new file mode 100644
index 00000000000..709590f436a
--- /dev/null
+++ b/vortex-jni/src/buffer/tests.rs
@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::sync::Arc;
+
+use arrow_array::Array;
+use arrow_array::Int32Array;
+use arrow_array::StructArray;
+use arrow_array::ffi::FFI_ArrowArray;
+use arrow_array::ffi::FFI_ArrowSchema;
+use arrow_array::ffi::from_ffi;
+use arrow_schema::DataType;
+use arrow_schema::Field;
+use vortex::error::VortexResult;
+
+use super::*;
+use crate::io::tests::assert_weak_ref_clears;
+use crate::io::tests::test_vm;
+
+#[test]
+fn test_replacing_owners_releases_unrelated_columns() -> VortexResult<()> {
+ let first = Int32Array::from(vec![Some(1), None, Some(3)]);
+ let second = Int32Array::from(vec![4, 5, 6]);
+ let first_data = first.to_data();
+ let first_values = first_data.buffers()[0].clone();
+ let first_nulls = first_data.nulls().unwrap().buffer().clone();
+ let second_values = second.to_data().buffers()[0].clone();
+ let root = StructArray::from(vec![
+ (
+ Arc::new(Field::new("a", DataType::Int32, true)),
+ Arc::new(first) as _,
+ ),
+ (
+ Arc::new(Field::new("b", DataType::Int32, false)),
+ Arc::new(second) as _,
+ ),
+ ]);
+ let data = root.to_data();
+ let exported = FFI_ArrowArray::new(&data);
+ let schema = FFI_ArrowSchema::try_from(data.data_type())?;
+ // SAFETY: the array and schema were exported above and are consumed exactly once.
+ let imported = unsafe { from_ffi(exported, &schema) }?;
+ drop(data);
+ drop(root);
+ drop(first_data);
+ let original = [first_values, first_nulls, second_values];
+ let baseline = original
+ .iter()
+ .map(Buffer::strong_count)
+ .collect::>();
+ let rebound = replace_owners(imported, &mut |buffer| {
+ let source = original
+ .iter()
+ .find(|b| b.as_ptr() == buffer.as_ptr())
+ .unwrap();
+ assert_eq!(source.len(), buffer.len());
+ Ok(source.clone())
+ })
+ .map_err(|error| vortex_err!("{error}"))?;
+ let kept = rebound.child_data()[0].buffers()[0].clone();
+ assert_eq!(kept.as_ptr(), original[0].as_ptr());
+ assert_eq!(rebound.child_data()[0].nulls().unwrap().null_count(), 1);
+ drop(rebound);
+ assert_eq!(original[0].strong_count(), 2);
+ assert_eq!(original[1].strong_count(), 1);
+ assert_eq!(original[2].strong_count(), 1);
+ assert!(baseline[2] > original[2].strong_count());
+ drop(kept);
+ assert_eq!(original[0].strong_count(), 1);
+ Ok(())
+}
+
+#[test]
+fn test_sliced_validity_preserves_offsets() -> VortexResult<()> {
+ let array = Int32Array::from(vec![Some(1), None, Some(3), Some(4)]);
+ let data = array.to_data().slice(1, 2);
+ let rebound = replace_owners(data.clone(), &mut |buffer| Ok(buffer.clone()))
+ .map_err(|error| vortex_err!("{error}"))?;
+ assert_eq!(rebound.offset(), data.offset());
+ assert_eq!(rebound.nulls(), data.nulls());
+ assert_eq!(rebound.buffers()[0].as_ptr(), data.buffers()[0].as_ptr());
+ assert_eq!(Int32Array::from(rebound), array.slice(1, 2));
+ Ok(())
+}
+
+#[test]
+fn test_release_from_native_thread() -> VortexResult<()> {
+ let Some(vm) = test_vm() else {
+ return Ok(());
+ };
+ // Semaphore.release() has the same JNI signature as NativeBuffer.release(), and
+ // its permit count lets us observe exactly how often the drop callback ran.
+ let (owner, semaphore, weak) = with_jvm(vm, |env| {
+ let object = env.new_object(
+ jni::jni_str!("java/util/concurrent/Semaphore"),
+ jni::jni_sig!("(I)V"),
+ &[jni::JValue::Int(0)],
+ )?;
+ let weak = env.new_object(
+ jni::jni_str!("java/lang/ref/WeakReference"),
+ jni::jni_sig!("(Ljava/lang/Object;)V"),
+ &[jni::JValue::Object(&object)],
+ )?;
+ Ok((
+ JavaBuffer {
+ vm: vm.clone(),
+ object: env.new_global_ref(&object)?,
+ capacity: 0,
+ },
+ env.new_global_ref(&object)?,
+ env.new_global_ref(&weak)?,
+ ))
+ })?;
+ std::thread::spawn(move || drop(owner))
+ .join()
+ .map_err(|_| vortex_err!("release thread panicked"))?;
+ let permits = with_jvm(vm, |env| {
+ Ok(env
+ .call_method(
+ &semaphore,
+ jni::jni_str!("availablePermits"),
+ jni::jni_sig!("()I"),
+ &[],
+ )?
+ .i()?)
+ })?;
+ assert_eq!(permits, 1);
+ drop(semaphore);
+ assert_weak_ref_clears(vm, &weak)
+}
diff --git a/vortex-jni/src/lib.rs b/vortex-jni/src/lib.rs
index fc013566802..86c1f6f1381 100644
--- a/vortex-jni/src/lib.rs
+++ b/vortex-jni/src/lib.rs
@@ -19,6 +19,7 @@ macro_rules! throw_runtime {
}
mod arrow_compat;
+mod buffer;
mod data_source;
mod errors;
mod expression;
diff --git a/vortex-jni/src/writer.rs b/vortex-jni/src/writer.rs
index fc7060195e3..8bd1a513408 100644
--- a/vortex-jni/src/writer.rs
+++ b/vortex-jni/src/writer.rs
@@ -15,6 +15,7 @@ use arrow_array::RecordBatch;
use arrow_array::StructArray;
use arrow_array::ffi::FFI_ArrowArray;
use arrow_array::ffi::FFI_ArrowSchema;
+use arrow_schema::DataType;
use arrow_schema::Schema;
use arrow_schema::SchemaRef;
use async_fs::File;
@@ -24,6 +25,7 @@ use jni::Env;
use jni::EnvUnowned;
use jni::objects::JClass;
use jni::objects::JObject;
+use jni::objects::JObjectArray;
use jni::objects::JString;
use jni::objects::JValue;
use jni::sys::JNI_FALSE;
@@ -68,6 +70,8 @@ use vortex::utils::aliases::hash_map::HashMap;
use vortex_arrow::ArrowSessionExt;
use crate::RUNTIME;
+use crate::buffer::exported_buffer_addresses;
+use crate::buffer::import_buffers;
use crate::errors::JNIError;
use crate::errors::try_or_throw;
use crate::file::extract_metadata;
@@ -530,6 +534,35 @@ pub extern "system" fn Java_dev_vortex_jni_NativeWriter_writeBatch(
})
}
+/// Import a Java Arrow batch with per-allocation ownership instead of a shared batch owner.
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_dev_vortex_jni_NativeWriter_writeBatchBuffers(
+ mut env: EnvUnowned,
+ _class: JClass,
+ writer_ptr: jlong,
+ arrow_array_addr: jlong,
+ buffers: JObjectArray,
+) -> jboolean {
+ if writer_ptr <= 0 {
+ return JNI_FALSE;
+ }
+ try_or_throw(&mut env, |env| {
+ // SAFETY: Java validates the root schema against the writer's schema and holds the
+ // writer monitor for this call. Moving the array clears its release callback.
+ let writer = unsafe { NativeWriter::from_ptr(writer_ptr) };
+ let array = unsafe { FFI_ArrowArray::from_raw(arrow_array_addr as *mut FFI_ArrowArray) };
+ let exported_addresses = exported_buffer_addresses(&array);
+ let dtype = DataType::Struct(writer.arrow_schema.fields().clone());
+ // SAFETY: the Java export keeps all buffers alive through the import, and the
+ // caller validated its schema before exporting the array.
+ let data = unsafe { arrow_array::ffi::from_ffi_and_data_type(array, dtype) }
+ .map_err(|e| JNIError::Vortex(vortex_err!("failed to import Arrow FFI data: {e}")))?;
+ let data = import_buffers(env, data, &exported_addresses, buffers)?;
+ writer.write_record_batch(RecordBatch::from(StructArray::from(data)))?;
+ Ok(JNI_TRUE)
+ })
+}
+
#[unsafe(no_mangle)]
pub extern "system" fn Java_dev_vortex_jni_NativeWriter_bytesWritten(
mut env: EnvUnowned,