From 62b73ec9822bd8a093f8ceb49503197dbbb17c29 Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Mon, 9 Nov 2026 20:53:51 +0000 Subject: [PATCH] perf(spark): avoid redundant metadata HEADs when opening Hadoop streams Signed-off-by: Christopher Wirt --- .../dev/vortex/spark/io/HadoopReadable.java | 24 +++- .../vortex/spark/io/HadoopReadableTest.java | 132 ++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/HadoopReadable.java b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/HadoopReadable.java index 26da27450b3..09f4831f69b 100644 --- a/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/HadoopReadable.java +++ b/java/vortex-spark/common/src/main/java/dev/vortex/spark/io/HadoopReadable.java @@ -5,13 +5,18 @@ import dev.vortex.io.PooledReadable; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.functional.FutureIO; /** A {@link dev.vortex.io.NativeReadable} that serves Vortex's reads from a Hadoop {@link FileSystem}. */ public final class HadoopReadable extends PooledReadable { @@ -46,7 +51,24 @@ private HadoopReadable(String name, Path path, FileSystem fs, long length) { @Override protected PositionalStream openStream() throws IOException { - return new HadoopStream(fs.open(path)); + // Given the length, S3A on Hadoop 3.3.5+ skips the metadata HEAD per stream; older Hadoop + // (Spark 3.5) ignores these keys. String keys because 3.3.4 lacks the OpenFileOptions constants. + CompletableFuture opened = fs.openFile(path) + .opt("fs.option.openfile.length", Long.toString(length())) + .opt("fs.option.openfile.read.policy", "random") + .build(); + try { + return new HadoopStream(opened.get()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + InterruptedIOException failure = new InterruptedIOException("Interrupted while opening " + path); + failure.initCause(e); + // Cancelling a future need not stop the underlying open. Close any stream it produces later. + var unused = opened.thenAccept(IOUtils::closeStream); + throw failure; + } catch (ExecutionException e) { + return FutureIO.raiseInnerCause(e); + } } private static final class HadoopStream implements PositionalStream { diff --git a/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/HadoopReadableTest.java b/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/HadoopReadableTest.java index afef7223b2a..588b3cbd83d 100644 --- a/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/HadoopReadableTest.java +++ b/java/vortex-spark/common/src/test/java/dev/vortex/spark/io/HadoopReadableTest.java @@ -5,21 +5,36 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import dev.vortex.io.NativeReadable; import java.io.EOFException; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.UncheckedIOException; +import java.net.URI; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FutureDataInputStreamBuilder; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -58,6 +73,74 @@ void suppliedLengthIsTrustedOverAnExtraStat() throws IOException { } } + @Test + void everyStreamUsesTheResolvedLengthAndRandomReadPolicy() throws IOException { + byte[] content = content(64); + Path file = write("builder.vortex", content); + for (long suppliedLength : new long[] {-1, content.length}) { + try (TrackingFileSystem fs = trackingFileSystem(); + HadoopReadable readable = + HadoopReadable.open(fs.getConf(), trackedPath(fs, file), suppliedLength)) { + assertEquals(suppliedLength < 0 ? 1 : 0, fs.stats); + // Open one stream directly and let a pooled read open a second. + try (var first = readable.openStream()) { + ByteBuffer buffer = ByteBuffer.allocateDirect(content.length); + readable.readFully(0, buffer); + assertArrayEquals(content, drain(buffer)); + } + assertEquals(2, fs.openOptions.size()); + for (Configuration options : fs.openOptions) { + assertEquals("64", options.get("fs.option.openfile.length")); + assertEquals("random", options.get("fs.option.openfile.read.policy")); + } + } + } + } + + @Test + void failedAsynchronousOpensPreserveTheirOriginalExceptions() throws IOException { + Path file = write("failed-open.vortex", content(64)); + IOException missing = new FileNotFoundException("missing object"); + for (Throwable failure : new Throwable[] { + missing, + new UncheckedIOException(missing), + new IllegalArgumentException("invalid option"), + new AssertionError("open failed") + }) { + try (TrackingFileSystem fs = trackingFileSystem(); + HadoopReadable readable = HadoopReadable.open(fs.getConf(), trackedPath(fs, file), 64)) { + fs.nextOpen = CompletableFuture.failedFuture(failure); + Throwable expected = failure instanceof UncheckedIOException ? missing : failure; + assertSame( + expected, + assertThrows(expected.getClass(), () -> readable.readFully(0, ByteBuffer.allocateDirect(1)))); + } + } + } + + @Test + void interruptedOpenPreservesInterruptAndClosesALateStream() throws IOException { + Path file = write("interrupted-open.vortex", content(64)); + try (TrackingFileSystem fs = trackingFileSystem(); + HadoopReadable readable = HadoopReadable.open(fs.getConf(), trackedPath(fs, file), 64)) { + CompletableFuture pending = new CompletableFuture<>(); + fs.nextOpen = pending; + Thread.currentThread().interrupt(); + try { + InterruptedIOException failure = assertThrows( + InterruptedIOException.class, () -> readable.readFully(0, ByteBuffer.allocateDirect(1))); + assertTrue(Thread.currentThread().isInterrupted()); + assertTrue(failure.getCause() instanceof InterruptedException); + } finally { + Thread.interrupted(); + } + assertFalse(pending.isCancelled()); + FSDataInputStream lateStream = fs.open(new org.apache.hadoop.fs.Path(trackedPath(fs, file))); + pending.complete(lateStream); + assertThrows(IOException.class, lateStream::read); + } + } + @Test void concurrentReadsEachGetTheirOwnStream() throws Exception { byte[] content = content(SIZE); @@ -148,6 +231,55 @@ void createReplacesAnExistingFile() throws IOException { assertArrayEquals(new byte[] {9}, Files.readAllBytes(file)); } + private static TrackingFileSystem trackingFileSystem() throws IOException { + Configuration trackingConf = new Configuration(); + trackingConf.setClass("fs.tracking.impl", TrackingFileSystem.class, FileSystem.class); + return (TrackingFileSystem) FileSystem.get(URI.create("tracking://" + UUID.randomUUID()), trackingConf); + } + + private static String trackedPath(TrackingFileSystem fs, Path file) { + return new org.apache.hadoop.fs.Path("tracking", fs.getUri().getAuthority(), file.toString()).toString(); + } + + /** Hadoop instantiates this filesystem through the test configuration. */ + public static final class TrackingFileSystem extends RawLocalFileSystem { + private URI uri; + private int stats; + private final List openOptions = new ArrayList<>(); + private CompletableFuture nextOpen; + + @Override + public void initialize(URI name, Configuration configuration) throws IOException { + uri = name; + super.initialize(name, configuration); + } + + @Override + public URI getUri() { + return uri != null ? uri : super.getUri(); + } + + @Override + public FileStatus getFileStatus(org.apache.hadoop.fs.Path path) throws IOException { + stats++; + return super.getFileStatus(path); + } + + @Override + public FutureDataInputStreamBuilder openFile(org.apache.hadoop.fs.Path path) { + return new FutureDataInputStreamBuilderImpl(this, path) { + @Override + public CompletableFuture build() throws IOException { + assertTrue(getMandatoryKeys().isEmpty(), "Open options must remain optional for other filesystems"); + openOptions.add(new Configuration(getOptions())); + return nextOpen != null + ? nextOpen + : CompletableFuture.completedFuture(TrackingFileSystem.this.open(path, getBufferSize())); + } + }; + } + } + private Path write(String name, byte[] content) throws IOException { Path file = tempDir.resolve(name); Files.write(file, content);