diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java
index 370affb0e38..5890e511536 100644
--- a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java
+++ b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java
@@ -145,6 +145,14 @@ public static Expression isNotNull(Expression child) {
return new Expression(NativeExpression.isNotNull(child.nativePointer()));
}
+ /**
+ * Null out {@code child} wherever {@code mask} is false, keeping its value where {@code mask} is true. The mask
+ * must be a non-nullable boolean.
+ */
+ public static Expression mask(Expression child, Expression mask) {
+ return new Expression(NativeExpression.mask(child.nativePointer(), mask.nativePointer()));
+ }
+
/**
* SQL {@code LIKE} pattern match.
*
diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java
index 58ae38be4d8..0943cd86e59 100644
--- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java
+++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java
@@ -37,6 +37,8 @@ private NativeExpression() {}
public static native long isNotNull(long childPointer);
+ public static native long mask(long childPointer, long maskPointer);
+
public static native long like(long childPointer, long patternPointer, boolean negated, boolean caseInsensitive);
public static native long between(
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/NestedSchemaPruning.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/NestedSchemaPruning.java
new file mode 100644
index 00000000000..2f13fdba979
--- /dev/null
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/NestedSchemaPruning.java
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+package dev.vortex.spark.read;
+
+import dev.vortex.api.Expression;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+/**
+ * Pushes Spark's nested schema pruning down to the native scan.
+ *
+ *
Only struct fields are pruned. Fields inside arrays and maps keep their full table type, since projecting each
+ * element needs a per-element expression the native scan does not expose yet.
+ */
+final class NestedSchemaPruning {
+ private NestedSchemaPruning() {}
+
+ /**
+ * Returns {@code tableType} pruned to the struct fields present in {@code requiredType}, in table order. Any type
+ * that is not a struct on both sides keeps its table type.
+ */
+ static DataType prune(DataType tableType, DataType requiredType) {
+ if (!(tableType instanceof StructType table) || !(requiredType instanceof StructType required)) {
+ return tableType;
+ }
+ List fields = new ArrayList<>();
+ for (StructField field : table.fields()) {
+ if (required.getFieldIndex(field.name()).isDefined()) {
+ DataType prunedType =
+ prune(field.dataType(), required.apply(field.name()).dataType());
+ fields.add(new StructField(field.name(), prunedType, field.nullable(), field.metadata()));
+ }
+ }
+ return fields.isEmpty() ? tableType : new StructType(fields.toArray(new StructField[0]));
+ }
+
+ /**
+ * Builds the native projection that returns {@code readSchema}, where each field is either its type in
+ * {@code tableSchema} or a {@link #prune pruned} version of it.
+ *
+ * When nothing is pruned this is a plain {@code select} of the top-level columns.
+ */
+ static Expression projection(StructType readSchema, StructType tableSchema) {
+ return structProjection(Expression.root(), false, readSchema, tableSchema);
+ }
+
+ private static Expression structProjection(Expression struct, boolean nullable, StructType read, StructType table) {
+ String[] names = read.fieldNames();
+ // `select` keeps the struct's own nulls, so use it when no child needs pruning.
+ if (!isPruned(read, table)) {
+ return Expression.select(names, struct);
+ }
+ Expression[] children = new Expression[names.length];
+ for (int i = 0; i < names.length; i++) {
+ Expression field = Expression.getItem(names[i], struct);
+ StructField readField = read.fields()[i];
+ DataType tableType = table.apply(names[i]).dataType();
+ children[i] = readField.dataType().equals(tableType)
+ ? field
+ : structProjection(
+ field, readField.nullable(), (StructType) readField.dataType(), (StructType) tableType);
+ }
+ Expression packed = Expression.pack(names, children, false);
+ // `pack` builds a struct with no nulls, so carry over the source struct's nulls.
+ return nullable ? Expression.mask(packed, Expression.isNotNull(struct)) : packed;
+ }
+
+ private static boolean isPruned(StructType read, StructType table) {
+ for (StructField field : read.fields()) {
+ if (!field.dataType().equals(table.apply(field.name()).dataType())) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java
index a1c4df239b7..b0ff943fd1a 100644
--- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java
@@ -26,12 +26,14 @@
import org.apache.spark.sql.connector.read.Batch;
import org.apache.spark.sql.connector.read.InputPartition;
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
+import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
/** Execution source for batch scans of Vortex file tables. */
public final class VortexBatchExec implements Batch {
private final List paths;
private final StructType readSchema;
+ private final StructType tableSchema;
private final Map formatOptions;
private final Predicate[] pushedPredicates;
private List resolvedPaths;
@@ -41,13 +43,19 @@ public final class VortexBatchExec implements Batch {
*
* @param paths the list of file paths to scan
* @param columns the list of columns to read from the files
+ * @param tableColumns the full table columns, which {@code columns} may prune nested fields from
* @param pushedPredicates predicates pushed down by Spark; converted to a single Vortex filter expression at read
* time
*/
public VortexBatchExec(
- List paths, List columns, Map formatOptions, Predicate[] pushedPredicates) {
+ List paths,
+ List columns,
+ List tableColumns,
+ Map formatOptions,
+ Predicate[] pushedPredicates) {
this.paths = List.copyOf(paths);
this.readSchema = CatalogV2Util.v2ColumnsToStructType(columns.toArray(new Column[0]));
+ this.tableSchema = CatalogV2Util.v2ColumnsToStructType(tableColumns.toArray(new Column[0]));
this.formatOptions = Map.copyOf(formatOptions);
this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone();
}
@@ -106,10 +114,14 @@ private Map measureFiles(List paths)
public PartitionReaderFactory createReaderFactory() {
List files = resolvedPaths != null ? resolvedPaths : resolvePaths();
Set partitionColumns = collectPartitionColumnNames(files);
- List dataColumnNames = Arrays.stream(readSchema.fieldNames())
- .filter(name -> !partitionColumns.contains(name))
- .collect(Collectors.toList());
- return new VortexPartitionReaderFactory(dataColumnNames, formatOptions, pushedPredicates);
+ StructType dataReadSchema = new StructType(Arrays.stream(readSchema.fields())
+ .filter(field -> !partitionColumns.contains(field.name()))
+ .toArray(StructField[]::new));
+ // Only the read columns' table types are needed to build the projection.
+ StructType dataTableSchema = new StructType(Arrays.stream(dataReadSchema.fieldNames())
+ .map(tableSchema::apply)
+ .toArray(StructField[]::new));
+ return new VortexPartitionReaderFactory(dataReadSchema, dataTableSchema, formatOptions, pushedPredicates);
}
private List resolvePaths() {
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java
index 7bc649ce67a..9880c31a19c 100644
--- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java
@@ -16,12 +16,12 @@
import dev.vortex.spark.VortexFilePartition;
import dev.vortex.spark.VortexSparkSession;
import java.io.IOException;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.PartitionReader;
import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.vectorized.ColumnVector;
import org.apache.spark.sql.vectorized.ColumnarBatch;
@@ -50,7 +50,8 @@ final class VortexPartitionReader implements PartitionReader {
VortexPartitionReader(
VortexFilePartition spark,
- List dataColumnNames,
+ StructType dataReadSchema,
+ StructType tableSchema,
Map formatOptions,
Predicate[] pushedPredicates) {
this.spark = spark;
@@ -60,9 +61,8 @@ final class VortexPartitionReader implements PartitionReader {
dataSource = DataSource.open(session, spark.paths(), formatOptions);
var options = ScanOptions.builder();
- if (!dataColumnNames.isEmpty()) {
- Expression projection = Expression.select(dataColumnNames.toArray(new String[0]), Expression.root());
- options.projection(projection);
+ if (dataReadSchema.fields().length > 0) {
+ options.projection(NestedSchemaPruning.projection(dataReadSchema, tableSchema));
}
if (pushedPredicates != null && pushedPredicates.length > 0) {
buildFilterExpression(pushedPredicates).ifPresent(options::filter);
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java
index e187e4863b1..0e0e1543c93 100644
--- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java
@@ -3,18 +3,17 @@
package dev.vortex.spark.read;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import dev.vortex.jni.NativeRuntime;
import dev.vortex.spark.VortexFilePartition;
import java.io.Serializable;
-import java.util.List;
import java.util.Map;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.InputPartition;
import org.apache.spark.sql.connector.read.PartitionReader;
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
+import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.vectorized.ColumnarBatch;
/**
@@ -28,13 +27,18 @@
public final class VortexPartitionReaderFactory implements PartitionReaderFactory, Serializable {
private static final long serialVersionUID = 1L;
- private final ImmutableList dataColumnNames;
+ private final StructType dataReadSchema;
+ private final StructType tableSchema;
private final ImmutableMap formatOptions;
private final Predicate[] pushedPredicates;
public VortexPartitionReaderFactory(
- List dataColumnNames, Map formatOptions, Predicate[] pushedPredicates) {
- this.dataColumnNames = ImmutableList.copyOf(dataColumnNames);
+ StructType dataReadSchema,
+ StructType tableSchema,
+ Map formatOptions,
+ Predicate[] pushedPredicates) {
+ this.dataReadSchema = dataReadSchema;
+ this.tableSchema = tableSchema;
this.formatOptions = ImmutableMap.copyOf(formatOptions);
this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone();
}
@@ -48,7 +52,7 @@ public PartitionReader createReader(InputPartition partition) {
public PartitionReader createColumnarReader(InputPartition partition) {
NativeRuntime.setWorkerThreads(Integer.parseInt(formatOptions.getOrDefault("vortex.workerThreads", "4")));
VortexFilePartition spark = (VortexFilePartition) partition;
- return new VortexPartitionReader(spark, dataColumnNames, formatOptions, pushedPredicates);
+ return new VortexPartitionReader(spark, dataReadSchema, tableSchema, formatOptions, pushedPredicates);
}
@Override
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java
index 6da2306ab69..8764ee6fe28 100644
--- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java
@@ -117,7 +117,7 @@ public String description() {
*/
@Override
public Batch toBatch() {
- return new VortexBatchExec(paths, readColumns, formatOptions, pushedPredicates);
+ return new VortexBatchExec(paths, readColumns, tableColumns, formatOptions, pushedPredicates);
}
/**
diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java
index 97da8d42302..8531cf4bf01 100644
--- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java
+++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java
@@ -130,22 +130,33 @@ public Scan build() {
/**
* Prunes the columns to only include those specified in the required schema.
*
- * Required columns keep their table type because the native scan returns whole top-level columns, even when
- * Spark prunes nested fields.
+ *
Columns stay in table order. Struct columns are pruned to the nested fields Spark requires, and the reader
+ * projects only those fields. See {@link NestedSchemaPruning} for which nested types are pruned.
*
* @param requiredSchema the schema specifying which columns are required
*/
@Override
public void pruneColumns(StructType requiredSchema) {
- Set required = Set.copyOf(Arrays.asList(requiredSchema.fieldNames()));
readColumns.clear();
for (Column column : tableColumns) {
- if (required.contains(column.name())) {
- readColumns.add(column);
+ if (requiredSchema.getFieldIndex(column.name()).isDefined()) {
+ DataType prunedType = NestedSchemaPruning.prune(
+ column.dataType(), requiredSchema.apply(column.name()).dataType());
+ readColumns.add(prunedType.equals(column.dataType()) ? column : withDataType(column, prunedType));
}
}
}
+ private static Column withDataType(Column column, DataType dataType) {
+ return Column.create(
+ column.name(),
+ dataType,
+ column.nullable(),
+ column.comment(),
+ column.defaultValue(),
+ column.metadataInJSON());
+ }
+
/**
* Splits the supplied predicates into pushed and not-pushed sets.
*
diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java
index 16bdc59a0f8..1d98b2264c2 100644
--- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java
+++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java
@@ -6,6 +6,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.nio.file.Path;
+import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.spark.sql.Column;
@@ -60,7 +61,27 @@ public void setUp(@TempDir Path tempDir) {
functions.map(functions.lit("k"), abc).alias("m"),
functions
.struct(functions.lit("x").alias("p"), abc.alias("inner"))
- .alias("outer"));
+ .alias("outer"),
+ nullWhen(
+ functions.col("id").equalTo(1),
+ functions.struct(
+ functions.lit("x").alias("p"),
+ nullWhen(functions.col("id").equalTo(2), abc)
+ .alias("inner")))
+ .alias("nullable_outer"),
+ nullWhen(
+ functions.col("id").mod(2).equalTo(1),
+ functions.struct(
+ functions.col("id").plus(100).alias("a"),
+ nullWhen(
+ functions
+ .col("id")
+ .equalTo(4),
+ functions
+ .col("id")
+ .plus(200))
+ .alias("b")))
+ .alias("n"));
String path = tempDir.resolve("nested").toUri().toString();
df.write()
@@ -81,30 +102,81 @@ public void tearDown() {
@Test
@DisplayName("Selecting a struct field other than the first returns that field")
public void structField() {
- assertEquals(C_VALUES, longsOf(functions.col("s.c")));
+ assertEquals(C_VALUES, valuesOf(table, functions.col("s.c")));
}
@Test
@DisplayName("Selecting a field of the structs in an array returns that field")
public void arrayElementField() {
- assertEquals(C_VALUES, longsOf(functions.element_at(functions.col("arr.c"), 1)));
+ assertEquals(C_VALUES, valuesOf(table, functions.element_at(functions.col("arr.c"), 1)));
}
@Test
@DisplayName("Selecting a field of the structs in a map returns that field")
public void mapValueField() {
- assertEquals(C_VALUES, longsOf(functions.col("m").getItem("k").getField("c")));
+ assertEquals(C_VALUES, valuesOf(table, functions.col("m").getItem("k").getField("c")));
}
@Test
@DisplayName("Selecting a field of a struct inside a struct returns that field")
public void structInStructField() {
- assertEquals(C_VALUES, longsOf(functions.col("outer.inner.c")));
+ assertEquals(C_VALUES, valuesOf(table, functions.col("outer.inner.c")));
}
- private List longsOf(Column column) {
- return table.orderBy("id").select(column).collectAsList().stream()
- .map(row -> row.getLong(0))
+ @Test
+ @DisplayName("Null structs and null fields stay null when the struct is pruned")
+ public void nullableStructField() {
+ assertEquals(Arrays.asList(200L, null, 202L, null, null), valuesOf(table, functions.col("n.b")));
+ assertEquals(
+ List.of(false, true, false, true, false),
+ valuesOf(table, functions.col("n").isNull()));
+ }
+
+ @Test
+ @DisplayName("Nulls at each level of a struct inside a struct survive pruning both levels")
+ public void nullableStructInStructField() {
+ assertEquals(
+ Arrays.asList(300L, null, null, 303L, 304L), valuesOf(table, functions.col("nullable_outer.inner.c")));
+ assertEquals(
+ List.of(false, true, false, false, false),
+ valuesOf(table, functions.col("nullable_outer").isNull()));
+ assertEquals(
+ List.of(false, true, true, false, false),
+ valuesOf(table, functions.col("nullable_outer.inner").isNull()));
+ }
+
+ @Test
+ @DisplayName("A pushed filter on a field the query does not select still applies")
+ public void filterOnPrunedField() {
+ assertEquals(
+ List.of(302L, 303L, 304L),
+ valuesOf(table.where(functions.col("s.a").gt(101)), functions.col("s.c")));
+ }
+
+ @Test
+ @DisplayName("Whole struct columns still read in full alongside pruned ones")
+ public void wholeAndPrunedStructs() {
+ Row row = table.where("id = 3")
+ .select(functions.col("s"), functions.col("outer.inner.a"))
+ .first();
+ assertEquals(
+ List.of(103L, 203L, 303L),
+ List.of(
+ row.getStruct(0).getLong(0),
+ row.getStruct(0).getLong(1),
+ row.getStruct(0).getLong(2)));
+ assertEquals(103L, row.getLong(1));
+ }
+
+ private static Column nullWhen(Column condition, Column value) {
+ return functions.when(condition, functions.lit(null)).otherwise(value);
+ }
+
+ /** Returns {@code column} for each row of {@code rows}, ordered by id. */
+ @SuppressWarnings("unchecked")
+ private static List valuesOf(Dataset rows, Column column) {
+ return rows.orderBy("id").select(column).collectAsList().stream()
+ .map(row -> (T) row.get(0))
.collect(Collectors.toList());
}
}
diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java
index ae67a475865..7b56522eb68 100644
--- a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java
+++ b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java
@@ -29,7 +29,7 @@ final class VortexBatchExecTest {
Column.create("name", org.apache.spark.sql.types.DataTypes.StringType));
private static VortexBatchExec execFor(List paths) {
- return new VortexBatchExec(paths, COLUMNS, Map.of(), new Predicate[0]);
+ return new VortexBatchExec(paths, COLUMNS, COLUMNS, Map.of(), new Predicate[0]);
}
@Test
@@ -89,7 +89,8 @@ void partitionValuesParsedPerFile() {
@DisplayName("Format options are propagated to every partition")
void formatOptionsPropagated() {
Map options = Map.of("vortex.workerThreads", "8");
- VortexBatchExec exec = new VortexBatchExec(List.of("/data/a.vortex"), COLUMNS, options, new Predicate[0]);
+ VortexBatchExec exec =
+ new VortexBatchExec(List.of("/data/a.vortex"), COLUMNS, COLUMNS, options, new Predicate[0]);
VortexFilePartition partition = (VortexFilePartition) exec.planInputPartitions()[0];
diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java
index 485c093daff..c043cc52895 100644
--- a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java
+++ b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java
@@ -15,7 +15,9 @@
import org.apache.spark.sql.connector.expressions.NamedReference;
import org.apache.spark.sql.connector.expressions.Transform;
import org.apache.spark.sql.connector.expressions.filter.Predicate;
+import org.apache.spark.sql.types.ArrayType;
import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.MapType;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.unsafe.types.UTF8String;
import org.junit.jupiter.api.DisplayName;
@@ -163,29 +165,63 @@ void pushedPredicatesReturnsCopy() {
// --- pruneColumns ---
+ private static final StructType ABC = new StructType()
+ .add("a", DataTypes.LongType)
+ .add("b", DataTypes.LongType)
+ .add("c", DataTypes.LongType);
+
@Test
- @DisplayName("Required columns keep their full table type, so readSchema matches what the reader returns")
- void pruneColumnsKeepsTableTypes() {
- StructType abc = new StructType()
- .add("a", DataTypes.LongType)
- .add("b", DataTypes.LongType)
- .add("c", DataTypes.LongType);
+ @DisplayName("Struct columns are pruned to the required fields, with columns and fields in table order")
+ void pruneColumnsPrunesStructFields() {
VortexScanBuilder builder = builderWithColumns(
new Transform[0],
Column.create("id", DataTypes.IntegerType),
- Column.create("s", abc),
+ Column.create("s", ABC),
Column.create("name", DataTypes.StringType));
- // With nested schema pruning, Spark asks for s with only the field the query uses.
builder.pruneColumns(new StructType()
- .add("s", new StructType().add("c", DataTypes.LongType))
+ .add("s", new StructType().add("c", DataTypes.LongType).add("a", DataTypes.LongType))
.add("id", DataTypes.IntegerType));
assertEquals(
- new StructType().add("id", DataTypes.IntegerType).add("s", abc),
+ new StructType()
+ .add("id", DataTypes.IntegerType)
+ .add("s", new StructType().add("a", DataTypes.LongType).add("c", DataTypes.LongType)),
builder.build().readSchema());
}
+ @Test
+ @DisplayName("Structs inside structs are pruned at every level")
+ void pruneColumnsPrunesNestedStructs() {
+ StructType outer = new StructType().add("p", DataTypes.StringType).add("inner", ABC);
+ VortexScanBuilder builder = builderWithColumns(new Transform[0], Column.create("outer", outer));
+
+ builder.pruneColumns(new StructType()
+ .add("outer", new StructType().add("inner", new StructType().add("b", DataTypes.LongType))));
+
+ assertEquals(
+ new StructType()
+ .add("outer", new StructType().add("inner", new StructType().add("b", DataTypes.LongType))),
+ builder.build().readSchema());
+ }
+
+ @Test
+ @DisplayName("Arrays and maps of structs keep their full table type")
+ void pruneColumnsKeepsArrayAndMapTypes() {
+ ArrayType arr = DataTypes.createArrayType(ABC);
+ MapType map = DataTypes.createMapType(DataTypes.StringType, ABC);
+ VortexScanBuilder builder =
+ builderWithColumns(new Transform[0], Column.create("arr", arr), Column.create("m", map));
+
+ StructType c = new StructType().add("c", DataTypes.LongType);
+ builder.pruneColumns(new StructType()
+ .add("arr", DataTypes.createArrayType(c))
+ .add("m", DataTypes.createMapType(DataTypes.StringType, c)));
+
+ assertEquals(
+ new StructType().add("arr", arr).add("m", map), builder.build().readSchema());
+ }
+
// --- build ---
@Test
diff --git a/vortex-jni/src/expression.rs b/vortex-jni/src/expression.rs
index afcefe990c4..21774b1c0fb 100644
--- a/vortex-jni/src/expression.rs
+++ b/vortex-jni/src/expression.rs
@@ -47,6 +47,7 @@ use vortex::expr::is_not_null;
use vortex::expr::is_null;
use vortex::expr::list_contains_opts;
use vortex::expr::lit;
+use vortex::expr::mask;
use vortex::expr::merge_opts;
use vortex::expr::not;
use vortex::expr::or_collect;
@@ -359,6 +360,18 @@ pub extern "system" fn Java_dev_vortex_jni_NativeExpression_isNotNull(
into_raw(is_not_null(child))
}
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_dev_vortex_jni_NativeExpression_mask(
+ _env: EnvUnowned,
+ _class: JClass,
+ child: jlong,
+ mask_expr: jlong,
+) -> jlong {
+ let child = unsafe { expr_ref(child) }.clone();
+ let mask_expr = unsafe { expr_ref(mask_expr) }.clone();
+ into_raw(mask(child, mask_expr))
+}
+
#[unsafe(no_mangle)]
pub extern "system" fn Java_dev_vortex_jni_NativeExpression_like(
_env: EnvUnowned,
diff --git a/vortex-layout/src/layouts/list/reader.rs b/vortex-layout/src/layouts/list/reader.rs
index fafa48346bd..208e32ebc39 100644
--- a/vortex-layout/src/layouts/list/reader.rs
+++ b/vortex-layout/src/layouts/list/reader.rs
@@ -616,8 +616,6 @@ fn apply_lengths_validity(
#[cfg(test)]
mod tests {
use std::ops::Range;
- use std::sync::atomic::AtomicUsize;
- use std::sync::atomic::Ordering;
use rstest::rstest;
use vortex_array::ArrayContext;
@@ -647,7 +645,7 @@ mod tests {
use crate::layouts::repartition::RepartitionStrategy;
use crate::layouts::repartition::RepartitionWriterOptions;
use crate::scan::split_by::SplitBy;
- use crate::segments::SegmentFuture;
+ use crate::segments::CountingSegmentSource;
use crate::segments::SegmentSource;
use crate::segments::TestSegments;
use crate::sequence::SequenceId;
@@ -1139,18 +1137,6 @@ mod tests {
))
}
- struct CountingSegmentSource {
- inner: Arc,
- request_count: Arc,
- }
-
- impl SegmentSource for CountingSegmentSource {
- fn request(&self, id: crate::segments::SegmentId) -> SegmentFuture {
- self.request_count.fetch_add(1, Ordering::Relaxed);
- self.inner.request(id)
- }
- }
-
/// The chunked-elements strategy must actually produce a chunked `elements` layout, otherwise
/// the reader would silently take the whole-chunk path and the bounded read would be untested.
#[tokio::test]
@@ -1175,12 +1161,8 @@ mod tests {
let ctx = LayoutReaderContext::new();
let (segments, layout, session) =
write_layout(&chunked_elements_list_strategy(), list.clone()).await?;
- let request_count = Arc::new(AtomicUsize::new(0));
- let source = Arc::new(CountingSegmentSource {
- inner: segments,
- request_count: Arc::clone(&request_count),
- });
- let reader = layout.new_reader("".into(), source, &session, &ctx)?;
+ let source = Arc::new(CountingSegmentSource::new(segments));
+ let reader = layout.new_reader("".into(), Arc::clone(&source) as _, &session, &ctx)?;
let mask = Mask::from_iter([true, false, false, false, false]);
let expr = root().bind(reader.dtype())?;
@@ -1191,7 +1173,7 @@ mod tests {
let expected = list.filter(mask)?;
let mut exec_ctx = session.create_execution_ctx();
assert_arrays_eq!(result, expected, &mut exec_ctx);
- assert_eq!(request_count.load(Ordering::Relaxed), 2);
+ assert_eq!(source.requests(), 2);
Ok(())
}
diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs
index 30547d298eb..63046e6e616 100644
--- a/vortex-layout/src/layouts/struct_/reader.rs
+++ b/vortex-layout/src/layouts/struct_/reader.rs
@@ -626,6 +626,7 @@ mod tests {
use vortex_array::expr::is_null;
use vortex_array::expr::lit;
use vortex_array::expr::lt;
+ use vortex_array::expr::mask;
use vortex_array::expr::not;
use vortex_array::expr::or;
use vortex_array::expr::pack;
@@ -648,6 +649,7 @@ mod tests {
use crate::layouts::table::TableStrategy;
use crate::layouts::zoned::writer::ZonedLayoutOptions;
use crate::layouts::zoned::writer::ZonedStrategy;
+ use crate::segments::CountingSegmentSource;
use crate::segments::SegmentSource;
use crate::segments::TestSegments;
use crate::sequence::SequenceId;
@@ -830,6 +832,142 @@ mod tests {
(segments, layout)
}
+ /// Writes `{id, s: {a, b, c}}`, where the struct `s` is null in row 0.
+ #[fixture]
+ fn table_with_null_struct_layout() -> (Arc, LayoutRef) {
+ let segments = Arc::new(TestSegments::default());
+ let (ptr, eof) = SequenceId::root().split();
+ let strategy = TableStrategy::new(
+ Arc::new(FlatLayoutStrategy::default()),
+ Arc::new(FlatLayoutStrategy::default()),
+ );
+ let s = StructArray::try_from_iter_with_validity(
+ [
+ ("a", buffer![7, 2, 3].into_array()),
+ ("b", buffer![4, 5, 6].into_array()),
+ ("c", buffer![8, 9, 10].into_array()),
+ ],
+ Validity::Array(BoolArray::from_iter([false, true, true]).into_array()),
+ )
+ .unwrap();
+ let table = StructArray::try_from_iter([
+ ("id", buffer![0, 1, 2].into_array()),
+ ("s", s.into_array()),
+ ])
+ .unwrap();
+ let segments2 = Arc::::clone(&segments);
+ let layout = block_on(|handle| async move {
+ let session = new_session().with_handle(handle);
+ strategy
+ .write_stream(
+ ArrayContext::empty().into(),
+ segments2,
+ table.into_array().to_array_stream().sequenced(ptr),
+ eof,
+ &session,
+ )
+ .await
+ })
+ .unwrap();
+
+ (segments, layout)
+ }
+
+ /// Projecting a nullable struct reads only the fields it keeps, plus the struct validity.
+ #[rstest]
+ #[case::select_one_field(select(["b"], root()), 2)]
+ #[case::whole_struct(root(), 4)]
+ fn test_nullable_struct_projection_request_count(
+ #[from(null_struct_layout)] (segments, layout): (Arc, LayoutRef),
+ #[case] expr: Expression,
+ #[case] expected_requests: usize,
+ ) -> VortexResult<()> {
+ let source = Arc::new(CountingSegmentSource::new(segments));
+ let reader = layout.new_reader(
+ "".into(),
+ Arc::clone(&source) as _,
+ &SESSION,
+ &Default::default(),
+ )?;
+ let expr = expr.bind(reader.dtype())?;
+ let project = reader.projection_evaluation(&(0..3), &expr, MaskFuture::new_true(3))?;
+ let result = block_on(move |_| project)?;
+
+ assert_eq!(source.requests(), expected_requests);
+ let mut ctx = SESSION.create_execution_ctx();
+ let result = result.execute::(&mut ctx)?;
+ assert_eq!(
+ result.validity()?.execute_mask(3, &mut ctx)?,
+ Mask::from_iter([false, true, true])
+ );
+ assert_arrays_eq!(
+ result.unmasked_field_by_name("b")?,
+ PrimitiveArray::from_iter([4i32, 5, 6]),
+ &mut ctx
+ );
+ Ok(())
+ }
+
+ /// Projecting a pruned nullable struct column, with `select` or a masked `pack`, reads only
+ /// the fields it keeps, plus the struct validity.
+ #[rstest]
+ #[case::select(
+ select(["b"], get_item("s", root())),
+ PrimitiveArray::from_iter([4i32, 5, 6]).into_array(),
+ )]
+ // Reading `b` through the nullable `s` also nulls it where `s` is null.
+ #[case::masked_pack(
+ mask(
+ pack([("b", get_item("b", get_item("s", root())))], Nullability::NonNullable),
+ is_not_null(get_item("s", root())),
+ ),
+ PrimitiveArray::from_option_iter([None, Some(5i32), Some(6)]).into_array(),
+ )]
+ fn test_pruned_nullable_struct_column_reads_only_kept_fields(
+ #[from(table_with_null_struct_layout)] (segments, layout): (
+ Arc,
+ LayoutRef,
+ ),
+ #[case] s_projection: Expression,
+ #[case] expected_b: ArrayRef,
+ ) -> VortexResult<()> {
+ let source = Arc::new(CountingSegmentSource::new(segments));
+ let reader = layout.new_reader(
+ "".into(),
+ Arc::clone(&source) as _,
+ &SESSION,
+ &Default::default(),
+ )?;
+ let expr = pack(
+ [("id", get_item("id", root())), ("s", s_projection)],
+ Nullability::NonNullable,
+ )
+ .bind(reader.dtype())?;
+ let project = reader.projection_evaluation(&(0..3), &expr, MaskFuture::new_true(3))?;
+ let result = block_on(move |_| project)?;
+
+ // `id`, `s.b` and the validity of `s`; `s.a` and `s.c` are never requested.
+ assert_eq!(source.requests(), 3);
+ let mut ctx = SESSION.create_execution_ctx();
+ let result = result.execute::(&mut ctx)?;
+ assert_arrays_eq!(
+ result.unmasked_field_by_name("id")?,
+ PrimitiveArray::from_iter([0i32, 1, 2]),
+ &mut ctx
+ );
+ let s = result
+ .unmasked_field_by_name("s")?
+ .clone()
+ .execute::(&mut ctx)?;
+ assert_eq!(s.names().as_ref(), ["b"].map(FieldName::from));
+ assert_eq!(
+ s.validity()?.execute_mask(3, &mut ctx)?,
+ Mask::from_iter([false, true, true])
+ );
+ assert_arrays_eq!(s.unmasked_field_by_name("b")?, expected_b, &mut ctx);
+ Ok(())
+ }
+
#[rstest]
fn test_struct_layout_or(
#[from(struct_layout)] (segments, layout): (Arc, LayoutRef),
diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs
index eff68dfe996..cfda7fdda15 100644
--- a/vortex-layout/src/scan/v2/tests.rs
+++ b/vortex-layout/src/scan/v2/tests.rs
@@ -5,8 +5,6 @@ use std::num::NonZeroUsize;
use std::ops::Range;
use std::sync::Arc;
use std::sync::LazyLock;
-use std::sync::atomic::AtomicUsize;
-use std::sync::atomic::Ordering;
use arrow_array::cast::AsArray;
use arrow_array::types::Int32Type;
@@ -74,8 +72,7 @@ use crate::scan::arrow::to_record_batch;
use crate::scan::execution::SegmentLocation;
use crate::scan::execution::row_chunks;
use crate::scan::scan_builder::ScanBuilder as V1ScanBuilder;
-use crate::segments::SegmentFuture;
-use crate::segments::SegmentId;
+use crate::segments::CountingSegmentSource;
use crate::segments::SegmentSource;
use crate::segments::TestSegments;
use crate::sequence::SequenceId;
@@ -503,18 +500,6 @@ fn smaller_chunks_keep_file_alignment() {
);
}
-struct CountingSegments {
- inner: Arc,
- reads: AtomicUsize,
-}
-
-impl SegmentSource for CountingSegments {
- fn request(&self, id: SegmentId) -> SegmentFuture {
- self.reads.fetch_add(1, Ordering::Relaxed);
- self.inner.request(id)
- }
-}
-
#[tokio::test]
async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> {
let session = new_session().with_tokio();
@@ -545,10 +530,7 @@ async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> {
.await?;
// Codes index the same values across fixed work chunks and stored chunk boundaries.
let layout = DictLayout::new(Arc::clone(&ints), ints).into_layout();
- let counted = Arc::new(CountingSegments {
- inner: segments,
- reads: AtomicUsize::new(0),
- });
+ let counted = Arc::new(CountingSegmentSource::new(segments));
let segments = Arc::clone(&counted) as Arc;
let scan = Arc::new(builder(&session, &segments, &layout)?.prepare()?);
let chunks = row_chunks(scan.rows().clone(), scan.chunk_rows);
@@ -557,9 +539,9 @@ async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> {
(STORED_CHUNKS * STORED_CHUNK_ROWS).div_ceil(MAX_ROWS_PER_CHUNK)
);
for (index, rows) in chunks.into_iter().enumerate() {
- let before = counted.reads.load(Ordering::Relaxed);
+ let before = counted.requests();
let arrays = run_rows(&scan, rows.clone()).await?;
- let reads = counted.reads.load(Ordering::Relaxed) - before;
+ let reads = counted.requests() - before;
let code_chunks =
(rows.end - 1) / STORED_CHUNK_ROWS as u64 - rows.start / STORED_CHUNK_ROWS as u64 + 1;
// The first chunk reads every values chunk once for the whole scan; its codes are one of
@@ -698,17 +680,14 @@ async fn scan_cache_reuses_values_between_scans(
let values = chunked_ints(&session, &segments).await?;
let codes = chunked_ints(&session, &segments).await?;
let layout = DictLayout::new(values, codes).into_layout();
- let counted = Arc::new(CountingSegments {
- inner: segments,
- reads: AtomicUsize::new(0),
- });
+ let counted = Arc::new(CountingSegmentSource::new(segments));
let segments = Arc::clone(&counted) as Arc;
let mut file = scan_file(&segments, &layout)?;
file.cache = cached.then(ScanCache::default);
let mut reads = Vec::new();
for _ in 0..2 {
- let before = counted.reads.load(Ordering::Relaxed);
+ let before = counted.requests();
let scan = ScanBuilder::new(session.clone(), file.clone())
.with_row_indices(StrictSortedBuffer::try_new(Buffer::from(vec![12u64, 15]))?);
let actual = collect_scan(layout.dtype().clone(), scan).await?;
@@ -717,7 +696,7 @@ async fn scan_cache_reuses_values_between_scans(
PrimitiveArray::from_iter([12i32, 15]),
&mut session.create_execution_ctx()
);
- reads.push(counted.reads.load(Ordering::Relaxed) - before);
+ reads.push(counted.requests() - before);
}
// Every values chunk and the one selected codes chunk, then only the codes chunk if cached.
assert_eq!(reads, vec![4, second_scan_reads]);
@@ -756,10 +735,7 @@ async fn repeated_projections_reuse_their_values() -> VortexResult<()> {
let values = chunked_ints(&session, &segments).await?;
let codes = chunked_ints(&session, &segments).await?;
let layout = DictLayout::new(values, codes).into_layout();
- let counted = Arc::new(CountingSegments {
- inner: segments,
- reads: AtomicUsize::new(0),
- });
+ let counted = Arc::new(CountingSegmentSource::new(segments));
let segments = Arc::clone(&counted) as Arc;
let mut file = scan_file(&segments, &layout)?;
file.cache = Some(ScanCache::default());
@@ -767,7 +743,7 @@ async fn repeated_projections_reuse_their_values() -> VortexResult<()> {
let projection = gt(root(), lit(15i32)).bind(layout.dtype())?;
let mut reads = Vec::new();
for _ in 0..3 {
- let before = counted.reads.load(Ordering::Relaxed);
+ let before = counted.requests();
let scan = ScanBuilder::new(session.clone(), file.clone())
.with_projection(projection.clone())
.with_row_indices(StrictSortedBuffer::try_new(Buffer::from(vec![12u64, 18]))?);
@@ -777,7 +753,7 @@ async fn repeated_projections_reuse_their_values() -> VortexResult<()> {
vortex_array::arrays::BoolArray::from_iter([false, true]),
&mut session.create_execution_ctx()
);
- reads.push(counted.reads.load(Ordering::Relaxed) - before);
+ reads.push(counted.requests() - before);
}
assert_eq!(reads, vec![4, 1, 1]);
Ok(())
@@ -841,10 +817,7 @@ async fn pruned_rows_get_no_chunk(
async fn shared_cache_prunes_zones_once() -> VortexResult<()> {
let session = new_session().with_tokio();
let (segments, layout) = write(&session, zoned_table().as_ref()).await?;
- let counted = Arc::new(CountingSegments {
- inner: segments,
- reads: AtomicUsize::new(0),
- });
+ let counted = Arc::new(CountingSegmentSource::new(segments));
let segments = Arc::clone(&counted) as Arc;
let mut file = scan_file(&segments, &layout)?;
let cache = ScanCache::default();
@@ -855,13 +828,13 @@ async fn shared_cache_prunes_zones_once() -> VortexResult<()> {
let mut reads = Vec::new();
for _ in 0..2 {
- let before = counted.reads.load(Ordering::Relaxed);
+ let before = counted.requests();
let scan = ScanBuilder::new(session.clone(), file.clone())
.with_projection(projection.clone())
.with_some_filter(Some(filter.clone()));
let actual = collect_scan(projection.dtype().clone(), scan).await?;
assert_eq!(actual.len(), 1_000);
- reads.push(counted.reads.load(Ordering::Relaxed) - before);
+ reads.push(counted.requests() - before);
}
let plans = cache
.filter_plans(&filter)
diff --git a/vortex-layout/src/segments/shared.rs b/vortex-layout/src/segments/shared.rs
index e2ec4cbcc47..6abc4145299 100644
--- a/vortex-layout/src/segments/shared.rs
+++ b/vortex-layout/src/segments/shared.rs
@@ -65,42 +65,23 @@ impl SegmentSource for SharedSegmentSource {
#[cfg(test)]
mod tests {
- use std::sync::atomic::AtomicUsize;
- use std::sync::atomic::Ordering;
-
use vortex_buffer::ByteBuffer;
use super::*;
+ use crate::segments::CountingSegmentSource;
use crate::segments::SegmentSink;
use crate::segments::TestSegments;
use crate::sequence::SequenceId;
- // Custom source that tracks how many times a segment is requested
- #[derive(Default, Clone)]
- struct CountingSegmentSource {
- segments: TestSegments,
- request_count: Arc,
- }
-
- impl SegmentSource for CountingSegmentSource {
- fn request(&self, id: SegmentId) -> SegmentFuture {
- self.request_count.fetch_add(1, Ordering::SeqCst);
- self.segments.request(id)
- }
- }
-
#[tokio::test]
async fn test_shared_source_deduplicates_concurrent_requests() {
- let source = CountingSegmentSource::default();
+ let segments = Arc::new(TestSegments::default());
+ let source = CountingSegmentSource::new(Arc::clone(&segments) as _);
// Add a segment to the test source
let data = ByteBuffer::from(vec![1, 2, 3, 4]);
let seq_id = SequenceId::root().downgrade();
- source
- .segments
- .write(seq_id, vec![data.clone()])
- .await
- .unwrap();
+ segments.write(seq_id, vec![data.clone()]).await.unwrap();
let shared_source = SharedSegmentSource::new(source.clone());
@@ -115,21 +96,18 @@ mod tests {
assert_eq!(result2.unwrap().unwrap_host(), data);
// The inner source should have been called only once
- assert_eq!(source.request_count.load(Ordering::Relaxed), 1);
+ assert_eq!(source.requests(), 1);
}
#[tokio::test]
async fn test_shared_source_handles_dropped_futures() {
- let source = CountingSegmentSource::default();
+ let segments = Arc::new(TestSegments::default());
+ let source = CountingSegmentSource::new(Arc::clone(&segments) as _);
// Add a segment
let data = ByteBuffer::from(vec![5, 6, 7, 8]);
let seq_id = SequenceId::root().downgrade();
- source
- .segments
- .write(seq_id, vec![data.clone()])
- .await
- .unwrap();
+ segments.write(seq_id, vec![data.clone()]).await.unwrap();
let shared_source = SharedSegmentSource::new(source.clone());
let id = SegmentId::from(0);
@@ -145,6 +123,6 @@ mod tests {
assert_eq!(result.unwrap().unwrap_host(), data);
// Should have made 2 requests since the first was dropped before completion
- assert_eq!(source.request_count.load(Ordering::Relaxed), 2);
+ assert_eq!(source.requests(), 2);
}
}
diff --git a/vortex-layout/src/segments/test.rs b/vortex-layout/src/segments/test.rs
index d880d15cc1a..024fecadc46 100644
--- a/vortex-layout/src/segments/test.rs
+++ b/vortex-layout/src/segments/test.rs
@@ -2,6 +2,8 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors
use std::sync::Arc;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
use async_trait::async_trait;
use futures::FutureExt;
@@ -58,3 +60,34 @@ impl SegmentSink for TestSegments {
Ok(segment_id)
}
}
+
+/// A [`SegmentSource`] that counts the segments requested from an inner source.
+///
+/// Clones share the count.
+#[derive(Clone)]
+pub struct CountingSegmentSource {
+ inner: Arc,
+ requests: Arc,
+}
+
+impl CountingSegmentSource {
+ /// Wraps `inner`, starting the count at zero.
+ pub fn new(inner: Arc) -> Self {
+ Self {
+ inner,
+ requests: Arc::new(AtomicUsize::new(0)),
+ }
+ }
+
+ /// Returns how many segments have been requested so far.
+ pub fn requests(&self) -> usize {
+ self.requests.load(Ordering::Relaxed)
+ }
+}
+
+impl SegmentSource for CountingSegmentSource {
+ fn request(&self, id: SegmentId) -> SegmentFuture {
+ self.requests.fetch_add(1, Ordering::Relaxed);
+ self.inner.request(id)
+ }
+}