diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java index 370affb0e38..5890e511536 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java @@ -145,6 +145,14 @@ public static Expression isNotNull(Expression child) { return new Expression(NativeExpression.isNotNull(child.nativePointer())); } + /** + * Null out {@code child} wherever {@code mask} is false, keeping its value where {@code mask} is true. The mask + * must be a non-nullable boolean. + */ + public static Expression mask(Expression child, Expression mask) { + return new Expression(NativeExpression.mask(child.nativePointer(), mask.nativePointer())); + } + /** * SQL {@code LIKE} pattern match. * diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java index 58ae38be4d8..0943cd86e59 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java @@ -37,6 +37,8 @@ private NativeExpression() {} public static native long isNotNull(long childPointer); + public static native long mask(long childPointer, long maskPointer); + public static native long like(long childPointer, long patternPointer, boolean negated, boolean caseInsensitive); public static native long between( diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/NestedSchemaPruning.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/NestedSchemaPruning.java new file mode 100644 index 00000000000..2f13fdba979 --- /dev/null +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/NestedSchemaPruning.java @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.spark.read; + +import dev.vortex.api.Expression; +import java.util.ArrayList; +import java.util.List; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +/** + * Pushes Spark's nested schema pruning down to the native scan. + * + *

Only struct fields are pruned. Fields inside arrays and maps keep their full table type, since projecting each + * element needs a per-element expression the native scan does not expose yet. + */ +final class NestedSchemaPruning { + private NestedSchemaPruning() {} + + /** + * Returns {@code tableType} pruned to the struct fields present in {@code requiredType}, in table order. Any type + * that is not a struct on both sides keeps its table type. + */ + static DataType prune(DataType tableType, DataType requiredType) { + if (!(tableType instanceof StructType table) || !(requiredType instanceof StructType required)) { + return tableType; + } + List fields = new ArrayList<>(); + for (StructField field : table.fields()) { + if (required.getFieldIndex(field.name()).isDefined()) { + DataType prunedType = + prune(field.dataType(), required.apply(field.name()).dataType()); + fields.add(new StructField(field.name(), prunedType, field.nullable(), field.metadata())); + } + } + return fields.isEmpty() ? tableType : new StructType(fields.toArray(new StructField[0])); + } + + /** + * Builds the native projection that returns {@code readSchema}, where each field is either its type in + * {@code tableSchema} or a {@link #prune pruned} version of it. + * + *

When nothing is pruned this is a plain {@code select} of the top-level columns. + */ + static Expression projection(StructType readSchema, StructType tableSchema) { + return structProjection(Expression.root(), false, readSchema, tableSchema); + } + + private static Expression structProjection(Expression struct, boolean nullable, StructType read, StructType table) { + String[] names = read.fieldNames(); + // `select` keeps the struct's own nulls, so use it when no child needs pruning. + if (!isPruned(read, table)) { + return Expression.select(names, struct); + } + Expression[] children = new Expression[names.length]; + for (int i = 0; i < names.length; i++) { + Expression field = Expression.getItem(names[i], struct); + StructField readField = read.fields()[i]; + DataType tableType = table.apply(names[i]).dataType(); + children[i] = readField.dataType().equals(tableType) + ? field + : structProjection( + field, readField.nullable(), (StructType) readField.dataType(), (StructType) tableType); + } + Expression packed = Expression.pack(names, children, false); + // `pack` builds a struct with no nulls, so carry over the source struct's nulls. + return nullable ? Expression.mask(packed, Expression.isNotNull(struct)) : packed; + } + + private static boolean isPruned(StructType read, StructType table) { + for (StructField field : read.fields()) { + if (!field.dataType().equals(table.apply(field.name()).dataType())) { + return true; + } + } + return false; + } +} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java index a1c4df239b7..b0ff943fd1a 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java @@ -26,12 +26,14 @@ import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; /** Execution source for batch scans of Vortex file tables. */ public final class VortexBatchExec implements Batch { private final List paths; private final StructType readSchema; + private final StructType tableSchema; private final Map formatOptions; private final Predicate[] pushedPredicates; private List resolvedPaths; @@ -41,13 +43,19 @@ public final class VortexBatchExec implements Batch { * * @param paths the list of file paths to scan * @param columns the list of columns to read from the files + * @param tableColumns the full table columns, which {@code columns} may prune nested fields from * @param pushedPredicates predicates pushed down by Spark; converted to a single Vortex filter expression at read * time */ public VortexBatchExec( - List paths, List columns, Map formatOptions, Predicate[] pushedPredicates) { + List paths, + List columns, + List tableColumns, + Map formatOptions, + Predicate[] pushedPredicates) { this.paths = List.copyOf(paths); this.readSchema = CatalogV2Util.v2ColumnsToStructType(columns.toArray(new Column[0])); + this.tableSchema = CatalogV2Util.v2ColumnsToStructType(tableColumns.toArray(new Column[0])); this.formatOptions = Map.copyOf(formatOptions); this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone(); } @@ -106,10 +114,14 @@ private Map measureFiles(List paths) public PartitionReaderFactory createReaderFactory() { List files = resolvedPaths != null ? resolvedPaths : resolvePaths(); Set partitionColumns = collectPartitionColumnNames(files); - List dataColumnNames = Arrays.stream(readSchema.fieldNames()) - .filter(name -> !partitionColumns.contains(name)) - .collect(Collectors.toList()); - return new VortexPartitionReaderFactory(dataColumnNames, formatOptions, pushedPredicates); + StructType dataReadSchema = new StructType(Arrays.stream(readSchema.fields()) + .filter(field -> !partitionColumns.contains(field.name())) + .toArray(StructField[]::new)); + // Only the read columns' table types are needed to build the projection. + StructType dataTableSchema = new StructType(Arrays.stream(dataReadSchema.fieldNames()) + .map(tableSchema::apply) + .toArray(StructField[]::new)); + return new VortexPartitionReaderFactory(dataReadSchema, dataTableSchema, formatOptions, pushedPredicates); } private List resolvePaths() { diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java index 7bc649ce67a..9880c31a19c 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java @@ -16,12 +16,12 @@ import dev.vortex.spark.VortexFilePartition; import dev.vortex.spark.VortexSparkSession; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -50,7 +50,8 @@ final class VortexPartitionReader implements PartitionReader { VortexPartitionReader( VortexFilePartition spark, - List dataColumnNames, + StructType dataReadSchema, + StructType tableSchema, Map formatOptions, Predicate[] pushedPredicates) { this.spark = spark; @@ -60,9 +61,8 @@ final class VortexPartitionReader implements PartitionReader { dataSource = DataSource.open(session, spark.paths(), formatOptions); var options = ScanOptions.builder(); - if (!dataColumnNames.isEmpty()) { - Expression projection = Expression.select(dataColumnNames.toArray(new String[0]), Expression.root()); - options.projection(projection); + if (dataReadSchema.fields().length > 0) { + options.projection(NestedSchemaPruning.projection(dataReadSchema, tableSchema)); } if (pushedPredicates != null && pushedPredicates.length > 0) { buildFilterExpression(pushedPredicates).ifPresent(options::filter); diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java index e187e4863b1..0e0e1543c93 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java @@ -3,18 +3,17 @@ package dev.vortex.spark.read; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import dev.vortex.jni.NativeRuntime; import dev.vortex.spark.VortexFilePartition; import java.io.Serializable; -import java.util.List; import java.util.Map; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnarBatch; /** @@ -28,13 +27,18 @@ public final class VortexPartitionReaderFactory implements PartitionReaderFactory, Serializable { private static final long serialVersionUID = 1L; - private final ImmutableList dataColumnNames; + private final StructType dataReadSchema; + private final StructType tableSchema; private final ImmutableMap formatOptions; private final Predicate[] pushedPredicates; public VortexPartitionReaderFactory( - List dataColumnNames, Map formatOptions, Predicate[] pushedPredicates) { - this.dataColumnNames = ImmutableList.copyOf(dataColumnNames); + StructType dataReadSchema, + StructType tableSchema, + Map formatOptions, + Predicate[] pushedPredicates) { + this.dataReadSchema = dataReadSchema; + this.tableSchema = tableSchema; this.formatOptions = ImmutableMap.copyOf(formatOptions); this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone(); } @@ -48,7 +52,7 @@ public PartitionReader createReader(InputPartition partition) { public PartitionReader createColumnarReader(InputPartition partition) { NativeRuntime.setWorkerThreads(Integer.parseInt(formatOptions.getOrDefault("vortex.workerThreads", "4"))); VortexFilePartition spark = (VortexFilePartition) partition; - return new VortexPartitionReader(spark, dataColumnNames, formatOptions, pushedPredicates); + return new VortexPartitionReader(spark, dataReadSchema, tableSchema, formatOptions, pushedPredicates); } @Override diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java index 6da2306ab69..8764ee6fe28 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java @@ -117,7 +117,7 @@ public String description() { */ @Override public Batch toBatch() { - return new VortexBatchExec(paths, readColumns, formatOptions, pushedPredicates); + return new VortexBatchExec(paths, readColumns, tableColumns, formatOptions, pushedPredicates); } /** diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index 97da8d42302..8531cf4bf01 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -130,22 +130,33 @@ public Scan build() { /** * Prunes the columns to only include those specified in the required schema. * - *

Required columns keep their table type because the native scan returns whole top-level columns, even when - * Spark prunes nested fields. + *

Columns stay in table order. Struct columns are pruned to the nested fields Spark requires, and the reader + * projects only those fields. See {@link NestedSchemaPruning} for which nested types are pruned. * * @param requiredSchema the schema specifying which columns are required */ @Override public void pruneColumns(StructType requiredSchema) { - Set required = Set.copyOf(Arrays.asList(requiredSchema.fieldNames())); readColumns.clear(); for (Column column : tableColumns) { - if (required.contains(column.name())) { - readColumns.add(column); + if (requiredSchema.getFieldIndex(column.name()).isDefined()) { + DataType prunedType = NestedSchemaPruning.prune( + column.dataType(), requiredSchema.apply(column.name()).dataType()); + readColumns.add(prunedType.equals(column.dataType()) ? column : withDataType(column, prunedType)); } } } + private static Column withDataType(Column column, DataType dataType) { + return Column.create( + column.name(), + dataType, + column.nullable(), + column.comment(), + column.defaultValue(), + column.metadataInJSON()); + } + /** * Splits the supplied predicates into pushed and not-pushed sets. * diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java index 16bdc59a0f8..1d98b2264c2 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.spark.sql.Column; @@ -60,7 +61,27 @@ public void setUp(@TempDir Path tempDir) { functions.map(functions.lit("k"), abc).alias("m"), functions .struct(functions.lit("x").alias("p"), abc.alias("inner")) - .alias("outer")); + .alias("outer"), + nullWhen( + functions.col("id").equalTo(1), + functions.struct( + functions.lit("x").alias("p"), + nullWhen(functions.col("id").equalTo(2), abc) + .alias("inner"))) + .alias("nullable_outer"), + nullWhen( + functions.col("id").mod(2).equalTo(1), + functions.struct( + functions.col("id").plus(100).alias("a"), + nullWhen( + functions + .col("id") + .equalTo(4), + functions + .col("id") + .plus(200)) + .alias("b"))) + .alias("n")); String path = tempDir.resolve("nested").toUri().toString(); df.write() @@ -81,30 +102,81 @@ public void tearDown() { @Test @DisplayName("Selecting a struct field other than the first returns that field") public void structField() { - assertEquals(C_VALUES, longsOf(functions.col("s.c"))); + assertEquals(C_VALUES, valuesOf(table, functions.col("s.c"))); } @Test @DisplayName("Selecting a field of the structs in an array returns that field") public void arrayElementField() { - assertEquals(C_VALUES, longsOf(functions.element_at(functions.col("arr.c"), 1))); + assertEquals(C_VALUES, valuesOf(table, functions.element_at(functions.col("arr.c"), 1))); } @Test @DisplayName("Selecting a field of the structs in a map returns that field") public void mapValueField() { - assertEquals(C_VALUES, longsOf(functions.col("m").getItem("k").getField("c"))); + assertEquals(C_VALUES, valuesOf(table, functions.col("m").getItem("k").getField("c"))); } @Test @DisplayName("Selecting a field of a struct inside a struct returns that field") public void structInStructField() { - assertEquals(C_VALUES, longsOf(functions.col("outer.inner.c"))); + assertEquals(C_VALUES, valuesOf(table, functions.col("outer.inner.c"))); } - private List longsOf(Column column) { - return table.orderBy("id").select(column).collectAsList().stream() - .map(row -> row.getLong(0)) + @Test + @DisplayName("Null structs and null fields stay null when the struct is pruned") + public void nullableStructField() { + assertEquals(Arrays.asList(200L, null, 202L, null, null), valuesOf(table, functions.col("n.b"))); + assertEquals( + List.of(false, true, false, true, false), + valuesOf(table, functions.col("n").isNull())); + } + + @Test + @DisplayName("Nulls at each level of a struct inside a struct survive pruning both levels") + public void nullableStructInStructField() { + assertEquals( + Arrays.asList(300L, null, null, 303L, 304L), valuesOf(table, functions.col("nullable_outer.inner.c"))); + assertEquals( + List.of(false, true, false, false, false), + valuesOf(table, functions.col("nullable_outer").isNull())); + assertEquals( + List.of(false, true, true, false, false), + valuesOf(table, functions.col("nullable_outer.inner").isNull())); + } + + @Test + @DisplayName("A pushed filter on a field the query does not select still applies") + public void filterOnPrunedField() { + assertEquals( + List.of(302L, 303L, 304L), + valuesOf(table.where(functions.col("s.a").gt(101)), functions.col("s.c"))); + } + + @Test + @DisplayName("Whole struct columns still read in full alongside pruned ones") + public void wholeAndPrunedStructs() { + Row row = table.where("id = 3") + .select(functions.col("s"), functions.col("outer.inner.a")) + .first(); + assertEquals( + List.of(103L, 203L, 303L), + List.of( + row.getStruct(0).getLong(0), + row.getStruct(0).getLong(1), + row.getStruct(0).getLong(2))); + assertEquals(103L, row.getLong(1)); + } + + private static Column nullWhen(Column condition, Column value) { + return functions.when(condition, functions.lit(null)).otherwise(value); + } + + /** Returns {@code column} for each row of {@code rows}, ordered by id. */ + @SuppressWarnings("unchecked") + private static List valuesOf(Dataset rows, Column column) { + return rows.orderBy("id").select(column).collectAsList().stream() + .map(row -> (T) row.get(0)) .collect(Collectors.toList()); } } diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java index ae67a475865..7b56522eb68 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexBatchExecTest.java @@ -29,7 +29,7 @@ final class VortexBatchExecTest { Column.create("name", org.apache.spark.sql.types.DataTypes.StringType)); private static VortexBatchExec execFor(List paths) { - return new VortexBatchExec(paths, COLUMNS, Map.of(), new Predicate[0]); + return new VortexBatchExec(paths, COLUMNS, COLUMNS, Map.of(), new Predicate[0]); } @Test @@ -89,7 +89,8 @@ void partitionValuesParsedPerFile() { @DisplayName("Format options are propagated to every partition") void formatOptionsPropagated() { Map options = Map.of("vortex.workerThreads", "8"); - VortexBatchExec exec = new VortexBatchExec(List.of("/data/a.vortex"), COLUMNS, options, new Predicate[0]); + VortexBatchExec exec = + new VortexBatchExec(List.of("/data/a.vortex"), COLUMNS, COLUMNS, options, new Predicate[0]); VortexFilePartition partition = (VortexFilePartition) exec.planInputPartitions()[0]; diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java index 485c093daff..c043cc52895 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/read/VortexScanBuilderTest.java @@ -15,7 +15,9 @@ import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.expressions.filter.Predicate; +import org.apache.spark.sql.types.ArrayType; import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.MapType; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; import org.junit.jupiter.api.DisplayName; @@ -163,29 +165,63 @@ void pushedPredicatesReturnsCopy() { // --- pruneColumns --- + private static final StructType ABC = new StructType() + .add("a", DataTypes.LongType) + .add("b", DataTypes.LongType) + .add("c", DataTypes.LongType); + @Test - @DisplayName("Required columns keep their full table type, so readSchema matches what the reader returns") - void pruneColumnsKeepsTableTypes() { - StructType abc = new StructType() - .add("a", DataTypes.LongType) - .add("b", DataTypes.LongType) - .add("c", DataTypes.LongType); + @DisplayName("Struct columns are pruned to the required fields, with columns and fields in table order") + void pruneColumnsPrunesStructFields() { VortexScanBuilder builder = builderWithColumns( new Transform[0], Column.create("id", DataTypes.IntegerType), - Column.create("s", abc), + Column.create("s", ABC), Column.create("name", DataTypes.StringType)); - // With nested schema pruning, Spark asks for s with only the field the query uses. builder.pruneColumns(new StructType() - .add("s", new StructType().add("c", DataTypes.LongType)) + .add("s", new StructType().add("c", DataTypes.LongType).add("a", DataTypes.LongType)) .add("id", DataTypes.IntegerType)); assertEquals( - new StructType().add("id", DataTypes.IntegerType).add("s", abc), + new StructType() + .add("id", DataTypes.IntegerType) + .add("s", new StructType().add("a", DataTypes.LongType).add("c", DataTypes.LongType)), builder.build().readSchema()); } + @Test + @DisplayName("Structs inside structs are pruned at every level") + void pruneColumnsPrunesNestedStructs() { + StructType outer = new StructType().add("p", DataTypes.StringType).add("inner", ABC); + VortexScanBuilder builder = builderWithColumns(new Transform[0], Column.create("outer", outer)); + + builder.pruneColumns(new StructType() + .add("outer", new StructType().add("inner", new StructType().add("b", DataTypes.LongType)))); + + assertEquals( + new StructType() + .add("outer", new StructType().add("inner", new StructType().add("b", DataTypes.LongType))), + builder.build().readSchema()); + } + + @Test + @DisplayName("Arrays and maps of structs keep their full table type") + void pruneColumnsKeepsArrayAndMapTypes() { + ArrayType arr = DataTypes.createArrayType(ABC); + MapType map = DataTypes.createMapType(DataTypes.StringType, ABC); + VortexScanBuilder builder = + builderWithColumns(new Transform[0], Column.create("arr", arr), Column.create("m", map)); + + StructType c = new StructType().add("c", DataTypes.LongType); + builder.pruneColumns(new StructType() + .add("arr", DataTypes.createArrayType(c)) + .add("m", DataTypes.createMapType(DataTypes.StringType, c))); + + assertEquals( + new StructType().add("arr", arr).add("m", map), builder.build().readSchema()); + } + // --- build --- @Test diff --git a/vortex-jni/src/expression.rs b/vortex-jni/src/expression.rs index afcefe990c4..21774b1c0fb 100644 --- a/vortex-jni/src/expression.rs +++ b/vortex-jni/src/expression.rs @@ -47,6 +47,7 @@ use vortex::expr::is_not_null; use vortex::expr::is_null; use vortex::expr::list_contains_opts; use vortex::expr::lit; +use vortex::expr::mask; use vortex::expr::merge_opts; use vortex::expr::not; use vortex::expr::or_collect; @@ -359,6 +360,18 @@ pub extern "system" fn Java_dev_vortex_jni_NativeExpression_isNotNull( into_raw(is_not_null(child)) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_dev_vortex_jni_NativeExpression_mask( + _env: EnvUnowned, + _class: JClass, + child: jlong, + mask_expr: jlong, +) -> jlong { + let child = unsafe { expr_ref(child) }.clone(); + let mask_expr = unsafe { expr_ref(mask_expr) }.clone(); + into_raw(mask(child, mask_expr)) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_dev_vortex_jni_NativeExpression_like( _env: EnvUnowned, diff --git a/vortex-layout/src/layouts/list/reader.rs b/vortex-layout/src/layouts/list/reader.rs index fafa48346bd..208e32ebc39 100644 --- a/vortex-layout/src/layouts/list/reader.rs +++ b/vortex-layout/src/layouts/list/reader.rs @@ -616,8 +616,6 @@ fn apply_lengths_validity( #[cfg(test)] mod tests { use std::ops::Range; - use std::sync::atomic::AtomicUsize; - use std::sync::atomic::Ordering; use rstest::rstest; use vortex_array::ArrayContext; @@ -647,7 +645,7 @@ mod tests { use crate::layouts::repartition::RepartitionStrategy; use crate::layouts::repartition::RepartitionWriterOptions; use crate::scan::split_by::SplitBy; - use crate::segments::SegmentFuture; + use crate::segments::CountingSegmentSource; use crate::segments::SegmentSource; use crate::segments::TestSegments; use crate::sequence::SequenceId; @@ -1139,18 +1137,6 @@ mod tests { )) } - struct CountingSegmentSource { - inner: Arc, - request_count: Arc, - } - - impl SegmentSource for CountingSegmentSource { - fn request(&self, id: crate::segments::SegmentId) -> SegmentFuture { - self.request_count.fetch_add(1, Ordering::Relaxed); - self.inner.request(id) - } - } - /// The chunked-elements strategy must actually produce a chunked `elements` layout, otherwise /// the reader would silently take the whole-chunk path and the bounded read would be untested. #[tokio::test] @@ -1175,12 +1161,8 @@ mod tests { let ctx = LayoutReaderContext::new(); let (segments, layout, session) = write_layout(&chunked_elements_list_strategy(), list.clone()).await?; - let request_count = Arc::new(AtomicUsize::new(0)); - let source = Arc::new(CountingSegmentSource { - inner: segments, - request_count: Arc::clone(&request_count), - }); - let reader = layout.new_reader("".into(), source, &session, &ctx)?; + let source = Arc::new(CountingSegmentSource::new(segments)); + let reader = layout.new_reader("".into(), Arc::clone(&source) as _, &session, &ctx)?; let mask = Mask::from_iter([true, false, false, false, false]); let expr = root().bind(reader.dtype())?; @@ -1191,7 +1173,7 @@ mod tests { let expected = list.filter(mask)?; let mut exec_ctx = session.create_execution_ctx(); assert_arrays_eq!(result, expected, &mut exec_ctx); - assert_eq!(request_count.load(Ordering::Relaxed), 2); + assert_eq!(source.requests(), 2); Ok(()) } diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index 30547d298eb..63046e6e616 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -626,6 +626,7 @@ mod tests { use vortex_array::expr::is_null; use vortex_array::expr::lit; use vortex_array::expr::lt; + use vortex_array::expr::mask; use vortex_array::expr::not; use vortex_array::expr::or; use vortex_array::expr::pack; @@ -648,6 +649,7 @@ mod tests { use crate::layouts::table::TableStrategy; use crate::layouts::zoned::writer::ZonedLayoutOptions; use crate::layouts::zoned::writer::ZonedStrategy; + use crate::segments::CountingSegmentSource; use crate::segments::SegmentSource; use crate::segments::TestSegments; use crate::sequence::SequenceId; @@ -830,6 +832,142 @@ mod tests { (segments, layout) } + /// Writes `{id, s: {a, b, c}}`, where the struct `s` is null in row 0. + #[fixture] + fn table_with_null_struct_layout() -> (Arc, LayoutRef) { + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + let strategy = TableStrategy::new( + Arc::new(FlatLayoutStrategy::default()), + Arc::new(FlatLayoutStrategy::default()), + ); + let s = StructArray::try_from_iter_with_validity( + [ + ("a", buffer![7, 2, 3].into_array()), + ("b", buffer![4, 5, 6].into_array()), + ("c", buffer![8, 9, 10].into_array()), + ], + Validity::Array(BoolArray::from_iter([false, true, true]).into_array()), + ) + .unwrap(); + let table = StructArray::try_from_iter([ + ("id", buffer![0, 1, 2].into_array()), + ("s", s.into_array()), + ]) + .unwrap(); + let segments2 = Arc::::clone(&segments); + let layout = block_on(|handle| async move { + let session = new_session().with_handle(handle); + strategy + .write_stream( + ArrayContext::empty().into(), + segments2, + table.into_array().to_array_stream().sequenced(ptr), + eof, + &session, + ) + .await + }) + .unwrap(); + + (segments, layout) + } + + /// Projecting a nullable struct reads only the fields it keeps, plus the struct validity. + #[rstest] + #[case::select_one_field(select(["b"], root()), 2)] + #[case::whole_struct(root(), 4)] + fn test_nullable_struct_projection_request_count( + #[from(null_struct_layout)] (segments, layout): (Arc, LayoutRef), + #[case] expr: Expression, + #[case] expected_requests: usize, + ) -> VortexResult<()> { + let source = Arc::new(CountingSegmentSource::new(segments)); + let reader = layout.new_reader( + "".into(), + Arc::clone(&source) as _, + &SESSION, + &Default::default(), + )?; + let expr = expr.bind(reader.dtype())?; + let project = reader.projection_evaluation(&(0..3), &expr, MaskFuture::new_true(3))?; + let result = block_on(move |_| project)?; + + assert_eq!(source.requests(), expected_requests); + let mut ctx = SESSION.create_execution_ctx(); + let result = result.execute::(&mut ctx)?; + assert_eq!( + result.validity()?.execute_mask(3, &mut ctx)?, + Mask::from_iter([false, true, true]) + ); + assert_arrays_eq!( + result.unmasked_field_by_name("b")?, + PrimitiveArray::from_iter([4i32, 5, 6]), + &mut ctx + ); + Ok(()) + } + + /// Projecting a pruned nullable struct column, with `select` or a masked `pack`, reads only + /// the fields it keeps, plus the struct validity. + #[rstest] + #[case::select( + select(["b"], get_item("s", root())), + PrimitiveArray::from_iter([4i32, 5, 6]).into_array(), + )] + // Reading `b` through the nullable `s` also nulls it where `s` is null. + #[case::masked_pack( + mask( + pack([("b", get_item("b", get_item("s", root())))], Nullability::NonNullable), + is_not_null(get_item("s", root())), + ), + PrimitiveArray::from_option_iter([None, Some(5i32), Some(6)]).into_array(), + )] + fn test_pruned_nullable_struct_column_reads_only_kept_fields( + #[from(table_with_null_struct_layout)] (segments, layout): ( + Arc, + LayoutRef, + ), + #[case] s_projection: Expression, + #[case] expected_b: ArrayRef, + ) -> VortexResult<()> { + let source = Arc::new(CountingSegmentSource::new(segments)); + let reader = layout.new_reader( + "".into(), + Arc::clone(&source) as _, + &SESSION, + &Default::default(), + )?; + let expr = pack( + [("id", get_item("id", root())), ("s", s_projection)], + Nullability::NonNullable, + ) + .bind(reader.dtype())?; + let project = reader.projection_evaluation(&(0..3), &expr, MaskFuture::new_true(3))?; + let result = block_on(move |_| project)?; + + // `id`, `s.b` and the validity of `s`; `s.a` and `s.c` are never requested. + assert_eq!(source.requests(), 3); + let mut ctx = SESSION.create_execution_ctx(); + let result = result.execute::(&mut ctx)?; + assert_arrays_eq!( + result.unmasked_field_by_name("id")?, + PrimitiveArray::from_iter([0i32, 1, 2]), + &mut ctx + ); + let s = result + .unmasked_field_by_name("s")? + .clone() + .execute::(&mut ctx)?; + assert_eq!(s.names().as_ref(), ["b"].map(FieldName::from)); + assert_eq!( + s.validity()?.execute_mask(3, &mut ctx)?, + Mask::from_iter([false, true, true]) + ); + assert_arrays_eq!(s.unmasked_field_by_name("b")?, expected_b, &mut ctx); + Ok(()) + } + #[rstest] fn test_struct_layout_or( #[from(struct_layout)] (segments, layout): (Arc, LayoutRef), diff --git a/vortex-layout/src/scan/v2/tests.rs b/vortex-layout/src/scan/v2/tests.rs index eff68dfe996..cfda7fdda15 100644 --- a/vortex-layout/src/scan/v2/tests.rs +++ b/vortex-layout/src/scan/v2/tests.rs @@ -5,8 +5,6 @@ use std::num::NonZeroUsize; use std::ops::Range; use std::sync::Arc; use std::sync::LazyLock; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; use arrow_array::cast::AsArray; use arrow_array::types::Int32Type; @@ -74,8 +72,7 @@ use crate::scan::arrow::to_record_batch; use crate::scan::execution::SegmentLocation; use crate::scan::execution::row_chunks; use crate::scan::scan_builder::ScanBuilder as V1ScanBuilder; -use crate::segments::SegmentFuture; -use crate::segments::SegmentId; +use crate::segments::CountingSegmentSource; use crate::segments::SegmentSource; use crate::segments::TestSegments; use crate::sequence::SequenceId; @@ -503,18 +500,6 @@ fn smaller_chunks_keep_file_alignment() { ); } -struct CountingSegments { - inner: Arc, - reads: AtomicUsize, -} - -impl SegmentSource for CountingSegments { - fn request(&self, id: SegmentId) -> SegmentFuture { - self.reads.fetch_add(1, Ordering::Relaxed); - self.inner.request(id) - } -} - #[tokio::test] async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> { let session = new_session().with_tokio(); @@ -545,10 +530,7 @@ async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> { .await?; // Codes index the same values across fixed work chunks and stored chunk boundaries. let layout = DictLayout::new(Arc::clone(&ints), ints).into_layout(); - let counted = Arc::new(CountingSegments { - inner: segments, - reads: AtomicUsize::new(0), - }); + let counted = Arc::new(CountingSegmentSource::new(segments)); let segments = Arc::clone(&counted) as Arc; let scan = Arc::new(builder(&session, &segments, &layout)?.prepare()?); let chunks = row_chunks(scan.rows().clone(), scan.chunk_rows); @@ -557,9 +539,9 @@ async fn dictionary_values_are_reused_between_splits() -> VortexResult<()> { (STORED_CHUNKS * STORED_CHUNK_ROWS).div_ceil(MAX_ROWS_PER_CHUNK) ); for (index, rows) in chunks.into_iter().enumerate() { - let before = counted.reads.load(Ordering::Relaxed); + let before = counted.requests(); let arrays = run_rows(&scan, rows.clone()).await?; - let reads = counted.reads.load(Ordering::Relaxed) - before; + let reads = counted.requests() - before; let code_chunks = (rows.end - 1) / STORED_CHUNK_ROWS as u64 - rows.start / STORED_CHUNK_ROWS as u64 + 1; // The first chunk reads every values chunk once for the whole scan; its codes are one of @@ -698,17 +680,14 @@ async fn scan_cache_reuses_values_between_scans( let values = chunked_ints(&session, &segments).await?; let codes = chunked_ints(&session, &segments).await?; let layout = DictLayout::new(values, codes).into_layout(); - let counted = Arc::new(CountingSegments { - inner: segments, - reads: AtomicUsize::new(0), - }); + let counted = Arc::new(CountingSegmentSource::new(segments)); let segments = Arc::clone(&counted) as Arc; let mut file = scan_file(&segments, &layout)?; file.cache = cached.then(ScanCache::default); let mut reads = Vec::new(); for _ in 0..2 { - let before = counted.reads.load(Ordering::Relaxed); + let before = counted.requests(); let scan = ScanBuilder::new(session.clone(), file.clone()) .with_row_indices(StrictSortedBuffer::try_new(Buffer::from(vec![12u64, 15]))?); let actual = collect_scan(layout.dtype().clone(), scan).await?; @@ -717,7 +696,7 @@ async fn scan_cache_reuses_values_between_scans( PrimitiveArray::from_iter([12i32, 15]), &mut session.create_execution_ctx() ); - reads.push(counted.reads.load(Ordering::Relaxed) - before); + reads.push(counted.requests() - before); } // Every values chunk and the one selected codes chunk, then only the codes chunk if cached. assert_eq!(reads, vec![4, second_scan_reads]); @@ -756,10 +735,7 @@ async fn repeated_projections_reuse_their_values() -> VortexResult<()> { let values = chunked_ints(&session, &segments).await?; let codes = chunked_ints(&session, &segments).await?; let layout = DictLayout::new(values, codes).into_layout(); - let counted = Arc::new(CountingSegments { - inner: segments, - reads: AtomicUsize::new(0), - }); + let counted = Arc::new(CountingSegmentSource::new(segments)); let segments = Arc::clone(&counted) as Arc; let mut file = scan_file(&segments, &layout)?; file.cache = Some(ScanCache::default()); @@ -767,7 +743,7 @@ async fn repeated_projections_reuse_their_values() -> VortexResult<()> { let projection = gt(root(), lit(15i32)).bind(layout.dtype())?; let mut reads = Vec::new(); for _ in 0..3 { - let before = counted.reads.load(Ordering::Relaxed); + let before = counted.requests(); let scan = ScanBuilder::new(session.clone(), file.clone()) .with_projection(projection.clone()) .with_row_indices(StrictSortedBuffer::try_new(Buffer::from(vec![12u64, 18]))?); @@ -777,7 +753,7 @@ async fn repeated_projections_reuse_their_values() -> VortexResult<()> { vortex_array::arrays::BoolArray::from_iter([false, true]), &mut session.create_execution_ctx() ); - reads.push(counted.reads.load(Ordering::Relaxed) - before); + reads.push(counted.requests() - before); } assert_eq!(reads, vec![4, 1, 1]); Ok(()) @@ -841,10 +817,7 @@ async fn pruned_rows_get_no_chunk( async fn shared_cache_prunes_zones_once() -> VortexResult<()> { let session = new_session().with_tokio(); let (segments, layout) = write(&session, zoned_table().as_ref()).await?; - let counted = Arc::new(CountingSegments { - inner: segments, - reads: AtomicUsize::new(0), - }); + let counted = Arc::new(CountingSegmentSource::new(segments)); let segments = Arc::clone(&counted) as Arc; let mut file = scan_file(&segments, &layout)?; let cache = ScanCache::default(); @@ -855,13 +828,13 @@ async fn shared_cache_prunes_zones_once() -> VortexResult<()> { let mut reads = Vec::new(); for _ in 0..2 { - let before = counted.reads.load(Ordering::Relaxed); + let before = counted.requests(); let scan = ScanBuilder::new(session.clone(), file.clone()) .with_projection(projection.clone()) .with_some_filter(Some(filter.clone())); let actual = collect_scan(projection.dtype().clone(), scan).await?; assert_eq!(actual.len(), 1_000); - reads.push(counted.reads.load(Ordering::Relaxed) - before); + reads.push(counted.requests() - before); } let plans = cache .filter_plans(&filter) diff --git a/vortex-layout/src/segments/shared.rs b/vortex-layout/src/segments/shared.rs index e2ec4cbcc47..6abc4145299 100644 --- a/vortex-layout/src/segments/shared.rs +++ b/vortex-layout/src/segments/shared.rs @@ -65,42 +65,23 @@ impl SegmentSource for SharedSegmentSource { #[cfg(test)] mod tests { - use std::sync::atomic::AtomicUsize; - use std::sync::atomic::Ordering; - use vortex_buffer::ByteBuffer; use super::*; + use crate::segments::CountingSegmentSource; use crate::segments::SegmentSink; use crate::segments::TestSegments; use crate::sequence::SequenceId; - // Custom source that tracks how many times a segment is requested - #[derive(Default, Clone)] - struct CountingSegmentSource { - segments: TestSegments, - request_count: Arc, - } - - impl SegmentSource for CountingSegmentSource { - fn request(&self, id: SegmentId) -> SegmentFuture { - self.request_count.fetch_add(1, Ordering::SeqCst); - self.segments.request(id) - } - } - #[tokio::test] async fn test_shared_source_deduplicates_concurrent_requests() { - let source = CountingSegmentSource::default(); + let segments = Arc::new(TestSegments::default()); + let source = CountingSegmentSource::new(Arc::clone(&segments) as _); // Add a segment to the test source let data = ByteBuffer::from(vec![1, 2, 3, 4]); let seq_id = SequenceId::root().downgrade(); - source - .segments - .write(seq_id, vec![data.clone()]) - .await - .unwrap(); + segments.write(seq_id, vec![data.clone()]).await.unwrap(); let shared_source = SharedSegmentSource::new(source.clone()); @@ -115,21 +96,18 @@ mod tests { assert_eq!(result2.unwrap().unwrap_host(), data); // The inner source should have been called only once - assert_eq!(source.request_count.load(Ordering::Relaxed), 1); + assert_eq!(source.requests(), 1); } #[tokio::test] async fn test_shared_source_handles_dropped_futures() { - let source = CountingSegmentSource::default(); + let segments = Arc::new(TestSegments::default()); + let source = CountingSegmentSource::new(Arc::clone(&segments) as _); // Add a segment let data = ByteBuffer::from(vec![5, 6, 7, 8]); let seq_id = SequenceId::root().downgrade(); - source - .segments - .write(seq_id, vec![data.clone()]) - .await - .unwrap(); + segments.write(seq_id, vec![data.clone()]).await.unwrap(); let shared_source = SharedSegmentSource::new(source.clone()); let id = SegmentId::from(0); @@ -145,6 +123,6 @@ mod tests { assert_eq!(result.unwrap().unwrap_host(), data); // Should have made 2 requests since the first was dropped before completion - assert_eq!(source.request_count.load(Ordering::Relaxed), 2); + assert_eq!(source.requests(), 2); } } diff --git a/vortex-layout/src/segments/test.rs b/vortex-layout/src/segments/test.rs index d880d15cc1a..024fecadc46 100644 --- a/vortex-layout/src/segments/test.rs +++ b/vortex-layout/src/segments/test.rs @@ -2,6 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use async_trait::async_trait; use futures::FutureExt; @@ -58,3 +60,34 @@ impl SegmentSink for TestSegments { Ok(segment_id) } } + +/// A [`SegmentSource`] that counts the segments requested from an inner source. +/// +/// Clones share the count. +#[derive(Clone)] +pub struct CountingSegmentSource { + inner: Arc, + requests: Arc, +} + +impl CountingSegmentSource { + /// Wraps `inner`, starting the count at zero. + pub fn new(inner: Arc) -> Self { + Self { + inner, + requests: Arc::new(AtomicUsize::new(0)), + } + } + + /// Returns how many segments have been requested so far. + pub fn requests(&self) -> usize { + self.requests.load(Ordering::Relaxed) + } +} + +impl SegmentSource for CountingSegmentSource { + fn request(&self, id: SegmentId) -> SegmentFuture { + self.requests.fetch_add(1, Ordering::Relaxed); + self.inner.request(id) + } +}