Skip to content
8 changes: 8 additions & 0 deletions java/vortex-jni/src/main/java/dev/vortex/api/Expression.java
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,14 @@ public static Expression isNotNull(Expression child) {
return new Expression(NativeExpression.isNotNull(child.nativePointer()));
}

/**
* Null out {@code child} wherever {@code mask} is false, keeping its value where {@code mask} is true. The mask
* must be a non-nullable boolean.
*/
public static Expression mask(Expression child, Expression mask) {
return new Expression(NativeExpression.mask(child.nativePointer(), mask.nativePointer()));
}

/**
* SQL {@code LIKE} pattern match.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ private NativeExpression() {}

public static native long isNotNull(long childPointer);

public static native long mask(long childPointer, long maskPointer);

public static native long like(long childPointer, long patternPointer, boolean negated, boolean caseInsensitive);

public static native long between(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

package dev.vortex.spark.read;

import dev.vortex.api.Expression;
import java.util.ArrayList;
import java.util.List;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

/**
* Pushes Spark's nested schema pruning down to the native scan.
*
* <p>Only struct fields are pruned. Fields inside arrays and maps keep their full table type, since projecting each
* element needs a per-element expression the native scan does not expose yet.
*/
final class NestedSchemaPruning {
private NestedSchemaPruning() {}

/**
* Returns {@code tableType} pruned to the struct fields present in {@code requiredType}, in table order. Any type
* that is not a struct on both sides keeps its table type.
*/
static DataType prune(DataType tableType, DataType requiredType) {
if (!(tableType instanceof StructType table) || !(requiredType instanceof StructType required)) {
return tableType;
}
List<StructField> fields = new ArrayList<>();
for (StructField field : table.fields()) {
if (required.getFieldIndex(field.name()).isDefined()) {
DataType prunedType =
prune(field.dataType(), required.apply(field.name()).dataType());
fields.add(new StructField(field.name(), prunedType, field.nullable(), field.metadata()));
}
}
return fields.isEmpty() ? tableType : new StructType(fields.toArray(new StructField[0]));
}

/**
* Builds the native projection that returns {@code readSchema}, where each field is either its type in
* {@code tableSchema} or a {@link #prune pruned} version of it.
*
* <p>When nothing is pruned this is a plain {@code select} of the top-level columns.
*/
static Expression projection(StructType readSchema, StructType tableSchema) {
return structProjection(Expression.root(), false, readSchema, tableSchema);
}

private static Expression structProjection(Expression struct, boolean nullable, StructType read, StructType table) {
String[] names = read.fieldNames();
// `select` keeps the struct's own nulls, so use it when no child needs pruning.
if (!isPruned(read, table)) {
return Expression.select(names, struct);
}
Expression[] children = new Expression[names.length];
for (int i = 0; i < names.length; i++) {
Expression field = Expression.getItem(names[i], struct);
StructField readField = read.fields()[i];
DataType tableType = table.apply(names[i]).dataType();
children[i] = readField.dataType().equals(tableType)
? field
: structProjection(
field, readField.nullable(), (StructType) readField.dataType(), (StructType) tableType);
}
Expression packed = Expression.pack(names, children, false);
// `pack` builds a struct with no nulls, so carry over the source struct's nulls.
return nullable ? Expression.mask(packed, Expression.isNotNull(struct)) : packed;
}

private static boolean isPruned(StructType read, StructType table) {
for (StructField field : read.fields()) {
if (!field.dataType().equals(table.apply(field.name()).dataType())) {
return true;
}
}
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,14 @@
import org.apache.spark.sql.connector.read.Batch;
import org.apache.spark.sql.connector.read.InputPartition;
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

/** Execution source for batch scans of Vortex file tables. */
public final class VortexBatchExec implements Batch {
private final List<String> paths;
private final StructType readSchema;
private final StructType tableSchema;
private final Map<String, String> formatOptions;
private final Predicate[] pushedPredicates;
private List<String> resolvedPaths;
Expand All @@ -41,13 +43,19 @@ public final class VortexBatchExec implements Batch {
*
* @param paths the list of file paths to scan
* @param columns the list of columns to read from the files
* @param tableColumns the full table columns, which {@code columns} may prune nested fields from
* @param pushedPredicates predicates pushed down by Spark; converted to a single Vortex filter expression at read
* time
*/
public VortexBatchExec(
List<String> paths, List<Column> columns, Map<String, String> formatOptions, Predicate[] pushedPredicates) {
List<String> paths,
List<Column> columns,
List<Column> tableColumns,
Map<String, String> formatOptions,
Predicate[] pushedPredicates) {
this.paths = List.copyOf(paths);
this.readSchema = CatalogV2Util.v2ColumnsToStructType(columns.toArray(new Column[0]));
this.tableSchema = CatalogV2Util.v2ColumnsToStructType(tableColumns.toArray(new Column[0]));
this.formatOptions = Map.copyOf(formatOptions);
this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone();
}
Expand Down Expand Up @@ -106,10 +114,14 @@ private Map<String, FileSplitPlanner.FileSize> measureFiles(List<String> paths)
public PartitionReaderFactory createReaderFactory() {
List<String> files = resolvedPaths != null ? resolvedPaths : resolvePaths();
Set<String> partitionColumns = collectPartitionColumnNames(files);
List<String> dataColumnNames = Arrays.stream(readSchema.fieldNames())
.filter(name -> !partitionColumns.contains(name))
.collect(Collectors.toList());
return new VortexPartitionReaderFactory(dataColumnNames, formatOptions, pushedPredicates);
StructType dataReadSchema = new StructType(Arrays.stream(readSchema.fields())
.filter(field -> !partitionColumns.contains(field.name()))
.toArray(StructField[]::new));
// Only the read columns' table types are needed to build the projection.
StructType dataTableSchema = new StructType(Arrays.stream(dataReadSchema.fieldNames())
.map(tableSchema::apply)
.toArray(StructField[]::new));
return new VortexPartitionReaderFactory(dataReadSchema, dataTableSchema, formatOptions, pushedPredicates);
}

private List<String> resolvePaths() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@
import dev.vortex.spark.VortexFilePartition;
import dev.vortex.spark.VortexSparkSession;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.PartitionReader;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.vectorized.ColumnVector;
import org.apache.spark.sql.vectorized.ColumnarBatch;

Expand Down Expand Up @@ -50,7 +50,8 @@ final class VortexPartitionReader implements PartitionReader<ColumnarBatch> {

VortexPartitionReader(
VortexFilePartition spark,
List<String> dataColumnNames,
StructType dataReadSchema,
StructType tableSchema,
Map<String, String> formatOptions,
Predicate[] pushedPredicates) {
this.spark = spark;
Expand All @@ -60,9 +61,8 @@ final class VortexPartitionReader implements PartitionReader<ColumnarBatch> {
dataSource = DataSource.open(session, spark.paths(), formatOptions);

var options = ScanOptions.builder();
if (!dataColumnNames.isEmpty()) {
Expression projection = Expression.select(dataColumnNames.toArray(new String[0]), Expression.root());
options.projection(projection);
if (dataReadSchema.fields().length > 0) {
options.projection(NestedSchemaPruning.projection(dataReadSchema, tableSchema));
}
if (pushedPredicates != null && pushedPredicates.length > 0) {
buildFilterExpression(pushedPredicates).ifPresent(options::filter);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,17 @@

package dev.vortex.spark.read;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import dev.vortex.jni.NativeRuntime;
import dev.vortex.spark.VortexFilePartition;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.connector.expressions.filter.Predicate;
import org.apache.spark.sql.connector.read.InputPartition;
import org.apache.spark.sql.connector.read.PartitionReader;
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.vectorized.ColumnarBatch;

/**
Expand All @@ -28,13 +27,18 @@
public final class VortexPartitionReaderFactory implements PartitionReaderFactory, Serializable {
private static final long serialVersionUID = 1L;

private final ImmutableList<String> dataColumnNames;
private final StructType dataReadSchema;
private final StructType tableSchema;
private final ImmutableMap<String, String> formatOptions;
private final Predicate[] pushedPredicates;

public VortexPartitionReaderFactory(
List<String> dataColumnNames, Map<String, String> formatOptions, Predicate[] pushedPredicates) {
this.dataColumnNames = ImmutableList.copyOf(dataColumnNames);
StructType dataReadSchema,
StructType tableSchema,
Map<String, String> formatOptions,
Predicate[] pushedPredicates) {
this.dataReadSchema = dataReadSchema;
this.tableSchema = tableSchema;
this.formatOptions = ImmutableMap.copyOf(formatOptions);
this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone();
}
Expand All @@ -48,7 +52,7 @@ public PartitionReader<InternalRow> createReader(InputPartition partition) {
public PartitionReader<ColumnarBatch> createColumnarReader(InputPartition partition) {
NativeRuntime.setWorkerThreads(Integer.parseInt(formatOptions.getOrDefault("vortex.workerThreads", "4")));
VortexFilePartition spark = (VortexFilePartition) partition;
return new VortexPartitionReader(spark, dataColumnNames, formatOptions, pushedPredicates);
return new VortexPartitionReader(spark, dataReadSchema, tableSchema, formatOptions, pushedPredicates);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public String description() {
*/
@Override
public Batch toBatch() {
return new VortexBatchExec(paths, readColumns, formatOptions, pushedPredicates);
return new VortexBatchExec(paths, readColumns, tableColumns, formatOptions, pushedPredicates);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,22 +130,33 @@ public Scan build() {
/**
* Prunes the columns to only include those specified in the required schema.
*
* <p>Required columns keep their table type because the native scan returns whole top-level columns, even when
* Spark prunes nested fields.
* <p>Columns stay in table order. Struct columns are pruned to the nested fields Spark requires, and the reader
* projects only those fields. See {@link NestedSchemaPruning} for which nested types are pruned.
*
* @param requiredSchema the schema specifying which columns are required
*/
@Override
public void pruneColumns(StructType requiredSchema) {
Set<String> required = Set.copyOf(Arrays.asList(requiredSchema.fieldNames()));
readColumns.clear();
for (Column column : tableColumns) {
if (required.contains(column.name())) {
readColumns.add(column);
if (requiredSchema.getFieldIndex(column.name()).isDefined()) {
DataType prunedType = NestedSchemaPruning.prune(
column.dataType(), requiredSchema.apply(column.name()).dataType());
readColumns.add(prunedType.equals(column.dataType()) ? column : withDataType(column, prunedType));
}
}
}

private static Column withDataType(Column column, DataType dataType) {
return Column.create(
column.name(),
dataType,
column.nullable(),
column.comment(),
column.defaultValue(),
column.metadataInJSON());
}

/**
* Splits the supplied predicates into pushed and not-pushed sets.
*
Expand Down
Loading
Loading