From 09dd97b792509ed4f639ec1e960eef225afff53f Mon Sep 17 00:00:00 2001 From: Christopher Wirt Date: Thu, 29 Oct 2026 10:00:56 +0000 Subject: [PATCH] test(spark): check struct nulls through the pruned read The null checks in VortexNestedSchemaPruningTest selected IS NULL on its own, so Spark read the whole struct and never exercised the pruned projection. Select the null checks next to a pruned field, so the scan reads the pruned struct, and assert the read schema to keep it that way. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Christopher Wirt --- .../spark/VortexNestedSchemaPruningTest.java | 52 +++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java index 1d98b2264c2..cf3e2f88200 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexNestedSchemaPruningTest.java @@ -9,6 +9,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -126,23 +127,37 @@ public void structInStructField() { @Test @DisplayName("Null structs and null fields stay null when the struct is pruned") public void nullableStructField() { - assertEquals(Arrays.asList(200L, null, 202L, null, null), valuesOf(table, functions.col("n.b"))); + // Selecting `n.b` next to `n IS NULL` makes Spark read the pruned `n`, so the null check sees its nulls. + Dataset rows = table.select( + functions.col("id"), functions.col("n.b"), functions.col("n").isNull()); + assertEquals("struct>", readSchema(rows)); assertEquals( - List.of(false, true, false, true, false), - valuesOf(table, functions.col("n").isNull())); + Arrays.asList( + Arrays.asList(0L, 200L, false), + Arrays.asList(1L, null, true), + Arrays.asList(2L, 202L, false), + Arrays.asList(3L, null, true), + Arrays.asList(4L, null, false)), + rowsOf(rows)); } @Test @DisplayName("Nulls at each level of a struct inside a struct survive pruning both levels") public void nullableStructInStructField() { + Dataset rows = table.select( + functions.col("id"), + functions.col("nullable_outer.inner.c"), + functions.col("nullable_outer").isNull(), + functions.col("nullable_outer.inner").isNull()); + assertEquals("struct>>", readSchema(rows)); assertEquals( - Arrays.asList(300L, null, null, 303L, 304L), valuesOf(table, functions.col("nullable_outer.inner.c"))); - assertEquals( - List.of(false, true, false, false, false), - valuesOf(table, functions.col("nullable_outer").isNull())); - assertEquals( - List.of(false, true, true, false, false), - valuesOf(table, functions.col("nullable_outer.inner").isNull())); + Arrays.asList( + Arrays.asList(0L, 300L, false, false), + Arrays.asList(1L, null, true, true), + Arrays.asList(2L, null, false, true), + Arrays.asList(3L, 303L, false, false), + Arrays.asList(4L, 304L, false, false)), + rowsOf(rows)); } @Test @@ -172,6 +187,23 @@ private static Column nullWhen(Column condition, Column value) { return functions.when(condition, functions.lit(null)).otherwise(value); } + /** Returns the schema Spark asks the Vortex scan for. */ + private static String readSchema(Dataset rows) { + return rows.queryExecution() + .optimizedPlan() + .collectLeaves() + .head() + .schema() + .simpleString(); + } + + /** Returns each row of {@code rows}, whose first column is the id, ordered by id. */ + private static List> rowsOf(Dataset rows) { + return rows.orderBy("id").collectAsList().stream() + .map(row -> IntStream.range(0, row.size()).mapToObj(row::get).collect(Collectors.toList())) + .collect(Collectors.toList()); + } + /** Returns {@code column} for each row of {@code rows}, ordered by id. */ @SuppressWarnings("unchecked") private static List valuesOf(Dataset rows, Column column) {