diff --git a/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/ContentDefinedChunkingWriteBenchmarks.java b/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/ContentDefinedChunkingWriteBenchmarks.java new file mode 100644 index 0000000000..2e5fba727e --- /dev/null +++ b/parquet-benchmarks/src/main/java/org/apache/parquet/benchmarks/ContentDefinedChunkingWriteBenchmarks.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.benchmarks; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +/** The same rows written with and without content defined chunking, to {@link BlackHoleOutputFile}. */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 3) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +public class ContentDefinedChunkingWriteBenchmarks { + + private static final MessageType SCHEMA = MessageTypeParser.parseMessageType("message m {" + + " required int64 l; optional double d; optional binary s (STRING);" + + " optional group list (LIST) { repeated group list { optional int64 element; } } }"); + + @Param({"false", "true"}) + public boolean chunking; + + private final List rows = new ArrayList<>(); + + @Setup + public void setup() { + Random random = new Random(TestDataFactory.DEFAULT_SEED); + SimpleGroupFactory factory = new SimpleGroupFactory(SCHEMA); + for (int i = 0; i < 100_000; i++) { + Group row = factory.newGroup().append("l", random.nextLong()); + if (random.nextInt(10) > 0) { + row.append("d", random.nextDouble()).append("s", Long.toString(random.nextLong(), 36)); + } + Group list = row.addGroup("list"); + for (int n = random.nextInt(9); n > 0; n--) { + list.addGroup("list").append("element", random.nextLong()); + } + rows.add(row); + } + } + + /** Dictionary encoding is off and the page row count limit lifted, so that the hashing dominates. */ + @Benchmark + public void write() throws IOException { + try (ParquetWriter writer = ExampleParquetWriter.builder(BlackHoleOutputFile.INSTANCE) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + .withType(SCHEMA) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .withDictionaryEncoding(false) + .withPageRowCountLimit(Integer.MAX_VALUE) + .withContentDefinedChunking(chunking) + .build()) { + for (Group row : rows) { + writer.write(row); + } + } + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 92eb080bfc..b348155d3f 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -21,6 +21,7 @@ import static org.apache.parquet.bytes.BytesUtils.getWidthFromMaxInt; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.OptionalDouble; @@ -33,6 +34,7 @@ import org.apache.parquet.bytes.HeapByteBufferAllocator; import org.apache.parquet.column.impl.ColumnWriteStoreV1; import org.apache.parquet.column.impl.ColumnWriteStoreV2; +import org.apache.parquet.column.impl.ContentDefinedChunker; import org.apache.parquet.column.page.PageWriteStore; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.alp.AlpConfig; @@ -72,6 +74,9 @@ public class ParquetProperties { public static final int DEFAULT_BLOOM_FILTER_CANDIDATES_NUMBER = 5; public static final boolean DEFAULT_STATISTICS_ENABLED = true; public static final boolean DEFAULT_SIZE_STATISTICS_ENABLED = true; + public static final long DEFAULT_CONTENT_DEFINED_CHUNKING_MIN_SIZE = 256 * 1024; + public static final long DEFAULT_CONTENT_DEFINED_CHUNKING_MAX_SIZE = 1024 * 1024; + public static final int DEFAULT_CONTENT_DEFINED_CHUNKING_NORM_LEVEL = 0; public static final boolean DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED = true; @@ -144,6 +149,10 @@ public static WriterVersion fromString(String name) { private final ColumnProperty sizeStatistics; private final ColumnProperty columnCodecs; private final ColumnProperty columnCompressionLevels; + private final boolean contentDefinedChunkingEnabled; + private final long contentDefinedChunkingMinSize; + private final long contentDefinedChunkingMaxSize; + private final int contentDefinedChunkingNormLevel; private ParquetProperties(Builder builder) { this.pageSizeThreshold = builder.pageSize; @@ -179,6 +188,15 @@ private ParquetProperties(Builder builder) { this.sizeStatistics = builder.sizeStatistics.build(); this.columnCodecs = builder.columnCodecs.build(); this.columnCompressionLevels = builder.columnCompressionLevels.build(); + this.contentDefinedChunkingEnabled = builder.contentDefinedChunkingEnabled; + this.contentDefinedChunkingMinSize = builder.contentDefinedChunkingMinSize; + this.contentDefinedChunkingMaxSize = builder.contentDefinedChunkingMaxSize; + this.contentDefinedChunkingNormLevel = builder.contentDefinedChunkingNormLevel; + if (contentDefinedChunkingEnabled) { + // Rejected here rather than once a writer has opened its file + ContentDefinedChunker.calculateMask( + contentDefinedChunkingMinSize, contentDefinedChunkingMaxSize, contentDefinedChunkingNormLevel); + } } public static Builder builder() { @@ -366,6 +384,30 @@ public ColumnWriteStore newColumnWriteStore( }; } + /** + * EXPERIMENTAL: The column write store of a row group, continuing the chunking of the file's earlier + * row groups. Required with content defined chunking. + * + * @param schema the schema of the row group + * @param pageStore the page store of the row group + * @param bloomFilterWriteStore the bloom filter store of the row group + * @param chunkers the file's {@link ContentDefinedChunker#forColumns chunkers}, or {@code null} + * without content defined chunking + * @return the column write store + */ + public ColumnWriteStore newColumnWriteStore( + MessageType schema, + PageWriteStore pageStore, + BloomFilterWriteStore bloomFilterWriteStore, + List chunkers) { + validateAlp(schema); + return switch (writerVersion) { + case PARQUET_1_0 -> new ColumnWriteStoreV1(schema, pageStore, bloomFilterWriteStore, this, chunkers); + case PARQUET_2_0 -> new ColumnWriteStoreV2(schema, pageStore, bloomFilterWriteStore, this, chunkers); + default -> throw new IllegalArgumentException("unknown version " + writerVersion); + }; + } + public int getMinRowCountForPageSizeCheck() { return minRowCountForPageSizeCheck; } @@ -394,6 +436,42 @@ public int getRowGroupRowCountLimit() { return rowGroupRowCountLimit; } + /** + * EXPERIMENTAL + * + * @return {@code true} if content defined chunking is enabled + */ + public boolean isContentDefinedChunkingEnabled() { + return contentDefinedChunkingEnabled; + } + + /** + * EXPERIMENTAL + * + * @return the minimum content defined chunk size in bytes + */ + public long getContentDefinedChunkingMinSize() { + return contentDefinedChunkingMinSize; + } + + /** + * EXPERIMENTAL + * + * @return the maximum content defined chunk size in bytes + */ + public long getContentDefinedChunkingMaxSize() { + return contentDefinedChunkingMaxSize; + } + + /** + * EXPERIMENTAL + * + * @return the normalization level of the content defined chunking's rolling hash mask + */ + public int getContentDefinedChunkingNormLevel() { + return contentDefinedChunkingNormLevel; + } + public int getPageRowCountLimit() { return pageRowCountLimit; } @@ -488,6 +566,12 @@ public String toString() { + "Bloom filter expected number of distinct values are: " + bloomFilterNDVs + '\n' + "Bloom filter false positive probabilities are: " + bloomFilterFPPs + '\n' + "Page row count limit to " + getPageRowCountLimit() + '\n' + + "Content defined chunking is: " + + (contentDefinedChunkingEnabled + ? contentDefinedChunkingMinSize + " to " + contentDefinedChunkingMaxSize + + " bytes, normalization level " + contentDefinedChunkingNormLevel + : "off") + + '\n' + "Writing page checksums is: " + (getPageWriteChecksumEnabled() ? "on" : "off") + '\n' + "Statistics enabled: " + statisticsEnabled + '\n' + "Size statistics enabled: " + sizeStatisticsEnabled + '\n' @@ -537,6 +621,10 @@ public static class Builder { private final ColumnProperty.Builder sizeStatistics; private final ColumnProperty.Builder columnCodecs; private final ColumnProperty.Builder columnCompressionLevels; + private boolean contentDefinedChunkingEnabled; + private long contentDefinedChunkingMinSize = DEFAULT_CONTENT_DEFINED_CHUNKING_MIN_SIZE; + private long contentDefinedChunkingMaxSize = DEFAULT_CONTENT_DEFINED_CHUNKING_MAX_SIZE; + private int contentDefinedChunkingNormLevel = DEFAULT_CONTENT_DEFINED_CHUNKING_NORM_LEVEL; private Builder() { enableDict = ColumnProperty.builder().withDefaultValue(DEFAULT_IS_DICTIONARY_ENABLED); @@ -590,6 +678,10 @@ private Builder(ParquetProperties toCopy) { this.sizeStatisticsEnabled = toCopy.sizeStatisticsEnabled; this.columnCodecs = ColumnProperty.builder(toCopy.columnCodecs); this.columnCompressionLevels = ColumnProperty.builder(toCopy.columnCompressionLevels); + this.contentDefinedChunkingEnabled = toCopy.contentDefinedChunkingEnabled; + this.contentDefinedChunkingMinSize = toCopy.contentDefinedChunkingMinSize; + this.contentDefinedChunkingMaxSize = toCopy.contentDefinedChunkingMaxSize; + this.contentDefinedChunkingNormLevel = toCopy.contentDefinedChunkingNormLevel; } /** @@ -899,6 +991,61 @@ public Builder withPageRowCountLimit(int rowCount) { return this; } + /** + * EXPERIMENTAL: Enable or disable content defined chunking, which also ends data pages where a + * rolling hash of the values matches, as Arrow C++ and arrow-rs do, so that files sharing a run of + * values share pages. Disabled by default. The page size and row count limits still apply. Column + * write stores must then come from + * {@link ParquetProperties#newColumnWriteStore(MessageType, PageWriteStore, BloomFilterWriteStore, + * List)}, and {@link #build()} rejects an invalid chunk size range. + * + *

With dictionary encoding, an edit that adds values renumbers the indices of values first seen + * after it. As parquet-java chooses between dictionary and plain encoding on the first page of a + * column chunk, which chunking can cut short, some column chunks fall back to plain encoding. + * + * @param enabled whether to enable content defined chunking + * @return this builder for method chaining. + */ + public Builder withContentDefinedChunking(boolean enabled) { + this.contentDefinedChunkingEnabled = enabled; + return this; + } + + /** + * EXPERIMENTAL: Set the minimum content defined chunk size, 256 KiB by default. Sizes count values + * and levels before encoding, at the width Arrow holds them in: an INT(16) value counts 2 bytes. + * + * @param minSize the minimum chunk size in bytes + * @return this builder for method chaining. + */ + public Builder withContentDefinedChunkingMinSize(long minSize) { + this.contentDefinedChunkingMinSize = minSize; + return this; + } + + /** + * EXPERIMENTAL: Set the maximum content defined chunk size, 1 MiB by default. + * + * @param maxSize the maximum chunk size in bytes + * @return this builder for method chaining. + */ + public Builder withContentDefinedChunkingMaxSize(long maxSize) { + this.contentDefinedChunkingMaxSize = maxSize; + return this; + } + + /** + * EXPERIMENTAL: Set the normalization level of the content defined chunking's rolling hash mask, 0 + * by default. Raising it makes chunk sizes more uniform at the cost of more small pages. + * + * @param normLevel the normalization level + * @return this builder for method chaining. + */ + public Builder withContentDefinedChunkingNormLevel(int normLevel) { + this.contentDefinedChunkingNormLevel = normLevel; + return this; + } + public Builder withPageWriteChecksumEnabled(boolean val) { this.pageWriteChecksumEnabled = val; return this; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java index 9bc7726491..a37181f904 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java @@ -23,10 +23,13 @@ import static java.util.Collections.unmodifiableMap; import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; +import org.apache.parquet.Preconditions; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ColumnWriteStore; import org.apache.parquet.column.ColumnWriter; @@ -62,6 +65,8 @@ private interface ColumnWriterProvider { // To be used by the deprecated constructor of ColumnWriteStoreV1 @Deprecated ColumnWriteStoreBase(final PageWriteStore pageWriteStore, final ParquetProperties props) { + Preconditions.checkState( + !props.isContentDefinedChunkingEnabled(), "Content defined chunking needs the chunkers of the file"); this.props = props; this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO); @@ -74,7 +79,7 @@ private interface ColumnWriterProvider { public ColumnWriter getColumnWriter(ColumnDescriptor path) { ColumnWriterBase column = columns.get(path); if (column == null) { - column = createColumnWriterBase(path, pageWriteStore.getPageWriter(path), null, props); + column = createColumnWriterBase(path, pageWriteStore.getPageWriter(path), null, null, props); columns.put(path, column); } return column; @@ -83,12 +88,14 @@ public ColumnWriter getColumnWriter(ColumnDescriptor path) { } ColumnWriteStoreBase(MessageType schema, PageWriteStore pageWriteStore, ParquetProperties props) { + Preconditions.checkState( + !props.isContentDefinedChunkingEnabled(), "Content defined chunking needs the chunkers of the file"); this.props = props; this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO); Map mcolumns = new TreeMap<>(); for (ColumnDescriptor path : schema.getColumns()) { PageWriter pageWriter = pageWriteStore.getPageWriter(path); - mcolumns.put(path, createColumnWriterBase(path, pageWriter, null, props)); + mcolumns.put(path, createColumnWriterBase(path, pageWriter, null, null, props)); } this.columns = unmodifiableMap(mcolumns); @@ -108,16 +115,32 @@ public ColumnWriter getColumnWriter(ColumnDescriptor path) { PageWriteStore pageWriteStore, BloomFilterWriteStore bloomFilterWriteStore, ParquetProperties props) { + this(schema, pageWriteStore, bloomFilterWriteStore, props, null); + } + + ColumnWriteStoreBase( + MessageType schema, + PageWriteStore pageWriteStore, + BloomFilterWriteStore bloomFilterWriteStore, + ParquetProperties props, + List chunkers) { + Iterator columnChunkers = null; + if (props.isContentDefinedChunkingEnabled()) { + Preconditions.checkState(chunkers != null, "Content defined chunking needs the chunkers of the file"); + Preconditions.checkArgument(chunkers.size() == schema.getColumns().size(), "Expected a chunker per column"); + columnChunkers = chunkers.iterator(); + } this.props = props; this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO); Map mcolumns = new TreeMap<>(); for (ColumnDescriptor path : schema.getColumns()) { PageWriter pageWriter = pageWriteStore.getPageWriter(path); + ContentDefinedChunker chunker = columnChunkers != null ? columnChunkers.next() : null; if (props.isBloomFilterEnabled(path)) { BloomFilterWriter bloomFilterWriter = bloomFilterWriteStore.getBloomFilterWriter(path); - mcolumns.put(path, createColumnWriterBase(path, pageWriter, bloomFilterWriter, props)); + mcolumns.put(path, createColumnWriterBase(path, pageWriter, bloomFilterWriter, chunker, props)); } else { - mcolumns.put(path, createColumnWriterBase(path, pageWriter, null, props)); + mcolumns.put(path, createColumnWriterBase(path, pageWriter, null, chunker, props)); } } this.columns = unmodifiableMap(mcolumns); @@ -136,14 +159,19 @@ private ColumnWriterBase createColumnWriterBase( ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, ParquetProperties props) { - ColumnWriterBase columnWriterBase = createColumnWriter(path, pageWriter, bloomFilterWriter, props); + ColumnWriterBase columnWriterBase = createColumnWriter(path, pageWriter, bloomFilterWriter, chunker, props); columnWriterBase.initStatusManager(statusManager); return columnWriterBase; } abstract ColumnWriterBase createColumnWriter( - ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, ParquetProperties props); + ColumnDescriptor path, + PageWriter pageWriter, + BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, + ParquetProperties props); @Override public ColumnWriter getColumnWriter(ColumnDescriptor path) { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java index 420e90e4d7..93f568890f 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java @@ -18,6 +18,7 @@ */ package org.apache.parquet.column.impl; +import java.util.List; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.page.PageWriteStore; @@ -45,12 +46,23 @@ public ColumnWriteStoreV1( super(schema, pageWriteStore, bloomFilterWriteStore, props); } + /** EXPERIMENTAL: A store continuing the content defined chunking of the file's {@code chunkers}. */ + public ColumnWriteStoreV1( + MessageType schema, + PageWriteStore pageWriteStore, + BloomFilterWriteStore bloomFilterWriteStore, + ParquetProperties props, + List chunkers) { + super(schema, pageWriteStore, bloomFilterWriteStore, props, chunkers); + } + @Override ColumnWriterBase createColumnWriter( ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, ParquetProperties props) { - return new ColumnWriterV1(path, pageWriter, bloomFilterWriter, props); + return new ColumnWriterV1(path, pageWriter, bloomFilterWriter, chunker, props); } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java index bdf0d0123f..c97ae976a8 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java @@ -18,6 +18,7 @@ */ package org.apache.parquet.column.impl; +import java.util.List; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.page.PageWriteStore; @@ -40,12 +41,23 @@ public ColumnWriteStoreV2( super(schema, pageWriteStore, bloomFilterWriteStore, props); } + /** EXPERIMENTAL: A store continuing the content defined chunking of the file's {@code chunkers}. */ + public ColumnWriteStoreV2( + MessageType schema, + PageWriteStore pageWriteStore, + BloomFilterWriteStore bloomFilterWriteStore, + ParquetProperties props, + List chunkers) { + super(schema, pageWriteStore, bloomFilterWriteStore, props, chunkers); + } + @Override ColumnWriterBase createColumnWriter( ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, ParquetProperties props) { - return new ColumnWriterV2(path, pageWriter, bloomFilterWriter, props); + return new ColumnWriterV2(path, pageWriter, bloomFilterWriter, chunker, props); } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java index 408627404d..8e7eff073d 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java @@ -60,15 +60,18 @@ abstract class ColumnWriterBase implements ColumnWriter { private StatusManager statusManager = StatusManager.create(); private final ColumnValueCollector collector; + // Null without content defined chunking + private final ContentDefinedChunker chunker; ColumnWriterBase(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) { - this(path, pageWriter, null, props); + this(path, pageWriter, null, null, props); } ColumnWriterBase( ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, ParquetProperties props) { this.path = path; this.pageWriter = pageWriter; @@ -78,6 +81,7 @@ abstract class ColumnWriterBase implements ColumnWriter { this.dataColumn = props.newValuesWriter(path); this.collector = new ColumnValueCollector(path, bloomFilterWriter, props); + this.chunker = chunker; } void initStatusManager(StatusManager statusManager) { @@ -114,6 +118,9 @@ private void repetitionLevel(int repetitionLevel) { public void writeNull(int repetitionLevel, int definitionLevel) { if (DEBUG) log(null, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); collector.writeNull(repetitionLevel, definitionLevel); @@ -152,6 +159,9 @@ public long getBufferedSizeInMemory() { public void write(double value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeDouble(value); @@ -174,6 +184,9 @@ public void write(double value, int repetitionLevel, int definitionLevel) { public void write(float value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeFloat(value); @@ -196,6 +209,9 @@ public void write(float value, int repetitionLevel, int definitionLevel) { public void write(Binary value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeBytes(value); @@ -218,6 +234,9 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) { public void write(boolean value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeBoolean(value); @@ -240,6 +259,9 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) { public void write(int value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeInteger(value); @@ -262,6 +284,9 @@ public void write(int value, int repetitionLevel, int definitionLevel) { public void write(long value, int repetitionLevel, int definitionLevel) { if (DEBUG) log(value, repetitionLevel, definitionLevel); try { + if (chunker != null && chunker.startsNewChunk(value, repetitionLevel, definitionLevel)) { + writePageIfNotEmpty(); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeLong(value); @@ -365,6 +390,15 @@ int getValueCount() { return this.valueCount; } + /** + * Writes the current data to a new page in the page store, unless there is none. + */ + private void writePageIfNotEmpty() { + if (valueCount > 0) { + writePage(); + } + } + /** * Writes the current data to a new page in the page store */ diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java index 882be23811..a27c113f35 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java @@ -43,8 +43,9 @@ public ColumnWriterV1( ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, ParquetProperties props) { - super(path, pageWriter, bloomFilterWriter, props); + super(path, pageWriter, bloomFilterWriter, chunker, props); } @Override diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java index e61bcef15d..06c9d247c8 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java @@ -65,8 +65,9 @@ public BytesInput getBytes() { ColumnDescriptor path, PageWriter pageWriter, BloomFilterWriter bloomFilterWriter, + ContentDefinedChunker chunker, ParquetProperties props) { - super(path, pageWriter, bloomFilterWriter, props); + super(path, pageWriter, bloomFilterWriter, chunker, props); } @Override diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ContentDefinedChunker.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ContentDefinedChunker.java new file mode 100644 index 0000000000..5ea0647a77 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ContentDefinedChunker.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.impl; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.parquet.Preconditions; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.io.ParquetEncodingException; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; + +/** + * EXPERIMENTAL: Decides the content defined chunk boundaries of one leaf column for a whole file: the + * column writer ends its page before a triplet {@code startsNewChunk} answers {@code true} for. + * + *

Ported from Arrow C++ {@code parquet/chunker_internal} and arrow-rs {@code column/chunker/cdc.rs}, + * with the same boundaries for the same values, except where Arrow holds them in another form, such as + * a DECIMAL, an INT96 or a dictionary array. + * + *

The chunkers {@link #forColumns} makes continue across the row groups of a file. Public for the + * file writers of other modules only. Not thread safe. + */ +public final class ContentDefinedChunker { + + private final long minChunkSize; + private final long maxChunkSize; + private final long rollingHashMask; + private final int maxDef; + private final int maxRep; + private final int byteWidth; + + private long rollingHash; + private boolean hasMatched; + private int nthRun; + private long chunkSize; + + /** Rolls the bytes written to it into the hash. */ + private final OutputStream byteRoller = new OutputStream() { + @Override + public void write(int b) { + write(new byte[] {(byte) b}, 0, 1); + } + + @Override + public void write(byte[] bytes, int offset, int length) { + long hash = rollingHash; + boolean matched = hasMatched; + long[] table = GearHashTable.TABLE[nthRun]; + for (int i = offset; i < offset + length; ++i) { + hash = (hash << 1) + table[bytes[i] & 0xFF]; + matched |= (hash & rollingHashMask) == 0; + } + rollingHash = hash; + hasMatched = matched; + } + }; + + /** + * EXPERIMENTAL: The chunkers of a file, one per leaf column of {@code schema}, for every row group. + * + * @param schema the schema of the file + * @param props the properties of the file + * @return the chunkers + */ + public static List forColumns(MessageType schema, ParquetProperties props) { + return schema.getColumns().stream() + .map(path -> new ContentDefinedChunker(path, props)) + .collect(Collectors.toList()); + } + + ContentDefinedChunker(ColumnDescriptor path, ParquetProperties props) { + this.minChunkSize = props.getContentDefinedChunkingMinSize(); + this.maxChunkSize = props.getContentDefinedChunkingMaxSize(); + this.rollingHashMask = calculateMask(minChunkSize, maxChunkSize, props.getContentDefinedChunkingNormLevel()); + this.maxDef = path.getMaxDefinitionLevel(); + this.maxRep = path.getMaxRepetitionLevel(); + this.byteWidth = byteWidth(path.getPrimitiveType()); + } + + /** + * The bytes Arrow holds an integer in: 1 for an INT(8), 2 for an INT(16), 4 for other INT32s and 8 for + * INT64s. Zero for other columns. + */ + private static int byteWidth(PrimitiveType type) { + if (type.getLogicalTypeAnnotation() instanceof IntLogicalTypeAnnotation integer) { + return integer.getBitWidth() / Byte.SIZE; + } + return switch (type.getPrimitiveTypeName()) { + case INT32 -> Integer.BYTES; + case INT64 -> Long.BYTES; + default -> 0; + }; + } + + /** + * EXPERIMENTAL: The rolling hash mask, bit for bit as Arrow C++ {@code CalculateMask} derives it: the + * top 15 bits for the default 256 KiB to 1 MiB. + * + * @param minChunkSize the minimum chunk size in bytes + * @param maxChunkSize the maximum chunk size in bytes + * @param normLevel the normalization level + * @return the rolling hash mask + * @throws IllegalArgumentException if the minimum is negative, the maximum not above it, or the + * mask would not have 1 to 63 bits + */ + public static long calculateMask(long minChunkSize, long maxChunkSize, int normLevel) { + Preconditions.checkArgument( + minChunkSize >= 0, "Invalid content defined chunking minimum chunk size (negative): %s", minChunkSize); + Preconditions.checkArgument( + maxChunkSize > minChunkSize, + "Invalid content defined chunking size range: maximum chunk size (%s) must be greater than minimum chunk size (%s)", + maxChunkSize, + minChunkSize); + // Halve before adding, so that the sum cannot overflow; the references do overflow there. + long avgChunkSize = minChunkSize / 2 + maxChunkSize / 2 + (minChunkSize % 2 + maxChunkSize % 2) / 2; + long targetSize = (avgChunkSize - minChunkSize) / GearHashTable.TABLE.length; + int targetBits = Long.SIZE - Long.numberOfLeadingZeros(targetSize); + int maskBits = targetBits == 0 ? 0 : targetBits - 1; + int effectiveBits = maskBits - normLevel; + + // Java takes a long's shift count modulo 64, so an out-of-range width must be rejected here. + Preconditions.checkArgument( + effectiveBits >= 1 && effectiveBits <= 63, + "The content defined chunking mask must be between 1 and 63 bits but was %s" + + " (minimum chunk size %s, maximum chunk size %s, normalization level %s)", + effectiveBits, + minChunkSize, + maxChunkSize, + normLevel); + return -1L << (Long.SIZE - effectiveBits); + } + + /** Rolls a triplet into the hash and answers whether a chunk starts with it, as only a record's first can. */ + boolean startsNewChunk(int value, int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + roll(value, byteWidth); + return repetitionLevel == 0 && needNewChunk(); + } + + boolean startsNewChunk(long value, int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + roll(value, byteWidth); + return repetitionLevel == 0 && needNewChunk(); + } + + boolean startsNewChunk(float value, int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + // Raw bits: floatToIntBits canonicalizes NaN payloads and would diverge from the references. + roll(Float.floatToRawIntBits(value), 4); + return repetitionLevel == 0 && needNewChunk(); + } + + boolean startsNewChunk(double value, int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + roll(Double.doubleToRawLongBits(value), 8); + return repetitionLevel == 0 && needNewChunk(); + } + + boolean startsNewChunk(boolean value, int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + // One byte per boolean, not one bit, as the references hash it. + roll(value ? 1 : 0, 1); + return repetitionLevel == 0 && needNewChunk(); + } + + boolean startsNewChunk(Binary value, int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + roll(value); + return repetitionLevel == 0 && needNewChunk(); + } + + /** The same for a triplet without a value: a null, or an omitted required field at {@code maxDef}. */ + boolean startsNewChunk(int repetitionLevel, int definitionLevel) { + rollLevels(repetitionLevel, definitionLevel); + return repetitionLevel == 0 && needNewChunk(); + } + + /** The definition level, then the repetition level, as two bytes each, if the column has them. */ + private void rollLevels(int repetitionLevel, int definitionLevel) { + if (maxDef > 0) { + roll(definitionLevel, 2); + } + if (maxRep > 0) { + roll(repetitionLevel, 2); + } + } + + /** Arrow C++ {@code Roll}: the {@code width} low-order bytes, little-endian. */ + private void roll(long bits, int width) { + chunkSize += width; + if (chunkSize < minChunkSize) { + return; + } + long hash = rollingHash; + boolean matched = hasMatched; + long[] table = GearHashTable.TABLE[nthRun]; + for (int i = 0; i < width; ++i, bits >>>= 8) { + hash = (hash << 1) + table[(int) bits & 0xFF]; + matched |= (hash & rollingHashMask) == 0; + } + rollingHash = hash; + hasMatched = matched; + } + + /** Arrow C++ {@code Roll(const uint8_t*, length)}: the value's bytes without a length prefix. */ + private void roll(Binary value) { + chunkSize += value.length(); + if (chunkSize < minChunkSize) { + return; + } + try { + // Hands over a heap value's bytes without copying them, as to the values writers + value.writeTo(byteRoller); + } catch (IOException e) { + throw new ParquetEncodingException("could not hash bytes", e); + } + } + + /** + * Arrow C++ {@code NeedNewChunk}: a chunk ends after a match against each of the eight gear hash + * tables in turn, or at {@code maxChunkSize}, which leaves the run counter alone. + */ + private boolean needNewChunk() { + if (hasMatched) { + hasMatched = false; + if (++nthRun >= GearHashTable.TABLE.length) { + nthRun = 0; + chunkSize = 0; + return true; + } + } + if (chunkSize >= maxChunkSize) { + chunkSize = 0; + return true; + } + return false; + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/GearHashTable.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/GearHashTable.java new file mode 100644 index 0000000000..5a81bdf37a --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/GearHashTable.java @@ -0,0 +1,567 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.impl; + +/** + * The eight gear hash tables of {@link ContentDefinedChunker}, copied from Arrow C++ + * {@code cpp/src/parquet/chunker_internal_generated.h}; do not edit. + */ +final class GearHashTable { + + private GearHashTable() {} + + static final long[][] TABLE = { + // seed = 0 + { + 0xf09f35a563783945L, 0x0dcc5b3bc5ae410aL, 0x63f1ea8d22554270L, 0xfbe5ee7bd05a7b61L, + 0x3f692ed5e9934abaL, 0xaab3755952250eb8L, 0xdefb168dc2888fa5L, 0x501b36f7c77a7d47L, + 0xd2fff45d1989642dL, 0x80217c1c600e30a6L, 0xb9469ee2e43df7acL, 0x3654b76a61999706L, + 0x6ea73dfe5de0c6b6L, 0xdfd662e1937a589dL, 0x0dbe0cc74b188a68L, 0xde45f4e6d73ffc6fL, + 0xcdf7a7759e70d87eL, 0x5d6a951b8d38c310L, 0xdc9423c3813fcf2cL, 0x25dc2976e167ffceL, + 0xc2555baa1d031c84L, 0x115bc3f2230a3ab6L, 0xd4b10260f350bedeL, 0xdfd3501ab447d723L, + 0x022e79217edaf167L, 0x1635e2255c5a7526L, 0xa0a750350cc77102L, 0xc027133e05d39f56L, + 0xd949459779cf0387L, 0xb92f1464f5c688c2L, 0xd9ac5f3e8b42f2f3L, 0xdf02bb6f5ecaac21L, + 0x8156f988fac7bfa4L, 0xe4580f97bede2ec8L, 0x44fe7d17a76fca32L, 0x885f59bd54c2014cL, + 0x435e63ec655ffae9L, 0x5ebc51930967b1f1L, 0x5428c2084ac29e47L, 0x9465938fec30e36bL, + 0xc7cb3de4977772cdL, 0x15692d7c201e8c3aL, 0x505ee65cdc4b17f4L, 0x7d9839a0a7aead6bL, + 0xeef5f5b6a0105291L, 0x76c2fb232ce7f5bfL, 0x5c13893c1c3ff3a9L, 0x65b6b547d4442f98L, + 0xb8ad7487c8c96fceL, 0x906bcf51c99974f8L, 0x2f56e48bb943a48cL, 0xbc9ab109f82d3a44L, + 0xcd5160cdc8c7e735L, 0xbe9acb9df3427732L, 0x386b91d477d7fadeL, 0x36be463621dd5af2L, + 0xcbe6a2faffd627a8L, 0x9c8fd528463a2f5aL, 0xb9b88c6bb802b184L, 0xb414b4e665c597c7L, + 0xbedb142568209556L, 0x5360d81c25429dceL, 0x63a69a960a952f37L, 0xc900d63899e1b503L, + 0x1abc63a8b37c7728L, 0xa8b3a8b6409080ebL, 0x495e391f662959f6L, 0xdf1e136f3e12229bL, + 0x33d5fc526b0dd38dL, 0x321221ae2abfac63L, 0x7fde18351fda7395L, 0xed79fe5c3a6aa4c3L, + 0x2dd6965a4867d8d4L, 0x54813ca20fe8799bL, 0x5d59ea6456465c39L, 0x0de0c294d1936b81L, + 0x4aaf0755002c588cL, 0x3530a1857ad04c6dL, 0xb8a64f4ce184442bL, 0xe0def10bceedfa17L, + 0x46e38d0a443757ecL, 0x9795a1c645ee16d7L, 0x7e531def245eac8aL, 0x683b25c43a0716cfL, + 0x884583d372da219dL, 0x5b06b62c910416e5L, 0x54b6902fbebd3dbeL, 0x931198d40a761a75L, + 0xead7d8e830013590L, 0x80b4d5dc99bfacedL, 0xf98272c8108a1ad2L, 0x1adce054289a0ec6L, + 0x7d53a1143c56b465L, 0x497fbe4f00c92b52L, 0x525e4cc2e81ebd69L, 0xc94478e0d5508ff6L, + 0xb8a5da83c196d07cL, 0x7667a921b65b0603L, 0xf236fabbdefe6cd1L, 0x53da978d19a92b98L, + 0xc604f6e97087124dL, 0x2cbd27221924b094L, 0x65cd1102c985b1d2L, 0x08c0755dc1a97eb4L, + 0x5e0419e921c0fef1L, 0x282d2c1196f84a29L, 0xe21117fcfc5793f7L, 0xcf4e985dc38e6c2eL, + 0xd521f4f264d55616L, 0xde69b04c485f2a10L, 0x59410e245305178aL, 0xceab1d477c943601L, + 0xa9805732d71ee5e9L, 0x054cd443896974f6L, 0xf2b517717a423a3eL, 0x09517937fa9fac95L, + 0x4938233e9ca871e3L, 0x9132cbaf56f83ec0L, 0x4703421ed1dd027dL, 0xfd9933f4e6f1ec4eL, + 0xf237c7fded2274a8L, 0xdf4616efe68cd7b4L, 0x5e46de0f39f0a380L, 0x3d41e0c6d8e095b0L, + 0xc5272f8a5bb2df09L, 0x68aa78e8301fb964L, 0xbf5b5b52c8e32ae0L, 0xbf28ed3df74bdcf7L, + 0xd6198f64c833815aL, 0x8cd99d2974267544L, 0xd90560ea4465ff2cL, 0x571d65ad7ad59261L, + 0x309453518baa367aL, 0xa60538377bc79fb2L, 0xace515da1ab4183cL, 0xf56d3c8d891d1c5bL, + 0x5b0d8370b59def49L, 0x775866ce7c83c762L, 0x3d76085695c8e18aL, 0xba064d1a9af1b114L, + 0xc84ef7cd7b98b521L, 0x90b9231681c2bc37L, 0x37e2b13e6f585b6bL, 0x1d0a34e55e0f369fL, + 0x86bb8019cf41447cL, 0x4b95c6ef55b3f71fL, 0x3b6ed1660732b310L, 0x617eee603d137f21L, + 0xf4f6278b464f3bbcL, 0xdfb763b720da205aL, 0x353478899b871cb7L, 0xe45fbbff574cc41eL, + 0x1a94b60847907d72L, 0xb10eef051eff67a5L, 0xf0e012ec6a284d40L, 0xcc1cd1a11b926d7cL, + 0xcf9d9c5453e19cadL, 0x270febcc0fc0e86bL, 0xd6567568778b781eL, 0x7323b98965eeb46bL, + 0xccecd374567086ffL, 0xef7b44bfc497a704L, 0xebc479c051a9f0a5L, 0xc9b7410e3e00a235L, + 0x1d084f7ecdf83dabL, 0xc8a9a97e33ba8ba3L, 0x8c75318f5b2350d6L, 0xaa3cd5d0c684bddaL, + 0xa81125fe0901bedfL, 0xf7bcd76020edfc93L, 0x834ee4c12e75874fL, 0xb2bb8a7beb44fa14L, + 0x32cd26f50a4f4e4dL, 0x0fc5817ca55d959aL, 0xd6e4ae2e3ae10718L, 0x074abdcceb8d6e38L, + 0xc0cc5f4f9b3a9c43L, 0x1115d364363595b2L, 0x69861db2eb19f2e8L, 0x59b8d804cf92bc67L, + 0x9bac9785e5e4b863L, 0x7fa0e17a41869561L, 0x10d3c9633f0c709cL, 0x534a03deee6bc44aL, + 0x73b1f7201257f581L, 0x46fd6a11e2e0706bL, 0x494abb554946e67aL, 0xb5d6da317864dc8eL, + 0x402ded9238f39687L, 0xd8fa37d2cbd6d290L, 0xcc818293fcb06791L, 0x6482ab344806cd4dL, + 0x0956e6ee9d8eb60bL, 0x01fee622d8465ac8L, 0xae7ece370cbd9c35L, 0x7ff09e937a177279L, + 0xa2c29ee7a33ca5f1L, 0x990e8dbee083923bL, 0x4a819b72f610863aL, 0xddecfad79d3f08beL, + 0x627372480fac20a7L, 0x802154d6eca2db4cL, 0x8fcf02e42f805e55L, 0x040a911ff8cea977L, + 0xbb544485bc64d0d4L, 0xaddde1aeb406d0fbL, 0xf6b35fae23dce66fL, 0xc07a9fb3645d2f9bL, + 0xccd113907e9c0fedL, 0xd17af369984fd213L, 0x9223823c59a083e7L, 0xe19d475606b81013L, + 0xe181ac116a90e57aL, 0x71f7b6258c6def4cL, 0x2246f34b45964f7cL, 0xd74aedaea2d31751L, + 0xb1add86e5dd305d1L, 0xeb9ba881f16d6471L, 0xef7600e036f5c6ffL, 0x1d50bc9735b8fb85L, + 0xe63942bd1f3e2969L, 0x9241ba9f8b3f4e72L, 0xee8bb2bca07d35b6L, 0x55cd55dab522654eL, + 0x94d0cfa7c1a6845dL, 0x02f9845d559884c3L, 0x8ce70ea21063b560L, 0xd70998028ef08b74L, + 0xdfdb5bbee310876bL, 0x4e21b2e348256d16L, 0xde007a981c13debcL, 0xe51950cbbddabfddL, + 0xd223301dbe9957c1L, 0x084b8634cc2cce4bL, 0x90e551378aa9d70cL, 0x833b533ac633e448L, + 0x7891e232882da57fL, 0xa1bf26f0163ce2b3L, 0xf33a0171eb9c68d5L, 0x2e7de18ca69b3fa2L, + 0x666fd6f175619199L, 0x1239d37edb5feb9fL, 0xfa9fc9382e61ff5cL, 0x3ca4ad427e3c126fL, + 0x37c6dd4c2c31ae6eL, 0x1f1bacb619d427b2L, 0x7dd09f5d10759afeL, 0xc8d941432327d733L, + 0x2b389ba25e1d43a7L, 0xa4e3030c3740ff21L, 0xcc56dae13fd37463L, 0x2481457c175b560fL, + 0x9deb35bde77c5c41L, 0x847aa6ea5549a0c3L, 0xcde01bb48b6e7f02L, 0x15a28844e64cb211L, + }, + // seed = 1 + { + 0xecfcba92fe5691a3L, 0x71377799fea34699L, 0xb284c9096fa614e5L, 0x54534170f40de6c8L, + 0xbbd804d45884fba3L, 0x44929a896388c8a1L, 0x79b712508e0fa3b1L, 0xeb53ab280af31054L, + 0x351ea23a6319da7aL, 0x2fbe55d9819d85a2L, 0x34f4b6568dcd28b1L, 0x8c94ea5e5d82967aL, + 0x09068d333a46d3c5L, 0x762ad4f64cb73381L, 0xd5c6db5ef0e22640L, 0x36d8ab5a36175680L, + 0xd41fe333cdc3525aL, 0xa1f51dbdf20ce781L, 0x1410a95e786c8be6L, 0x96b7499a670c2b41L, + 0x3912e1037835d893L, 0x272c5bd83e1e9115L, 0x2ea7f91cad82a0d6L, 0xcd10e85662ce9931L, + 0xedad49be8d5e8b74L, 0x7ccd8fe0f37d12bcL, 0xfac0482005eed593L, 0x4513991681f6c8b0L, + 0x2804d612eb0ad37dL, 0x7cca9e8412b81d34L, 0x85ffd6707192b7b8L, 0xea0560aeea954411L, + 0x0122d28226102bbaL, 0xf51c47cdbd22fdd1L, 0x3707d851183ff17cL, 0xaef5a1465f3e902dL, + 0xbcb38c2d8736a04fL, 0x4025317e864bef15L, 0x8d3f66d86e1ea58fL, 0xc16759a3d97ed79aL, + 0x1c62abdc0659f2f5L, 0x23b3eb4e699bd28fL, 0x5083c4fceed3ccafL, 0xa65bf34562cc989cL, + 0xaa5865932fd79064L, 0xf24d08d268c24593L, 0x7fbd00a215196999L, 0x7812cd366d752964L, + 0x62e8dcb27ef3d945L, 0xf08b7984e1b946dcL, 0x547d23ad9a5c1dcfL, 0x496b1fb249b27fb7L, + 0xcd692e1db5f3b3baL, 0x41931e39f1e1bc61L, 0x286c6a7d7edae82bL, 0x17ef6638b6c4ca6eL, + 0x609beb5a2576a934L, 0xcc5e16fe4a69b83cL, 0xbbd14d08b078fc24L, 0x2a617680f481cb94L, + 0x81dbbd5f86e6d039L, 0xeb8205e1fc8ecc3cL, 0xe5e3bb576faa8042L, 0x5d6f1eb9d9df01b5L, + 0x9a47b8739c10fb44L, 0x398a7caad7ea7696L, 0x9c0fc1d7c46adde6L, 0x67cd6de0a51978a6L, + 0x68ccc4b77a21cca4L, 0x1e067066b82f415cL, 0xf7ddade6535e1819L, 0xf2185c884291751bL, + 0xc322b7381fcbe34fL, 0x242f593e88290b9bL, 0x8e11ccc0ea5e84a3L, 0x40e3a2e3346db8a2L, + 0xf18bfc3ad2931a2cL, 0x2468397394b00144L, 0xeae199cce14e6817L, 0x05b462686c75a1aeL, + 0xda096cb859c51673L, 0xd87aeb967a906befL, 0xaabc74493cb02fe6L, 0x74d48fc2e7da143eL, + 0x6ec1c8fed3f2c1fdL, 0xe01e0704b463f18eL, 0xc3d88a4d3a8056e4L, 0xd01ae0ffab6c8f3fL, + 0x881ba052620ae7c7L, 0xcea033aef0a823a5L, 0x8d2cad91d83df1e3L, 0x18746d205e66dbe9L, + 0x3061f8e58d046650L, 0xd819c59f0ce2cf8bL, 0x144e89e93635e870L, 0x3415e88279b21651L, + 0xd6f7ab944b86c3faL, 0x45f1dd15d0f67bdcL, 0xbf0d97c7f4fa24f4L, 0x34a7de520a57fcd2L, + 0x4ba86fda03e9e2bcL, 0xa7995265a025b552L, 0x698f6819d5f51cf7L, 0xd07dbe9d8a156981L, + 0x2683945373857fc1L, 0x116f8a84f96167deL, 0x8bc832bd85595ebfL, 0xb206519d74fdfafaL, + 0xde9519b2e9b5cc5fL, 0x16fdd6f2da1d8163L, 0x7ba32bd48ef56f11L, 0x6f4e4d7ee8b29717L, + 0xd31576dde7468aadL, 0x023bb08848676045L, 0xf6dcc083178160b7L, 0x42035f426250e683L, + 0x343732993cfed89fL, 0x0640a870a22d3d58L, 0x65cff80b53b4ae6aL, 0x27996fa17ab05215L, + 0xfd5db01401b21a04L, 0x894508784bc1673cL, 0x5bfcf43a2380e27dL, 0x4cd6dcc2715583b7L, + 0xa43b3763e7d4c902L, 0x6da83e12ef0c1257L, 0xfe80a602b0335affL, 0x293a7d8f4ff344deL, + 0xb4ae7c2b8956bf5aL, 0x6b45432d38254b4dL, 0xd086acbdf15d9455L, 0xa4d19e43f41ea87bL, + 0xf01f13ba4bb87fbfL, 0xca582cf301a299ffL, 0x0ddad3d45298fa7dL, 0x0646a130459c3999L, + 0xc08e3af3747e2ceeL, 0xfc7db8aa9ed67295L, 0x783b329e7bd79d5fL, 0x732dbc607957af7bL, + 0x8e446ac19fb26555L, 0xff1dfa4d61dc89a5L, 0xb6fbc46bd8d011d8L, 0x185147ec5779f0d7L, + 0x6eb2cf6149a5380fL, 0xb0e773df803a1eaeL, 0xc07706c5519bfce5L, 0xc35abcf54fa95f14L, + 0x40a01d99a38608eaL, 0x776dcd6f603c277fL, 0x6ae12389b1d6d0bbL, 0x8bd981448df92bb9L, + 0x426a6a7ca21a2c16L, 0x87efd5b71c1bad26L, 0x71fb7fc4cd41de48L, 0xdd9033c45619d463L, + 0x40eaab322654cef7L, 0xe077fffed6f3e3a2L, 0x375a4dbef9384447L, 0x2066b009d2c4a100L, + 0xeca4a5794a068447L, 0x2128f64bddf341a1L, 0x738b4bb1be90bd61L, 0x433772cf3813d52eL, + 0x9540c88add8e4474L, 0x0b6d5decd21d3519L, 0x654ead966745642dL, 0xe1bfb03c3b4bdb4cL, + 0x0b977a9937515b1fL, 0x0a4587509ef63870L, 0xe89f0de1d9cfd44aL, 0x23a91390272e7f68L, + 0xd92defbc9096b8d8L, 0x004db87174612539L, 0xc88ecaabdd1a71f1L, 0x050de38393073346L, + 0x8af1426d7964e038L, 0xf352c4fef8ad5c87L, 0x6f26bc7408e26548L, 0x0d41543fd9bf3084L, + 0xfc4e07553a840fc6L, 0x5ef117de86a555a9L, 0x1f11c42dffb5ae1bL, 0x4147648f07490fa5L, + 0x09b35fd7671b21aaL, 0x1453b14f7ccca481L, 0x944f6fcce4c9b2baL, 0x5b08dd2e3583dc06L, + 0xe0220df78dc9c22dL, 0x1c200b9506cbf666L, 0x8a0b7465eadb523bL, 0xfbcb43a91a1e2d80L, + 0xe697f44be3c36a58L, 0x2f8a8e48fb7e350dL, 0x7baba71b8920d55fL, 0x10edc0216105bc96L, + 0x52db07c79d7a7a63L, 0x1916e8cef9452ac3L, 0x5cbbbf21f867b6ccL, 0xadd583365a690a4bL, + 0x4e4ca2c8bffc2fdbL, 0xf5fe3416d2eebcfeL, 0x839af8b85e452476L, 0x8496c0c54ad44e16L, + 0x6c46f1ecad4482bfL, 0xb794cad76ae18715L, 0x67b762eec7c62985L, 0x52dc9e68df5b3a53L, + 0x0cc7e444b422a5f9L, 0xadbfe90841c112b0L, 0xfe37b136f0ca5c34L, 0xcfe9e47948a8d73eL, + 0xee90572b86a30d91L, 0x549e72d8262830aaL, 0x3361564b469f32c6L, 0x1e6eba9e0d2648e2L, + 0x5f8e2b2ac5fcb4ebL, 0xe4224fa5f71f7cc6L, 0x7357a9230c76757bL, 0xcad70f74aaf6b702L, + 0xeef28ced23894cc2L, 0x753fdd3352aefd68L, 0x1fed6ba90bbeb9d2L, 0x05316f4ab4034b4bL, + 0x3396df022b9f63d6L, 0x82d7125a7cfd0935L, 0x3519a71caf1f87f0L, 0xd1dfb7a5cc3974beL, + 0xbfae40ecbdbbcc2aL, 0x152c11778e08dd54L, 0x4a96566a6c848554L, 0x3a84d621c340cdd7L, + 0xfd47aa1887e2fb03L, 0xa63cae94b2f1d099L, 0xed61783f3e5b75e0L, 0xefd44864106019beL, + 0x145ff78b80b081aaL, 0x34670e5fcea9230eL, 0x876ef976328db371L, 0x4221f3a5269942a6L, + 0x95315cbd85c648f4L, 0x3ca344dc7c3b1600L, 0x38421ea39ff28780L, 0x31dbeee967c0435cL, + 0x27437c3e268402e7L, 0xdd0cf8343312a654L, 0x965ab9dad1d8aa29L, 0xf871706dd3e23509L, + 0xce23d06c7a25e699L, 0x1b37d59382b27589L, 0x3407f004723d6324L, 0x56efb69cdb5deaa1L, + 0xf46cdd2b9fd604e0L, 0xcad3ca79fdac69bdL, 0x7252802a574e63cbL, 0xc281fb8acc6ec1d3L, + }, + // seed = 2 + { + 0xdd16cb672ba6979cL, 0x3954eaa9ec41ae41L, 0x52cb802771d2966dL, 0xf57ed8eb0d0294f2L, + 0x768be23c71da2219L, 0x6131e22d95a84ad3L, 0xd849e4e49bb15842L, 0x18e8e5c4978cf00dL, + 0x3af5e5867ce1f9bdL, 0x06c75a9fffe83d63L, 0xe8de75a00b58a065L, 0x0a773251bc0d755aL, + 0x629dc21e54548329L, 0x2a168f5e5a883e70L, 0x33547375f0996c86L, 0xdfcb4c7680451322L, + 0x55c1ecaaaa57e397L, 0x4546c346c24f5a31L, 0x6f8f0401dfabc86cL, 0x7760d2d36ee340b4L, + 0xf6448e48bdeb229dL, 0xba70e1633b4dba65L, 0x069cda561e273054L, 0xa010b6a84aebf340L, + 0x5c23b8229eee34b6L, 0xea63c926d90153afL, 0x7d7de27b3e43ec1bL, 0xea119541eddc3491L, + 0xf1259daeddfc724cL, 0x2873ca9a67730647L, 0xa1e7710dade32607L, 0x758de030b61d43fdL, + 0xd2c9bcbfa475edb4L, 0x18ade47bb8a0aa29L, 0xf7a74af0ff1aea88L, 0x6f8873274a987162L, + 0x6963e8d876f4d282L, 0xd435d4fe448c6c5bL, 0x93ec80ba404cafffL, 0xcf90d24c509e41e7L, + 0x5f0fc8a62923e36eL, 0x9224878fe458f3a4L, 0xd9a039edf1945bcdL, 0x0877d1892c288441L, + 0x75205491f4b4740bL, 0x30f9d2d523a9085bL, 0x4b7f4029fa097c99L, 0x170bb013745709d4L, + 0x7087af537f11ef2eL, 0x28c62b88e08fc464L, 0x84bbcb3e0bb56271L, 0x485a4b099165c681L, + 0x357c63357caa9292L, 0x819eb7d1aee2d27eL, 0xdaa759eb9c0f8c9dL, 0x42cdc36729cc3db5L, + 0x9489aa852eddbb06L, 0x8161e4f85a84e6d4L, 0xa964863fdad3eb29L, 0xcc095ddbce1a6702L, + 0x3ecfadbb8dc2ce58L, 0x971316509b95a231L, 0xc8f484d1dbc38427L, 0xae9c510c463574c0L, + 0xdf2b31179600c21aL, 0x440de87bada4dfa3L, 0xbd8d30f3f6fb7522L, 0x84e6d7f678a0e2d0L, + 0x0ec4d74323e15975L, 0xf6947610dad6d9abL, 0x73a55a95d73fe3a5L, 0x3e5f623024d37edaL, + 0x8d99a728d95d9344L, 0x8b82a7956c4acdc4L, 0x7faeaea4385b27f6L, 0x540625ff4aa2ff21L, + 0x4aa43b3ebd92ce2bL, 0x899646a6df2da807L, 0x49225115780942d7L, 0xe16606636af89525L, + 0xb980bcf893888e33L, 0xf9ed57695291b0d8L, 0x5c6dd14464619afaL, 0x50606d69b733d4f3L, + 0x7fb1af465b990f97L, 0x3fab2634c8bbd936L, 0x556da6168838b902L, 0x0f15975902a30e1fL, + 0xb29d782ae9e1991fL, 0xae00e26ff8f7e739L, 0xd3da86458bb292d5L, 0x4528ee0afb27e4ceL, + 0x49882d5ba49fabadL, 0x7e873b6a7cf875eeL, 0x777edd535113c912L, 0x94ed05e7ff149594L, + 0x0b8f95fc4211df43L, 0x9135c2b42426fef2L, 0x411e6c2b47307073L, 0x503207d1af0c8cf8L, + 0xd76f8619059f9a79L, 0x64d24617855dee45L, 0xf7bc7a877923196aL, 0xd6cc42ed6a65be79L, + 0xe3912ff09d4fc574L, 0x4192d03b2bc2460aL, 0xa0dcc37dad98af85L, 0xfc59049b2a5818a4L, + 0x2128bae90a5b975fL, 0xbe7067ca05ea3294L, 0x5bab7e7753064c4fL, 0x42cbf0949ef88443L, + 0x564df4bbd017492cL, 0xf2c2eb500cf80564L, 0x5b92e67eb00e92afL, 0x8c4103eef59c0341L, + 0x83412122b8284998L, 0x888daf2da0636b6dL, 0x4d54b10303dd07d6L, 0x201190e7c1e7b5edL, + 0x3797510bb53a5771L, 0x03f7bc598b570b79L, 0xdc1e15d67d94f73eL, 0x721e8b499ebe02c1L, + 0x71f954f606d13fa0L, 0x0c7a2e408c168bf0L, 0x07df2ef14f69c89dL, 0xe295096f46b4baafL, + 0x7a2037916438737eL, 0xd1e861aeaf8676eaL, 0xb36ebdce368b8108L, 0xb7e53b090ddb5d25L, + 0x5a606607b390b1aaL, 0x475e52994f4a2471L, 0xbcc2038ba55b2078L, 0x28b8a6b6c80df694L, + 0xb5f0130ec972c9a2L, 0x7a87cd2a93276b54L, 0x4d0eec7ecf92d625L, 0xac1a8ce16269a42eL, + 0xa4ca0237ca9637b8L, 0xd8dc8ff91202b6ffL, 0x75b29846799d7678L, 0x761b11a5edd9c757L, + 0xf2581db294ef3307L, 0xe3173c2b6a48e20fL, 0xe46fd7d486d65b3cL, 0x1352024303580d1fL, + 0x2d665dae485c1d6dL, 0x4e0905c825d74d3bL, 0x14ff470c331c229eL, 0xbdc656b8613d8805L, + 0x36de38e396345721L, 0xaae682c1aa8ff13bL, 0x57eb28d7b85a1052L, 0xf3145290231d443aL, + 0xd0f68095e23cbe39L, 0x67f99b3c2570b33dL, 0x54575285f3017a83L, 0x9b2f7bb03d836a79L, + 0xa57b209d303367a9L, 0x7ccb545dd0939c79L, 0x1392b79a37f4716dL, 0x6e81bb91a3c79bcdL, + 0x2c2cd80307dddf81L, 0xb949e119e2a16cbbL, 0x69625382c4c7596fL, 0xf19c6d97204fb95cL, + 0x1b2ea42a24b6b05eL, 0x8976f83cd43d20acL, 0x7149dd3de44c9872L, 0xc79f1ae2d2623059L, + 0xca17a4f143a414e1L, 0x66d7a1a21b6f0185L, 0xed2c6198fe73f113L, 0x16a5f0295cbe06afL, + 0x5f27162e38d98013L, 0xf54d9f295bdc0f76L, 0x9ba7d562073ef77bL, 0xa4a24daaa2cfc571L, + 0x49884cf486da43cdL, 0x74c641c0e2148a24L, 0xbff9dcbff504c482L, 0xf8fc2d9403c837abL, + 0x6ccc44828af0bb1eL, 0xbcf0d69b4c19dfdbL, 0x8fe0d962d47abf8fL, 0xa65f1d9d5514271dL, + 0x26ff393e62ef6a03L, 0xc7153500f283e8fcL, 0xea5ed99cdd9d15cdL, 0xfc16ac2ba8b48bb7L, + 0xf49694b70041c67aL, 0xbd35dd30f5d15f72L, 0xcf10ad7385f83f98L, 0x709e52e27339cdc2L, + 0xe9505cb3ec893b71L, 0x2ffa610e4a229af7L, 0x12e1bc774d1f0e52L, 0xe301a3bb7eacccc8L, + 0x1fdd3b6dcd877ebfL, 0x56a7e8bda59c05aaL, 0x99acd421035d6ab4L, 0xfd21e401cecd2808L, + 0x9a89d23df8b8d46fL, 0x4e26b1f1eb297b9cL, 0x9df24d973e1eae07L, 0xe6cdc74da62a6318L, + 0xfc360d74df992db0L, 0xf4eca0a739514c98L, 0x481c515ba9bf5215L, 0xce89cce80f5f3022L, + 0xf487a10fc80e4777L, 0x235b379a87e41832L, 0x76f72e028371f194L, 0xd044d4a201325a7dL, + 0x47d8e855e0ffbddeL, 0x268ae196fe7334b0L, 0x123f2b26db46faa8L, 0x11741175b86eb083L, + 0x72ee185a423e6e31L, 0x8da113dfe6f6df89L, 0x286b72e338bbd548L, 0xa922246204973592L, + 0x7237b4f939a6b629L, 0x31babda9bedf039aL, 0xb2e8f18c6aeec258L, 0x0f5f6ce6dd65a45eL, + 0x8f9071a0f23e57d3L, 0x71307115ba598423L, 0xcbe70264c0e1768cL, 0x1c23729f955681a8L, + 0xfbc829099bc2fc24L, 0x9619355cbc37d5d6L, 0xea694d4e59b59a74L, 0xb41cf8d3a7c4f638L, + 0xae1e792df721cd0bL, 0x7cd855d28aac11f6L, 0xca11ba0efec11238L, 0x7c433e554ce261d8L, + 0xe3140366f042b6baL, 0x8a59d68642b3b18cL, 0x094fcdd5d7bccac2L, 0x9517d80356362c37L, + 0x4a20a9949c6c74e8L, 0xc25bcf1699d3b326L, 0xa8893f1d1ed2f340L, 0x9b58986e0e8a886eL, + 0x29d78c647587ce41L, 0x3b210181df471767L, 0xd45e8e807627849dL, 0x1ec56bc3f2b653e3L, + 0x974ff23068558b00L, 0xdb72bdac5d34262cL, 0x23225143bb206b57L, 0xd0a34cfe027cbb7eL, + }, + // seed = 3 + { + 0x39209fb3eb541043L, 0xee0cd3754563088fL, 0x36c05fc545bf8abeL, 0x842cb6381a9d396bL, + 0xd5059dcb443ce3bfL, 0xe92545a8dfa7097eL, 0xb9d47558d8049174L, 0xc6389e426f4c2fc0L, + 0xd8e0a6e4c0b850d3L, 0x7730e54360bd0d0dL, 0x6ecb4d4c50d050d5L, 0x07a16584d4eb229fL, + 0x13305d05f4a92267L, 0xb278ddd75db4baecL, 0x32381b774138608fL, 0x61fe7a7163948057L, + 0x460c58a9092efee6L, 0x553bf895d9b5ff62L, 0x899daf2dabfd0189L, 0xf388ab9c1c4b6f70L, + 0xd600fe47027ea4cdL, 0x16d527ec2b5ef355L, 0x5ac1f58ff6908c81L, 0xa08d79ff8ee9ffe8L, + 0xc1060a80b7a5e117L, 0x14b2c23118c60bdaL, 0x8cc0defbb890df8fL, 0xe29540fd94c6d28bL, + 0xa604f003f82d5b71L, 0xa67583d4eb066d18L, 0xd62cbd796322b3fcL, 0x070cfe244cdcccf3L, + 0x73557c30b3af47e5L, 0x2e544e31153a2163L, 0x996eef7464d5beadL, 0xbc71cb5ab0586cdcL, + 0x0bfcb6c1b517ed69L, 0x62b4f1fcc82e8ca0L, 0x0edbc68f544965c5L, 0x40fa39baa24af412L, + 0xf39aeb2413dab165L, 0x17e6013e7afee738L, 0x8109bff1c8d42a9dL, 0x3cd99863390989b5L, + 0x02021a4cc9c336c8L, 0xa06060778cb60aa4L, 0xd96591db60bc1e06L, 0xd2727175183f4022L, + 0xcdc1f1c5bce3e7ceL, 0xb393ccc447872a37L, 0xdf6efe63257ead3aL, 0x20729d0340dbceb6L, + 0x9f3d2d26fc0ea0d7L, 0xf392e0885189bd79L, 0xdf2ee01eb212b8b6L, 0x6e103a0c0f97e2c3L, + 0x96c604a763bd841bL, 0x9fc590c43bba0169L, 0xf92dcd5ddc248c40L, 0x113a8b54446941dcL, + 0x5943eda146b46bb8L, 0xbf657901a36a39a7L, 0x5a4e0e7ea6568971L, 0xb94c635bae9f9117L, + 0x2626fb65b3a4ef81L, 0xa59bfd5478ce97deL, 0x79112ba9cc1a1c63L, 0xf41f102f002cf39cL, + 0x0a589bcbfb7ff1c8L, 0xa1478c53540c4fa1L, 0x60d55e72c86dfacaL, 0x312e7b6840ea7a39L, + 0x8aae72dcccfe1f75L, 0xff2f51f55bf0247aL, 0x3c2e4b109edb4a90L, 0x5c6d73f6525c7637L, + 0xe49acb04a199f61cL, 0x27860642d966df7fL, 0x541ce75fb1e21c30L, 0xd9fcd6f90806c7ccL, + 0xb87c27bc93a7969bL, 0x92f77a1179b8f8dcL, 0xb1f29379deb89ed4L, 0x7e63ead35808efe7L, + 0x13545183d7fa5420L, 0x575f593e34cf029dL, 0x27f1199fb07344aeL, 0xe67f95f7dc741455L, + 0x49b478b761ab850bL, 0xd7bedf794adfc21eL, 0xdc788dcd2dda40aeL, 0x14673eb9f4d8ad35L, + 0x0cced3c71ecf5eb1L, 0xe62d4e6c84471180L, 0xdfe1b9e2cb4ada7dL, 0x70185a8fce980426L, + 0x0ce2db5e8f9553d6L, 0x1fedc57bb37b7264L, 0xb9310a2e970b3760L, 0x989ff8ab9805e87dL, + 0x0b912d7eb712d9eeL, 0x1fe272830379e67cL, 0x16e6a73aff4738fbL, 0xeed196d98ba43866L, + 0x7088ca12d356cbe2L, 0x23539aa43a71eee0L, 0xed52f0311fa0f7adL, 0xa12b16233f302eeaL, + 0xc477786f0870ecb4L, 0xd603674717a93920L, 0x4abe0ae17fa62a4cL, 0xa18f1ad79e4edc8dL, + 0xc49fe6db967c6981L, 0xcc154d7e3c1271e9L, 0xdd075d640013c0c0L, 0xc026cd797d10922aL, + 0xead7339703f95572L, 0x4342f6f11739eb4bL, 0x9862f4657d15c197L, 0x4f3cb1d4d392f9ffL, + 0xe35bffa018b97d03L, 0x600c755031939ad3L, 0xb8c6557ffea83abfL, 0x14c9e7f2f8a122eaL, + 0x0a2eb9285ee95a7cL, 0x8823fec19840c46fL, 0x2c4c445c736ed1d0L, 0x83181dff233449f1L, + 0x15ed3fca3107bef5L, 0x305e9adb688a4c71L, 0x7dbef196f68a3e2eL, 0x93e47ece3e249187L, + 0x8353c5e890ead93cL, 0xea8a7ae66abafdf7L, 0xf956dbb6becf7f74L, 0x9f37c494fbfdb6e4L, + 0x11c6cbaa2485dd32L, 0x206f336fcca11320L, 0x9befe9a59135d8feL, 0x5f3ef8b8db92c7dbL, + 0xbb305e556ce0ce9aL, 0xf26bdafb1305887fL, 0xcbf28abe23f08c61L, 0x0bc64173b914e00bL, + 0x9168da52e983f54aL, 0x6ea41d09c3574a3eL, 0x78aa44d4a74459aeL, 0x2931422878387bf5L, + 0x018f64a3a92c2d9cL, 0x9be43f6752e66b34L, 0xae378890decd1152L, 0x07325329a1cb7623L, + 0x3b96f4ee3dd9c525L, 0x2d6ebcdbe77d61a3L, 0x10e32b0e975f510cL, 0xffc007b9da959bf9L, + 0x38bf66c6559e5d90L, 0xbe22bdf0bf8899feL, 0x87807d7a991632a8L, 0x149a0d702816766aL, + 0x026f723db057e9abL, 0xeeecb83625ec6798L, 0xcec2ed5984208148L, 0xd985a78e97f03c84L, + 0xf96c279e7927b116L, 0x99d5027b3204f6e2L, 0x13a84878c3d34c55L, 0x5cf5ec96229e9676L, + 0x0bc36b07e4f8e289L, 0xbed33b80a069914dL, 0x2fbfbdd1ff4b9396L, 0xab352bb6982da90fL, + 0x154d219e4fa3f62bL, 0x4d087512bb6b9be7L, 0xc582e31775ee400eL, 0x7dadb002ae8c4a4eL, + 0xaae2957375c1aee2L, 0x5f36ca643356625bL, 0xf87cf8eb76e07fb7L, 0x46f432a755e02cc3L, + 0x36087e07aba09642L, 0xe5642c1e4ebb9939L, 0xb9152d22338eefadL, 0xf7ba44278a22cf7fL, + 0xd3b8013502acd838L, 0x7761511da6482659L, 0xb0857621638e8e50L, 0x552eddb4a8b1d5f5L, + 0xc43d9861e812c3eaL, 0xd765c2aada47910cL, 0x21c935b68f552b19L, 0x6256d5641a2b47dcL, + 0xab711d8e6c94bc79L, 0xa8d0b91a2a01ab81L, 0x5e6d66141e8d632aL, 0x7638285124d5d602L, + 0x794876dbca3e471fL, 0x951937d8682670ceL, 0x0f99cb1f52ed466aL, 0x8c7cd205543b804cL, + 0x2fd24d74a9c33783L, 0xe5dcb7b7762e5af1L, 0x45e6749cca4af77cL, 0x540ac7ee61f2259fL, + 0x89c505c72802ce86L, 0xeab83b9d2d8000d1L, 0x9f01d5e76748d005L, 0xc740aaef3035b6d0L, + 0x49afcd31d582d054L, 0xcba5dc4c1efb5ddcL, 0xc0a4c07434350ca1L, 0xfc8dfaddcc65ee80L, + 0x157c9780f6e4b2d9L, 0x9762a872e1797617L, 0xc4afae2cf3c7e1bdL, 0x71cde14591b595d4L, + 0x8843c3e0e641f3b9L, 0xd92ecd91dce28750L, 0x1474e7a1742cb19fL, 0xec198e22764fa06bL, + 0x39394edb47330c7dL, 0x00ba1d925242533dL, 0xaed8702536c6fb30L, 0x6d3618e531c2967aL, + 0x77f7cedcd7cc0411L, 0xbc1e2ab82be5b752L, 0x07b0cf9223676977L, 0x596c693b099edd53L, + 0xbb7f570f5b9b2811L, 0x96bfdad3c4a6840cL, 0x668015e79b60c534L, 0x3ad38d72123f1366L, + 0x6b994d81d2fcbb09L, 0x70885f022c5052d8L, 0xc891ee79d9306a7bL, 0x2c4df05c0ed02497L, + 0x19ebc13816898be2L, 0xea7c64df11c392a2L, 0xb7663e88dd12e1bdL, 0x79f768cb8e154c21L, + 0x1fb21b12e945933bL, 0xe6a9045643f6906eL, 0x544c47acd7e15371L, 0xb7709b14f727e3d1L, + 0x326ee36a46942971L, 0x477f1cf7b0e2d847L, 0x88b8f6b82b3b0c24L, 0x18bc357b80e3cd5cL, + 0x3333de70e4d66e0bL, 0x4fd4c5e148583cf6L, 0xae1b62f3008c0af3L, 0xc49f419b6ab29cf5L, + 0x2c29fa65afc3fa28L, 0x4b19d93734d03009L, 0x7dd6c09e589276adL, 0x1cece97f30de48adL, + }, + // seed = 4 + { + 0x58bdf4338602e4fbL, 0x71a5620b02c926d5L, 0x3811c960129c2d9fL, 0x29c2fb11fccac567L, + 0x0d6b1ea7780f1352L, 0xcc4d3ddfae3f87b3L, 0xfdd30257362a586bL, 0xabc948fde69f25f1L, + 0x51b3523469d30f7bL, 0xe0f0322724405aceL, 0xd3729266d896da1eL, 0xb10c37e5147915bfL, + 0x8b577039f9fa32a3L, 0xe677c6a9cbfb44b3L, 0x7317a756ebb51a03L, 0xf8e988ef37359485L, + 0x600fc1ef3f469ff3L, 0xbf0b8f8520444e01L, 0x3711168b08b63d73L, 0x34146f2944a6cb36L, + 0x717feb263862cddeL, 0x7185f8347db00412L, 0x900798d82127e693L, 0x84089e976a473268L, + 0x10f8308c0d293719L, 0xf62a618d4e5719b8L, 0x8bdbd257a1a9516fL, 0xf49f666fd7a75110L, + 0xbaf45e2db7864339L, 0xe4efa1ea0c627697L, 0x3e71d4c82a09fe10L, 0x54a2a51cf12127bbL, + 0xa0592c9f54ba14cdL, 0x27dd627a101c7a42L, 0x3d2ceb44b3d20d72L, 0x7ee1f94a68ca8f5dL, + 0x7e8cb8651b006c36L, 0xbd9fa7ca3a475259L, 0x856de173586a7b34L, 0xcedb291b594cb1b5L, + 0xa3d6e462fd21cddcL, 0x74561d10af9118e4L, 0x13a3d389fc2d4b36L, 0xeea8594a4a054856L, + 0xf56d7474d9ba4b13L, 0x25ddce2f6490b2fdL, 0x920653ff3a8d830bL, 0xcd8c0c9cdac740d1L, + 0x2c348a738db9c4a0L, 0x2967ccbe8ea44c22L, 0x47963f69adb049f8L, 0xf9d01eb5b4cf7eb6L, + 0x7a5c26eb63a86bd2L, 0x62ad8b7a71fa0566L, 0xb373213179f250aeL, 0x589d4e9a88245a4dL, + 0x433dafebe2d558a8L, 0x521fbef2c8fe4399L, 0x62a31f9ff9ccd46bL, 0x51602203eba7c1a6L, + 0x9afc8c451b06c99fL, 0xb529085bdbaffceaL, 0xac251825cc75892bL, 0x94976a5bce23d58eL, + 0xdd17925b6c71b515L, 0x568fd07a57bce92eL, 0xefac31200d8bd340L, 0x716c3e466b540ef9L, + 0x3d2c9e380063c69bL, 0x14168f9a3662dd83L, 0xd298c7504dbc412fL, 0x74490a94f016719fL, + 0x0e0da431e1ab80c8L, 0xe321f63dc6b169aeL, 0xf08671544febc95aL, 0x39324450cc394b3bL, + 0xea6e3d35f1aa3a70L, 0x8ef8a886508ce486L, 0xdc1a631ef0a17f06L, 0xfda2b3fbcd79e87bL, + 0xd75bcae936403b10L, 0xf88b5bd9f035f875L, 0xc43efec2e3792dd4L, 0xe9fac21a9d47cd94L, + 0xc2876f0c4b7d47c3L, 0xaba156cf49f368b4L, 0x5ccda2170fa58bf9L, 0xadc92c879ed18df7L, + 0x110c1b227354e6c8L, 0x298ee7a603249200L, 0xde92142ede0e8ee7L, 0x88e4a4610644ba9eL, + 0xbb62d277e7641d3aL, 0xb9be1985b7bf8073L, 0x29024e5426cdb0d1L, 0xf6aefd01f3092ab8L, + 0x2a07087b313133aaL, 0x6d71f445d6dfc839L, 0x1e2412ff12e5526bL, 0xed5cdeba6617b9e1L, + 0x20b1d0d5e5f8760eL, 0x12ff15705c368260L, 0x7bf4338b7c387203L, 0x34ff25f00cd06185L, + 0x1148c706c518cf28L, 0x5c04f0623388f025L, 0xcb9d649275d87d79L, 0x9b5f0c24fabc42ecL, + 0x1a7b5e7964e33858L, 0x2a81bbd8efdc6793L, 0x8d05431ffe42752eL, 0x83915cd511002677L, + 0x580ed4d791837b31L, 0x5982e041d19ff306L, 0xcad0d08fa5d864caL, 0x867bee6efe1afa63L, + 0x26467b0320f23009L, 0xd842414dfda4ec36L, 0x047fcdcbc0a76725L, 0xbddb340a3768aecaL, + 0xef4ce6fa6e99ab45L, 0x88c5b66c7762bf9bL, 0x5679f1c51ffb225dL, 0xdab79048317d77eeL, + 0xf14e9b8a8ba03803L, 0xe77f07f7731184c1L, 0x4c2aab9a108c1ef5L, 0xa137795718e6ad97L, + 0x8d6c7cc73350b88bL, 0x5c34e2ae74131a49L, 0xd4828f579570a056L, 0xb7862594da5336fcL, + 0x6fd590a4a2bed7a5L, 0x138d327de35e0ec1L, 0xe8290eb33d585b0bL, 0xcee01d52cdf88833L, + 0x165c7c76484f160eL, 0x7232653da72fc7f6L, 0x66600f13445ca481L, 0x6bbdf0a01f7b127dL, + 0xd7b71d6a1992c73bL, 0xcf259d37ae3fda4aL, 0xf570c70d05895acfL, 0x1e01e6a3e8f60155L, + 0x2dacbb83c2bd3671L, 0x9c291f5a5bca81afL, 0xd976826c68b4ee90L, 0x95112eec1f6310a2L, + 0x11ebc7f623bc4c9aL, 0x18471781b1122b30L, 0x48f7c65414b00187L, 0x6834b03efa2f5c30L, + 0x0875ef5c2c56b164L, 0x45248d4f2a60ba71L, 0x5a7d466e7f7ba830L, 0x2bebe6a5e42c4a1dL, + 0xd871d8483db51d10L, 0x6ee37decd2fd392fL, 0x7d724392010cede3L, 0x8e96ef11e1c9bcc8L, + 0x804a61d86b89d178L, 0xbb1b83ce956055ecL, 0xcb44e107410ff64fL, 0xc426bb09ee0ba955L, + 0x057c08f42c3dd7f1L, 0x40ea1ec148602bdfL, 0xc24688deeb65d7f1L, 0xd8bcc53c768ba4e4L, + 0x16e0e3af65c1106cL, 0xfc12f7e7d647218bL, 0x70d6e1d3ee93cef4L, 0x01d2a505c4541ef9L, + 0x1ef79e16e764d5c3L, 0x0363d14d13870b98L, 0xb56ef64345d06b11L, 0xe653d557ebb7c346L, + 0x8304a8597c2b2706L, 0x1536e1322ce7e7bbL, 0x525aec08a65af822L, 0x91f66d6e98d28e43L, + 0xe65af12c0b5c0274L, 0xdf6ae56b7d5ea4c2L, 0x5cef621cedf3c81cL, 0x41e8b1ffd4889944L, + 0xb5c0f452c213c3e5L, 0x77af86f3e67e499bL, 0xe20e76ea5b010704L, 0xbdc205ab0c889ec0L, + 0xc76d93eb0469cd83L, 0x17ac27f65cab0034L, 0xd49ec4531fd62133L, 0x07a873ea2f1b9984L, + 0xbff270dfef0032eeL, 0x1764dbe91592f255L, 0xe40363126f79e859L, 0xa06cad3ab46971f6L, + 0x0be596e90dedd875L, 0x3387cce5c1658461L, 0x44246acf88a9585eL, 0xe0ad82b92d5ecb2cL, + 0x2177491c9a1600a6L, 0x16e7c4aac0f02422L, 0x75792eeeec15c4e1L, 0x2309cd359d08ee30L, + 0x7cd9831dd1b83b0aL, 0x374914a7c4ee8cf0L, 0x0dd17765c9ac2e54L, 0xb7847470ba9a7688L, + 0xfba4f4bbe2991173L, 0x422b203fc3de040eL, 0x63bfcaf2ecf2ab0eL, 0x0c5559f3a192946eL, + 0xfdf80675c1847695L, 0xf5f570accab842c9L, 0x65cc5a448767afeaL, 0x1efeb0a7ee234f2fL, + 0x9b05f03d81e7b5d2L, 0xe7c31317a8626cf4L, 0x620f2a53081d0398L, 0x1b6de96cdd9943aeL, + 0x8c226a436777d303L, 0xa08fbbd50fafb10dL, 0x6a64c5ec20104883L, 0x9c9c653502c0f671L, + 0x678a02b2174f52a0L, 0x68e008ba16bbad4bL, 0xa317c16d2efb860fL, 0xeab2075d17ed714cL, + 0x565eeeddf0c4ea15L, 0x8ec8e94d242a6c19L, 0x139e8e27d9000faeL, 0xc977a7ff1b33d2f5L, + 0x1d0accca84420346L, 0xc9e82602cd436e03L, 0x6a2231da53d2ccd3L, 0xb44b12d917826e2aL, + 0x4f4567c6a74cf0b9L, 0xd8e115a42fc6da8fL, 0xb6bbe79d95742a74L, 0x5686c647f1707dabL, + 0xa70d58eb6c008fc5L, 0xaaedc2dbe4418026L, 0x6661e2267bdcfd3dL, 0x4882a6eda7706f9eL, + 0xf6c2d2c912dafdd0L, 0x2f2298c142fd61f9L, 0x31d75afeb17143a8L, 0x1f9b96580a2a982fL, + 0xa6cd3e5604a8ad49L, 0x0dae2a80aad17419L, 0xdb9a9d12868124acL, 0x66b6109f80877facL, + 0x9a81d9c703a94029L, 0xbd3b381b1e03c647L, 0xe88bc07b70f31083L, 0x4e17878356a55822L, + }, + // seed = 5 + { + 0xb3c58c2483ad5eadL, 0x6570847428cdcf6cL, 0x2b38adbf813ac866L, 0x8cb9945d37eb9ad3L, + 0xf5b409ec3d1aed1cL, 0xa35f4bffc9bb5a93L, 0x5db89cde3c9e9340L, 0xff1225231b2afb2bL, + 0x157b0b212b9cc47dL, 0xf03faf97a2b2e04dL, 0x86fdab8544a20f87L, 0xfcb8732744ae5c1cL, + 0xd91744c0787986d5L, 0x5f8db2a76d65ad05L, 0xcff605cbed17a90dL, 0xf80284980a3164e7L, + 0x59cc24e713fccc7dL, 0x268982cada117ce4L, 0xcd020e63896e730eL, 0xe760dc46e9fe9885L, + 0x6aaece8ab49c6b5dL, 0x7451194d597aae3eL, 0x35d4385900332457L, 0xa40fb563a096583dL, + 0xa797b612f7f11b76L, 0x2fed6eb68e6a2b9bL, 0x2f06ee64aeffd943L, 0x9dd0e49d9ca45330L, + 0x97d48f08bd7f1d8fL, 0x1cfa7fe3ebe4d8eeL, 0x2a2ba076bd397d42L, 0x68c4344f7472f333L, + 0xce21ec31987d74b5L, 0xb73dabdc91d84088L, 0x801aadee592222feL, 0xaf41345398ebc3f5L, + 0x8a8f653d7f15ee46L, 0xce2d065ff2ba2965L, 0x4e05da515da2adb7L, 0xa6dbdb8aa25f0fd4L, + 0xca9f9666bbd2d5a9L, 0x6b917ce50bd46408L, 0x1550cc564ba6c84dL, 0xb3063ae043506504L, + 0x84e5f96bb796653dL, 0xe2364798096cf6e3L, 0x3b0dfedf6d3a53d0L, 0xb7e4c7c77bde8d93L, + 0xe99545bac9ab418aL, 0xa0e31f96889507bbL, 0x883c74f80c346885L, 0xf674ae0b039fd341L, + 0x8bb6ce2d5e8d1c75L, 0x0c48737966a7ed7cL, 0x04fcdf897b34c61cL, 0xe96ac181bacbd4d6L, + 0x5a9c55a6106a9c01L, 0x2520f020de4f45d3L, 0x935730955e94d208L, 0xce5ad4d7f3f67d3bL, + 0xa4b6d107fe2d81caL, 0x4f0033f50ae7944eL, 0x32c5d28dd8a645a7L, 0x57ce018223ef1039L, + 0x2cbab15a661ab68eL, 0x6de08798c0b5bec2L, 0xee197fb2c5c007c6L, 0x31b630ac63e7bda2L, + 0xab98785aefe9efe3L, 0xa36006158a606bf7L, 0x7b20376b9f4af635L, 0xa40762fdc3c08680L, + 0x943b5faffd0ebee2L, 0x7f39f41d0b81f06eL, 0x7c4b399b116a90f8L, 0x24e1662ac92bc9f3L, + 0xcf586fc4e8e6c7dbL, 0xe46e0d047eeb12d7L, 0xe8021076e4ea9958L, 0x11fc13492e3ca22aL, + 0xd61eae01410397e3L, 0x7e8c4a58036a8e9fL, 0x068a6de267970745L, 0x64faab129bef1a41L, + 0xb4a6f720943dad01L, 0x631491058d73a9d5L, 0xdad4fe95eab3ec02L, 0x0a8b141c5c3a44f6L, + 0x9fc69d4c2b335b98L, 0x94d5f84a07d6e4cdL, 0x1b73965de143c608L, 0x443932c2dda54bccL, + 0x7397818fb0b04cd2L, 0xef4ab03a1202b277L, 0xf3d2ee459c0c2b92L, 0x182d4daf8b058a87L, + 0x90e63035d7b51368L, 0xba4cd8b9a95d45fdL, 0x12a7392c76731090L, 0x890d264ec5d082d2L, + 0xeeaf5c363da4994eL, 0xd6aad756902123fbL, 0xb531ebebdb28f191L, 0xe71ce659fc59babdL, + 0x37c1b94f63f2dcb5L, 0xe4e3abeb311f9b96L, 0x4a31b72ccb8695d3L, 0x52cae1f0629fdce4L, + 0xe5b0475e2ed71369L, 0x2724e8c3506414fbL, 0xbab0367920672debL, 0x0161a781c305449fL, + 0x37b70f40f5bb60beL, 0xddd1094c50251a01L, 0x3b28283afd17224eL, 0x06dec0cfe889fc6bL, + 0x47608ea95bb4902dL, 0xad883ebc12c00e82L, 0x9e8d7ae0f7a8df29L, 0xa79443e9f7c013a1L, + 0xcfa26f68b7c68b71L, 0x33ae6cc19bda1f23L, 0xd9741e22b407887fL, 0xf2bff78066d46b1cL, + 0x794123191c9d32d4L, 0x56cb6b903764ec76L, 0x98775d0ef91e1a5aL, 0xae7b713bc15c1db9L, + 0x3b4c1a7870ed7a0dL, 0x46666965f305cc34L, 0x0ea0c3b2e9c6b3cdL, 0x4dc387039a143bffL, + 0x5f38bb9229ef9477L, 0xea5d39ba72af7850L, 0x69a5ed0174ce2b6dL, 0x06969a36bfe7594dL, + 0x0adee8e4065ccaa3L, 0x908a581d57113718L, 0x64822d6c5a8190edL, 0x8c5068b56ace4e4cL, + 0x88ba3b4fb4e30befL, 0xa6ec0b8bb5896cfeL, 0x4e23fcc6b47996fdL, 0xe18e75b0dd549c7aL, + 0xcd90f17e106cf939L, 0x1666fdfb2ef7c52fL, 0x4fae325f206dd88cL, 0xe7bc1160e25b062dL, + 0x3cc999cb246db950L, 0xc5930a7326cd5c37L, 0xb008a48a211367bdL, 0xc5559da145a88fd4L, + 0x1e3ad46655fac69cL, 0x7834266b4841bfd7L, 0xa764450fbffc58ccL, 0x54d8cf93a939c667L, + 0x93c51f11b21b2d9dL, 0x0964112082ed65ccL, 0x4c2df21213e7fb03L, 0xf0405bc877468615L, + 0x17b4fc835d116ab4L, 0xa6b112ae5f3cb4efL, 0x23cfc8a7fd38a46eL, 0x8e0a360dc2774808L, + 0x24ca9c8092105ad5L, 0xafd3f75524f2e0d5L, 0x4f39ed7dbaddc24cL, 0xe5e362c7679a7875L, + 0x00914a916b07b389L, 0xdfe1119b7d5ab5daL, 0xabd6ed9940e46161L, 0x630ed2044171e22cL, + 0xdecc244157dd1601L, 0x777e6d5b4b4868d5L, 0x9b3530bee67017d8L, 0xd2faf08b291fdcb9L, + 0x006e99455d6523deL, 0xd559b5817f6955b5L, 0xefcc1063b0088c61L, 0xed73145ae0f00ae7L, + 0xab2af402cf5b7421L, 0x897767f537644926L, 0x26c9c0473ca83695L, 0x192e34e1881b2962L, + 0xf7cf666ec3b3d020L, 0x27f9b79c7404afb7L, 0xe533e8bed3010767L, 0xe5817838e11d05d3L, + 0x65659c531bd36517L, 0xd427c5e0a23836fdL, 0xf3eab7ea58fa3528L, 0x07683adae1289f35L, + 0x201d6af7e896dd32L, 0xd5da938b9a21ad88L, 0x843fb73ad67bc316L, 0x1782ec7d5feef21bL, + 0x943f66f6ec772877L, 0x7e9112e7b26da097L, 0xeac8161f8663c2c7L, 0xe8600db480a9ebf4L, + 0x07807fc90f6eaf5fL, 0xe0e4c9deb41abf83L, 0xbdf533db271f9c15L, 0xb398411b0497afe2L, + 0xdebb45ef25448940L, 0xe7a5decefcd376c4L, 0xaf1ef3c728c83735L, 0xb8b83a99355cb15aL, + 0x6444a0344f1611e4L, 0xe8bb7f5cf3c60179L, 0x77ab5c5177e75ff7L, 0xc38fd6fa849d585dL, + 0x390d57d53029060aL, 0xa66327eb7b8b593cL, 0x6350a14f6fcd5ac9L, 0x2c08125bcd7008b4L, + 0x2d00c299a6a6bf8eL, 0x6b0039c1f68d1445L, 0x0035150c5d06f143L, 0xa34d01628cc927e1L, + 0xdf5b3164d7b2ede1L, 0x8167db1d0583d72eL, 0x4e13b341cd2ae8bcL, 0xa693d9b1f416e306L, + 0xc15ed7ca0bc67609L, 0xdc344313c1c4f0afL, 0x88b6887ccf772bb4L, 0x6326d8f93ca0b20eL, + 0x6964fad667dc2f11L, 0xe9783dd38fc6d515L, 0x359ed258fa022718L, 0x27ac934d1f7fd60aL, + 0xd68130437294dbccL, 0xaf5f869921f8f416L, 0x2b8f149b4ab4bf9fL, 0xc41caca607e421cbL, + 0x7746976904238ef9L, 0x604cb5529b1532f0L, 0x1c94cd17c4c4e4abL, 0xe833274b734d6bbeL, + 0xe9f1d3ef674539ceL, 0x64f56ed68d193c6aL, 0xe34192343d8ecfc1L, 0xcb162f6c3aa71fe8L, + 0x99eaf25f4c0f8fa4L, 0x92f11e7361cb8d02L, 0xb89170cddff37197L, 0x4f86e68a51e071e3L, + 0x31abf6afd911a75bL, 0x6d20cf259c269333L, 0x4150b9f88fcb6513L, 0x705063989ebf7451L, + 0x559231d927c84410L, 0x1ca8ec4b098bc687L, 0xebed22405c9180e0L, 0xaa815b37d052af59L, + }, + // seed = 6 + { + 0x946ac62246e04460L, 0x9cebee264fcbc1aeL, 0x8af54943a415652bL, 0x2b327ed3b17b8682L, + 0x983fde47b3c3847eL, 0x10a3013f99a2ad33L, 0x6e230bb92d2721efL, 0x1cf8b8369e5c5c50L, + 0x7f64017f2b7b3738L, 0xd393248a62417fa1L, 0x9ff01c0b20a372c5L, 0xb0e44abce7e7c220L, + 0xcebb9f88d48a815fL, 0xdb7df6bd09033886L, 0x7844fc82b6fa9091L, 0x72d095449863b8ecL, + 0xc13e678c89da2c7eL, 0x6caf4d5ad231d12fL, 0x2e0ab7b5fcf35c49L, 0xf410720cb932a70fL, + 0xd66ea581f16fce06L, 0x175c9f002f57dc98L, 0xccbcfd0d32988775L, 0xfde4c407d3b0a232L, + 0x5db2931ae7e97223L, 0x6e07e2173085809fL, 0x6e1d1ec0f9cad73cL, 0xb2fc251a7f802619L, + 0xbc1fc17f04f342deL, 0x8de8f21ec658e078L, 0x72c0f40cbee53fd6L, 0x0678244411fc17a1L, + 0x1d5837ca166b9bbdL, 0xc8cada003c554345L, 0x6a2fe2bfb2e58652L, 0xfca9d797a6f7988bL, + 0x6699e24ac737948bL, 0x69623ffcb05789baL, 0x946429c529d95b75L, 0x0d14df0b2a13970fL, + 0x593d8592c440dfecL, 0x2ee176f3d7e74b94L, 0xae003f1da3be9e26L, 0x0c7b02c4c0f6764aL, + 0x3117e2fa1f632462L, 0xf0f23265b6f1eaebL, 0x3111255d9b10c137L, 0xc82745e509a00397L, + 0xbd1d04037005fea7L, 0xe104ab0dd22a9036L, 0x51b27ce50851ac7aL, 0xb2cb9fb21b471b15L, + 0x29d298074c5a3e26L, 0x6ebdf2058b737418L, 0xc4a974041431b96fL, 0x1ec5a30ccb6bdaacL, + 0xe818beede9bf4425L, 0x4b69b1bce67a5555L, 0xf5c35f1eb0d62698L, 0xf4509bbd8e99867cL, + 0xb17206debd52e1bcL, 0x35785668c770b3beL, 0xe9343987ff5863bcL, 0x2ee768499ac73114L, + 0x5132bb3426eeaaf4L, 0x471bce2c6833c5ffL, 0xbb9a2d5428e6f6f9L, 0xd5678943c595792dL, + 0xab2a65e7f81e479cL, 0xa82407bb23990b31L, 0xdae321383984923cL, 0x01823bb22648e6f1L, + 0xda6e8df4214a8b04L, 0x0e172bb88e03d94fL, 0x552da6c22e362777L, 0x7ce67329fb0e90cbL, + 0x7b2d7f287ede7ebfL, 0xd44f8222500651bdL, 0x4acca1ef58fbb8abL, 0x428ecf058df9656bL, + 0xd7e1ec6a8987c185L, 0x365be6a54b253246L, 0x168849be1e271ee8L, 0x6a00f3c4151a8db2L, + 0x37602727ca94b33dL, 0xf6b50f18504fa9ceL, 0x1c10817f6bc872deL, 0x4bfe1fe42b0f3638L, + 0x135fad4b8ef6143bL, 0x1b25ad2bafc25f58L, 0x41e37f85cf321f92L, 0xfc73f75d9d5b9beaL, + 0x9eb3694d1e9cb7e1L, 0x601d51f08fa83b90L, 0x234a2a9b88366f41L, 0x63fe903e16f2c3bfL, + 0x1cdbd34fa751c0b0L, 0x0ce4fc6747c0558cL, 0x51ed72afb8bb49aaL, 0x20313ba13ca12c96L, + 0x271fa38f9ebd54c1L, 0x3696a5ac03a8eddeL, 0x05602be7df625702L, 0x11f1ac73790f7a9fL, + 0xa2836c099f0810bdL, 0xe5ac2e47caa532faL, 0xd9c000a66d39f681L, 0xd93d900e6f3d9d5fL, + 0x792c81c65b7900f2L, 0x5c5dce790ee20da1L, 0x74ff1950edec1aeeL, 0x71fc85fa1e277d8fL, + 0x0e77df17d6546cbcL, 0x07debad44816c3b4L, 0xbafa721581e92a70L, 0x8ab6fbe2ed27bba8L, + 0xe83243a20dea304aL, 0xaa85a63a84c00a07L, 0xde0e79917fc4153aL, 0x21bb445e83537896L, + 0xeedcac49fc0b433aL, 0xffb2926a810ae57aL, 0xf724be1f41d28702L, 0x79cb95746039bb3bL, + 0x5a54fe3742a00900L, 0xda4768d64922c04fL, 0x420396a84a339daeL, 0xa171e26ee5e8724eL, + 0x4c8da7c5d289c20aL, 0x9ebd79a1a8e94742L, 0x39235232b97e9782L, 0xb75df0be9bba7d80L, + 0x0c1d204dd87d48fcL, 0x8f81f3e7177266e8L, 0xe4a460b39e78d72bL, 0x50b98fa151e65351L, + 0xb7cb585c3ee1eddcL, 0x11cdad9a76ee1dc4L, 0xa38054a78595dc1cL, 0x92f09e2ec4978edcL, + 0xa8f0061b5efdabaaL, 0x04bcc4abc224d230L, 0xc58606738e692d46L, 0xdd2b27b565952433L, + 0x19e6ed1b740beec0L, 0xceadd49b2ef9891fL, 0x328178c28fe95cadL, 0xe5ad4c43afe02848L, + 0x03c0cb538cd967c0L, 0xec4352526d19a630L, 0x4c7e99389d39b031L, 0xf65dd05362c2deb6L, + 0xd1e70daf6879d28dL, 0xbe9f57db6309b265L, 0xa4b66f370b872bb7L, 0xe26896fbc6ee1fd5L, + 0xac705e661bfcf7c5L, 0xab4d0d07d7f09940L, 0x976417c06aeb6267L, 0x8161c684a6bd468cL, + 0xf77b6b9976dc4601L, 0xc6489b779a39c12cL, 0xb2aa58d5681cea1aL, 0x043b1b40f8c3e04cL, + 0x681fcbfadc845430L, 0xab8896c921ba8defL, 0x57aaf172606f37b2L, 0xc3735048cd5eb8d7L, + 0xa7078b96955631bdL, 0xdd6b3543aa187f33L, 0xc7103ea4a2a697fdL, 0x8d7b95f6ff1f7407L, + 0xe44f419e84709530L, 0xf340caa9132cbb0aL, 0x2ba407283143c66cL, 0xe1be240ca636c844L, + 0x90d32f2877ac08bcL, 0x5d26e6294b2c8673L, 0x4a6b2f5b27c87a44L, 0x961fb9043f76d34fL, + 0x0afee02d8d3c55d2L, 0x6228e3f48c42e5dcL, 0xc338e69ee6593675L, 0x853f74b16efb7bddL, + 0xd062f40bdd22e687L, 0x647164b9ab4c4190L, 0xf94689f67d598369L, 0x8e4b29d87a5012d7L, + 0xaf02b8b925656fbdL, 0x7a722a767179a630L, 0xb5c8afe937a75aceL, 0xfdb8e8d02d279372L, + 0x887ef700cb25fae1L, 0xcfe9bd912f72cabeL, 0xb1d4dedc24f978deL, 0x517522d38319cc2aL, + 0x7dd87b2b36aab798L, 0x579c4ff3046b5a04L, 0xf5c5975c5028b7a7L, 0x7094579d1000ec84L, + 0xbc8d5b1ea70a5291L, 0x161b2d783be8855cL, 0xd26d0b0d6d18279fL, 0x0be1945f02a78bd5L, + 0xb822a5a9e045415bL, 0x2fe9d68b1ccc3562L, 0xb2e375960033d14fL, 0x26aca04e49b4ff22L, + 0x732a81c862112aeaL, 0x8bd901ed6e4260b8L, 0xe839532c561ad5b0L, 0x8fb6e4d517a79b12L, + 0x0dd37f8c0be9b429L, 0xc8ad87ad12f1b1b0L, 0xc51f3aa62b90318bL, 0x031a7e8b86c1cefcL, + 0xa95547af2b70fc76L, 0x9cb3615c5a98801eL, 0xa387e3c3341d7032L, 0xa087ea52a1debaefL, + 0x16325ec9a2e6e835L, 0x587944a484c585ebL, 0xc8879033bde22eccL, 0xa39dbfce709c464aL, + 0x7acc010f99208774L, 0x98dd2973a096c5adL, 0x26458b51139f198cL, 0x2f5d19575e8c4f02L, + 0x726643f0d38af352L, 0x44d879b6d73e6e94L, 0xa68a03885c980abeL, 0x06048acd161c40c0L, + 0xa4dab8f89d405d28L, 0x7120c880cb04be18L, 0xa062ace22a1cf0cfL, 0x3901a9daf29704f4L, + 0xff08f3ed989db30aL, 0x6d22b13e874c67e9L, 0x80c6f35518d73f4dL, 0xc23c2a521aac6f29L, + 0x2e708fd83aaa42e0L, 0x7fc3780f55f1b0fdL, 0xabb3075c98cf87f2L, 0xb4df3f40f7c61143L, + 0x2a04418098a76d75L, 0x0d9eeee9509b2d37L, 0x6be8ae51f4b59cdcL, 0xe746cc7c00e4a2abL, + 0x785bc6df9cac597cL, 0x33cb6620ce8adc48L, 0xc1ba30739bffcef7L, 0x6d95771f18e503f7L, + 0xf7be3ae2e62652ffL, 0xc8d82ffd2a73c62bL, 0x8725a3ba5b110973L, 0x67ed6b9c724757ecL, + }, + // seed = 7 + { + 0xc0272d42c19ff3aeL, 0x4694228b43ea043bL, 0x5709a6ef8a462841L, 0xc9210a1e538805c9L, + 0x279b171196113ec2L, 0x859b769fc2d9e815L, 0x0d5d3125a2bf14d3L, 0x22bca1cfefa878baL, + 0x481b6bf58037bd83L, 0x4933ba8647728d22L, 0xf08c7b6b56f6e1b6L, 0x374e8af5a15407c7L, + 0xa95c4dc3d2487a5cL, 0x9b832808ff11e751L, 0xf2048507e9da01d5L, 0xa9c576189f544a4aL, + 0xf6c2a45b2e9d2b41L, 0x9b9874c9f10ecc2fL, 0x37d9b5f51f8c149eL, 0x93aead54c9de9467L, + 0x59cf0b4af262da23L, 0xe7e9929af18194b2L, 0x9df2644e33eb0178L, 0xde4122d6f0671938L, + 0xf005786c07f4800bL, 0xb1fc9d254b5d1039L, 0x0bf1088631f6dd7bL, 0x665623f0a4b8f0c7L, + 0x60f0113a9187db7cL, 0xfd7cceda4f0d23a6L, 0x26c01e9d89955940L, 0x33afa1dfc0f5a6a0L, + 0xeb77daf215e9283cL, 0xc7575214bf85edb4L, 0xeb0d804bf297e616L, 0x84bff4ffd564f747L, + 0xc4ac33189246f620L, 0x43ef61213ecc1005L, 0xcbbb0dea6cd96acdL, 0x8ed27abfa8cfcb05L, + 0x543b61529cb996b6L, 0xa5f987ca41ea5e59L, 0x3c50e0ac5254cb7aL, 0x4192b0446c06d1e6L, + 0x3e86592e21b45388L, 0xdb766f06fcc6e51eL, 0x0448ee36efe632dbL, 0x663c9db689253e35L, + 0x72e0bd4985331dd4L, 0xff501b5bf7d94e74L, 0xe911ce758e2113a8L, 0xec3a8d03a75a6ba4L, + 0xaf6b4b72f56edc83L, 0xf284857936c0a391L, 0x5ba6feff407d46f4L, 0x9d689c26de9d6702L, + 0x28c04a9083726b5dL, 0x2ccf4a627a029730L, 0x7b4719500d4f0c71L, 0x76470a9a7da250a8L, + 0xcc48409404a1c890L, 0xccefbdc7ec9a8055L, 0xe0db91bff3cc42d3L, 0x0532436426141254L, + 0xf2ee9325e6f0ff0bL, 0x149c20a5fbb28d9dL, 0xe71624cd8d2d14d4L, 0x8f01d4dc8cc2dd77L, + 0x29cf409b333015b7L, 0xba8bebd211884dd1L, 0xc3396635e8c8db1dL, 0x8ed0f6208d0528b8L, + 0x0d90b43fdd0ee334L, 0xd73c9a3333a044c7L, 0xa2595cd208dbdc38L, 0xae93cb264f940c09L, + 0x8e0538d8afb07a97L, 0x19115ec881385ba2L, 0xa886f9e6a8039c6aL, 0xcd5d62147ce3ecacL, + 0xaecdf9e0bb4969f7L, 0x2ddd631c53dcad10L, 0x73ad1c97b3412054L, 0xb08915fa2722efc6L, + 0x97966047e5067eb0L, 0x337f1675ed91445cL, 0xb3a833d150b96a0dL, 0x5940a98fe35e5e2eL, + 0xfd03cc354ed0d8ffL, 0x4e65b98291a8644aL, 0x14a259f2852a60b2L, 0x7648e3478c1e8e5fL, + 0xbc0fbef6d9a919b4L, 0xbec4302081346cf1L, 0x57d2ce7aa1c7c511L, 0x234c209d8f4e1ac3L, + 0x87cf80cc933ce443L, 0x7c262c616931e94eL, 0xc5e33b049cf9eddfL, 0x1a80790ed03ae51bL, + 0xf2e8b9494f7220cfL, 0x124cb59c14fff3ffL, 0xa8a06cbfdb86ce18L, 0x9068ef1f80b37653L, + 0x0c55417b8d90338fL, 0xcd579a523f6bcd30L, 0xa31bfe2476a8d2a9L, 0x1f8d142208094223L, + 0x332dc40a5203cfadL, 0xf8792fe5b2d33b4cL, 0x443bd9668bf9461eL, 0xc9019db0ace1409eL, + 0x781bea919a113e8bL, 0xb0f11d866abfbeecL, 0xcfe139a60db0c26aL, 0x869ab8721e6aa39eL, + 0xdb48a4977717837aL, 0x588a5ff151065b18L, 0xe4a251ea0028864dL, 0x7f0e43ba408a77c3L, + 0x65f66dd50a536135L, 0x6f49e934d9331c3eL, 0xb8d742e0f0fa6b09L, 0xe4e9b272deca2348L, + 0xaee132ff902f773cL, 0x43f658f7c2a0c90aL, 0x28cb4dbc76cc53eaL, 0x7d92253aa99ac39bL, + 0x4fea3d832370baabL, 0xb29e36936e51d78eL, 0xea10778712321064L, 0xff4f21f8ef274be2L, + 0x84eff18ddfa0933fL, 0xd0ec6a9f86c758a0L, 0xaf82e5973c431ae0L, 0x352023c00c045425L, + 0xad34d7bc4a2f8961L, 0xbdb4a02a24d4dee0L, 0x354a4846d97447cfL, 0x331a8b944d5bc19fL, + 0x5ce04f8e17909035L, 0x6497581bad8f4aabL, 0x07c503bba647111eL, 0x85f412ba78e1f7ffL, + 0x7f3b920fd20f4cffL, 0x424e1a9a4ce34e2fL, 0x3035e2d62e1b9f0aL, 0xef63114bff7b729aL, + 0xe86a05889ab6bb60L, 0xee0830cf095585a1L, 0x4a54f7fa47d9c94bL, 0x17daeece9fcb556aL, + 0xc506d3f391834c6fL, 0xb3f24be362e1af64L, 0xc435e4e23608efddL, 0xeeba9caaa4cc1768L, + 0x5a71f306daddc22dL, 0x18e5205f41eba1a0L, 0x7b29b4d1f6610925L, 0x065cb65a0258d9a9L, + 0x3e5ac8faa9fd1f95L, 0x3b362362c1ea0470L, 0xce0e4f6434db7a2eL, 0xf327341098de52f2L, + 0xcfca3b9e2a1992c3L, 0x7483bf9401233e41L, 0xbafbac531c6f9281L, 0x4b52dd71b2c106f8L, + 0xdf73b66e50b5a1f7L, 0x237aec0202a20283L, 0x23dd5be23dffdf2bL, 0xea9730731ee122efL, + 0x5cb3f846014fbcd3L, 0xc3b21c8ffdce9201L, 0x06a99a02f91a8760L, 0x721a81fa8fd7b7a3L, + 0x6aafcdddc53cbcd8L, 0xd03b464005a93bccL, 0x8212edc1b1669dcbL, 0x71f4c31364c31bc7L, + 0xfeeec0eba8772307L, 0x1948d00a13d88cf1L, 0x19064fd6d943ada8L, 0x4ec8d31722697bfdL, + 0x596d9a953a516609L, 0xc4cb4bff53507da2L, 0x1d59f3c5be36e4caL, 0xe5b4fc5bf6044c9bL, + 0x1bb74e052232f735L, 0x04e8a0db611ddd5dL, 0x8d04eaa009b421bfL, 0xa7878ae0ac0e6d58L, + 0x28c1030217cab2b3L, 0x827943767e56a883L, 0x28fce5fa02d22809L, 0xb30c322fffc8c58eL, + 0x1ca5a6a9f8066c5bL, 0xb24db5f1462b2513L, 0x02f653b89b7e5f6cL, 0xe31f8fb5d5f78eeeL, + 0x266acc514ed93501L, 0x936879d1c6fddcc4L, 0xcd51be3636af1952L, 0x3fdbb6fc332c78c8L, + 0x9eb656379fa73094L, 0x056146cc92fa0f96L, 0xed6c4f1836c027c3L, 0x021e0bb5d2113f2aL, + 0x8983e42ec1c626b3L, 0x73ea9bc6513ad9c9L, 0x0c904903b24f4247L, 0xacbac1e6243e2525L, + 0x0b1069a0c230fb06L, 0x77d709fca3fc1ce5L, 0x87ad0f65020947e6L, 0x555302641c53f4e6L, + 0x65ea87871fa9aaeeL, 0x58aaf4ecc1067bb4L, 0x1a66c48cc4c65b3fL, 0xca96aca48b2ea969L, + 0xa68eb70bad14de2bL, 0x5ccdb3d7e00a6f6eL, 0xe178fbfec73fe72fL, 0x2b63d6a16b83e890L, + 0x32fdb7a5330fbae0L, 0x2ab5803c8d1bf32cL, 0xda838388c1527c94L, 0x16a50bdc4de24acbL, + 0xe561301f134c074aL, 0xd7ae63d2816b4db1L, 0x036aabd4df0dd741L, 0xc5e0db8783435b9dL, + 0x9c4386cf0a07f3b2L, 0x6a72ac1aa56a13a1L, 0x299bbdb04bb20a23L, 0x138c1018fda16b81L, + 0x0e354f0b3bda49dfL, 0x9f4c295b23127437L, 0xd133ceb2bd561341L, 0xd8b4bfd5a526ac29L, + 0xcdd0a70ddc1c7bbdL, 0x81dce595bf572225L, 0x1c6f925c05f6efd7L, 0x8ae5097553856ea0L, + 0x3aabeaeef248f60dL, 0xd9005809d19a69e2L, 0x2a3a1a314311cc27L, 0x89bb2dc76b2b624aL, + 0x50a2a95d0412e289L, 0x9def8df564e68581L, 0xf49010a9b2e2ea5cL, 0x8602ae175d9ff3f0L, + 0xbf037e245369a618L, 0x8038164365f6e2b5L, 0xe2e1f6163b4e8d08L, 0x8df9314914f0857eL, + }, + }; +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java b/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java index 79336d90a4..34a119cb66 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/TestParquetProperties.java @@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.apache.parquet.column.impl.ContentDefinedChunker; import org.apache.parquet.column.page.mem.MemPageStore; import org.apache.parquet.column.values.alp.AlpConfig; import org.apache.parquet.schema.MessageType; @@ -192,6 +193,20 @@ public void alp_onNonFloatOrDoubleColumn_fails() { .hasMessageContaining("only supports FLOAT and DOUBLE"); } + @Test + public void alp_onNonFloatOrDoubleColumn_failsAlsoWithContentDefinedChunking() { + ParquetProperties props = ParquetProperties.builder() + .withAlp("col_b") + .withContentDefinedChunking(true) + .build(); + + assertThatThrownBy(() -> props.newColumnWriteStore( + SCHEMA, new MemPageStore(0), column -> null, ContentDefinedChunker.forColumns(SCHEMA, props))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("col_b") + .hasMessageContaining("only supports FLOAT and DOUBLE"); + } + @Test public void alp_withByteStreamSplitOnTheSameColumn_fails() { ParquetProperties props = ParquetProperties.builder() @@ -338,4 +353,53 @@ public void columnCompressionLevel_explicitNullStillResolvesToNull() { assertThat(props.getColumnCompressionLevel(colB)).isEqualTo(7); assertThat(props.getColumnCompressionLevel(colC)).isNull(); } + + @Test + public void contentDefinedChunking_byDefault_isDisabled() { + assertThat(ParquetProperties.builder().build().isContentDefinedChunkingEnabled()) + .isFalse(); + } + + @Test + public void contentDefinedChunking_sizes_surviveCopy() { + ParquetProperties original = ParquetProperties.builder() + .withContentDefinedChunking(true) + .withContentDefinedChunkingMinSize(4 * 1024) + .withContentDefinedChunkingMaxSize(16 * 1024) + .withContentDefinedChunkingNormLevel(-1) + .build(); + ParquetProperties copy = ParquetProperties.copy(original).build(); + + assertThat(copy.isContentDefinedChunkingEnabled()).isTrue(); + assertThat(copy.getContentDefinedChunkingMinSize()).isEqualTo(4 * 1024); + assertThat(copy.getContentDefinedChunkingMaxSize()).isEqualTo(16 * 1024); + assertThat(copy.getContentDefinedChunkingNormLevel()).isEqualTo(-1); + assertThat(ParquetProperties.copy(original) + .withContentDefinedChunking(false) + .build() + .isContentDefinedChunkingEnabled()) + .isFalse(); + } + + @Test + public void contentDefinedChunking_defaultSizes_matchArrow() { + ParquetProperties props = ParquetProperties.builder().build(); + + assertThat(props.getContentDefinedChunkingMinSize()).isEqualTo(256 * 1024); + assertThat(props.getContentDefinedChunkingMaxSize()).isEqualTo(1024 * 1024); + assertThat(props.getContentDefinedChunkingNormLevel()).isZero(); + } + + @Test + public void contentDefinedChunking_invalidSizes_failOnlyWhileEnabled() { + ParquetProperties.Builder builder = ParquetProperties.builder() + .withContentDefinedChunkingMinSize(16 * 1024) + .withContentDefinedChunkingMaxSize(4 * 1024); + + assertThat(builder.build().isContentDefinedChunkingEnabled()).isFalse(); + assertThatThrownBy(() -> builder.withContentDefinedChunking(true).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid content defined chunking size range: maximum chunk size (4096) must be greater " + + "than minimum chunk size (16384)"); + } } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/impl/TestContentDefinedChunker.java b/parquet-column/src/test/java/org/apache/parquet/column/impl/TestContentDefinedChunker.java new file mode 100644 index 0000000000..63a5733cc7 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/impl/TestContentDefinedChunker.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.impl; + +import static org.apache.parquet.column.impl.ContentDefinedChunker.calculateMask; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.column.page.mem.MemPageStore; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +/** Arrow C++'s TestCDC in {@code parquet/chunker_internal_test.cc}. */ +public class TestContentDefinedChunker { + + private static final String MASK_OUT_OF_RANGE = "The content defined chunking mask must be between 1 and 63 bits"; + + @ParameterizedTest + @CsvSource({ + "262144, 1048576, 0", + "0, 32, 0", + "32, 64, 0", + "1048576, 2097152, 0", + "1073741824, 2147483648, 0", + "1, 65, 1", + "0, 128, 2" + }) + public void chunkSizeParametersAreAccepted(long min, long max, int normLevel) { + assertThatCode(() -> properties(min, max, normLevel)).doesNotThrowAnyException(); + } + + /** With norm level 0, 1 or 2 the maximum must exceed the minimum by at least 32, 64 or 128 bytes. */ + @ParameterizedTest + @CsvSource({ + "0, -1, 0", + "1024, 512, 0", + "-1, 0, 0", + "0, 0, 0", + "0, 16, 0", + "-16, -16, 0", + "16, 0, 0", + "32, 32, 0", + "32, 48, 0", + "1, -1, 1", + "-1, 1, 1", + "1, 1, 1", + "1, 32, 1", + "1, 33, 1", + "0, 123, 2" + }) + public void chunkSizeParametersAreRejected(long min, long max, int normLevel) { + assertThatThrownBy(() -> properties(min, max, normLevel)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("content defined chunking"); + } + + @Test + public void rollingHashMaskCalculation() { + long min = 256 * 1024; + long max = 1024 * 1024; + assertThat(calculateMask(min, max, 0)).isEqualTo(0xFFFE000000000000L); + assertThat(calculateMask(min, max, 1)).isEqualTo(0xFFFC000000000000L); + assertThat(calculateMask(min, max, 2)).isEqualTo(0xFFF8000000000000L); + assertThat(calculateMask(min, max, 3)).isEqualTo(0xFFF0000000000000L); + assertThat(calculateMask(min, max, -1)).isEqualTo(0xFFFF000000000000L); + + // The mask bits must be between 1 and 63 once adjusted for the 8 gear hash tables + assertThatThrownBy(() -> calculateMask(0, 16, 0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith(MASK_OUT_OF_RANGE); + assertThat(calculateMask(0, 32, 0)).isEqualTo(0x8000000000000000L); + assertThatThrownBy(() -> calculateMask(0, 32, 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith(MASK_OUT_OF_RANGE); + assertThat(calculateMask(0, 64, 0)).isEqualTo(0xC000000000000000L); + assertThat(calculateMask(0, 16, -1)).isEqualTo(0x8000000000000000L); + + // Unrealistic, but checks the validation + assertThatThrownBy(() -> calculateMask(128, 384, -60)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith(MASK_OUT_OF_RANGE); + assertThat(calculateMask(128, 384, -59)).isEqualTo(0xFFFFFFFFFFFFFFFEL); + + // Unlike Arrow C++, the average chunk size does not overflow + assertThat(calculateMask(256 * 1024, Long.MAX_VALUE, 0)).isEqualTo(0xFFFFFFFFFFFFFFC0L); + } + + /** + * Arrow C++'s WriteSingleColumnParquetFile: a column write store made without the chunkers of the + * file, which every row group of the file must continue, is rejected. + */ + @Test + public void storeWithoutTheChunkersOfTheFileIsRejected() { + MessageType schema = MessageTypeParser.parseMessageType("message m { required int32 number; }"); + ParquetProperties props = + ParquetProperties.builder().withContentDefinedChunking(true).build(); + String message = "Content defined chunking needs the chunkers of the file"; + + assertThatThrownBy(() -> props.newColumnWriteStore(schema, new MemPageStore(1), column -> null)) + .isInstanceOf(IllegalStateException.class) + .hasMessage(message); + assertThatThrownBy(() -> props.newColumnWriteStore(schema, new MemPageStore(1))) + .isInstanceOf(IllegalStateException.class) + .hasMessage(message); + assertThatThrownBy(() -> new ColumnWriteStoreV1(new MemPageStore(1), props)) + .isInstanceOf(IllegalStateException.class) + .hasMessage(message); + assertThatThrownBy(() -> new ColumnWriteStoreV2(schema, new MemPageStore(1), column -> null, props, null)) + .isInstanceOf(IllegalStateException.class) + .hasMessage(message); + assertThatThrownBy(() -> props.newColumnWriteStore(schema, new MemPageStore(1), column -> null, List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Expected a chunker per column"); + } + + private static ParquetProperties properties(long min, long max, int normLevel) { + return ParquetProperties.builder() + .withContentDefinedChunking(true) + .withContentDefinedChunkingMinSize(min) + .withContentDefinedChunkingMaxSize(max) + .withContentDefinedChunkingNormLevel(normLevel) + .build(); + } +} diff --git a/parquet-hadoop/README.md b/parquet-hadoop/README.md index 2025243143..0003a38af0 100644 --- a/parquet-hadoop/README.md +++ b/parquet-hadoop/README.md @@ -284,6 +284,41 @@ true if the reader is using a `DirectByteBufferAllocator` --- +**Property:** `parquet.content-defined-chunking.enabled` +**Description:** EXPERIMENTAL: Whether data pages also end at boundaries derived from a rolling hash of the +column's values, so that files sharing a run of values share pages, which content addressable storage can +deduplicate. Readers need no support. The boundaries are those of Arrow C++ and arrow-rs for the same values, +except where Arrow holds them in another form, such as decimals, INT96 and dictionary arrays. The page size and +row count limits still apply; a page the size limit splits may deduplicate less, as the cut can move after an edit. +With dictionary encoding, an edit that adds values renumbers the indices of values first seen after it. As +parquet-java chooses between dictionary and plain encoding on the first page of a column chunk, which chunking can +cut short, some column chunks fall back to plain encoding. +**Default value:** `false` + +--- + +**Property:** `parquet.content-defined-chunking.min.size` +**Description:** EXPERIMENTAL: The minimum content defined chunk size in bytes, at least 0. Read only when content +defined chunking is enabled. +**Default value:** `262144` (256 KiB) + +--- + +**Property:** `parquet.content-defined-chunking.max.size` +**Description:** EXPERIMENTAL: The maximum content defined chunk size in bytes, greater than the minimum. Read only +when content defined chunking is enabled. +**Default value:** `1048576` (1 MiB) + +--- + +**Property:** `parquet.content-defined-chunking.norm.level` +**Description:** EXPERIMENTAL: The normalization level of the rolling hash mask. Raising it makes chunk sizes +more uniform at the cost of more small pages. At level 0, 1 or 2 the maximum must exceed the minimum by at least +32, 64 or 128 bytes. Read only when content defined chunking is enabled. +**Default value:** `0` + +--- + **Property:** `parquet.page.write-checksum.enabled` **Description:** Whether to write out page level checksums. **Default value:** `true` diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java index 95f422ea6d..5d996b074a 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java @@ -23,12 +23,14 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ColumnWriteStore; import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.column.impl.ContentDefinedChunker; import org.apache.parquet.column.values.bloomfilter.BloomFilterWriteStore; import org.apache.parquet.compression.CompressionCodecFactory.BytesInputCompressor; import org.apache.parquet.crypto.InternalFileEncryptor; @@ -71,6 +73,8 @@ class InternalParquetRecordWriter { private long lastRowGroupEndPos = 0; private ColumnWriteStore columnStore; + // Continued by every row group of the file; null unless content defined chunking is enabled. + private final List contentDefinedChunkers; private ColumnChunkPageWriteStore pageStore; private BloomFilterWriteStore bloomFilterWriteStore; private RecordConsumer recordConsumer; @@ -107,6 +111,8 @@ public InternalParquetRecordWriter( this.props = props; this.fileEncryptor = parquetFileWriter.getEncryptor(); this.rowGroupOrdinal = 0; + this.contentDefinedChunkers = + props.isContentDefinedChunkingEnabled() ? ContentDefinedChunker.forColumns(schema, props) : null; initStore(); recordCountForNextMemCheck = props.getMinRowCountForPageSizeCheck(); } @@ -128,7 +134,7 @@ private void initStore() { pageStore = columnChunkPageWriteStore; bloomFilterWriteStore = columnChunkPageWriteStore; - columnStore = props.newColumnWriteStore(schema, pageStore, bloomFilterWriteStore); + columnStore = props.newColumnWriteStore(schema, pageStore, bloomFilterWriteStore, contentDefinedChunkers); MessageColumnIO columnIO = new ColumnIOFactory(validating).getColumnIO(schema); this.recordConsumer = columnIO.getRecordWriter(columnStore); writeSupport.prepareForWrite(recordConsumer); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 6cc19acca7..4676d6f95e 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -163,6 +163,11 @@ public static enum JobSummaryLevel { public static final String STATISTICS_ENABLED = "parquet.column.statistics.enabled"; public static final String SIZE_STATISTICS_ENABLED = "parquet.size.statistics.enabled"; public static final String COLUMN_COMPRESSION_LEVEL_PREFIX = "parquet.compression.level"; + // EXPERIMENTAL: see ParquetProperties.Builder#withContentDefinedChunking + public static final String CONTENT_DEFINED_CHUNKING_ENABLED = "parquet.content-defined-chunking.enabled"; + public static final String CONTENT_DEFINED_CHUNKING_MIN_SIZE = "parquet.content-defined-chunking.min.size"; + public static final String CONTENT_DEFINED_CHUNKING_MAX_SIZE = "parquet.content-defined-chunking.max.size"; + public static final String CONTENT_DEFINED_CHUNKING_NORM_LEVEL = "parquet.content-defined-chunking.norm.level"; public static JobSummaryLevel getJobSummaryLevel(Configuration conf) { String level = conf.get(JOB_SUMMARY_LEVEL); @@ -535,6 +540,19 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .withPageRowCountLimit(getPageRowCountLimit(conf)) .withPageWriteChecksumEnabled(getPageWriteChecksumEnabled(conf)) .withStatisticsEnabled(getStatisticsEnabled(conf)); + if (conf.getBoolean(CONTENT_DEFINED_CHUNKING_ENABLED, false)) { + propsBuilder + .withContentDefinedChunking(true) + .withContentDefinedChunkingMinSize(conf.getLong( + CONTENT_DEFINED_CHUNKING_MIN_SIZE, + ParquetProperties.DEFAULT_CONTENT_DEFINED_CHUNKING_MIN_SIZE)) + .withContentDefinedChunkingMaxSize(conf.getLong( + CONTENT_DEFINED_CHUNKING_MAX_SIZE, + ParquetProperties.DEFAULT_CONTENT_DEFINED_CHUNKING_MAX_SIZE)) + .withContentDefinedChunkingNormLevel(conf.getInt( + CONTENT_DEFINED_CHUNKING_NORM_LEVEL, + ParquetProperties.DEFAULT_CONTENT_DEFINED_CHUNKING_NORM_LEVEL)); + } new ColumnConfigParser() .withColumnConfig( ENABLE_DICTIONARY, key -> conf.getBoolean(key, false), propsBuilder::withDictionaryEncoding) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java index e140563453..eca298d6e4 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java @@ -684,6 +684,50 @@ public SELF withPageRowCountLimit(int rowCount) { return self(); } + /** + * EXPERIMENTAL: See {@link ParquetProperties.Builder#withContentDefinedChunking(boolean)}. + * + * @param enabled whether to enable content defined chunking + * @return this builder for method chaining + */ + public SELF withContentDefinedChunking(boolean enabled) { + encodingPropsBuilder.withContentDefinedChunking(enabled); + return self(); + } + + /** + * EXPERIMENTAL: See {@link ParquetProperties.Builder#withContentDefinedChunkingMinSize(long)}. + * + * @param minSize the minimum chunk size in bytes + * @return this builder for method chaining + */ + public SELF withContentDefinedChunkingMinSize(long minSize) { + encodingPropsBuilder.withContentDefinedChunkingMinSize(minSize); + return self(); + } + + /** + * EXPERIMENTAL: See {@link ParquetProperties.Builder#withContentDefinedChunkingMaxSize(long)}. + * + * @param maxSize the maximum chunk size in bytes + * @return this builder for method chaining + */ + public SELF withContentDefinedChunkingMaxSize(long maxSize) { + encodingPropsBuilder.withContentDefinedChunkingMaxSize(maxSize); + return self(); + } + + /** + * EXPERIMENTAL: See {@link ParquetProperties.Builder#withContentDefinedChunkingNormLevel(int)}. + * + * @param normLevel the normalization level + * @return this builder for method chaining + */ + public SELF withContentDefinedChunkingNormLevel(int normLevel) { + encodingPropsBuilder.withContentDefinedChunkingNormLevel(normLevel); + return self(); + } + /** * Set the Parquet format dictionary page size used by the constructed * writer. diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestContentDefinedChunking.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestContentDefinedChunking.java new file mode 100644 index 0000000000..ee328df91b --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestContentDefinedChunking.java @@ -0,0 +1,1031 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0; +import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_2_0; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.parquet.FixedBinaryTestUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.example.ExampleOutputFormat; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.hadoop.example.GroupWriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.LocalInputFile; +import org.apache.parquet.io.LocalOutputFile; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Arrow C++'s {@code parquet/chunker_internal_test.cc}: pages weigh between the minimum and maximum + * chunk size, and an edit changes only the pages around it. + */ +public class TestContentDefinedChunking { + + @TempDir + private Path tempDir; + + /** Arrow C++'s TestFindDifferences. */ + @Nested + class FindDifferences { + + @Test + public void basic() { + assertThat(findDifferences(List.of(1, 2, 3, 4, 5), List.of(1, 7, 8, 4, 5))) + .containsExactly(new ChunkDiff(List.of(2, 3), List.of(7, 8))); + } + + @Test + public void multipleDifferences() { + assertThat(findDifferences(List.of(1, 2, 3, 4, 5, 6, 7), List.of(1, 8, 9, 4, 10, 6, 11))) + .containsExactly( + new ChunkDiff(List.of(2, 3), List.of(8, 9)), + new ChunkDiff(List.of(5), List.of(10)), + new ChunkDiff(List.of(7), List.of(11))); + } + + @Test + public void differentLengths() { + assertThat(findDifferences(List.of(1, 2, 3), List.of(1, 2, 3, 4, 5))) + .containsExactly(new ChunkDiff(List.of(), List.of(4, 5))); + } + + @Test + public void changesAtBothEnds() { + assertThat(findDifferences(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9), List.of(0, 0, 2, 3, 4, 5, 7, 7, 8))) + .containsExactly( + new ChunkDiff(List.of(1), List.of(0, 0)), + new ChunkDiff(List.of(6), List.of(7)), + new ChunkDiff(List.of(9), List.of())); + } + + @Test + public void emptyArrays() { + assertThat(findDifferences(List.of(), List.of())).isEmpty(); + } + + @Test + public void longSequenceWithSingleDifference() { + assertThat(findDifferences( + List.of(1994, 2193, 2700, 1913, 2052), List.of(2048, 43, 2080, 2700, 1913, 2052))) + .containsExactly(new ChunkDiff(List.of(1994, 2193), List.of(2048, 43, 2080))); + } + + @Test + public void longSequenceWithMiddleChanges() { + assertThat(findDifferences( + List.of(2169, 1976, 2180, 2147, 1934, 1772, 1914, 2075, 2154, 1940, 1934, 1970), + List.of(2169, 1976, 2180, 2147, 2265, 1804, 1717, 1925, 2122, 1940, 1934, 1970))) + .containsExactly(new ChunkDiff( + List.of(1934, 1772, 1914, 2075, 2154), List.of(2265, 1804, 1717, 1925, 2122))); + } + + @Test + public void additionalCase() { + assertThat(findDifferences( + List.of(445, 312, 393, 401, 410, 138, 558, 457), + List.of(445, 312, 393, 393, 410, 138, 558, 457))) + .containsExactly(new ChunkDiff(List.of(401), List.of(393))); + } + } + + /** Arrow C++'s TestCDCSingleRowGroup, for each of its column types that Parquet stores apart. */ + @Nested + @ParameterizedClass + @MethodSource("cases") + class SingleRowGroup { + + private static final int PART_SIZE = 128 * 1024; + private static final int EDIT_SIZE = 128; + private static final CdcOptions OPTIONS = new CdcOptions(4 * 1024, 16 * 1024, 0); + private static final int ROW_GROUP_LENGTH = 1024 * 1024; + + private final Case param; + private final Table part1, part2, part3, part4, part5, part6, part7; + + SingleRowGroup(Case param) { + this.param = param; + // The chunk sizes are constant, so the record count follows from the size of a record, a hand + // picked one for variable length and nested types + int bytesPerRecord = param.field() instanceof Primitive field && field.type().byteWidth > 0 + ? field.type().byteWidth + (field.nullable() ? 2 : 0) + : 16; + int partLength = PART_SIZE / bytesPerRecord; + int editLength = EDIT_SIZE / bytesPerRecord; + List fields = List.of(param.field()); + part1 = generateTable(fields, partLength, 0); + part2 = generateTable(fields, editLength, 1); + part3 = generateTable(fields, partLength, partLength); + part4 = generateTable(fields, editLength, 2); + part5 = generateTable(fields, partLength, 2L * partLength); + part6 = generateTable(fields, editLength, 3); + part7 = generateTable(fields, editLength, 4); + } + + static Stream cases() { + return Stream.of( + // Boolean + new Case(new Primitive("f0", Type.BOOLEAN, false), PARQUET_1_0), + // Numeric + new Case(new Primitive("f0", Type.UINT8, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.UINT16, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.UINT32, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.UINT64, true), PARQUET_1_0), + new Case(new Primitive("f0", Type.INT8, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.INT16, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.INT32, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.INT64, true), PARQUET_1_0), + new Case(new Primitive("f0", Type.FLOAT16, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.FLOAT, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.DOUBLE, true), PARQUET_1_0), + new Case(new Primitive("f0", Type.DECIMAL_18_6, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.DECIMAL_40_6, false), PARQUET_1_0), + // Binary-like + new Case(new Primitive("f0", Type.STRING, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.BINARY, true), PARQUET_1_0), + new Case(new Primitive("f0", Type.FIXED_LEN_BYTE_ARRAY, true), PARQUET_1_0), + // Temporal + new Case(new Primitive("f0", Type.DATE, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.TIME_MILLIS, true), PARQUET_1_0), + new Case(new Primitive("f0", Type.TIME_NANOS, false), PARQUET_1_0), + new Case(new Primitive("f0", Type.TIMESTAMP_NANOS, true), PARQUET_1_0), + // Nested types + new Case(new ListOf("f0", new Primitive("element", Type.INT16, true), false), PARQUET_1_0), + new Case(new ListOf("f0", new Primitive("element", Type.INT32, true), true), PARQUET_1_0), + new Case(new ListOf("f0", new Primitive("element", Type.STRING, true), true), PARQUET_1_0), + new Case(new StructOf("f0", new Primitive("f0", Type.INT32, true), false), PARQUET_1_0), + new Case(new StructOf("f0", new Primitive("f0", Type.DOUBLE, true), true), PARQUET_1_0), + new Case( + new ListOf( + "f0", new StructOf("element", new Primitive("f0", Type.INT32, true), true), false), + PARQUET_1_0), + // V2 data pages + new Case(new Primitive("f0", Type.BINARY, false), PARQUET_2_0), + new Case(new ListOf("f0", new Primitive("element", Type.STRING, true), true), PARQUET_2_0)); + } + + @Test + public void deleteOnce() throws IOException { + List original = write(Table.concat(part1, part2, part3)); + List modified = write(Table.concat(part1, part3)); + // A single row group, so no other page changes + assertPageLengthDifferences(original, modified, 0, 0, 1, editLength(part2)); + } + + @Test + public void deleteTwice() throws IOException { + List original = write(Table.concat(part1, part2, part3, part4, part5)); + List modified = write(Table.concat(part1, part3, part5)); + // part2 and part4 have as many values + assertPageLengthDifferences(original, modified, 0, 0, 2, editLength(part2)); + } + + @Test + public void updateOnce() throws IOException { + List original = write(Table.concat(part1, part2, part3)); + List modified = write(Table.concat(part1, part4, part3)); + assertPageLengthDifferences(original, modified, 1); + } + + @Test + public void updateTwice() throws IOException { + List original = write(Table.concat(part1, part2, part3, part4, part5)); + List modified = write(Table.concat(part1, part6, part3, part7, part5)); + assertPageLengthDifferences(original, modified, 2); + } + + @Test + public void insertOnce() throws IOException { + List original = write(Table.concat(part1, part3)); + List modified = write(Table.concat(part1, part2, part3)); + assertPageLengthDifferences(original, modified, 0, 1, 0, editLength(part2)); + } + + @Test + public void insertTwice() throws IOException { + List original = write(Table.concat(part1, part3, part5)); + List modified = write(Table.concat(part1, part2, part3, part4, part5)); + assertPageLengthDifferences(original, modified, 0, 2, 0, editLength(part2)); + } + + @Test + public void prepend() throws IOException { + List original = write(Table.concat(part1, part2, part3)); + List modified = write(Table.concat(part4, part1, part2, part3)); + // As many pages or more at the start, and no later page larger + assertThat(modified).hasSizeGreaterThanOrEqualTo(original.size()); + assertPageLengthDifferences(original, modified, 0, 1, 0, editLength(part4)); + } + + @Test + public void append() throws IOException { + List original = write(Table.concat(part1, part2, part3)); + List modified = write(Table.concat(part1, part2, part3, part4)); + // More pages, or a larger last page; every other page is the same + assertThat(modified).hasSizeGreaterThanOrEqualTo(original.size()); + assertThat(modified.subList(0, original.size() - 1)) + .containsExactlyElementsOf(original.subList(0, original.size() - 1)); + assertThat(modified.get(original.size() - 1)).isGreaterThanOrEqualTo(original.get(original.size() - 1)); + } + + @Test + public void emptyTable() throws IOException { + Table empty = generateTable(List.of(param.field()), 0, 0); + Path file = writeTable(empty, OPTIONS, ROW_GROUP_LENGTH, param.version()); + + // Arrow C++ writes an empty row group, parquet-java none + assertThat(pageLengths(file, 0)).isEmpty(); + } + + /** + * Writes the table into a single row group, and checks that the values of its pages weigh between + * the minimum and maximum chunk size. + */ + private List write(Table table) throws IOException { + List> pages = pageLengths(writeTable(table, OPTIONS, ROW_GROUP_LENGTH, param.version()), 0); + assertThat(pages).hasSize(1); + assertContentDefinedChunkSizes(table, pages.get(0)); + return pages.get(0); + } + + /** + * Arrow C++'s AssertContentDefinedChunkSizes, for primitive columns only: with chunks of 4 to 16 + * KiB, required int32 pages of 1024, 4096 and 100 values (4 KiB, 16 KiB, 400 B) pass, as only the + * last page may be under the minimum. + */ + private void assertContentDefinedChunkSizes(Table table, List pageLengths) { + if (!(param.field() instanceof Primitive field)) { + return; + } + int offset = 0; + for (int page = 0; page < pageLengths.size(); page++) { + long size = 0; + for (Group row : table.rows().subList(offset, offset + pageLengths.get(page))) { + if (row.getFieldRepetitionCount(field.name()) > 0) { + size += field.type() == Type.BOOLEAN + ? 1 + : field.type().byteWidth > 0 + ? field.type().byteWidth + : row.getBinary(field.name(), 0).length(); + } + // Of nullable columns, the definition level too + size += field.nullable() ? 2 : 0; + } + offset += pageLengths.get(page); + if (page < pageLengths.size() - 1) { + assertThat(size) + .as("chunk size of page %s", page) + .isBetween(OPTIONS.minChunkSize(), OPTIONS.maxChunkSize()); + } else { + assertThat(size).as("chunk size of the last page").isLessThanOrEqualTo(OPTIONS.maxChunkSize()); + } + } + assertThat(offset).as("values of the pages").isEqualTo(table.rows().size()); + } + + /** How many levels an edit adds or removes: the lists null, [], [a] and [b, c, d] are 1 + 1 + 1 + 3. */ + private int editLength(Table edit) { + if (!(param.field() instanceof ListOf field)) { + return edit.rows().size(); + } + int length = 0; + for (Group row : edit.rows()) { + int elements = row.getFieldRepetitionCount(field.name()) == 0 + ? 0 + : row.getGroup(field.name(), 0).getFieldRepetitionCount("list"); + length += Math.max(1, elements); + } + return length; + } + } + + /** Arrow C++'s TestCDCMultipleRowGroups. */ + @Nested + class MultipleRowGroups { + + private static final int PART_LENGTH = 128 * 1024; + private static final int EDIT_LENGTH = 128; + private static final int ROW_GROUP_LENGTH = 64 * 1024; + private static final CdcOptions OPTIONS = new CdcOptions(4 * 1024, 16 * 1024, 0); + + private final List fields = List.of( + new Primitive("int32", Type.INT32, true), + new Primitive("float64", Type.DOUBLE, true), + new Primitive("bool", Type.BOOLEAN, false)); + private final Table part1 = generateTable(fields, PART_LENGTH, 0); + private final Table part2 = generateTable(fields, PART_LENGTH, 2); + private final Table part3 = generateTable(fields, PART_LENGTH, 4); + private final Table edit1 = generateTable(fields, EDIT_LENGTH, 1); + private final Table edit2 = generateTable(fields, EDIT_LENGTH, 3); + private final Table edit3 = generateTable(fields, EDIT_LENGTH, 5); + + /** + * An insertion makes a larger difference in the row group it is in, and a smaller one at its end, as + * the row groups have a fixed length; the later row groups shift likewise, and the last one grows. + */ + @Test + public void insertOnce() throws IOException { + Path original = write(Table.concat(part1, edit1, part2, part3)); + Path modified = write(Table.concat(part1, edit1, edit2, part2, part3)); + + for (int column = 0; column < fields.size(); column++) { + List> originalPages = pageLengths(original, column); + List> modifiedPages = pageLengths(modified, column); + assertThat(originalPages).hasSize(7); + assertThat(modifiedPages).hasSize(7); + // Each part is two row groups, and the first is not edited + assertThat(modifiedPages.subList(0, 2)).containsExactlyElementsOf(originalPages.subList(0, 2)); + for (int i = 2; i < 6; i++) { + assertPageLengthDifferences(originalPages.get(i), modifiedPages.get(i), 0, 1, 1, EDIT_LENGTH); + } + assertPageLengthDifferences(originalPages.get(6), modifiedPages.get(6), 0, 1, 0, EDIT_LENGTH); + } + } + + /** A deletion makes a smaller difference, and the shift a larger one at the end of each row group. */ + @Test + public void deleteOnce() throws IOException { + Path original = write(Table.concat(part1, edit1, part2, part3, edit2)); + Path modified = write(Table.concat(part1, part2, part3, edit2)); + + for (int column = 0; column < fields.size(); column++) { + List> originalPages = pageLengths(original, column); + List> modifiedPages = pageLengths(modified, column); + assertThat(originalPages).hasSize(7); + assertThat(modifiedPages).hasSize(7); + assertThat(modifiedPages.subList(0, 2)).containsExactlyElementsOf(originalPages.subList(0, 2)); + for (int i = 2; i < 6; i++) { + assertPageLengthDifferences(originalPages.get(i), modifiedPages.get(i), 0, 1, 1, EDIT_LENGTH); + } + assertPageLengthDifferences(originalPages.get(6), modifiedPages.get(6), 0, 0, 1, EDIT_LENGTH); + } + } + + /** An update shifts nothing, so it changes pages of one row group at most. */ + @Test + public void updateOnce() throws IOException { + Path original = write(Table.concat(part1, edit1, part2, part3, edit2)); + Path modified = write(Table.concat(part1, edit3, part2, part3, edit2)); + + for (int column = 0; column < fields.size(); column++) { + List> originalPages = pageLengths(original, column); + List> modifiedPages = pageLengths(modified, column); + assertThat(originalPages).hasSize(7); + assertThat(modifiedPages).hasSize(7); + assertThat(modifiedPages.subList(0, 2)).containsExactlyElementsOf(originalPages.subList(0, 2)); + assertPageLengthDifferences(originalPages.get(2), modifiedPages.get(2), 1); + assertThat(modifiedPages.subList(2, 7)).containsExactlyElementsOf(originalPages.subList(2, 7)); + } + } + + @Test + public void append() throws IOException { + Path original = write(Table.concat(part1, edit1, part2, part3)); + Path modified = write(Table.concat(part1, edit1, part2, part3, edit2)); + + for (int column = 0; column < fields.size(); column++) { + List> originalPages = pageLengths(original, column); + List> modifiedPages = pageLengths(modified, column); + assertThat(originalPages).hasSize(7); + assertThat(modifiedPages).hasSize(7); + assertThat(modifiedPages.subList(0, 6)).containsExactlyElementsOf(originalPages.subList(0, 6)); + // Only the last row group grows, and of it only the last page + List originalLast = originalPages.get(6); + List modifiedLast = modifiedPages.get(6); + assertThat(modifiedLast).hasSameSizeAs(originalLast); + assertThat(modifiedLast.subList(0, modifiedLast.size() - 1)) + .containsExactlyElementsOf(originalLast.subList(0, originalLast.size() - 1)); + assertThat(modifiedLast.get(modifiedLast.size() - 1)) + .isGreaterThan(originalLast.get(originalLast.size() - 1)); + } + } + + /** + * Splitting the data into row groups moves no page boundary but adds one at the end of each row + * group: if the pages of a file of one row group end at rows 100, 250 and 400, then with row groups + * of 200 rows they end at rows 100, 200, 250 and 400. + */ + @Test + public void independentOfRowGroupBoundaries() throws IOException { + Table table = Table.concat(part1, part2, part3); + int rows = table.rows().size(); + int rowGroupLength = rows / 6; + Path single = writeTable(table, OPTIONS, rows, PARQUET_1_0); + Path multi = writeTable(table, OPTIONS, rowGroupLength, PARQUET_1_0); + + assertThat(pageLengths(single, 0)).hasSize(1); + assertThat(pageLengths(multi, 0)).hasSize(6); + for (int column = 0; column < fields.size(); column++) { + List singleEnds = pageEnds(single, column); + List expected = new ArrayList<>(singleEnds); + for (int i = 1; i < 6; i++) { + expected.add(i * rowGroupLength); + } + Collections.sort(expected); + + assertThat(pageEnds(multi, column)).as("column %s", column).containsExactlyElementsOf(expected); + } + } + + /** + * The page lengths a build of Arrow C++'s main branch, which has the unreleased apache/arrow#51685, + * writes for this table: {@code WriteTable} with row groups of 64Ki rows, {@code CdcOptions{64 KiB, + * 256 KiB, 0}}, V1 pages, no dictionary or compression, and no page size or rows per page limit. + */ + @Test + public void pageLengthsMatchArrowCpp() throws IOException { + Path file = writeTable( + Table.concat(part1, edit1, part2, part3), + new CdcOptions(64 * 1024, 256 * 1024, 0), + ROW_GROUP_LENGTH, + PARQUET_1_0); + + assertThat(pageLengths(file, 0)) + .as("int32") + .containsExactly( + List.of(28446, 22751, 14339), + List.of(8214, 23968, 20647, 12707), + List.of(14647, 33235, 17654), + List.of(2454, 29854, 20647, 12581), + List.of(14645, 33235, 17656), + List.of(2452, 29854, 20647, 12583), + List.of(128)); + assertThat(pageLengths(file, 1)) + .as("float64") + .containsExactly( + List.of(13867, 13300, 9396, 10801, 15077, 3095), + List.of(14371, 15918, 20443, 14285, 519), + List.of(10507, 13212, 12970, 10801, 15077, 2969), + List.of(14497, 15918, 20443, 14285, 393), + List.of(13991, 13300, 9396, 10801, 15077, 2971), + List.of(14495, 15918, 20443, 14285, 395), + List.of(128)); + assertThat(pageLengths(file, 2)) + .as("bool") + .containsExactly( + List.of(65536), + List.of(65536), + List.of(16580, 48956), + List.of(65536), + List.of(16578, 48958), + List.of(65536), + List.of(128)); + } + + private Path write(Table table) throws IOException { + return writeTable(table, OPTIONS, ROW_GROUP_LENGTH, PARQUET_1_0); + } + } + + /** + * As {@link MultipleRowGroups#pageLengthsMatchArrowCpp}, for a nullable list of strings and NaNs of + * differing payloads in one row group, with {@code CdcOptions{1 KiB, 2 KiB, -1}}, so that the NaN + * pages of 256 values end at the maximum chunk size. + */ + @Test + public void pageLengthsOfListsAndNaNsMatchArrowCpp() throws IOException { + Table table = generateTable( + List.of( + new ListOf("strings", new Primitive("element", Type.STRING, true), true), + new Primitive("nan", Type.NAN, false)), + 2048, + 0); + Path file = + writeTable(table, new CdcOptions(1024, 2048, -1), table.rows().size(), PARQUET_1_0); + + assertThat(pageLengths(file, 0)) + .as("strings") + .containsExactly(List.of( + 116, 60, 115, 87, 114, 63, 114, 63, 108, 120, 84, 116, 60, 115, 57, 120, 66, 108, 72, 86, 106, + 116, 88, 116, 70, 116, 79, 117, 84, 111, 81, 111, 33)); + assertThat(pageLengths(file, 1)) + .as("nan") + .containsExactly(List.of(224, 233, 256, 217, 242, 256, 210, 221, 189)); + } + + /** + * A chunk may end before the first value of a page, which leaves no page to end: 16-byte values end a + * chunk of at most 16 bytes before each value, so row groups of 5 values make 5 pages of 1 value. + */ + @Test + public void chunkEndingBeforeAPageStartsWritesNoEmptyPage() throws IOException { + Table table = generateTable(List.of(new Primitive("f0", Type.FIXED_LEN_BYTE_ARRAY, false)), 10, 0); + Path file = writeTable(table, new CdcOptions(0, 16, -1), 5, PARQUET_1_0); + + assertThat(pageLengths(file, 0)).containsExactly(List.of(1, 1, 1, 1, 1), List.of(1, 1, 1, 1, 1)); + } + + /** + * The record writer's page size checks, which also flush held back nulls, only add page ends: with + * chunks ending after 100 and 250 values, pages may end after 40, 100, 180 and 250, never after 40, + * 120 and 250. + */ + @Test + public void pageSizeCheckScheduleMovesNoChunkBoundary() throws Exception { + Table table = + generateTable(List.of(new ListOf("f0", new Primitive("element", Type.STRING, true), true)), 40_000, 0); + CdcOptions options = new CdcOptions(16 * 1024, 64 * 1024, 0); + Configuration conf = outputFormatConf(table.schema()); + conf.setBoolean(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_ENABLED, true); + conf.setLong(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_MIN_SIZE, options.minChunkSize()); + conf.setLong(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_MAX_SIZE, options.maxChunkSize()); + conf.setInt(ParquetOutputFormat.PAGE_SIZE, 8 * 1024); + + List chunks = pageEnds(writeTable(table, options, table.rows().size(), PARQUET_1_0), 0); + assertThat(pageEnds(writeThroughOutputFormat(conf, table, "scheduled"), 0)) + .containsAll(chunks) + .hasSizeGreaterThan(2 * chunks.size()); + } + + /** Chunk sizes such as "8m", which the keys do not accept, or a minimum above the maximum. */ + @Test + public void invalidOptionsFailOnlyWhileEnabled() { + Table empty = generateTable(List.of(new Primitive("f0", Type.INT32, false)), 0, 0); + Configuration conf = outputFormatConf(empty.schema()); + conf.set(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_MIN_SIZE, "8m"); + + conf.setBoolean(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_ENABLED, false); + assertThatCode(() -> writeThroughOutputFormat(conf, empty, "off")).doesNotThrowAnyException(); + + conf.setLong(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_MIN_SIZE, 8 * 1024 * 1024); + conf.setBoolean(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_ENABLED, true); + assertThatThrownBy(() -> writeThroughOutputFormat(conf, empty, "on")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid content defined chunking size range: maximum chunk size (1048576) must be greater " + + "than minimum chunk size (8388608)"); + assertThat(tempDir.resolve("on.parquet")).doesNotExist(); + } + + @Test + public void configurationKeysReachTheWriter() throws Exception { + Table table = generateTable(List.of(new Primitive("f0", Type.STRING, true)), 40_000, 0); + Configuration conf = outputFormatConf(table.schema()); + CdcOptions options = new CdcOptions(16 * 1024, 64 * 1024, 1); + conf.setBoolean(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_ENABLED, true); + conf.setLong(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_MIN_SIZE, options.minChunkSize()); + conf.setLong(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_MAX_SIZE, options.maxChunkSize()); + conf.setInt(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_NORM_LEVEL, options.normLevel()); + + List> pages = pageLengths(writeThroughOutputFormat(conf, table, "configured"), 0); + assertThat(pages) + .containsExactlyElementsOf( + pageLengths(writeTable(table, options, table.rows().size(), PARQUET_1_0), 0)); + assertThat(pages.get(0)).hasSizeGreaterThan(1); + + conf.setBoolean(ParquetOutputFormat.CONTENT_DEFINED_CHUNKING_ENABLED, false); + assertThat(pageLengths(writeThroughOutputFormat(conf, table, "unconfigured"), 0)) + .containsExactly(List.of(table.rows().size())); + } + + private Path path(String label) { + return tempDir.resolve(label + ".parquet"); + } + + /** + * Arrow C++'s WriteTableToBuffer, checking that the rows read back are those written. Without page + * limits, pages end only at chunk and row group ends; dictionary encoding is off, as it moves none. + */ + private Path writeTable(Table table, CdcOptions options, int rowGroupLength, WriterVersion version) + throws IOException { + Path file = path(UUID.randomUUID().toString()); + try (ParquetWriter writer = ExampleParquetWriter.builder(new LocalOutputFile(file)) + .withType(table.schema()) + .withContentDefinedChunking(true) + .withContentDefinedChunkingMinSize(options.minChunkSize()) + .withContentDefinedChunkingMaxSize(options.maxChunkSize()) + .withContentDefinedChunkingNormLevel(options.normLevel()) + .withRowGroupRowCountLimit(rowGroupLength) + .withDictionaryEncoding(false) + .withWriterVersion(version) + .withPageSize(Integer.MAX_VALUE) + .withPageRowCountLimit(Integer.MAX_VALUE) + .build()) { + for (Group row : table.rows()) { + writer.write(row); + } + } + try (ParquetReader reader = ParquetReader.builder( + new GroupReadSupport(), new org.apache.hadoop.fs.Path(file.toUri())) + .build()) { + for (Group row : table.rows()) { + assertThat(reader.read()).as("row read back").asString().isEqualTo(row.toString()); + } + assertThat(reader.read()) + .as("row read back beyond the written ones") + .isNull(); + } + return file; + } + + /** The configuration of a {@link ParquetOutputFormat} without the page size and row count limits. */ + private static Configuration outputFormatConf(MessageType schema) { + Configuration conf = new Configuration(); + GroupWriteSupport.setSchema(schema, conf); + conf.setInt(ParquetOutputFormat.PAGE_ROW_COUNT_LIMIT, Integer.MAX_VALUE); + conf.setInt(ParquetOutputFormat.PAGE_SIZE, Integer.MAX_VALUE); + conf.setBoolean(ParquetOutputFormat.ENABLE_DICTIONARY, false); + return conf; + } + + private Path writeThroughOutputFormat(Configuration conf, Table table, String label) throws Exception { + Path file = path(label); + RecordWriter writer = new ExampleOutputFormat() + .getRecordWriter(conf, new org.apache.hadoop.fs.Path(file.toUri()), CompressionCodecName.UNCOMPRESSED); + try { + for (Group row : table.rows()) { + writer.write(null, row); + } + } finally { + writer.close(null); + } + return file; + } + + /** Arrow C++'s CdcOptions. */ + record CdcOptions(long minChunkSize, long maxChunkSize, int normLevel) {} + + /** + * Arrow C++'s GetColumnParquetInfo: the value counts of the data pages of a column, in each row + * group, none of them empty. + */ + private static List> pageLengths(Path file, int column) throws IOException { + List> rowGroups = new ArrayList<>(); + try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(file))) { + ColumnDescriptor path = + reader.getFileMetaData().getSchema().getColumns().get(column); + for (PageReadStore rowGroup = reader.readNextRowGroup(); + rowGroup != null; + rowGroup = reader.readNextRowGroup()) { + List pages = new ArrayList<>(); + PageReader pageReader = rowGroup.getPageReader(path); + for (DataPage page = pageReader.readPage(); page != null; page = pageReader.readPage()) { + assertThat(page.getValueCount()).as("values of a data page").isPositive(); + pages.add(page.getValueCount()); + } + rowGroups.add(pages); + } + } + return rowGroups; + } + + /** Arrow C++'s GetPageEnds: where the pages of a column end, counted in values across its row groups. */ + private static List pageEnds(Path file, int column) throws IOException { + List ends = new ArrayList<>(); + int end = 0; + for (List rowGroup : pageLengths(file, column)) { + for (int length : rowGroup) { + end += length; + ends.add(end); + } + } + return ends; + } + + /** A run of page lengths that differ, like a hunk of a diff: [2, 3] to [7, 8]. */ + record ChunkDiff(List first, List second) { + + /** How many values the run gains: 10 from [2, 3] to [7, 8]. */ + int growth() { + return second.stream().mapToInt(Integer::intValue).sum() + - first.stream().mapToInt(Integer::intValue).sum(); + } + } + + /** + * Arrow C++'s FindDifferences, aligning the page lengths by their longest common subsequence: [1, 2, + * 3, 4, 5] and [1, 7, 8, 4, 5] differ in the run [2, 3] to [7, 8]. A run only inserting next to one + * only deleting is merged: [] to [9] and [6] to [] make [6] to [9]. + */ + static List findDifferences(List first, List second) { + int n = first.size(); + int m = second.size(); + int[][] common = new int[n + 1][m + 1]; + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + common[i + 1][j + 1] = first.get(i).equals(second.get(j)) + ? common[i][j] + 1 + : Math.max(common[i + 1][j], common[i][j + 1]); + } + } + + List matches = new ArrayList<>(); + for (int i = n, j = m; i > 0 && j > 0; ) { + if (first.get(i - 1).equals(second.get(j - 1))) { + matches.add(new int[] {--i, --j}); + } else if (common[i - 1][j] >= common[i][j - 1]) { + i--; + } else { + j--; + } + } + Collections.reverse(matches); + + List diffs = new ArrayList<>(); + int lastI = 0; + int lastJ = 0; + for (int[] match : matches) { + if (match[0] > lastI || match[1] > lastJ) { + diffs.add(new ChunkDiff(first.subList(lastI, match[0]), second.subList(lastJ, match[1]))); + } + lastI = match[0] + 1; + lastJ = match[1] + 1; + } + if (lastI < n || lastJ < m) { + diffs.add(new ChunkDiff(first.subList(lastI, n), second.subList(lastJ, m))); + } + + List merged = new ArrayList<>(); + for (ChunkDiff diff : diffs) { + ChunkDiff previous = merged.isEmpty() ? null : merged.get(merged.size() - 1); + if (previous != null && previous.first().isEmpty() && diff.second().isEmpty()) { + merged.set(merged.size() - 1, new ChunkDiff(diff.first(), previous.second())); + } else if (previous != null + && previous.second().isEmpty() + && diff.first().isEmpty()) { + merged.set(merged.size() - 1, new ChunkDiff(previous.first(), diff.second())); + } else { + merged.add(diff); + } + } + return merged; + } + + /** + * Arrow C++'s AssertPageLengthDifferences: deleting 128 values turns [100, 200, 300] into [100, 72, + * 300], so no run keeps its size, none grows and one shrinks, by the edit length. + */ + private static void assertPageLengthDifferences( + List original, + List modified, + int equalDiffs, + int largerDiffs, + int smallerDiffs, + int editLength) { + List growths = findDifferences(original, modified).stream() + .map(ChunkDiff::growth) + .toList(); + String description = "growth of each run of pages from " + original + " to " + modified; + + assertThat(growths).as(description).filteredOn(growth -> growth == 0).hasSize(equalDiffs); + assertThat(growths).as(description).filteredOn(growth -> growth > 0).hasSize(largerDiffs); + assertThat(growths).as(description).filteredOn(growth -> growth < 0).hasSize(smallerDiffs); + assertThat(growths) + .as(description) + .allSatisfy(growth -> assertThat(Math.abs(growth)).isIn(0, editLength)); + } + + /** + * Arrow C++'s AssertPageLengthDifferences for updates, which shift no value: [100, 200, 300] to [100, + * 150, 50, 300] is one run, holding as many values. + */ + private static void assertPageLengthDifferences(List original, List modified, int maxEqualDiffs) { + List growths = findDifferences(original, modified).stream() + .map(ChunkDiff::growth) + .toList(); + + assertThat(growths) + .as("growth of each run of pages from %s to %s", original, modified) + .hasSizeLessThanOrEqualTo(maxEqualDiffs) + .allMatch(growth -> growth == 0); + } + + /** Arrow C++'s hash, which generates deterministic and platform independent data. */ + static long hash(long seed, long index) { + long h = (index + seed) * 0xc4ceb9fe1a85ec53L; + h ^= h >>> 33; + h *= 0xff51afd7ed558ccdL; + h ^= h >>> 33; + h *= 0xc4ceb9fe1a85ec53L; + h ^= h >>> 33; + return h; + } + + /** Arrow C++'s GenerateTable: rows of the fields, each field generated from the next seed. */ + static Table generateTable(List fields, int length, long seed) { + MessageType schema = MessageTypeParser.parseMessageType( + "message m { " + fields.stream().map(Field::declaration).collect(Collectors.joining(" ")) + " }"); + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + List rows = new ArrayList<>(length); + for (int index = 0; index < length; index++) { + Group row = factory.newGroup(); + for (int i = 0; i < fields.size(); i++) { + fields.get(i).append(row, seed + 1 + i, index); + } + rows.add(row); + } + return new Table(schema, rows); + } + + /** Rows, and the schema they follow. */ + record Table(MessageType schema, List rows) { + + /** Arrow C++'s ConcatAndCombine. */ + static Table concat(Table... parts) { + return new Table( + parts[0].schema(), + Arrays.stream(parts).flatMap(part -> part.rows().stream()).toList()); + } + } + + /** A case of Arrow C++'s TestCDCSingleRowGroup: the column, and the version of its data pages. */ + record Case(Field field, WriterVersion version) { + + @Override + public String toString() { + return field.declaration() + " " + version; + } + } + + /** A field, and how Arrow C++'s GenerateArray generates its values. */ + interface Field { + + String declaration(); + + /** Appends the value of the field in row {@code index}, unless it is null. */ + void append(Group group, long seed, int index); + } + + /** A field of a primitive type: one in ten values is null if the field is nullable. */ + record Primitive(String name, Type type, boolean nullable) implements Field { + + @Override + public String declaration() { + return (nullable ? "optional " : "required ") + String.format(type.declaration, name) + ";"; + } + + @Override + public void append(Group group, long seed, int index) { + long value = hash(seed, index); + if (!nullable || Long.remainderUnsigned(value, 10) != 0) { + type.writer.append(group, name, value); + } + } + } + + /** + * A list field, its rows as Arrow C++ generates them: null, [], [e0], [e1, e2, e3], null, [], [e4], + * [e5, e6, e7] and so on, with [] for null if the field is not nullable. + */ + record ListOf(String name, Field element, boolean nullable) implements Field { + + // Arrow C++'s list offsets in each run of four lists, from the run's first element + private static final int[] OFFSETS = {0, 0, 0, 1, 4}; + + @Override + public String declaration() { + return (nullable ? "optional" : "required") + " group " + name + " (LIST) { repeated group list { " + + element.declaration() + " } }"; + } + + @Override + public void append(Group group, long seed, int index) { + int kind = index % 4; + if (kind == 0 && nullable) { + return; + } + Group list = group.addGroup(name); + int start = index - kind; + for (int element = start + OFFSETS[kind]; element < start + OFFSETS[kind + 1]; element++) { + this.element.append(list.addGroup("list"), seed, element); + } + } + } + + /** A struct field of one field, never null. */ + record StructOf(String name, Field field, boolean nullable) implements Field { + + @Override + public String declaration() { + return (nullable ? "optional" : "required") + " group " + name + " { " + field.declaration() + " }"; + } + + @Override + public void append(Group group, long seed, int index) { + field.append(group.addGroup(name), seed, index); + } + } + + /** + * The primitive types, with the value Arrow C++'s GenerateArray derives from a hash and the width + * Arrow holds it in: INT16 appends {@code (short) hash} and weighs 2 bytes, as Arrow's int16. Booleans, + * which Arrow packs into bits, and binaries, whose width varies, weigh 0. + */ + enum Type { + BOOLEAN("boolean %s", 0, (group, field, value) -> group.append(field, (value & 1) == 0)), + UINT8("int32 %s (INTEGER(8,false))", 1, (group, field, value) -> group.append(field, (int) (value & 0xFF))), + UINT16("int32 %s (INTEGER(16,false))", 2, (group, field, value) -> group.append(field, (int) (value & 0xFFFF))), + UINT32("int32 %s (INTEGER(32,false))", 4, (group, field, value) -> group.append(field, (int) value)), + UINT64("int64 %s (INTEGER(64,false))", 8, (group, field, value) -> group.append(field, value)), + INT8("int32 %s (INTEGER(8,true))", 1, (group, field, value) -> group.append(field, (int) (byte) value)), + INT16("int32 %s (INTEGER(16,true))", 2, (group, field, value) -> group.append(field, (int) (short) value)), + INT32("int32 %s", 4, (group, field, value) -> group.append(field, (int) value)), + INT64("int64 %s", 8, (group, field, value) -> group.append(field, value)), + FLOAT16( + "fixed_len_byte_array(2) %s (FLOAT16)", + 2, (group, field, value) -> { + int bits = (int) Long.remainderUnsigned(value, 1000); + group.append(field, Binary.fromConstantByteArray(new byte[] {(byte) bits, (byte) (bits >>> 8)})); + }), + FLOAT( + "float %s", + 4, (group, field, value) -> group.append(field, (float) Long.remainderUnsigned(value, 1000) / 1000.0f)), + DOUBLE( + "double %s", + 8, + (group, field, value) -> group.append(field, (double) Long.remainderUnsigned(value, 100000) / 1000.0)), + // Arrow's decimal128(18, 6), which parquet-java stores and hashes as an INT64 + DECIMAL_18_6( + "int64 %s (DECIMAL(18,6))", + 8, (group, field, value) -> group.append(field, Long.remainderUnsigned(value, 999_999_999_999L))), + // Arrow's decimal256(40, 6), its values limited to 9 digits + DECIMAL_40_6( + "fixed_len_byte_array(17) %s (DECIMAL(40,6))", + 17, + (group, field, value) -> group.append( + field, + FixedBinaryTestUtils.getFixedBinary( + 17, BigInteger.valueOf(Long.remainderUnsigned(value, 999_999_999L))))), + STRING( + "binary %s (STRING)", + 0, (group, field, value) -> group.append(field, "str_" + Long.toUnsignedString(value))), + BINARY( + "binary %s", + 0, + (group, field, value) -> group.append(field, Binary.fromString("bin_" + Long.toUnsignedString(value)))), + FIXED_LEN_BYTE_ARRAY( + "fixed_len_byte_array(16) %s", + 16, + (group, field, value) -> group.append( + field, Binary.fromString(("bin_" + Long.toUnsignedString(value)).substring(0, 16)))), + DATE("int32 %s (DATE)", 4, (group, field, value) -> group.append(field, (int) value)), + TIME_MILLIS( + "int32 %s (TIME(MILLIS,true))", + 4, (group, field, value) -> group.append(field, Math.abs((int) value % 86_400_000))), + TIME_NANOS( + "int64 %s (TIME(NANOS,true))", + 8, (group, field, value) -> group.append(field, Math.abs(value % 86_400_000_000L))), + TIMESTAMP_NANOS("int64 %s (TIMESTAMP(NANOS,true))", 8, (group, field, value) -> group.append(field, value)), + // Not one of Arrow C++'s: NaNs with the hash as their payload, which Arrow C++ hashes as it is + NAN( + "double %s", + 8, (group, field, value) -> group.append(field, Double.longBitsToDouble(0x7FF8000000000000L | value))); + + private final String declaration; + private final int byteWidth; + private final ValueWriter writer; + + Type(String declaration, int byteWidth, ValueWriter writer) { + this.declaration = declaration; + this.byteWidth = byteWidth; + this.writer = writer; + } + } + + interface ValueWriter { + void append(Group group, String field, long value); + } +}