diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0e100a37..dc087966 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -658,6 +658,7 @@ add_custom_command(
# Library config
set(LIBPARQUET_SRCS
+ src/parquet/arrow/deserializer.cc
src/parquet/arrow/reader.cc
src/parquet/arrow/record_reader.cc
src/parquet/arrow/schema.cc
diff --git a/data/dremel.parquet b/data/dremel.parquet
new file mode 100644
index 00000000..93c8075f
Binary files /dev/null and b/data/dremel.parquet differ
diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc
index 47226a3c..a4eeca67 100644
--- a/src/parquet/arrow/arrow-reader-writer-test.cc
+++ b/src/parquet/arrow/arrow-reader-writer-test.cc
@@ -42,6 +42,7 @@
#include "parquet/util/test-common.h"
#include "arrow/api.h"
+#include "arrow/pretty_print.h"
#include "arrow/test-util.h"
#include "arrow/type_traits.h"
#include "arrow/util/decimal.h"
@@ -427,7 +428,8 @@ void CheckSimpleRoundtrip(const std::shared_ptr
& table, int64_t row_group
const std::shared_ptr& arrow_properties =
default_arrow_writer_properties()) {
std::shared_ptr result;
- DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties);
+ ASSERT_NO_FATAL_FAILURE(
+ DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties));
ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result, false));
}
@@ -508,8 +510,8 @@ class TestParquetIO : public ::testing::Test {
std::shared_ptr out;
std::unique_ptr reader;
- ReaderFromSink(&reader);
- ReadSingleColumnFile(std::move(reader), &out);
+ ASSERT_NO_FATAL_FAILURE(ReaderFromSink(&reader));
+ ASSERT_NO_FATAL_FAILURE(ReadSingleColumnFile(std::move(reader), &out));
internal::AssertArraysEqual(values, *out);
}
@@ -554,8 +556,8 @@ class TestParquetIO : public ::testing::Test {
void ReadAndCheckSingleColumnTable(const std::shared_ptr& values) {
std::shared_ptr<::arrow::Table> out;
std::unique_ptr reader;
- ReaderFromSink(&reader);
- ReadTableFromFile(std::move(reader), &out);
+ ASSERT_NO_FATAL_FAILURE(ReaderFromSink(&reader));
+ ASSERT_NO_FATAL_FAILURE(ReadTableFromFile(std::move(reader), &out));
ASSERT_EQ(1, out->num_columns());
ASSERT_EQ(values->length(), out->num_rows());
@@ -1043,8 +1045,8 @@ TEST_F(TestNullParquetIO, NullListColumn) {
std::shared_ptr out;
std::unique_ptr reader;
- this->ReaderFromSink(&reader);
- this->ReadTableFromFile(std::move(reader), &out);
+ ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
+ ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
ASSERT_EQ(1, out->num_columns());
ASSERT_EQ(offsets.size() - 1, out->num_rows());
@@ -1132,7 +1134,7 @@ class TestPrimitiveParquetIO : public TestParquetIO {
ASSERT_NO_FATAL_FAILURE(MakeTestFile(values, num_chunks, &file_reader));
std::shared_ptr out;
- this->ReadTableFromFile(std::move(file_reader), &out);
+ ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(file_reader), &out));
ASSERT_EQ(1, out->num_columns());
ASSERT_EQ(SMALL_SIZE, out->num_rows());
@@ -1244,6 +1246,136 @@ TEST(TestArrowReadWrite, DateTimeTypes) {
ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result));
}
+TEST(TestArrowReadWrite, DremelExampleTest) {
+ std::string dir_string(test::get_data_dir());
+ std::stringstream ss;
+ ss << dir_string << "/dremel.parquet";
+ auto parquet_reader = ParquetFileReader::OpenFile(ss.str());
+ parquet::arrow::FileReader reader(::arrow::default_memory_pool(),
+ std::move(parquet_reader));
+ std::shared_ptr<::arrow::Table> out;
+ ASSERT_OK(reader.ReadTable(&out));
+
+ // Dremel paper uses int64, but the sample file
+ // stored them as int32
+ auto forward = std::make_shared<::arrow::Int32Builder>();
+ auto backward = std::make_shared<::arrow::Int32Builder>();
+ auto docId = std::make_shared<::arrow::Int32Builder>();
+ auto country = std::make_shared<::arrow::StringBuilder>();
+ auto code = std::make_shared<::arrow::StringBuilder>();
+ auto url = std::make_shared<::arrow::StringBuilder>();
+
+ std::vector> language_fields;
+ language_fields.emplace_back(
+ std::make_shared<::arrow::Field>("Code", ::arrow::utf8(), false));
+ language_fields.emplace_back(
+ std::make_shared<::arrow::Field>("Country", ::arrow::utf8()));
+
+ auto language_type = std::make_shared<::arrow::StructType>(language_fields);
+ std::vector> language_builders = {code, country};
+ auto language = std::make_shared<::arrow::StructBuilder>(
+ language_type, ::arrow::default_memory_pool(), std::move(language_builders));
+ auto language_list =
+ std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), language);
+
+ auto backward_list =
+ std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), backward);
+ auto forward_list =
+ std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), forward);
+ std::vector> links_fields;
+ links_fields.emplace_back(
+ std::make_shared<::arrow::Field>("Backward_Outer", backward_list->type(), false));
+ links_fields.emplace_back(
+ std::make_shared<::arrow::Field>("Forward_Outer", backward_list->type(), false));
+ auto links_type = std::make_shared<::arrow::StructType>(links_fields);
+ ::arrow::StructBuilder links(links_type, ::arrow::default_memory_pool(),
+ {forward_list, backward_list});
+
+ std::vector> name_fields;
+ name_fields.emplace_back(
+ std::make_shared<::arrow::Field>("Language_Outer", language_list->type(), false));
+ name_fields.emplace_back(std::make_shared<::arrow::Field>("Url", url->type()));
+
+ auto name_type = std::make_shared<::arrow::StructType>(name_fields);
+ std::vector> name_builders = {language_list,
+ url};
+ auto name = std::make_shared<::arrow::StructBuilder>(
+ name_type, ::arrow::default_memory_pool(), std::move(name_builders));
+ ::arrow::ListBuilder name_list(::arrow::default_memory_pool(), name);
+
+ // Row 1
+ docId->Append(10);
+ links.Append();
+ name_list.Append();
+
+ // 3 forward links
+ forward_list->Append();
+ forward->Append(20);
+ forward->Append(40);
+ forward->Append(60);
+ // No backward links
+ backward_list->Append();
+
+ // First name
+ name->Append();
+ language_list->Append();
+ language->Append();
+ code->Append("en-us");
+ country->Append("us");
+
+ language->Append();
+ code->Append("en");
+ country->AppendNull();
+
+ url->Append("http://A");
+
+ // Second name
+ name->Append();
+ language_list->Append();
+ url->Append("http://B");
+
+ // Third name
+ name->Append();
+ language_list->Append();
+ language->Append();
+ code->Append("en-gb");
+ country->Append("gb");
+ url->AppendNull();
+
+ // Row 2
+ docId->Append(20);
+ links.Append();
+ name_list.Append();
+
+ backward_list->Append();
+ backward->Append(10);
+ backward->Append(30);
+ forward_list->Append();
+ forward->Append(80);
+
+ name->Append();
+ language_list->Append();
+ url->Append("http://C");
+
+ std::shared_ptr<::arrow::Array> docIdArray, nameArray, linksArray;
+
+ ASSERT_OK(docId->Finish(&docIdArray));
+ ASSERT_OK(name_list.Finish(&nameArray));
+ ASSERT_OK(links.Finish(&linksArray));
+
+ std::vector> fields;
+ fields.emplace_back(
+ std::make_shared<::arrow::Field>("DocId", docIdArray->type(), false));
+ fields.emplace_back(std::make_shared<::arrow::Field>("Links", linksArray->type()));
+ fields.emplace_back(std::make_shared<::arrow::Field>("Name", nameArray->type(), false));
+
+ auto schema = std::make_shared<::arrow::Schema>(fields);
+
+ auto expected = ::arrow::Table::Make(schema, {docIdArray, linksArray, nameArray});
+
+ ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*expected, *out));
+}
+
TEST(TestArrowReadWrite, CoerceTimestamps) {
using ::arrow::ArrayFromVector;
using ::arrow::field;
@@ -1445,8 +1577,8 @@ TEST(TestArrowReadWrite, CoerceTimestampsAndSupportDeprecatedInt96) {
using ::arrow::Schema;
using ::arrow::Table;
using ::arrow::TimeUnit;
- using ::arrow::TimestampType;
using ::arrow::TimestampBuilder;
+ using ::arrow::TimestampType;
using ::arrow::default_memory_pool;
auto timestamp_type = std::make_shared(TimeUnit::NANO);
@@ -2161,6 +2293,7 @@ TEST_F(TestNestedSchemaRead, ReadIntoTableFull) {
std::shared_ptr table;
ASSERT_OK_NO_THROW(reader_->ReadTable(&table));
+
ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS);
ASSERT_EQ(table->num_columns(), 2);
ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 2);
@@ -2228,9 +2361,11 @@ TEST_F(TestNestedSchemaRead, ReadTablePartial) {
}
TEST_F(TestNestedSchemaRead, StructAndListTogetherUnsupported) {
+#if 0
ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::REPEATED));
std::shared_ptr table;
ASSERT_RAISES(NotImplemented, reader_->ReadTable(&table));
+#endif
}
TEST_P(TestNestedSchemaRead, DeepNestedSchemaRead) {
diff --git a/src/parquet/arrow/arrow-schema-test.cc b/src/parquet/arrow/arrow-schema-test.cc
index 5c16c044..cb2b8508 100644
--- a/src/parquet/arrow/arrow-schema-test.cc
+++ b/src/parquet/arrow/arrow-schema-test.cc
@@ -62,8 +62,8 @@ class TestConvertParquetSchema : public ::testing::Test {
for (int i = 0; i < expected_schema->num_fields(); ++i) {
auto lhs = result_schema_->field(i);
auto rhs = expected_schema->field(i);
- EXPECT_TRUE(lhs->Equals(rhs)) << i << " " << lhs->ToString()
- << " != " << rhs->ToString();
+ EXPECT_TRUE(lhs->Equals(rhs))
+ << i << " " << lhs->ToString() << " != " << rhs->ToString();
}
}
@@ -607,9 +607,10 @@ TEST_F(TestConvertParquetSchema, ParquetRepeatedNestedSchema) {
auto inner_group_type = std::make_shared<::arrow::StructType>(inner_group_fields);
auto outer_group_fields = {
std::make_shared("leaf2", INT32, true),
- std::make_shared("innerGroup", ::arrow::list(std::make_shared(
- "innerGroup", inner_group_type, false)),
- false)};
+ std::make_shared(
+ "innerGroup",
+ ::arrow::list(std::make_shared("innerGroup", inner_group_type, false)),
+ false)};
auto outer_group_type = std::make_shared<::arrow::StructType>(outer_group_fields);
arrow_fields.push_back(std::make_shared("leaf1", INT32, true));
diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc
new file mode 100644
index 00000000..d8dce5eb
--- /dev/null
+++ b/src/parquet/arrow/deserializer.cc
@@ -0,0 +1,2383 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "parquet/arrow/deserializer.h"
+
+#include
+#include
+
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
+#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/parallel.h"
+#include "parquet/arrow/schema.h"
+#include "parquet/file_reader.h"
+#include "parquet/types.h"
+
+namespace arrow {
+
+class Array;
+}
+
+using arrow::Array;
+using arrow::Status;
+using arrow::Table;
+
+using parquet::schema::GroupNode;
+using parquet::schema::Node;
+using parquet::schema::PrimitiveNode;
+
+namespace parquet {
+
+namespace arrow {
+
+constexpr int64_t kJulianToUnixEpochDays = 2440588LL;
+constexpr int64_t kMillisecondsInADay = 86400000LL;
+constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL;
+
+static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timestamp) {
+ int64_t days_since_epoch = impala_timestamp.value[2] - kJulianToUnixEpochDays;
+ int64_t nanoseconds = *(reinterpret_cast(&(impala_timestamp.value)));
+ return days_since_epoch * kNanosecondsInADay + nanoseconds;
+}
+
+// This is a "tagged" integer type for definition/repetition levels.
+// The tag gives us strong typing so that we can differentiate between
+// definition and repeition level arguments in the DeserializerBuilder
+// build function calls.
+template
+class Level {
+ public:
+ Level() : value_(0) {}
+
+ Level& operator++() {
+ ++value_;
+ return *this;
+ }
+
+ int16_t operator*() const { return value_; }
+
+ private:
+ int16_t value_;
+};
+
+struct Repetition;
+struct Definition;
+
+using RepetitionLevel = Level;
+using DefinitionLevel = Level;
+
+//
+// This is a utility class that is used by primitive node deserializer
+// nodes. It's similar to a std::list, but stores its data as a contiguous
+// array of memory because it's passed to calls to ReadBatch.
+template
+class PrimitiveDeserializerArray {
+ public:
+ PrimitiveDeserializerArray(::arrow::MemoryPool* pool, int64_t capacity)
+ : pool_(pool), capacity_(capacity), size_(0), index_(0) {}
+
+ ~PrimitiveDeserializerArray() {
+ pool_->Free(reinterpret_cast(data_), capacity_);
+ }
+
+ Status Initialize() {
+ uint8_t* out;
+ RETURN_NOT_OK(pool_->Allocate(sizeof(T) * capacity_, &out));
+ data_ = reinterpret_cast(out);
+ return Status::OK();
+ }
+
+ bool const Empty() const { return index_ >= size_; }
+
+ T Front() const { return data_[index_]; }
+
+ void PopFront() { ++index_; }
+
+ void Reset() {
+ index_ = 0;
+ size_ = 0;
+ }
+
+ ::arrow::MemoryPool* pool_;
+
+ T* data_;
+
+ int64_t capacity_;
+
+ int64_t size_;
+
+ int64_t index_;
+};
+
+//
+// Interface for implementations that are able to deserialize
+// Parquet primitive and group nodes. Successive calls to
+// Deserialize will cause this node to append values or nulls
+// to its builder.
+//
+class DeserializerNode {
+ public:
+ using LevelSet = std::set;
+
+ //
+ // \brief Deserializes the next set of values.
+ // \param[in] the current repeition level. When this is first called
+ // when deserializing a row, it should be zero. It may be
+ // non-zero when a list node is calling a child node
+ // during a repetition. In those cases, a child node may
+ // do nothing because it's next value doesn't repeat at
+ // the current repeition level.
+ // \param[out] this is used to find out what the next set of
+ // repetitions will be. This is used by list
+ // (aka - repeated) nodes in their Deserialize method
+ // to determine whether or not they need to repeat
+ // and call Deserialize for their child nodes
+ // again before popping back to their parent node.
+ // \return error status if there is no more data for this node
+ virtual Status Deserialize(RepetitionLevel current_repetition_level,
+ LevelSet& next_repetitions) = 0;
+
+ // \brief Skips deserialization. This is used by group and list
+ // nodes when they are nullable, in which they call this
+ // method on their child nodes so that they do not append
+ // any values to their builders.
+ // \param[out] this is used to find out what the next set of
+ // repetitions will be. This is used by list
+ // (aka - repeated) nodes in their Deserialize method
+ // to determine whether or not they need to repeat
+ // and call Deserialize for their child nodes
+ // again before popping back to their parent node.
+ // \return error status if there is no more data for this node
+ virtual Status Skip(LevelSet& next_repetitions) = 0;
+
+ // \brief returns the maximum definition level of all of this
+ // node's descendants. In the case of a primitive node
+ // it will be the next definition level in the file (if
+ // one exists) or zero otherwise. In the case of a group
+ // node, it will simply take the maximum of all of its
+ // child nodes.
+ // \param[out] the maximum definition level for this nodes and its
+ // descendants
+ // \return error status if there is no more data for this node
+ virtual Status CurrentMaxDefinitionLevel(int16_t& level) = 0;
+
+ // \brief returns the array buidler that was used by this node
+ // to append values and null in calls to Deserialize
+ // \return the array builder
+ virtual std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() = 0;
+
+ // \brief initializes this node for a new row group. This is used
+ // by primitive node deserializers to obtain a column reader
+ // for that row group
+ // \return status error if this node is a primitive node and the
+ // column index is out of bounds
+ virtual Status InitializeRowGroup(
+ std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) = 0;
+};
+
+//
+// The PrimitiveAppender class is used adapt the reading of parquet
+// values from a ColumnReader to calls to a builder's append method.
+// The reason this is needed is that the signature of the builder's
+// Append method varies by type. For example, byte arrays take
+// a pointer and a length whereas primitive types take just a value.
+// By using partial specialization for various types, the deserializer
+// code for primitive types does not need specialization but instead
+// delegates to these set of classes to handle it.
+//
+template
+class PrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, BuilderType& builder,
+ ParquetValueType value) {
+ using value_type = typename BuilderType::value_type;
+ return builder.Append(static_cast(value));
+ }
+};
+
+template
+class PrimitiveAppender<::parquet::ByteArray, BuilderType> {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, BuilderType& builder,
+ ParquetValueType value) {
+ return builder.Append(value.ptr, value.len);
+ }
+};
+
+template
+class PrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, BuilderType& builder,
+ ParquetValueType value) {
+ return builder.Append(value.ptr);
+ }
+};
+
+template <>
+class PrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder,
+ ParquetValueType value) {
+ ::arrow::Decimal128 converted;
+ RETURN_NOT_OK(::arrow::Decimal128::FromBigEndian(value.ptr, value.len, &converted));
+ return builder.Append(converted);
+ }
+};
+
+template <>
+class PrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder,
+ ParquetValueType value) {
+ ::arrow::Decimal128 converted;
+ RETURN_NOT_OK(::arrow::Decimal128::FromBigEndian(value.ptr, descriptor->type_length(),
+ &converted));
+ return builder.Append(converted);
+ }
+};
+
+template
+class PrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder,
+ ParquetValueType value) {
+ return builder.Append(::arrow::Decimal128(value));
+ }
+};
+
+template <>
+class PrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, ::arrow::TimestampBuilder& builder,
+ ParquetValueType value) {
+ return builder.Append(impala_timestamp_to_nanoseconds(value));
+ }
+};
+
+class NullPrimitiveAppender {
+ public:
+ template
+ Status Append(const ColumnDescriptor* descriptor, ::arrow::NullBuilder& builder,
+ ParquetValueType value) {
+ std::ostringstream stream;
+ stream << "Unexpected call to NullPrimitiveAppender::Append";
+ return Status::Invalid(stream.str());
+ }
+};
+
+template
+class PrimitiveAppender
+ : public NullPrimitiveAppender {};
+
+// These specializations avoid ambiguity with other ones where FLBA is
+// specified but the builder type isn't (ambiguous with the class right above)
+template <>
+class PrimitiveAppender : public NullPrimitiveAppender {};
+
+template <>
+class PrimitiveAppender : public NullPrimitiveAppender {
+};
+
+//
+// This is the base class for primitive types deserializers.
+// The ColumnReaderType should be an instantiation of
+// TypeColumnReader (see column_reader.h). The ArrayBuilderType
+// should be an instantiation of one of the Arrow builders
+// (e.g. - Int32Builder). This class is used for initializing
+// the column reader and appending values, whereas the sub-classes
+// handle reading of repetition and definition levels and implement
+// the DeserializerNode virtual functions.
+//
+template
+class PrimitiveDeserializerNode : public DeserializerNode {
+ public:
+ PrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index,
+ int64_t buffer_size,
+ std::shared_ptr const& builder,
+ ::arrow::MemoryPool* pool)
+ : descriptor_(descriptor),
+ column_index_(column_index),
+ values_(pool, buffer_size),
+ builder_(builder) {}
+
+ std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() { return builder_; }
+
+ // \brief initializes the column reader for this deserializer
+ // \param[in] the reader for the row group from which the column reader
+ // is obtained
+ // \return error status if the column index is out of bounds
+ Status InitializeRowGroup(
+ std::shared_ptr const& row_group_reader) final {
+ if (column_index_ < row_group_reader->metadata()->num_columns()) {
+ column_reader_ = std::static_pointer_cast(
+ row_group_reader->Column(column_index_));
+ } else {
+ std::ostringstream stream;
+ stream << "Invalid column index " << column_index_ << " for row group ";
+ return Status::Invalid(stream.str());
+ }
+ return Status::OK();
+ }
+
+ virtual Status Initialize() { return values_.Initialize(); }
+
+ protected:
+ // \brief appends the next value in this node's values array. It is
+ // assumed that the caller has checked to make sure that the
+ // values buffer is not empty
+ // \return error status if the call to the builder's Append function fails
+ Status Append() {
+ auto status = appender_.Append(descriptor_, *builder_, values_.Front());
+ values_.PopFront();
+ return status;
+ }
+
+ const ColumnDescriptor* const descriptor_;
+
+ int const column_index_;
+
+ PrimitiveDeserializerArray values_;
+
+ std::shared_ptr builder_;
+
+ PrimitiveAppender appender_;
+
+ std::shared_ptr column_reader_;
+};
+
+// This enum is used to differentiate between the three
+// different types of primitive deserializers
+enum PrimitiveDeserializerType {
+ // This means that the node in the tree has no ancestors (including
+ // itself) that are repeated or optional
+ REQUIRED,
+ // This means that the node in the tree has only ancestors
+ // (including itself) that are required or optional, and therefore
+ // doesn't need to store repetition levels but does need to store
+ // definition levels
+ OPTIONAL,
+ // This means that the node in the tree has at least one ancestor
+ // (including itself)
+ REPEATED
+};
+
+//
+// The TypedPrimitiveDeserializerNode does the heavy lifting of
+// reading repetition and definition levels and implementing
+// the DeserializerNode interface for primitive types. There are
+// three specializations based on whether or not the field
+// has definition levels or repetition levels (e.g. - a required
+// field at depth 1 doesn't have repeition or definition levels
+// and they are not stored in the parquet file). By using
+// specialization in this manner, we save space and also don't
+// have to do things like check if the max repetition or
+// max definition level is zero. Unfortunately, this pattern is not
+// yet being used in the column reader classes, so those checks
+// still happen in calls to ReadBatch.
+//
+template
+class TypedPrimitiveDeserializerNode;
+
+// Specialization for a field that has definition levels but no
+// repetition levels. For example, the following schemas would
+// create this type of node:
+//
+// message root {
+// optional int32 field;
+// }
+//
+// message root {
+// optional group contact {
+// required bytearray name (UTF8);
+// optional int32 phone_number;
+// }
+//
+// Both name and phone_number would be serialized with this type of
+// node because they have a max definition level of 1 and 2 respectively.
+//
+template
+class TypedPrimitiveDeserializerNode
+ : public PrimitiveDeserializerNode {
+ public:
+ TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor,
+ int const column_index, std::size_t buffer_size,
+ std::shared_ptr const& builder,
+ ::arrow::MemoryPool* pool)
+ : PrimitiveDeserializerNode(
+ descriptor, column_index, buffer_size, builder, pool),
+ definition_levels_(pool, buffer_size) {}
+
+ Status Initialize() final {
+ RETURN_NOT_OK(definition_levels_.Initialize());
+ return PrimitiveDeserializerNode::Initialize();
+ }
+
+ Status Skip(DeserializerNode::LevelSet& next_repetitions) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to skip data";
+ return Status::IOError(stream.str());
+ }
+ if (definition_levels_.Front() == this->descriptor_->max_definition_level()) {
+ this->values_.PopFront();
+ }
+ definition_levels_.PopFront();
+ return Status::OK();
+ }
+
+ Status CurrentMaxDefinitionLevel(int16_t& level) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to read current max definition level";
+ return Status::IOError(stream.str());
+ }
+ level = definition_levels_.Front();
+ return Status::OK();
+ }
+
+ Status Deserialize(RepetitionLevel current_repetition_level,
+ DeserializerNode::LevelSet& next_repetitions) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to deserialize value";
+ return Status::IOError(stream.str());
+ }
+ if (definition_levels_.Front() == this->descriptor_->max_definition_level()) {
+ // The value is not null
+ RETURN_NOT_OK(this->Append());
+ } else {
+ RETURN_NOT_OK(this->builder_->AppendNull());
+ }
+ definition_levels_.PopFront();
+ return Status::OK();
+ }
+
+ private:
+ bool Fill() {
+ if (!definition_levels_.Empty()) {
+ return true;
+ }
+ definition_levels_.Reset();
+ this->values_.Reset();
+ definition_levels_.size_ = this->column_reader_->ReadBatch(
+ definition_levels_.capacity_, definition_levels_.data_, 0, this->values_.data_,
+ &this->values_.size_);
+ return !definition_levels_.Empty();
+ }
+
+ PrimitiveDeserializerArray definition_levels_;
+};
+
+//
+// Specialization for a field that has both definition and repetition levels.
+// For example, the following schema would create this type of node:
+//
+// message root {
+// required group list_outer (LIST) {
+// repeated group list_inner {
+// required int32 value;
+// }
+// }
+// }
+template
+class TypedPrimitiveDeserializerNode
+ : public PrimitiveDeserializerNode {
+ public:
+ TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor,
+ int column_index, std::size_t buffer_size,
+ std::shared_ptr const& builder,
+ ::arrow::MemoryPool* pool)
+ : PrimitiveDeserializerNode(
+ descriptor, column_index, buffer_size, builder, pool),
+ definition_levels_(pool, buffer_size),
+ repetition_levels_(pool, buffer_size) {}
+
+ Status Initialize() final {
+ RETURN_NOT_OK(definition_levels_.Initialize());
+ RETURN_NOT_OK(repetition_levels_.Initialize());
+ return PrimitiveDeserializerNode::Initialize();
+ }
+
+ Status CurrentMaxDefinitionLevel(int16_t& level) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to determine current max definition level";
+ return Status::IOError(stream.str());
+ }
+ level = definition_levels_.Front();
+ return Status::OK();
+ }
+
+ Status Skip(DeserializerNode::LevelSet& next_repetitions) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to determine skip data";
+ return Status::IOError(stream.str());
+ }
+ if (definition_levels_.Front() == this->descriptor_->max_definition_level()) {
+ this->values_.PopFront();
+ }
+ definition_levels_.PopFront();
+ AddRepetitions(next_repetitions);
+ return Status::OK();
+ }
+
+ Status Deserialize(RepetitionLevel current_repetition_level,
+ DeserializerNode::LevelSet& next_repetitions) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to fill values when deserializing";
+ return Status::IOError(stream.str());
+ }
+ // See Dremel paper: second entry for Links.Backward would
+ // apply in this case and return
+ if (repetition_levels_.Front() != *current_repetition_level) {
+ return Status::OK();
+ }
+
+ if (definition_levels_.Front() == this->descriptor_->max_definition_level()) {
+ // The value is not null
+ RETURN_NOT_OK(this->Append());
+ } else {
+ // Insert a null
+ RETURN_NOT_OK(this->builder_->AppendNull());
+ }
+ definition_levels_.PopFront();
+ AddRepetitions(next_repetitions);
+ return Status::OK();
+ }
+
+ private:
+ // This is factored into its own method because it's used in both
+ // Skip and Deserialize
+ void AddRepetitions(DeserializerNode::LevelSet& next_repetitions) {
+ repetition_levels_.PopFront();
+ if (Fill()) {
+ next_repetitions.insert(repetition_levels_.Front());
+ }
+ }
+
+ bool Fill() {
+ if (!repetition_levels_.Empty()) {
+ return true;
+ }
+ definition_levels_.Reset();
+ repetition_levels_.Reset();
+ this->values_.Reset();
+ auto levels_read = this->column_reader_->ReadBatch(
+ definition_levels_.capacity_, definition_levels_.data_, repetition_levels_.data_,
+ this->values_.data_, &this->values_.size_);
+ definition_levels_.size_ = levels_read;
+ repetition_levels_.size_ = levels_read;
+
+ return !definition_levels_.Empty();
+ }
+
+ PrimitiveDeserializerArray definition_levels_;
+
+ PrimitiveDeserializerArray repetition_levels_;
+};
+
+//
+// Specialization for a field that has neither definition nor repetition levels.
+// For example, all of the primitive fields in the following schema
+// would create this type of node:
+//
+// message root {
+// required group contact {
+// required bytearray name (UTF8);
+// required group address {
+// required bytearray street (UTF8);
+// required int32 number;
+// required bytearray city (UTF8);
+// required int8 zip;
+// }
+// }
+// }
+template
+class TypedPrimitiveDeserializerNode
+ : public PrimitiveDeserializerNode {
+ public:
+ TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor,
+ int column_index, std::size_t buffer_size,
+ std::shared_ptr const& builder,
+ ::arrow::MemoryPool* pool)
+ : PrimitiveDeserializerNode(
+ descriptor, column_index, buffer_size, builder, pool) {}
+
+ Status CurrentMaxDefinitionLevel(int16_t& level) final {
+ level = this->descriptor_->max_definition_level();
+ return Status::OK();
+ }
+
+ Status Skip(DeserializerNode::LevelSet& next_repetitions) final {
+ std::ostringstream stream;
+ stream << "Call to Skip made when it shouldn't have";
+ return Status::Invalid(stream.str());
+ }
+
+ Status Deserialize(RepetitionLevel current_repetition_level,
+ DeserializerNode::LevelSet& next_repetitions) final {
+ if (!Fill()) {
+ std::ostringstream stream;
+ stream << "Unable to load more values in call to Deserialize";
+ return Status::IOError(stream.str());
+ }
+ return this->Append();
+ }
+
+ private:
+ bool Fill() {
+ if (!this->values_.Empty()) {
+ return true;
+ }
+ this->values_.Reset();
+ // Ignore the return value here
+ this->column_reader_->ReadBatch(this->values_.capacity_, 0, 0, this->values_.data_,
+ &this->values_.size_);
+ return !this->values_.Empty();
+ }
+};
+
+//
+// Deserializer for lists that are required. The Parquet spec on list
+// types requires that the schema for a required list looks likes this:
+//
+// required group list_outer (LIST) {
+// repeated group list_inner {
+// (required|optional) element ...
+// }
+// }
+//
+// The reason for the outer level of nesting is that without it
+// you wouldn't be able to differentiate a list of a single null
+// from an empty list. By having this extra level, an empty list
+// is indicated when the max definition level of the list_outer's
+// descendants are equal to list_outer's max definition level, and
+// a list of null would be indicated by a definition level of
+// list_inner's descendants that is equal to list_inner's max
+// definition level.
+//
+// The constructor of this class takes the max_definition_level of
+// list_outer and the max_repetition_level of list_inner
+class ListDeserializerNode : public DeserializerNode {
+ public:
+ ListDeserializerNode(std::unique_ptr&& node_reader,
+ ::arrow::MemoryPool* pool, RepetitionLevel max_repetition_level,
+ DefinitionLevel max_definition_level)
+ : node_reader_(std::move(node_reader)),
+ max_repetition_level_(max_repetition_level),
+ max_definition_level_(max_definition_level),
+ builder_(std::make_shared<::arrow::ListBuilder>(
+ pool, node_reader_->GetArrayBuilder())) {}
+
+ Status CurrentMaxDefinitionLevel(int16_t& level) final {
+ return node_reader_->CurrentMaxDefinitionLevel(level);
+ }
+
+ Status Skip(LevelSet& next_repetitions) final {
+ return node_reader_->Skip(next_repetitions);
+ }
+
+ Status Deserialize(RepetitionLevel current_repetition_level,
+ LevelSet& next_repetitions) {
+ RETURN_NOT_OK(builder_->Append());
+ int16_t current_max_definition_level;
+ RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level));
+ if (current_max_definition_level == *max_definition_level_) {
+ // This is just an empty list
+ return node_reader_->Skip(next_repetitions);
+ } else {
+ // Deserialize the first one
+ RETURN_NOT_OK(
+ node_reader_->Deserialize(current_repetition_level, next_repetitions));
+ // Deserialize the repetitions
+ return DeserializeRepetitions(next_repetitions);
+ }
+ }
+
+ std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; }
+
+ Status InitializeRowGroup(std::shared_ptr const& row_group_reader) {
+ return node_reader_->InitializeRowGroup(row_group_reader);
+ }
+
+ protected:
+ // \brief keeps appending elements to the current list as long as
+ // the descendants of this node have repetitions at this list's
+ // repetition level
+ // \param[in] the set of next repetitions from the last call to
+ // children's Deserialize functions.
+ // \return status error if calls to child node Deserialize functions
+ // produce an error
+ Status DeserializeRepetitions(LevelSet& next_repetitions) {
+ bool repetition_at_this_level = true;
+ while (!next_repetitions.empty() && repetition_at_this_level) {
+ // next_repetitions is a sorted set, so end
+ // is the highest value. If it's lower than our
+ // max_repetition_level_, we just return from this
+ // function. Otherwise, we erase highest, descend again,
+ // and then re-check if we repeated at this level
+ auto highest = next_repetitions.end();
+ --highest;
+ repetition_at_this_level = (*highest == *max_repetition_level_);
+ if (repetition_at_this_level) {
+ next_repetitions.erase(highest);
+ RETURN_NOT_OK(node_reader_->Deserialize(max_repetition_level_, next_repetitions));
+ }
+ }
+ return Status::OK();
+ }
+
+ std::unique_ptr node_reader_;
+
+ RepetitionLevel max_repetition_level_;
+
+ DefinitionLevel max_definition_level_;
+
+ std::shared_ptr<::arrow::ListBuilder> builder_;
+};
+
+//
+// Deserializer for lists that are optional. The Parquet spec on list
+// types requires that the schema for an optional list looks likes this:
+//
+// optional group list_outer (LIST) {
+// repeated group list_inner {
+// (required|optional) element ...
+// }
+// }
+//
+// When deserializing, if the max definition level of this node's
+// children is less than list_outer's max definition level, then
+// we append a null. If it's equal to it, then we create an empty
+// list. Otherwise, we create a non-empty list.
+//
+// The constructor of this class takes the max_definition_level of
+// list_outer and the max_repetition_level of list_inner
+class NullableListDeserializerNode : public ListDeserializerNode {
+ public:
+ NullableListDeserializerNode(std::unique_ptr&& node_reader,
+ ::arrow::MemoryPool* pool,
+ RepetitionLevel max_repetition_level,
+ DefinitionLevel max_definition_level)
+ : ListDeserializerNode(std::forward>(node_reader),
+ pool, max_repetition_level, max_definition_level) {}
+
+ Status Deserialize(RepetitionLevel current_repetition_level,
+ LevelSet& next_repetitions) final {
+ int16_t current_max_definition_level;
+ RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level));
+ if (current_max_definition_level < *max_definition_level_) {
+ // It's null
+ RETURN_NOT_OK(builder_->AppendNull());
+ RETURN_NOT_OK(node_reader_->Skip(next_repetitions));
+ } else if (current_max_definition_level == *max_definition_level_) {
+ // Empty list
+ RETURN_NOT_OK(builder_->Append());
+ RETURN_NOT_OK(node_reader_->Skip(next_repetitions));
+ } else {
+ RETURN_NOT_OK(builder_->Append());
+ RETURN_NOT_OK(
+ node_reader_->Deserialize(current_repetition_level, next_repetitions));
+ RETURN_NOT_OK(DeserializeRepetitions(next_repetitions));
+ }
+ return Status::OK();
+ }
+};
+
+//
+// Serializer for a group (aka struct) node. The schema for such a node
+// would look like this;
+//
+// (repeated|required) group pair {
+// required int32 key;
+// required bytearray value (UTF8);
+// }
+//
+// This has an Optional template parameter so that we can remove unnecessary checks
+// for required nodes at compile time.
+//
+template
+class NestedDeserializerNode : public DeserializerNode {
+ public:
+ NestedDeserializerNode(DefinitionLevel max_definition_level,
+ std::vector>&& children,
+ std::vector>& fields,
+ ::arrow::MemoryPool* pool)
+ : children_(std::move(children)), max_definition_level_(max_definition_level) {
+ std::vector> child_builders;
+ for (auto const& child : children_) {
+ child_builders.emplace_back(child->GetArrayBuilder());
+ }
+ auto arrow_type = std::make_shared<::arrow::StructType>(fields);
+ builder_ = std::make_shared<::arrow::StructBuilder>(arrow_type, pool,
+ std::move(child_builders));
+ }
+
+ Status Skip(LevelSet& next_repetitions) {
+ for (auto& child : children_) {
+ RETURN_NOT_OK(child->Skip(next_repetitions));
+ }
+ return Status::OK();
+ }
+
+ Status CurrentMaxDefinitionLevel(int16_t& max) final {
+ max = 0;
+ for (auto& child : children_) {
+ int16_t current;
+ RETURN_NOT_OK(child->CurrentMaxDefinitionLevel(current));
+ if (current > max) {
+ max = current;
+ }
+ }
+ return Status::OK();
+ }
+
+ Status Deserialize(RepetitionLevel current_repetition_level,
+ LevelSet& next_repetitions) final {
+ // TODO: detect this at tree construction time. If all of
+ // the fields inside this nested node are required, then
+ // there is no need to do this check.
+ auto has_data = !Optional;
+ if (Optional) {
+ int16_t current_max_definition_level;
+ RETURN_NOT_OK(CurrentMaxDefinitionLevel(current_max_definition_level));
+ has_data = current_max_definition_level >= *max_definition_level_;
+ }
+ if (has_data) {
+ RETURN_NOT_OK(builder_->Append());
+ } else {
+ RETURN_NOT_OK(builder_->AppendNull());
+ }
+ // For structs, we call Deserialize even when they don't have data.
+ // This ensures that the length of the child arrays match
+ // the length of the struct bitmap.
+ for (auto& child : children_) {
+ RETURN_NOT_OK(child->Deserialize(current_repetition_level, next_repetitions));
+ }
+ return Status::OK();
+ }
+
+ std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; }
+
+ Status InitializeRowGroup(
+ std::shared_ptr const& row_group_reader) final {
+ for (auto& child : children_) {
+ RETURN_NOT_OK(child->InitializeRowGroup(row_group_reader));
+ }
+ return Status::OK();
+ }
+
+ private:
+ std::vector> children_;
+
+ DefinitionLevel max_definition_level_;
+
+ std::shared_ptr<::arrow::StructBuilder> builder_;
+};
+
+// The following set of classes are used to drive the
+// iteration over the input data in a particular manner.
+//
+// TODO: except for the batched array serializer, it might
+// be possible to collapse these classes through the
+// use of iterators.
+ArrayDeserializer::ArrayDeserializer(std::unique_ptr&& node)
+ : node_(std::move(node)) {}
+
+ArrayDeserializer::~ArrayDeserializer() {}
+
+// This class will deserialize a single array from the entire
+// file. The node that is passed to the constructor may
+// contain multiple primitive nodes as its descendants.
+class FileArrayDeserializer : public ArrayDeserializer {
+ public:
+ FileArrayDeserializer(std::unique_ptr&& node,
+ std::shared_ptr const& file_reader)
+ : ArrayDeserializer(std::forward>(node)),
+ file_reader_(file_reader) {}
+
+ ~FileArrayDeserializer() {}
+
+ ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final {
+ DeserializerNode::LevelSet repetitions;
+ auto const num_row_groups = file_reader_->metadata()->num_row_groups();
+ for (int ii = 0; ii < num_row_groups; ++ii) {
+ auto row_group_reader = file_reader_->RowGroup(ii);
+ RETURN_NOT_OK(node_->InitializeRowGroup(row_group_reader));
+ auto const num_rows = row_group_reader->metadata()->num_rows();
+ for (int jj = 0; jj < num_rows; ++jj) {
+ RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions));
+ repetitions.clear();
+ }
+ }
+ return node_->GetArrayBuilder()->Finish(&array);
+ }
+
+ private:
+ std::shared_ptr file_reader_;
+};
+
+ArrayBatchedDeserializer::~ArrayBatchedDeserializer() {}
+
+// This class will deserialize a single array from the entire
+// file in batches. The node that is passed to the constructor may
+// contain multiple primitive nodes as its descendants.
+class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer {
+ public:
+ FileArrayBatchedDeserializer(std::unique_ptr&& node,
+ std::shared_ptr const& file_reader)
+ : node_(std::move(node)),
+ file_reader_(file_reader),
+ current_row_group_(file_reader->RowGroup(0)),
+ current_row_group_index_(0),
+ current_row_(0) {}
+
+ ~FileArrayBatchedDeserializer() {}
+
+ Status Initialize() { return node_->InitializeRowGroup(current_row_group_); }
+
+ ::arrow::Status DeserializeBatch(int64_t num_records,
+ std::shared_ptr<::arrow::Array>& array) final {
+ DeserializerNode::LevelSet repetitions;
+ int64_t records_read = 0;
+ auto const num_row_groups = file_reader_->metadata()->num_row_groups();
+ while (records_read < num_records && current_row_group_index_ < num_row_groups) {
+ auto const num_rows = current_row_group_->metadata()->num_rows();
+ for (; current_row_ < num_rows && records_read < num_records;
+ ++current_row_, ++records_read) {
+ RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions));
+ repetitions.clear();
+ }
+ if (current_row_ >= num_rows) {
+ ++current_row_group_index_;
+ if (current_row_group_index_ < num_row_groups) {
+ current_row_group_ = file_reader_->RowGroup(current_row_group_index_);
+ RETURN_NOT_OK(node_->InitializeRowGroup(current_row_group_));
+ current_row_ = 0;
+ }
+ }
+ }
+ return node_->GetArrayBuilder()->Finish(&array);
+ }
+
+ private:
+ std::unique_ptr node_;
+
+ std::shared_ptr<::parquet::ParquetFileReader> file_reader_;
+
+ std::shared_ptr<::parquet::RowGroupReader> current_row_group_;
+
+ int current_row_group_index_;
+
+ int current_row_;
+};
+
+// This class will deserialize a single array from a single
+// row group for a single node. The DeserializeBuilder only
+// constructs this for a node that has at most one primitive
+// descendant, but it could just as well be used for nodes that
+// contain multiple primitive descendants.
+class ColumnChunkDeserializer : public ArrayDeserializer {
+ public:
+ ColumnChunkDeserializer(
+ std::unique_ptr&& node,
+ std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader)
+ : ArrayDeserializer(std::move(node)), row_group_reader_(row_group_reader) {}
+
+ Status Initialize() { return node_->InitializeRowGroup(row_group_reader_); }
+
+ ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final {
+ DeserializerNode::LevelSet repetitions;
+ auto const num_rows = row_group_reader_->metadata()->num_rows();
+ for (int ii = 0; ii < num_rows; ++ii) {
+ RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions));
+ repetitions.clear();
+ }
+ return node_->GetArrayBuilder()->Finish(&array);
+ }
+
+ private:
+ std::shared_ptr<::parquet::RowGroupReader> row_group_reader_;
+};
+
+TableDeserializer::TableDeserializer(std::shared_ptr<::arrow::Schema> const& schema)
+ : schema_(schema) {}
+
+TableDeserializer::~TableDeserializer() {}
+
+//
+// A table deserializer that does it's deserialization
+// in a single thread. This is in contrast to
+// ParallelTableDeserializer which deserializes each
+// top level array in the schema in a separate thread.
+class SerialTableDeserializer : public TableDeserializer {
+ protected:
+ SerialTableDeserializer(std::vector>&& nodes,
+ std::shared_ptr<::arrow::Schema> const& schema)
+ : TableDeserializer(schema), nodes_(std::move(nodes)) {}
+
+ // \brief Convenience method which iterates over the nodes in this
+ // deserializer and constructs a table from them, using
+ // the schema passed to this object's constructor
+ // \return error status if there is an issue constructing the table
+ ::arrow::Status MakeTable(std::shared_ptr<::arrow::Table>& result) {
+ std::vector> arrays;
+ for (auto const& node : nodes_) {
+ std::shared_ptr<::arrow::Array> array;
+ RETURN_NOT_OK(node->GetArrayBuilder()->Finish(&array));
+ arrays.emplace_back(std::move(array));
+ }
+ result = Table::Make(schema_, arrays);
+ return Status::OK();
+ }
+
+ // \brief Convenience method which iterates over all rows in the rows
+ // group and deserializes each node
+ // \param[in] the row group reader
+ // \return status error if there is a problem deserializing the row group
+ Status DeserializeRowGroup(
+ std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) {
+ for (auto& node : nodes_) {
+ RETURN_NOT_OK(node->InitializeRowGroup(row_group_reader));
+ }
+ DeserializerNode::LevelSet repetitions;
+ auto const num_rows = row_group_reader->metadata()->num_rows();
+ for (int ii = 0; ii < num_rows; ++ii) {
+ for (auto& node : nodes_) {
+ RETURN_NOT_OK(node->Deserialize(RepetitionLevel(), repetitions));
+ repetitions.clear();
+ }
+ }
+ return Status::OK();
+ }
+
+ private:
+ std::vector> nodes_;
+};
+
+//
+// This is a deserializer that can deserialize columns in
+// parallel using multiple threads. It takes a vector of
+// ArrayDeserializer's and calls DeserializeArray separately
+// in each thread. Each of the ArrayDeserializer's will either
+// have a separate row group reader, or will create row group
+// readers as it iterates through the file. This ensures that
+// there is no sharing of data across the threads.
+//
+class ParallelTableDeserializer : public TableDeserializer {
+ public:
+ ParallelTableDeserializer(
+ std::shared_ptr<::arrow::Schema> const& schema,
+ std::vector>&& array_deserializers,
+ int num_threads)
+ : TableDeserializer(schema),
+ num_threads_(num_threads),
+ array_deserializers_(std::move(array_deserializers)),
+ arrays_(array_deserializers_.size()) {}
+
+ ~ParallelTableDeserializer() {}
+
+ ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final {
+ // For now, the parallelism is done a column basis. Other possible
+ // approaches are deserializing row groups or column chunks in parallel.
+ RETURN_NOT_OK(::arrow::ParallelFor(
+ num_threads_, static_cast(array_deserializers_.size()), [this](int index) {
+ auto const array_index = static_cast(index);
+ return array_deserializers_[array_index]->DeserializeArray(
+ arrays_[array_index]);
+ }));
+ table = Table::Make(schema_, arrays_);
+ return Status::OK();
+ }
+
+ private:
+ int num_threads_;
+
+ std::vector> array_deserializers_;
+
+ // Used to hold the output.
+ std::vector> arrays_;
+};
+
+// This class will deserialize all of the specified nodes from a single row group.
+class RowGroupDeserializer : public SerialTableDeserializer {
+ public:
+ RowGroupDeserializer(std::vector>&& nodes,
+ std::shared_ptr<::arrow::Schema> const& schema,
+ std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader)
+ : SerialTableDeserializer(
+ std::forward>>(nodes), schema),
+ row_group_reader_(row_group_reader) {}
+
+ ~RowGroupDeserializer() {}
+
+ ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final {
+ DeserializeRowGroup(row_group_reader_);
+ return MakeTable(table);
+ }
+
+ private:
+ std::shared_ptr<::parquet::RowGroupReader> row_group_reader_;
+};
+
+// This class wil deserialize all of the specified nodes from a Parquet file
+class FileTableDeserializer : public SerialTableDeserializer {
+ public:
+ FileTableDeserializer(std::vector>&& nodes,
+ std::shared_ptr<::arrow::Schema> const& schema,
+ std::shared_ptr<::parquet::ParquetFileReader> const& file_reader)
+ : SerialTableDeserializer(
+ std::forward>>(nodes), schema),
+ file_reader_(file_reader) {}
+
+ ~FileTableDeserializer() {}
+
+ ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) {
+ auto num_row_groups = file_reader_->metadata()->num_row_groups();
+ for (int ii = 0; ii < num_row_groups; ++ii) {
+ DeserializeRowGroup(file_reader_->RowGroup(ii));
+ }
+ return MakeTable(table);
+ }
+
+ private:
+ std::shared_ptr<::parquet::ParquetFileReader> file_reader_;
+};
+
+// This class does the main work of constructing deserializer
+// nodes from Parquet schema nodes.
+class DeserializerBuilder::Impl {
+ public:
+ Impl(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader,
+ ::arrow::MemoryPool* pool, int64_t buffer_size);
+
+ ::arrow::Status BuildSchemaNodeDeserializer(int schema_index,
+ std::unique_ptr& result);
+
+ ::arrow::Status BuildColumnDeserializer(int column_index,
+ std::unique_ptr& result);
+
+ ::arrow::Status BuildColumnChunkDeserializer(
+ int column_index, int row_group_index, std::unique_ptr& result);
+
+ ::arrow::Status BuildRowGroupDeserializer(int row_group_index,
+ const std::vector& column_indices,
+ int num_threads,
+ std::unique_ptr& result);
+
+ ::arrow::Status BuildFileDeserializer(const std::vector& column_indices,
+ int num_threads,
+ std::unique_ptr& result);
+
+ ::arrow::Status BuildArrayBatchedDeserializer(
+ int column_index, std::unique_ptr& result);
+
+ private:
+ std::vector const FillColumnIndices() const {
+ auto const num_columns = file_reader_->metadata()->num_columns();
+ std::vector indices(num_columns);
+ for (int ii = 0; ii < num_columns; ++ii) {
+ indices[ii] = ii;
+ }
+ return indices;
+ }
+
+ // \brief calls BuildNode for each node in the parquet file's
+ // schema
+ // \param[in] indices that are used to filter indicate which
+ // primitive columns to include. The indices correspond
+ // to the index you would obtain from calling
+ // SchemaDescriptor::Column
+ // \param[out] the deserializer nodes
+ // \param[in] if specified, the nodes will appear in the result
+ // list based on the order in which they appeared in
+ // column_indices. For example, if the Parquet schema has two
+ // primitive nodes and the column_indices given is {1, 0},
+ // then the resulting table will have the fields reversed
+ // in the Arrow schema.
+ // \return error status if all of the column indices were out of bounds
+ // or any of the converted/physical type specifications in the
+ // Parquet schema are unsupported
+ Status BuildNodes(std::vector const& column_indices,
+ std::vector>& nodes,
+ bool in_index_order = true);
+
+ // \brief builds a deserializer node based on the Parquet node.
+ // \param[in] the parquet node
+ // \param[in] indices that are used to filter indicate which
+ // primitive columns to include. The indices correspond
+ // to the index you would obtain from calling
+ // SchemaDescriptor::Column
+ // \param[in] these indicate which indices were used in building
+ // this node. This is used to re-order the arrays in
+ // the schema of the deserialized Table for that it
+ // matches the order in which the indices were specified.
+ // \param[out] the deserializer node
+ // \param[in] repetition levels seen so far. Since this function is
+ // called recursively for group nodes, the value is
+ // incremented as the tree is traversed. Not specifying
+ // this value in the call should only be done when the
+ // node is the root node.
+ // \param[in] definition levels seen so far. Used in the same manner
+ // as the repetition levels.
+ // \return error status if all of the column indices were out of bounds
+ // or any of the converted/physical type specifications in the
+ // Parquet schema are unsupported
+ Status BuildNode(std::shared_ptr const& node,
+ std::vector const& indices, std::vector& indices_seen,
+ std::unique_ptr& result,
+ RepetitionLevel repetition_levels = RepetitionLevel(),
+ DefinitionLevel definition_levels = DefinitionLevel());
+
+ // \brief builds a primitive deserializer node based on the Parquet node.
+ // \param[in] the parquet node, which must point to a PrimitiveNode
+ // \param[in] indices that are used to filter indicate which
+ // primitive columns to include. The indices correspond
+ // to the index you would obtain from calling
+ // SchemaDescriptor::Column
+ // \param[in] these indicate which indices were used in building
+ // this node. This is used to re-order the arrays in
+ // the schema of the deserialized Table for that it
+ // matches the order in which the indices were specified.
+ // \param[out] the deserializer node
+ // \param[in] repetition levels seen so far. Since this function is
+ // called recursively for group nodes, the value is
+ // incremented as the tree is traversed. Not specifying
+ // this value in the call should only be done when the
+ // node is the root node.
+ // \param[in] definition levels seen so far. Used in the same manner
+ // as the repetition levels.
+ // \return error status if all of the column indices were out of bounds
+ // or any of the converted/physical type specifications in the
+ // Parquet schema are unsupported
+ Status BuildPrimitiveNode(std::shared_ptr const& node,
+ RepetitionLevel repetition_levels,
+ DefinitionLevel definition_levels,
+ std::vector const& indices,
+ std::vector& indices_seen,
+ std::unique_ptr& result);
+
+ // \brief builds a group deserializer node based on the Parquet node.
+ // \param[in] the parquet node, which must point to a GroupNode
+ // \param[in] indices that are used to filter indicate which
+ // primitive columns to include. The indices correspond
+ // to the index you would obtain from calling
+ // SchemaDescriptor::Column
+ // \param[in] these indicate which indices were used in building
+ // this node. This is used to re-order the arrays in
+ // the schema of the deserialized Table for that it
+ // matches the order in which the indices were specified.
+ // \param[out] the deserializer node
+ // \param[in] repetition levels seen so far. Since this function is
+ // called recursively for group nodes, the value is
+ // incremented as the tree is traversed. Not specifying
+ // this value in the call should only be done when the
+ // node is the root node.
+ // \param[in] definition levels seen so far. Used in the same manner
+ // as the repetition levels.
+ // \return error status if all of the column indices were out of bounds
+ // or any of the converted/physical type specifications in the
+ // Parquet schema are unsupported
+ Status BuildGroupNode(std::shared_ptr const& node,
+ RepetitionLevel repetition_levels,
+ DefinitionLevel definition_levels,
+ std::vector const& indices, std::vector& indices_seen,
+ std::unique_ptr& result);
+
+ // \brief builds a list deserializer node based on the Parquet node.
+ // \param[in] the parquet node, which must point to a GroupNode which
+ // has a LIST annotation.
+ // \param[in] indices that are used to filter indicate which
+ // primitive columns to include. The indices correspond
+ // to the index you would obtain from calling
+ // SchemaDescriptor::Column
+ // \param[in] these indicate which indices were used in building
+ // this node. This is used to re-order the arrays in
+ // the schema of the deserialized Table for that it
+ // matches the order in which the indices were specified.
+ // \param[out] the deserializer node
+ // \param[in] repetition levels seen so far. Since this function is
+ // called recursively for group nodes, the value is
+ // incremented as the tree is traversed. Not specifying
+ // this value in the call should only be done when the
+ // node is the root node.
+ // \param[in] definition levels seen so far. Used in the same manner
+ // as the repetition levels.
+ // \return error status if all of the column indices were out of bounds
+ // or any of the converted/physical type specifications in the
+ // Parquet schema are unsupported
+ Status BuildListNode(std::shared_ptr const& node,
+ RepetitionLevel repetition_levels,
+ DefinitionLevel definition_levels, std::vector const& indices,
+ std::vector& indices_seen,
+ std::unique_ptr& result);
+
+ // \brief builds a typed primitive node. This is the main entry point
+ // for a number of SFINAE based calls.
+ // \param[in] the parquet node, which must point to a PrimitiveNode
+ // \param[in] the column descriptor, which is used in error
+ // messages to report what node had an unsupported type
+ // \parma[in] the index of the column, which is stored in the node
+ // and then later used to lookup the column reader from
+ // the row group reader
+ // \param[out] the deserializer node
+ // \return error status if any of the converted/physical type
+ // specifications in the Parquet schema are unsupported
+ template
+ Status MakeTypedPrimitiveDeserializerNode(
+ std::shared_ptr const& node,
+ const ColumnDescriptor* const column_descriptor, int column_index,
+ std::unique_ptr& result) {
+ auto primitive_node = std::static_pointer_cast(node);
+ switch (primitive_node->physical_type()) {
+ case Type::BOOLEAN:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::INT32:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::INT64:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::INT96:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::FLOAT:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::DOUBLE:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::BYTE_ARRAY:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ case Type::FIXED_LEN_BYTE_ARRAY:
+ return MakeLogicalPrimitiveDeserializerNode(
+ primitive_node, column_descriptor, column_index, result);
+ default:
+ std::ostringstream stream;
+ stream << "Unrecognized type when creating a primitive node reader";
+ return Status::Invalid(stream.str());
+ }
+ }
+
+ // \brief builds a typed primitive node, taking into account it's
+ // logical (aka converted) type.
+ // \param[in] the parquet node, which must point to a PrimitiveNode
+ // \param[in] the column descriptor, which is used in error
+ // messages to report what node had an unsupported type
+ // \parma[in] the index of the column, which is stored in the node
+ // and then later used to lookup the column reader from
+ // the row group reader
+ // \param[out] the deserializer node
+ // \return error status if any of the converted/physical type
+ // specifications in the Parquet schema are unsupported
+ template
+ Status MakeLogicalPrimitiveDeserializerNode(
+ std::shared_ptr const& node,
+ const ColumnDescriptor* const column_descriptor, int column_index,
+ std::unique_ptr& result) {
+ switch (node->logical_type()) {
+ case parquet::LogicalType::DATE:
+ return MakeDateDeserializerNode(
+ column_descriptor, column_index, result);
+ case parquet::LogicalType::TIME_MILLIS:
+ return MakeTimeMillisDeserializerNode(
+ column_descriptor, column_index, result);
+ case parquet::LogicalType::TIME_MICROS:
+ return MakeTimeMicrosDeserializerNode(
+ column_descriptor, column_index, result);
+ case parquet::LogicalType::TIMESTAMP_MILLIS:
+ return MakeTimestampDeserializerNode(
+ column_descriptor, column_index, ::arrow::TimeUnit::MILLI, result);
+ case parquet::LogicalType::TIMESTAMP_MICROS:
+ return MakeTimestampDeserializerNode(
+ column_descriptor, column_index, ::arrow::TimeUnit::MICRO, result);
+ case parquet::LogicalType::DECIMAL:
+ return MakeDecimalDeserializerNode(
+ column_descriptor, column_index, result);
+ case parquet::LogicalType::UINT_8:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::UINT_16:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::UINT_32:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::UINT_64:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::INT_8:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::INT_16:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::INT_32:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::INT_64:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case parquet::LogicalType::UTF8:
+ return MakeUTF8DeserializerNode(
+ column_descriptor, column_index, result);
+ case parquet::LogicalType::NA:
+ return MakeNullDeserializerNode(
+ column_descriptor, column_index, result);
+ default:
+ switch (node->physical_type()) {
+ case Type::BOOLEAN:
+ return MakeBooleanDeserializerNode(
+ column_descriptor, column_index, result);
+ case Type::INT32:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case Type::INT64:
+ return MakeIntegerDeserializerNode(column_descriptor,
+ column_index, result);
+ case Type::FLOAT:
+ return MakeFloatDeserializerNode(
+ column_descriptor, column_index, result);
+ case Type::DOUBLE:
+ return MakeDoubleDeserializerNode(
+ column_descriptor, column_index, result);
+ case Type::BYTE_ARRAY:
+ return MakeByteArrayDeserializerNode(
+ column_descriptor, column_index, result);
+ case Type::FIXED_LEN_BYTE_ARRAY:
+ return MakeFixedLengthByteArrayDeserializerNode(
+ column_descriptor, column_index, result);
+ case Type::INT96:
+ return MakeImpalaTimestampDeserializerNode(
+ column_descriptor, column_index, result);
+ default:
+ std::ostringstream stream;
+ stream << "Unsupported physical type " << node->physical_type() << std::endl;
+ return Status::Invalid(stream.str());
+ }
+ }
+ }
+
+ //
+ // The functions below create the various logical types of primitive nodes.
+ // Since the ParquetType template parameter is determined at run-time and we
+ // bind that earlier in the call stack, we can use SFINAE to limit the
+ // allowable types for a particular field (e.g. - Timestamp types can only
+ // be constructed from an int64 physical type).
+ //
+ // Most of the code is boiler-plate, except for the fact that there are
+ // different builders for the various types.
+
+ template
+ Status MakeNullDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ auto arrow_type = std::make_shared<::arrow::TimestampType>(::arrow::TimeUnit::NANO);
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, arrow_type, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a nano timestamp field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an int96";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ struct SupportsDecimal {
+ static constexpr auto value =
+ (std::is_same::value ||
+ std::is_same::value ||
+ std::is_same::value ||
+ std::is_same::value);
+ };
+
+ template
+ typename std::enable_if::value, Status>::type
+ MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ auto arrow_type = std::make_shared<::arrow::Decimal128Type>(
+ column_descriptor->type_precision(), column_descriptor->type_scale());
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, arrow_type, pool_);
+ }
+
+ template
+ typename std::enable_if::value, Status>::type
+ MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a decimal field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an int32, int64, or fixed length byte array ";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a boolean field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an boolean ";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, std::unique_ptr& result) {
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a TIME_MILLIS field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an int32 ";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ auto arrow_type = std::make_shared<::arrow::Time32Type>(::arrow::TimeUnit::MILLI);
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, arrow_type, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a TIME_MILLIS field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an int32 ";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ auto arrow_type = std::make_shared<::arrow::Time64Type>(::arrow::TimeUnit::MICRO);
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, arrow_type, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a TIME_MICROS field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an int64";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, ::arrow::TimeUnit::type time_unit,
+ std::unique_ptr& result) {
+ auto arrow_type = std::make_shared<::arrow::TimestampType>(time_unit);
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, arrow_type, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, ::arrow::TimeUnit::type time_unit,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make a TIMESTAMP field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an int64";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ struct CanCastInteger {
+ static constexpr auto parquet_is_int =
+ std::is_integral::value;
+
+ static constexpr auto arrow_is_int =
+ std::is_integral::value;
+
+ static constexpr auto parquet_size = sizeof(typename ParquetType::c_type);
+
+ static constexpr auto arrow_size = sizeof(typename ArrayBuilderType::value_type);
+
+ static constexpr auto both_ints = parquet_is_int && arrow_is_int;
+
+ static constexpr auto value = (both_ints && parquet_size >= arrow_size);
+ };
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, std::unique_ptr& result)
+
+ {
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index,
+ std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make an integer field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not an integer or not the correct size ";
+ return Status::Invalid(stream.str());
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, std::unique_ptr& result)
+
+ {
+ return MakeTypedPrimitiveNodeWithBuilder(
+ column_descriptor, column_index, result, pool_);
+ }
+
+ template
+ typename std::enable_if::value,
+ Status>::type
+ MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor,
+ int column_index, std::unique_ptr& result) {
+ std::ostringstream stream;
+ stream << "Type mismatch: attempt to make an UTF8 field for "
+ << column_descriptor->path()->ToDotString();
+ stream << ", but it was not a byte array";
+ return Status::Invalid(stream.str());
+ }
+
+ template