diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e100a37..dc087966 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -658,6 +658,7 @@ add_custom_command( # Library config set(LIBPARQUET_SRCS + src/parquet/arrow/deserializer.cc src/parquet/arrow/reader.cc src/parquet/arrow/record_reader.cc src/parquet/arrow/schema.cc diff --git a/data/dremel.parquet b/data/dremel.parquet new file mode 100644 index 00000000..93c8075f Binary files /dev/null and b/data/dremel.parquet differ diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc index 47226a3c..a4eeca67 100644 --- a/src/parquet/arrow/arrow-reader-writer-test.cc +++ b/src/parquet/arrow/arrow-reader-writer-test.cc @@ -42,6 +42,7 @@ #include "parquet/util/test-common.h" #include "arrow/api.h" +#include "arrow/pretty_print.h" #include "arrow/test-util.h" #include "arrow/type_traits.h" #include "arrow/util/decimal.h" @@ -427,7 +428,8 @@ void CheckSimpleRoundtrip(const std::shared_ptr& table, int64_t row_group const std::shared_ptr& arrow_properties = default_arrow_writer_properties()) { std::shared_ptr
result; - DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties); + ASSERT_NO_FATAL_FAILURE( + DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties)); ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result, false)); } @@ -508,8 +510,8 @@ class TestParquetIO : public ::testing::Test { std::shared_ptr out; std::unique_ptr reader; - ReaderFromSink(&reader); - ReadSingleColumnFile(std::move(reader), &out); + ASSERT_NO_FATAL_FAILURE(ReaderFromSink(&reader)); + ASSERT_NO_FATAL_FAILURE(ReadSingleColumnFile(std::move(reader), &out)); internal::AssertArraysEqual(values, *out); } @@ -554,8 +556,8 @@ class TestParquetIO : public ::testing::Test { void ReadAndCheckSingleColumnTable(const std::shared_ptr& values) { std::shared_ptr<::arrow::Table> out; std::unique_ptr reader; - ReaderFromSink(&reader); - ReadTableFromFile(std::move(reader), &out); + ASSERT_NO_FATAL_FAILURE(ReaderFromSink(&reader)); + ASSERT_NO_FATAL_FAILURE(ReadTableFromFile(std::move(reader), &out)); ASSERT_EQ(1, out->num_columns()); ASSERT_EQ(values->length(), out->num_rows()); @@ -1043,8 +1045,8 @@ TEST_F(TestNullParquetIO, NullListColumn) { std::shared_ptr
out; std::unique_ptr reader; - this->ReaderFromSink(&reader); - this->ReadTableFromFile(std::move(reader), &out); + ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader)); + ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out)); ASSERT_EQ(1, out->num_columns()); ASSERT_EQ(offsets.size() - 1, out->num_rows()); @@ -1132,7 +1134,7 @@ class TestPrimitiveParquetIO : public TestParquetIO { ASSERT_NO_FATAL_FAILURE(MakeTestFile(values, num_chunks, &file_reader)); std::shared_ptr
out; - this->ReadTableFromFile(std::move(file_reader), &out); + ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(file_reader), &out)); ASSERT_EQ(1, out->num_columns()); ASSERT_EQ(SMALL_SIZE, out->num_rows()); @@ -1244,6 +1246,136 @@ TEST(TestArrowReadWrite, DateTimeTypes) { ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result)); } +TEST(TestArrowReadWrite, DremelExampleTest) { + std::string dir_string(test::get_data_dir()); + std::stringstream ss; + ss << dir_string << "/dremel.parquet"; + auto parquet_reader = ParquetFileReader::OpenFile(ss.str()); + parquet::arrow::FileReader reader(::arrow::default_memory_pool(), + std::move(parquet_reader)); + std::shared_ptr<::arrow::Table> out; + ASSERT_OK(reader.ReadTable(&out)); + + // Dremel paper uses int64, but the sample file + // stored them as int32 + auto forward = std::make_shared<::arrow::Int32Builder>(); + auto backward = std::make_shared<::arrow::Int32Builder>(); + auto docId = std::make_shared<::arrow::Int32Builder>(); + auto country = std::make_shared<::arrow::StringBuilder>(); + auto code = std::make_shared<::arrow::StringBuilder>(); + auto url = std::make_shared<::arrow::StringBuilder>(); + + std::vector> language_fields; + language_fields.emplace_back( + std::make_shared<::arrow::Field>("Code", ::arrow::utf8(), false)); + language_fields.emplace_back( + std::make_shared<::arrow::Field>("Country", ::arrow::utf8())); + + auto language_type = std::make_shared<::arrow::StructType>(language_fields); + std::vector> language_builders = {code, country}; + auto language = std::make_shared<::arrow::StructBuilder>( + language_type, ::arrow::default_memory_pool(), std::move(language_builders)); + auto language_list = + std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), language); + + auto backward_list = + std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), backward); + auto forward_list = + std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), forward); + std::vector> links_fields; + links_fields.emplace_back( + std::make_shared<::arrow::Field>("Backward_Outer", backward_list->type(), false)); + links_fields.emplace_back( + std::make_shared<::arrow::Field>("Forward_Outer", backward_list->type(), false)); + auto links_type = std::make_shared<::arrow::StructType>(links_fields); + ::arrow::StructBuilder links(links_type, ::arrow::default_memory_pool(), + {forward_list, backward_list}); + + std::vector> name_fields; + name_fields.emplace_back( + std::make_shared<::arrow::Field>("Language_Outer", language_list->type(), false)); + name_fields.emplace_back(std::make_shared<::arrow::Field>("Url", url->type())); + + auto name_type = std::make_shared<::arrow::StructType>(name_fields); + std::vector> name_builders = {language_list, + url}; + auto name = std::make_shared<::arrow::StructBuilder>( + name_type, ::arrow::default_memory_pool(), std::move(name_builders)); + ::arrow::ListBuilder name_list(::arrow::default_memory_pool(), name); + + // Row 1 + docId->Append(10); + links.Append(); + name_list.Append(); + + // 3 forward links + forward_list->Append(); + forward->Append(20); + forward->Append(40); + forward->Append(60); + // No backward links + backward_list->Append(); + + // First name + name->Append(); + language_list->Append(); + language->Append(); + code->Append("en-us"); + country->Append("us"); + + language->Append(); + code->Append("en"); + country->AppendNull(); + + url->Append("http://A"); + + // Second name + name->Append(); + language_list->Append(); + url->Append("http://B"); + + // Third name + name->Append(); + language_list->Append(); + language->Append(); + code->Append("en-gb"); + country->Append("gb"); + url->AppendNull(); + + // Row 2 + docId->Append(20); + links.Append(); + name_list.Append(); + + backward_list->Append(); + backward->Append(10); + backward->Append(30); + forward_list->Append(); + forward->Append(80); + + name->Append(); + language_list->Append(); + url->Append("http://C"); + + std::shared_ptr<::arrow::Array> docIdArray, nameArray, linksArray; + + ASSERT_OK(docId->Finish(&docIdArray)); + ASSERT_OK(name_list.Finish(&nameArray)); + ASSERT_OK(links.Finish(&linksArray)); + + std::vector> fields; + fields.emplace_back( + std::make_shared<::arrow::Field>("DocId", docIdArray->type(), false)); + fields.emplace_back(std::make_shared<::arrow::Field>("Links", linksArray->type())); + fields.emplace_back(std::make_shared<::arrow::Field>("Name", nameArray->type(), false)); + + auto schema = std::make_shared<::arrow::Schema>(fields); + + auto expected = ::arrow::Table::Make(schema, {docIdArray, linksArray, nameArray}); + + ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*expected, *out)); +} + TEST(TestArrowReadWrite, CoerceTimestamps) { using ::arrow::ArrayFromVector; using ::arrow::field; @@ -1445,8 +1577,8 @@ TEST(TestArrowReadWrite, CoerceTimestampsAndSupportDeprecatedInt96) { using ::arrow::Schema; using ::arrow::Table; using ::arrow::TimeUnit; - using ::arrow::TimestampType; using ::arrow::TimestampBuilder; + using ::arrow::TimestampType; using ::arrow::default_memory_pool; auto timestamp_type = std::make_shared(TimeUnit::NANO); @@ -2161,6 +2293,7 @@ TEST_F(TestNestedSchemaRead, ReadIntoTableFull) { std::shared_ptr
table; ASSERT_OK_NO_THROW(reader_->ReadTable(&table)); + ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS); ASSERT_EQ(table->num_columns(), 2); ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 2); @@ -2228,9 +2361,11 @@ TEST_F(TestNestedSchemaRead, ReadTablePartial) { } TEST_F(TestNestedSchemaRead, StructAndListTogetherUnsupported) { +#if 0 ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::REPEATED)); std::shared_ptr
table; ASSERT_RAISES(NotImplemented, reader_->ReadTable(&table)); +#endif } TEST_P(TestNestedSchemaRead, DeepNestedSchemaRead) { diff --git a/src/parquet/arrow/arrow-schema-test.cc b/src/parquet/arrow/arrow-schema-test.cc index 5c16c044..cb2b8508 100644 --- a/src/parquet/arrow/arrow-schema-test.cc +++ b/src/parquet/arrow/arrow-schema-test.cc @@ -62,8 +62,8 @@ class TestConvertParquetSchema : public ::testing::Test { for (int i = 0; i < expected_schema->num_fields(); ++i) { auto lhs = result_schema_->field(i); auto rhs = expected_schema->field(i); - EXPECT_TRUE(lhs->Equals(rhs)) << i << " " << lhs->ToString() - << " != " << rhs->ToString(); + EXPECT_TRUE(lhs->Equals(rhs)) + << i << " " << lhs->ToString() << " != " << rhs->ToString(); } } @@ -607,9 +607,10 @@ TEST_F(TestConvertParquetSchema, ParquetRepeatedNestedSchema) { auto inner_group_type = std::make_shared<::arrow::StructType>(inner_group_fields); auto outer_group_fields = { std::make_shared("leaf2", INT32, true), - std::make_shared("innerGroup", ::arrow::list(std::make_shared( - "innerGroup", inner_group_type, false)), - false)}; + std::make_shared( + "innerGroup", + ::arrow::list(std::make_shared("innerGroup", inner_group_type, false)), + false)}; auto outer_group_type = std::make_shared<::arrow::StructType>(outer_group_fields); arrow_fields.push_back(std::make_shared("leaf1", INT32, true)); diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc new file mode 100644 index 00000000..d8dce5eb --- /dev/null +++ b/src/parquet/arrow/deserializer.cc @@ -0,0 +1,2383 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/arrow/deserializer.h" + +#include +#include + +#include "arrow/builder.h" +#include "arrow/memory_pool.h" +#include "arrow/table.h" +#include "arrow/type.h" +#include "arrow/util/bit-util.h" +#include "arrow/util/decimal.h" +#include "arrow/util/logging.h" +#include "arrow/util/parallel.h" +#include "parquet/arrow/schema.h" +#include "parquet/file_reader.h" +#include "parquet/types.h" + +namespace arrow { + +class Array; +} + +using arrow::Array; +using arrow::Status; +using arrow::Table; + +using parquet::schema::GroupNode; +using parquet::schema::Node; +using parquet::schema::PrimitiveNode; + +namespace parquet { + +namespace arrow { + +constexpr int64_t kJulianToUnixEpochDays = 2440588LL; +constexpr int64_t kMillisecondsInADay = 86400000LL; +constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL; + +static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timestamp) { + int64_t days_since_epoch = impala_timestamp.value[2] - kJulianToUnixEpochDays; + int64_t nanoseconds = *(reinterpret_cast(&(impala_timestamp.value))); + return days_since_epoch * kNanosecondsInADay + nanoseconds; +} + +// This is a "tagged" integer type for definition/repetition levels. +// The tag gives us strong typing so that we can differentiate between +// definition and repeition level arguments in the DeserializerBuilder +// build function calls. +template +class Level { + public: + Level() : value_(0) {} + + Level& operator++() { + ++value_; + return *this; + } + + int16_t operator*() const { return value_; } + + private: + int16_t value_; +}; + +struct Repetition; +struct Definition; + +using RepetitionLevel = Level; +using DefinitionLevel = Level; + +// +// This is a utility class that is used by primitive node deserializer +// nodes. It's similar to a std::list, but stores its data as a contiguous +// array of memory because it's passed to calls to ReadBatch. +template +class PrimitiveDeserializerArray { + public: + PrimitiveDeserializerArray(::arrow::MemoryPool* pool, int64_t capacity) + : pool_(pool), capacity_(capacity), size_(0), index_(0) {} + + ~PrimitiveDeserializerArray() { + pool_->Free(reinterpret_cast(data_), capacity_); + } + + Status Initialize() { + uint8_t* out; + RETURN_NOT_OK(pool_->Allocate(sizeof(T) * capacity_, &out)); + data_ = reinterpret_cast(out); + return Status::OK(); + } + + bool const Empty() const { return index_ >= size_; } + + T Front() const { return data_[index_]; } + + void PopFront() { ++index_; } + + void Reset() { + index_ = 0; + size_ = 0; + } + + ::arrow::MemoryPool* pool_; + + T* data_; + + int64_t capacity_; + + int64_t size_; + + int64_t index_; +}; + +// +// Interface for implementations that are able to deserialize +// Parquet primitive and group nodes. Successive calls to +// Deserialize will cause this node to append values or nulls +// to its builder. +// +class DeserializerNode { + public: + using LevelSet = std::set; + + // + // \brief Deserializes the next set of values. + // \param[in] the current repeition level. When this is first called + // when deserializing a row, it should be zero. It may be + // non-zero when a list node is calling a child node + // during a repetition. In those cases, a child node may + // do nothing because it's next value doesn't repeat at + // the current repeition level. + // \param[out] this is used to find out what the next set of + // repetitions will be. This is used by list + // (aka - repeated) nodes in their Deserialize method + // to determine whether or not they need to repeat + // and call Deserialize for their child nodes + // again before popping back to their parent node. + // \return error status if there is no more data for this node + virtual Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) = 0; + + // \brief Skips deserialization. This is used by group and list + // nodes when they are nullable, in which they call this + // method on their child nodes so that they do not append + // any values to their builders. + // \param[out] this is used to find out what the next set of + // repetitions will be. This is used by list + // (aka - repeated) nodes in their Deserialize method + // to determine whether or not they need to repeat + // and call Deserialize for their child nodes + // again before popping back to their parent node. + // \return error status if there is no more data for this node + virtual Status Skip(LevelSet& next_repetitions) = 0; + + // \brief returns the maximum definition level of all of this + // node's descendants. In the case of a primitive node + // it will be the next definition level in the file (if + // one exists) or zero otherwise. In the case of a group + // node, it will simply take the maximum of all of its + // child nodes. + // \param[out] the maximum definition level for this nodes and its + // descendants + // \return error status if there is no more data for this node + virtual Status CurrentMaxDefinitionLevel(int16_t& level) = 0; + + // \brief returns the array buidler that was used by this node + // to append values and null in calls to Deserialize + // \return the array builder + virtual std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() = 0; + + // \brief initializes this node for a new row group. This is used + // by primitive node deserializers to obtain a column reader + // for that row group + // \return status error if this node is a primitive node and the + // column index is out of bounds + virtual Status InitializeRowGroup( + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) = 0; +}; + +// +// The PrimitiveAppender class is used adapt the reading of parquet +// values from a ColumnReader to calls to a builder's append method. +// The reason this is needed is that the signature of the builder's +// Append method varies by type. For example, byte arrays take +// a pointer and a length whereas primitive types take just a value. +// By using partial specialization for various types, the deserializer +// code for primitive types does not need specialization but instead +// delegates to these set of classes to handle it. +// +template +class PrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, + ParquetValueType value) { + using value_type = typename BuilderType::value_type; + return builder.Append(static_cast(value)); + } +}; + +template +class PrimitiveAppender<::parquet::ByteArray, BuilderType> { + public: + template + Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, + ParquetValueType value) { + return builder.Append(value.ptr, value.len); + } +}; + +template +class PrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, + ParquetValueType value) { + return builder.Append(value.ptr); + } +}; + +template <> +class PrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + ParquetValueType value) { + ::arrow::Decimal128 converted; + RETURN_NOT_OK(::arrow::Decimal128::FromBigEndian(value.ptr, value.len, &converted)); + return builder.Append(converted); + } +}; + +template <> +class PrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + ParquetValueType value) { + ::arrow::Decimal128 converted; + RETURN_NOT_OK(::arrow::Decimal128::FromBigEndian(value.ptr, descriptor->type_length(), + &converted)); + return builder.Append(converted); + } +}; + +template +class PrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + ParquetValueType value) { + return builder.Append(::arrow::Decimal128(value)); + } +}; + +template <> +class PrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, ::arrow::TimestampBuilder& builder, + ParquetValueType value) { + return builder.Append(impala_timestamp_to_nanoseconds(value)); + } +}; + +class NullPrimitiveAppender { + public: + template + Status Append(const ColumnDescriptor* descriptor, ::arrow::NullBuilder& builder, + ParquetValueType value) { + std::ostringstream stream; + stream << "Unexpected call to NullPrimitiveAppender::Append"; + return Status::Invalid(stream.str()); + } +}; + +template +class PrimitiveAppender + : public NullPrimitiveAppender {}; + +// These specializations avoid ambiguity with other ones where FLBA is +// specified but the builder type isn't (ambiguous with the class right above) +template <> +class PrimitiveAppender : public NullPrimitiveAppender {}; + +template <> +class PrimitiveAppender : public NullPrimitiveAppender { +}; + +// +// This is the base class for primitive types deserializers. +// The ColumnReaderType should be an instantiation of +// TypeColumnReader (see column_reader.h). The ArrayBuilderType +// should be an instantiation of one of the Arrow builders +// (e.g. - Int32Builder). This class is used for initializing +// the column reader and appending values, whereas the sub-classes +// handle reading of repetition and definition levels and implement +// the DeserializerNode virtual functions. +// +template +class PrimitiveDeserializerNode : public DeserializerNode { + public: + PrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, + int64_t buffer_size, + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) + : descriptor_(descriptor), + column_index_(column_index), + values_(pool, buffer_size), + builder_(builder) {} + + std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() { return builder_; } + + // \brief initializes the column reader for this deserializer + // \param[in] the reader for the row group from which the column reader + // is obtained + // \return error status if the column index is out of bounds + Status InitializeRowGroup( + std::shared_ptr const& row_group_reader) final { + if (column_index_ < row_group_reader->metadata()->num_columns()) { + column_reader_ = std::static_pointer_cast( + row_group_reader->Column(column_index_)); + } else { + std::ostringstream stream; + stream << "Invalid column index " << column_index_ << " for row group "; + return Status::Invalid(stream.str()); + } + return Status::OK(); + } + + virtual Status Initialize() { return values_.Initialize(); } + + protected: + // \brief appends the next value in this node's values array. It is + // assumed that the caller has checked to make sure that the + // values buffer is not empty + // \return error status if the call to the builder's Append function fails + Status Append() { + auto status = appender_.Append(descriptor_, *builder_, values_.Front()); + values_.PopFront(); + return status; + } + + const ColumnDescriptor* const descriptor_; + + int const column_index_; + + PrimitiveDeserializerArray values_; + + std::shared_ptr builder_; + + PrimitiveAppender appender_; + + std::shared_ptr column_reader_; +}; + +// This enum is used to differentiate between the three +// different types of primitive deserializers +enum PrimitiveDeserializerType { + // This means that the node in the tree has no ancestors (including + // itself) that are repeated or optional + REQUIRED, + // This means that the node in the tree has only ancestors + // (including itself) that are required or optional, and therefore + // doesn't need to store repetition levels but does need to store + // definition levels + OPTIONAL, + // This means that the node in the tree has at least one ancestor + // (including itself) + REPEATED +}; + +// +// The TypedPrimitiveDeserializerNode does the heavy lifting of +// reading repetition and definition levels and implementing +// the DeserializerNode interface for primitive types. There are +// three specializations based on whether or not the field +// has definition levels or repetition levels (e.g. - a required +// field at depth 1 doesn't have repeition or definition levels +// and they are not stored in the parquet file). By using +// specialization in this manner, we save space and also don't +// have to do things like check if the max repetition or +// max definition level is zero. Unfortunately, this pattern is not +// yet being used in the column reader classes, so those checks +// still happen in calls to ReadBatch. +// +template +class TypedPrimitiveDeserializerNode; + +// Specialization for a field that has definition levels but no +// repetition levels. For example, the following schemas would +// create this type of node: +// +// message root { +// optional int32 field; +// } +// +// message root { +// optional group contact { +// required bytearray name (UTF8); +// optional int32 phone_number; +// } +// +// Both name and phone_number would be serialized with this type of +// node because they have a max definition level of 1 and 2 respectively. +// +template +class TypedPrimitiveDeserializerNode + : public PrimitiveDeserializerNode { + public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, + int const column_index, std::size_t buffer_size, + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) + : PrimitiveDeserializerNode( + descriptor, column_index, buffer_size, builder, pool), + definition_levels_(pool, buffer_size) {} + + Status Initialize() final { + RETURN_NOT_OK(definition_levels_.Initialize()); + return PrimitiveDeserializerNode::Initialize(); + } + + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to skip data"; + return Status::IOError(stream.str()); + } + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + this->values_.PopFront(); + } + definition_levels_.PopFront(); + return Status::OK(); + } + + Status CurrentMaxDefinitionLevel(int16_t& level) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to read current max definition level"; + return Status::IOError(stream.str()); + } + level = definition_levels_.Front(); + return Status::OK(); + } + + Status Deserialize(RepetitionLevel current_repetition_level, + DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to deserialize value"; + return Status::IOError(stream.str()); + } + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + // The value is not null + RETURN_NOT_OK(this->Append()); + } else { + RETURN_NOT_OK(this->builder_->AppendNull()); + } + definition_levels_.PopFront(); + return Status::OK(); + } + + private: + bool Fill() { + if (!definition_levels_.Empty()) { + return true; + } + definition_levels_.Reset(); + this->values_.Reset(); + definition_levels_.size_ = this->column_reader_->ReadBatch( + definition_levels_.capacity_, definition_levels_.data_, 0, this->values_.data_, + &this->values_.size_); + return !definition_levels_.Empty(); + } + + PrimitiveDeserializerArray definition_levels_; +}; + +// +// Specialization for a field that has both definition and repetition levels. +// For example, the following schema would create this type of node: +// +// message root { +// required group list_outer (LIST) { +// repeated group list_inner { +// required int32 value; +// } +// } +// } +template +class TypedPrimitiveDeserializerNode + : public PrimitiveDeserializerNode { + public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, + int column_index, std::size_t buffer_size, + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) + : PrimitiveDeserializerNode( + descriptor, column_index, buffer_size, builder, pool), + definition_levels_(pool, buffer_size), + repetition_levels_(pool, buffer_size) {} + + Status Initialize() final { + RETURN_NOT_OK(definition_levels_.Initialize()); + RETURN_NOT_OK(repetition_levels_.Initialize()); + return PrimitiveDeserializerNode::Initialize(); + } + + Status CurrentMaxDefinitionLevel(int16_t& level) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to determine current max definition level"; + return Status::IOError(stream.str()); + } + level = definition_levels_.Front(); + return Status::OK(); + } + + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to determine skip data"; + return Status::IOError(stream.str()); + } + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + this->values_.PopFront(); + } + definition_levels_.PopFront(); + AddRepetitions(next_repetitions); + return Status::OK(); + } + + Status Deserialize(RepetitionLevel current_repetition_level, + DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to fill values when deserializing"; + return Status::IOError(stream.str()); + } + // See Dremel paper: second entry for Links.Backward would + // apply in this case and return + if (repetition_levels_.Front() != *current_repetition_level) { + return Status::OK(); + } + + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + // The value is not null + RETURN_NOT_OK(this->Append()); + } else { + // Insert a null + RETURN_NOT_OK(this->builder_->AppendNull()); + } + definition_levels_.PopFront(); + AddRepetitions(next_repetitions); + return Status::OK(); + } + + private: + // This is factored into its own method because it's used in both + // Skip and Deserialize + void AddRepetitions(DeserializerNode::LevelSet& next_repetitions) { + repetition_levels_.PopFront(); + if (Fill()) { + next_repetitions.insert(repetition_levels_.Front()); + } + } + + bool Fill() { + if (!repetition_levels_.Empty()) { + return true; + } + definition_levels_.Reset(); + repetition_levels_.Reset(); + this->values_.Reset(); + auto levels_read = this->column_reader_->ReadBatch( + definition_levels_.capacity_, definition_levels_.data_, repetition_levels_.data_, + this->values_.data_, &this->values_.size_); + definition_levels_.size_ = levels_read; + repetition_levels_.size_ = levels_read; + + return !definition_levels_.Empty(); + } + + PrimitiveDeserializerArray definition_levels_; + + PrimitiveDeserializerArray repetition_levels_; +}; + +// +// Specialization for a field that has neither definition nor repetition levels. +// For example, all of the primitive fields in the following schema +// would create this type of node: +// +// message root { +// required group contact { +// required bytearray name (UTF8); +// required group address { +// required bytearray street (UTF8); +// required int32 number; +// required bytearray city (UTF8); +// required int8 zip; +// } +// } +// } +template +class TypedPrimitiveDeserializerNode + : public PrimitiveDeserializerNode { + public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, + int column_index, std::size_t buffer_size, + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) + : PrimitiveDeserializerNode( + descriptor, column_index, buffer_size, builder, pool) {} + + Status CurrentMaxDefinitionLevel(int16_t& level) final { + level = this->descriptor_->max_definition_level(); + return Status::OK(); + } + + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { + std::ostringstream stream; + stream << "Call to Skip made when it shouldn't have"; + return Status::Invalid(stream.str()); + } + + Status Deserialize(RepetitionLevel current_repetition_level, + DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + std::ostringstream stream; + stream << "Unable to load more values in call to Deserialize"; + return Status::IOError(stream.str()); + } + return this->Append(); + } + + private: + bool Fill() { + if (!this->values_.Empty()) { + return true; + } + this->values_.Reset(); + // Ignore the return value here + this->column_reader_->ReadBatch(this->values_.capacity_, 0, 0, this->values_.data_, + &this->values_.size_); + return !this->values_.Empty(); + } +}; + +// +// Deserializer for lists that are required. The Parquet spec on list +// types requires that the schema for a required list looks likes this: +// +// required group list_outer (LIST) { +// repeated group list_inner { +// (required|optional) element ... +// } +// } +// +// The reason for the outer level of nesting is that without it +// you wouldn't be able to differentiate a list of a single null +// from an empty list. By having this extra level, an empty list +// is indicated when the max definition level of the list_outer's +// descendants are equal to list_outer's max definition level, and +// a list of null would be indicated by a definition level of +// list_inner's descendants that is equal to list_inner's max +// definition level. +// +// The constructor of this class takes the max_definition_level of +// list_outer and the max_repetition_level of list_inner +class ListDeserializerNode : public DeserializerNode { + public: + ListDeserializerNode(std::unique_ptr&& node_reader, + ::arrow::MemoryPool* pool, RepetitionLevel max_repetition_level, + DefinitionLevel max_definition_level) + : node_reader_(std::move(node_reader)), + max_repetition_level_(max_repetition_level), + max_definition_level_(max_definition_level), + builder_(std::make_shared<::arrow::ListBuilder>( + pool, node_reader_->GetArrayBuilder())) {} + + Status CurrentMaxDefinitionLevel(int16_t& level) final { + return node_reader_->CurrentMaxDefinitionLevel(level); + } + + Status Skip(LevelSet& next_repetitions) final { + return node_reader_->Skip(next_repetitions); + } + + Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) { + RETURN_NOT_OK(builder_->Append()); + int16_t current_max_definition_level; + RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level)); + if (current_max_definition_level == *max_definition_level_) { + // This is just an empty list + return node_reader_->Skip(next_repetitions); + } else { + // Deserialize the first one + RETURN_NOT_OK( + node_reader_->Deserialize(current_repetition_level, next_repetitions)); + // Deserialize the repetitions + return DeserializeRepetitions(next_repetitions); + } + } + + std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } + + Status InitializeRowGroup(std::shared_ptr const& row_group_reader) { + return node_reader_->InitializeRowGroup(row_group_reader); + } + + protected: + // \brief keeps appending elements to the current list as long as + // the descendants of this node have repetitions at this list's + // repetition level + // \param[in] the set of next repetitions from the last call to + // children's Deserialize functions. + // \return status error if calls to child node Deserialize functions + // produce an error + Status DeserializeRepetitions(LevelSet& next_repetitions) { + bool repetition_at_this_level = true; + while (!next_repetitions.empty() && repetition_at_this_level) { + // next_repetitions is a sorted set, so end + // is the highest value. If it's lower than our + // max_repetition_level_, we just return from this + // function. Otherwise, we erase highest, descend again, + // and then re-check if we repeated at this level + auto highest = next_repetitions.end(); + --highest; + repetition_at_this_level = (*highest == *max_repetition_level_); + if (repetition_at_this_level) { + next_repetitions.erase(highest); + RETURN_NOT_OK(node_reader_->Deserialize(max_repetition_level_, next_repetitions)); + } + } + return Status::OK(); + } + + std::unique_ptr node_reader_; + + RepetitionLevel max_repetition_level_; + + DefinitionLevel max_definition_level_; + + std::shared_ptr<::arrow::ListBuilder> builder_; +}; + +// +// Deserializer for lists that are optional. The Parquet spec on list +// types requires that the schema for an optional list looks likes this: +// +// optional group list_outer (LIST) { +// repeated group list_inner { +// (required|optional) element ... +// } +// } +// +// When deserializing, if the max definition level of this node's +// children is less than list_outer's max definition level, then +// we append a null. If it's equal to it, then we create an empty +// list. Otherwise, we create a non-empty list. +// +// The constructor of this class takes the max_definition_level of +// list_outer and the max_repetition_level of list_inner +class NullableListDeserializerNode : public ListDeserializerNode { + public: + NullableListDeserializerNode(std::unique_ptr&& node_reader, + ::arrow::MemoryPool* pool, + RepetitionLevel max_repetition_level, + DefinitionLevel max_definition_level) + : ListDeserializerNode(std::forward>(node_reader), + pool, max_repetition_level, max_definition_level) {} + + Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) final { + int16_t current_max_definition_level; + RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level)); + if (current_max_definition_level < *max_definition_level_) { + // It's null + RETURN_NOT_OK(builder_->AppendNull()); + RETURN_NOT_OK(node_reader_->Skip(next_repetitions)); + } else if (current_max_definition_level == *max_definition_level_) { + // Empty list + RETURN_NOT_OK(builder_->Append()); + RETURN_NOT_OK(node_reader_->Skip(next_repetitions)); + } else { + RETURN_NOT_OK(builder_->Append()); + RETURN_NOT_OK( + node_reader_->Deserialize(current_repetition_level, next_repetitions)); + RETURN_NOT_OK(DeserializeRepetitions(next_repetitions)); + } + return Status::OK(); + } +}; + +// +// Serializer for a group (aka struct) node. The schema for such a node +// would look like this; +// +// (repeated|required) group pair { +// required int32 key; +// required bytearray value (UTF8); +// } +// +// This has an Optional template parameter so that we can remove unnecessary checks +// for required nodes at compile time. +// +template +class NestedDeserializerNode : public DeserializerNode { + public: + NestedDeserializerNode(DefinitionLevel max_definition_level, + std::vector>&& children, + std::vector>& fields, + ::arrow::MemoryPool* pool) + : children_(std::move(children)), max_definition_level_(max_definition_level) { + std::vector> child_builders; + for (auto const& child : children_) { + child_builders.emplace_back(child->GetArrayBuilder()); + } + auto arrow_type = std::make_shared<::arrow::StructType>(fields); + builder_ = std::make_shared<::arrow::StructBuilder>(arrow_type, pool, + std::move(child_builders)); + } + + Status Skip(LevelSet& next_repetitions) { + for (auto& child : children_) { + RETURN_NOT_OK(child->Skip(next_repetitions)); + } + return Status::OK(); + } + + Status CurrentMaxDefinitionLevel(int16_t& max) final { + max = 0; + for (auto& child : children_) { + int16_t current; + RETURN_NOT_OK(child->CurrentMaxDefinitionLevel(current)); + if (current > max) { + max = current; + } + } + return Status::OK(); + } + + Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) final { + // TODO: detect this at tree construction time. If all of + // the fields inside this nested node are required, then + // there is no need to do this check. + auto has_data = !Optional; + if (Optional) { + int16_t current_max_definition_level; + RETURN_NOT_OK(CurrentMaxDefinitionLevel(current_max_definition_level)); + has_data = current_max_definition_level >= *max_definition_level_; + } + if (has_data) { + RETURN_NOT_OK(builder_->Append()); + } else { + RETURN_NOT_OK(builder_->AppendNull()); + } + // For structs, we call Deserialize even when they don't have data. + // This ensures that the length of the child arrays match + // the length of the struct bitmap. + for (auto& child : children_) { + RETURN_NOT_OK(child->Deserialize(current_repetition_level, next_repetitions)); + } + return Status::OK(); + } + + std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } + + Status InitializeRowGroup( + std::shared_ptr const& row_group_reader) final { + for (auto& child : children_) { + RETURN_NOT_OK(child->InitializeRowGroup(row_group_reader)); + } + return Status::OK(); + } + + private: + std::vector> children_; + + DefinitionLevel max_definition_level_; + + std::shared_ptr<::arrow::StructBuilder> builder_; +}; + +// The following set of classes are used to drive the +// iteration over the input data in a particular manner. +// +// TODO: except for the batched array serializer, it might +// be possible to collapse these classes through the +// use of iterators. +ArrayDeserializer::ArrayDeserializer(std::unique_ptr&& node) + : node_(std::move(node)) {} + +ArrayDeserializer::~ArrayDeserializer() {} + +// This class will deserialize a single array from the entire +// file. The node that is passed to the constructor may +// contain multiple primitive nodes as its descendants. +class FileArrayDeserializer : public ArrayDeserializer { + public: + FileArrayDeserializer(std::unique_ptr&& node, + std::shared_ptr const& file_reader) + : ArrayDeserializer(std::forward>(node)), + file_reader_(file_reader) {} + + ~FileArrayDeserializer() {} + + ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final { + DeserializerNode::LevelSet repetitions; + auto const num_row_groups = file_reader_->metadata()->num_row_groups(); + for (int ii = 0; ii < num_row_groups; ++ii) { + auto row_group_reader = file_reader_->RowGroup(ii); + RETURN_NOT_OK(node_->InitializeRowGroup(row_group_reader)); + auto const num_rows = row_group_reader->metadata()->num_rows(); + for (int jj = 0; jj < num_rows; ++jj) { + RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); + repetitions.clear(); + } + } + return node_->GetArrayBuilder()->Finish(&array); + } + + private: + std::shared_ptr file_reader_; +}; + +ArrayBatchedDeserializer::~ArrayBatchedDeserializer() {} + +// This class will deserialize a single array from the entire +// file in batches. The node that is passed to the constructor may +// contain multiple primitive nodes as its descendants. +class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { + public: + FileArrayBatchedDeserializer(std::unique_ptr&& node, + std::shared_ptr const& file_reader) + : node_(std::move(node)), + file_reader_(file_reader), + current_row_group_(file_reader->RowGroup(0)), + current_row_group_index_(0), + current_row_(0) {} + + ~FileArrayBatchedDeserializer() {} + + Status Initialize() { return node_->InitializeRowGroup(current_row_group_); } + + ::arrow::Status DeserializeBatch(int64_t num_records, + std::shared_ptr<::arrow::Array>& array) final { + DeserializerNode::LevelSet repetitions; + int64_t records_read = 0; + auto const num_row_groups = file_reader_->metadata()->num_row_groups(); + while (records_read < num_records && current_row_group_index_ < num_row_groups) { + auto const num_rows = current_row_group_->metadata()->num_rows(); + for (; current_row_ < num_rows && records_read < num_records; + ++current_row_, ++records_read) { + RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); + repetitions.clear(); + } + if (current_row_ >= num_rows) { + ++current_row_group_index_; + if (current_row_group_index_ < num_row_groups) { + current_row_group_ = file_reader_->RowGroup(current_row_group_index_); + RETURN_NOT_OK(node_->InitializeRowGroup(current_row_group_)); + current_row_ = 0; + } + } + } + return node_->GetArrayBuilder()->Finish(&array); + } + + private: + std::unique_ptr node_; + + std::shared_ptr<::parquet::ParquetFileReader> file_reader_; + + std::shared_ptr<::parquet::RowGroupReader> current_row_group_; + + int current_row_group_index_; + + int current_row_; +}; + +// This class will deserialize a single array from a single +// row group for a single node. The DeserializeBuilder only +// constructs this for a node that has at most one primitive +// descendant, but it could just as well be used for nodes that +// contain multiple primitive descendants. +class ColumnChunkDeserializer : public ArrayDeserializer { + public: + ColumnChunkDeserializer( + std::unique_ptr&& node, + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) + : ArrayDeserializer(std::move(node)), row_group_reader_(row_group_reader) {} + + Status Initialize() { return node_->InitializeRowGroup(row_group_reader_); } + + ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final { + DeserializerNode::LevelSet repetitions; + auto const num_rows = row_group_reader_->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); + repetitions.clear(); + } + return node_->GetArrayBuilder()->Finish(&array); + } + + private: + std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; +}; + +TableDeserializer::TableDeserializer(std::shared_ptr<::arrow::Schema> const& schema) + : schema_(schema) {} + +TableDeserializer::~TableDeserializer() {} + +// +// A table deserializer that does it's deserialization +// in a single thread. This is in contrast to +// ParallelTableDeserializer which deserializes each +// top level array in the schema in a separate thread. +class SerialTableDeserializer : public TableDeserializer { + protected: + SerialTableDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema) + : TableDeserializer(schema), nodes_(std::move(nodes)) {} + + // \brief Convenience method which iterates over the nodes in this + // deserializer and constructs a table from them, using + // the schema passed to this object's constructor + // \return error status if there is an issue constructing the table + ::arrow::Status MakeTable(std::shared_ptr<::arrow::Table>& result) { + std::vector> arrays; + for (auto const& node : nodes_) { + std::shared_ptr<::arrow::Array> array; + RETURN_NOT_OK(node->GetArrayBuilder()->Finish(&array)); + arrays.emplace_back(std::move(array)); + } + result = Table::Make(schema_, arrays); + return Status::OK(); + } + + // \brief Convenience method which iterates over all rows in the rows + // group and deserializes each node + // \param[in] the row group reader + // \return status error if there is a problem deserializing the row group + Status DeserializeRowGroup( + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) { + for (auto& node : nodes_) { + RETURN_NOT_OK(node->InitializeRowGroup(row_group_reader)); + } + DeserializerNode::LevelSet repetitions; + auto const num_rows = row_group_reader->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + for (auto& node : nodes_) { + RETURN_NOT_OK(node->Deserialize(RepetitionLevel(), repetitions)); + repetitions.clear(); + } + } + return Status::OK(); + } + + private: + std::vector> nodes_; +}; + +// +// This is a deserializer that can deserialize columns in +// parallel using multiple threads. It takes a vector of +// ArrayDeserializer's and calls DeserializeArray separately +// in each thread. Each of the ArrayDeserializer's will either +// have a separate row group reader, or will create row group +// readers as it iterates through the file. This ensures that +// there is no sharing of data across the threads. +// +class ParallelTableDeserializer : public TableDeserializer { + public: + ParallelTableDeserializer( + std::shared_ptr<::arrow::Schema> const& schema, + std::vector>&& array_deserializers, + int num_threads) + : TableDeserializer(schema), + num_threads_(num_threads), + array_deserializers_(std::move(array_deserializers)), + arrays_(array_deserializers_.size()) {} + + ~ParallelTableDeserializer() {} + + ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final { + // For now, the parallelism is done a column basis. Other possible + // approaches are deserializing row groups or column chunks in parallel. + RETURN_NOT_OK(::arrow::ParallelFor( + num_threads_, static_cast(array_deserializers_.size()), [this](int index) { + auto const array_index = static_cast(index); + return array_deserializers_[array_index]->DeserializeArray( + arrays_[array_index]); + })); + table = Table::Make(schema_, arrays_); + return Status::OK(); + } + + private: + int num_threads_; + + std::vector> array_deserializers_; + + // Used to hold the output. + std::vector> arrays_; +}; + +// This class will deserialize all of the specified nodes from a single row group. +class RowGroupDeserializer : public SerialTableDeserializer { + public: + RowGroupDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema, + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) + : SerialTableDeserializer( + std::forward>>(nodes), schema), + row_group_reader_(row_group_reader) {} + + ~RowGroupDeserializer() {} + + ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final { + DeserializeRowGroup(row_group_reader_); + return MakeTable(table); + } + + private: + std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; +}; + +// This class wil deserialize all of the specified nodes from a Parquet file +class FileTableDeserializer : public SerialTableDeserializer { + public: + FileTableDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema, + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader) + : SerialTableDeserializer( + std::forward>>(nodes), schema), + file_reader_(file_reader) {} + + ~FileTableDeserializer() {} + + ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) { + auto num_row_groups = file_reader_->metadata()->num_row_groups(); + for (int ii = 0; ii < num_row_groups; ++ii) { + DeserializeRowGroup(file_reader_->RowGroup(ii)); + } + return MakeTable(table); + } + + private: + std::shared_ptr<::parquet::ParquetFileReader> file_reader_; +}; + +// This class does the main work of constructing deserializer +// nodes from Parquet schema nodes. +class DeserializerBuilder::Impl { + public: + Impl(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, int64_t buffer_size); + + ::arrow::Status BuildSchemaNodeDeserializer(int schema_index, + std::unique_ptr& result); + + ::arrow::Status BuildColumnDeserializer(int column_index, + std::unique_ptr& result); + + ::arrow::Status BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result); + + ::arrow::Status BuildRowGroupDeserializer(int row_group_index, + const std::vector& column_indices, + int num_threads, + std::unique_ptr& result); + + ::arrow::Status BuildFileDeserializer(const std::vector& column_indices, + int num_threads, + std::unique_ptr& result); + + ::arrow::Status BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result); + + private: + std::vector const FillColumnIndices() const { + auto const num_columns = file_reader_->metadata()->num_columns(); + std::vector indices(num_columns); + for (int ii = 0; ii < num_columns; ++ii) { + indices[ii] = ii; + } + return indices; + } + + // \brief calls BuildNode for each node in the parquet file's + // schema + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[out] the deserializer nodes + // \param[in] if specified, the nodes will appear in the result + // list based on the order in which they appeared in + // column_indices. For example, if the Parquet schema has two + // primitive nodes and the column_indices given is {1, 0}, + // then the resulting table will have the fields reversed + // in the Arrow schema. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported + Status BuildNodes(std::vector const& column_indices, + std::vector>& nodes, + bool in_index_order = true); + + // \brief builds a deserializer node based on the Parquet node. + // \param[in] the parquet node + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported + Status BuildNode(std::shared_ptr const& node, + std::vector const& indices, std::vector& indices_seen, + std::unique_ptr& result, + RepetitionLevel repetition_levels = RepetitionLevel(), + DefinitionLevel definition_levels = DefinitionLevel()); + + // \brief builds a primitive deserializer node based on the Parquet node. + // \param[in] the parquet node, which must point to a PrimitiveNode + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported + Status BuildPrimitiveNode(std::shared_ptr const& node, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, + std::vector const& indices, + std::vector& indices_seen, + std::unique_ptr& result); + + // \brief builds a group deserializer node based on the Parquet node. + // \param[in] the parquet node, which must point to a GroupNode + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported + Status BuildGroupNode(std::shared_ptr const& node, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, + std::vector const& indices, std::vector& indices_seen, + std::unique_ptr& result); + + // \brief builds a list deserializer node based on the Parquet node. + // \param[in] the parquet node, which must point to a GroupNode which + // has a LIST annotation. + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported + Status BuildListNode(std::shared_ptr const& node, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, + std::vector& indices_seen, + std::unique_ptr& result); + + // \brief builds a typed primitive node. This is the main entry point + // for a number of SFINAE based calls. + // \param[in] the parquet node, which must point to a PrimitiveNode + // \param[in] the column descriptor, which is used in error + // messages to report what node had an unsupported type + // \parma[in] the index of the column, which is stored in the node + // and then later used to lookup the column reader from + // the row group reader + // \param[out] the deserializer node + // \return error status if any of the converted/physical type + // specifications in the Parquet schema are unsupported + template + Status MakeTypedPrimitiveDeserializerNode( + std::shared_ptr const& node, + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) { + auto primitive_node = std::static_pointer_cast(node); + switch (primitive_node->physical_type()) { + case Type::BOOLEAN: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::INT32: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::INT64: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::INT96: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::FLOAT: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::DOUBLE: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::BYTE_ARRAY: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::FIXED_LEN_BYTE_ARRAY: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + default: + std::ostringstream stream; + stream << "Unrecognized type when creating a primitive node reader"; + return Status::Invalid(stream.str()); + } + } + + // \brief builds a typed primitive node, taking into account it's + // logical (aka converted) type. + // \param[in] the parquet node, which must point to a PrimitiveNode + // \param[in] the column descriptor, which is used in error + // messages to report what node had an unsupported type + // \parma[in] the index of the column, which is stored in the node + // and then later used to lookup the column reader from + // the row group reader + // \param[out] the deserializer node + // \return error status if any of the converted/physical type + // specifications in the Parquet schema are unsupported + template + Status MakeLogicalPrimitiveDeserializerNode( + std::shared_ptr const& node, + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) { + switch (node->logical_type()) { + case parquet::LogicalType::DATE: + return MakeDateDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::TIME_MILLIS: + return MakeTimeMillisDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::TIME_MICROS: + return MakeTimeMicrosDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::TIMESTAMP_MILLIS: + return MakeTimestampDeserializerNode( + column_descriptor, column_index, ::arrow::TimeUnit::MILLI, result); + case parquet::LogicalType::TIMESTAMP_MICROS: + return MakeTimestampDeserializerNode( + column_descriptor, column_index, ::arrow::TimeUnit::MICRO, result); + case parquet::LogicalType::DECIMAL: + return MakeDecimalDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::UINT_8: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::UINT_16: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::UINT_32: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::UINT_64: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::INT_8: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::INT_16: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::INT_32: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::INT_64: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case parquet::LogicalType::UTF8: + return MakeUTF8DeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::NA: + return MakeNullDeserializerNode( + column_descriptor, column_index, result); + default: + switch (node->physical_type()) { + case Type::BOOLEAN: + return MakeBooleanDeserializerNode( + column_descriptor, column_index, result); + case Type::INT32: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case Type::INT64: + return MakeIntegerDeserializerNode(column_descriptor, + column_index, result); + case Type::FLOAT: + return MakeFloatDeserializerNode( + column_descriptor, column_index, result); + case Type::DOUBLE: + return MakeDoubleDeserializerNode( + column_descriptor, column_index, result); + case Type::BYTE_ARRAY: + return MakeByteArrayDeserializerNode( + column_descriptor, column_index, result); + case Type::FIXED_LEN_BYTE_ARRAY: + return MakeFixedLengthByteArrayDeserializerNode( + column_descriptor, column_index, result); + case Type::INT96: + return MakeImpalaTimestampDeserializerNode( + column_descriptor, column_index, result); + default: + std::ostringstream stream; + stream << "Unsupported physical type " << node->physical_type() << std::endl; + return Status::Invalid(stream.str()); + } + } + } + + // + // The functions below create the various logical types of primitive nodes. + // Since the ParquetType template parameter is determined at run-time and we + // bind that earlier in the call stack, we can use SFINAE to limit the + // allowable types for a particular field (e.g. - Timestamp types can only + // be constructed from an int64 physical type). + // + // Most of the code is boiler-plate, except for the fact that there are + // different builders for the various types. + + template + Status MakeNullDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::TimestampType>(::arrow::TimeUnit::NANO); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a nano timestamp field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int96"; + return Status::Invalid(stream.str()); + } + + template + struct SupportsDecimal { + static constexpr auto value = + (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value); + }; + + template + typename std::enable_if::value, Status>::type + MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::Decimal128Type>( + column_descriptor->type_precision(), column_descriptor->type_scale()); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, pool_); + } + + template + typename std::enable_if::value, Status>::type + MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a decimal field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int32, int64, or fixed length byte array "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a boolean field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an boolean "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIME_MILLIS field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int32 "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::Time32Type>(::arrow::TimeUnit::MILLI); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIME_MILLIS field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int32 "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::Time64Type>(::arrow::TimeUnit::MICRO); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIME_MICROS field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int64"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, ::arrow::TimeUnit::type time_unit, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::TimestampType>(time_unit); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, ::arrow::TimeUnit::type time_unit, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIMESTAMP field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int64"; + return Status::Invalid(stream.str()); + } + + template + struct CanCastInteger { + static constexpr auto parquet_is_int = + std::is_integral::value; + + static constexpr auto arrow_is_int = + std::is_integral::value; + + static constexpr auto parquet_size = sizeof(typename ParquetType::c_type); + + static constexpr auto arrow_size = sizeof(typename ArrayBuilderType::value_type); + + static constexpr auto both_ints = parquet_is_int && arrow_is_int; + + static constexpr auto value = (both_ints && parquet_size >= arrow_size); + }; + + template + typename std::enable_if::value, + Status>::type + MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make an integer field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an integer or not the correct size "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make an UTF8 field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a byte array"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a byte array field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a byte array"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeFixedLengthByteArrayDeserializerNode( + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) + + { + auto arrow_type = + std::make_shared<::arrow::FixedSizeBinaryType>(column_descriptor->type_length()); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeFixedLengthByteArrayDeserializerNode( + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a byte array field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a fixed length byte array"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a double field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a double type"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); + } + + template + typename std::enable_if::value, + Status>::type + MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a float field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a float type"; + return Status::Invalid(stream.str()); + } + + // \brief The primary function for creating a primitive node. This will + // create an array builder of the specified type, passing + // args to its constructor, and finally pass that builder + // to the TypedPrimitiveDeserializerNode constructor + // \param[in] the column descriptor, which is used in error + // messages to report what node had an unsupported type + // \parma[in] the index of the column, which is stored in the node + // and then later used to lookup the column reader from + // the row group reader + // \param[in] arguments that are passed to the BuilderType's + // constructor + // \param[out] the deserializer node + // \return error status if any of the converted/physical type + // specifications in the Parquet schema are unsupported + template + Status MakeTypedPrimitiveNodeWithBuilder( + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result, Args&&... args) { + auto builder = std::make_shared(std::forward(args)...); + using ReturnType = TypedPrimitiveDeserializerNode, + ArrayBuilderType, DeserializerType>; + auto node = std::unique_ptr( + new ReturnType(column_descriptor, column_index, buffer_size_, builder, pool_)); + RETURN_NOT_OK(node->Initialize()); + result = std::move(node); + return Status::OK(); + } + + // This method is pulled out here instead of inside BuildNode because + // we skip the "middle" node when constructing List nodes. + void IncrementLevels(std::shared_ptr const& node, + RepetitionLevel& repetition_levels, + DefinitionLevel& definition_levels) { + if (!node->is_required()) { + ++definition_levels; + } + if (node->is_repeated()) { + ++repetition_levels; + } + } + + std::shared_ptr<::parquet::ParquetFileReader> file_reader_; + + ::arrow::MemoryPool* pool_; + + int64_t buffer_size_; +}; + +DeserializerBuilder::Impl::Impl( + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, int64_t buffer_size) + : file_reader_(file_reader), pool_(pool), buffer_size_(buffer_size) {} + +::arrow::Status DeserializerBuilder::Impl::BuildSchemaNodeDeserializer( + int schema_index, std::unique_ptr& result) { + std::vector> nodes; + // Pass false to in_index_order so that the nodes are created + // in schema order so we can look it up that way + auto const column_indices = FillColumnIndices(); + RETURN_NOT_OK(BuildNodes(column_indices, nodes, false /* in_index_order */)); + if (static_cast(schema_index) >= nodes.size()) { + std::ostringstream stream; + stream << "Could not find schema node with index "; + stream << schema_index << ", there were only " << nodes.size(); + stream << "nodes"; + return Status::Invalid(stream.str()); + } + // nodes is guaranteed to be non-empty after calling BuildNodes + result.reset(new FileArrayDeserializer(std::move(nodes[schema_index]), file_reader_)); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildColumnDeserializer( + int column_index, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes({column_index}, nodes)); + // nodes is guaranteed to be non-empty after calling BuildNodes + if (nodes.size() > 1) { + std::ostringstream stream; + stream << "Unable to build a deserializer for column index "; + stream << column_index << ", multiple deserializer nodes "; + stream << "were created"; + return Status::Invalid(stream.str()); + } + result.reset(new FileArrayDeserializer(std::move(nodes[0]), file_reader_)); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes({column_index}, nodes)); + // nodes is guaranteed to be non-empty after calling BuildNodes + if (nodes.size() > 1) { + std::ostringstream stream; + stream << "Unable to build an array batched deserializer for column index "; + stream << column_index << ", multiple deserializer nodes "; + stream << "were created"; + return Status::Invalid(stream.str()); + } + std::unique_ptr deserializer( + new FileArrayBatchedDeserializer(std::move(nodes[0]), file_reader_)); + RETURN_NOT_OK(deserializer->Initialize()); + result = std::move(deserializer); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes({column_index}, nodes)); + // nodes is guaranteed to be non-empty after calling BuildNodes + if (nodes.size() > 1) { + std::ostringstream stream; + stream << "Unable to build a deserializer for column chunk "; + stream << "with column index " << column_index << " and "; + stream << "row group " << row_group_index << ", multiple deserializer nodes "; + stream << "were created"; + return Status::Invalid(stream.str()); + } + std::unique_ptr deserializer(new ColumnChunkDeserializer( + std::move(nodes[0]), file_reader_->RowGroup(row_group_index))); + RETURN_NOT_OK(deserializer->Initialize()); + result = std::move(deserializer); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildRowGroupDeserializer( + int row_group_index, const std::vector& indices, int num_threads, + std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes(indices, nodes)); + std::shared_ptr<::arrow::Schema> schema; + RETURN_NOT_OK(FromParquetSchema(file_reader_->metadata()->schema(), indices, &schema)); + if (num_threads <= 1) { + result.reset(new RowGroupDeserializer(std::move(nodes), schema, + file_reader_->RowGroup(row_group_index))); + } else { + // When reading in parallel, we construct a + // ColumnChunkDeserializer for each node (aka column). The + // ParallelTableDeserializer simply invokes the array deserializer + // in a separate thread and holds onto its result + std::vector> array_deserializers; + for (auto&& node : nodes) { + // Note: the call to RowGroup should be done for each node, + // so it has its own RowGroupReader. Otherwise, it would be + // shared across threads and would induce a race. + std::unique_ptr current(new ColumnChunkDeserializer( + std::move(node), file_reader_->RowGroup(row_group_index))); + RETURN_NOT_OK(current->Initialize()); + array_deserializers.push_back(std::move(current)); + } + result.reset(new ParallelTableDeserializer(schema, std::move(array_deserializers), + num_threads)); + } + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildFileDeserializer( + const std::vector& column_indices, int num_threads, + std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes(column_indices, nodes)); + std::shared_ptr<::arrow::Schema> schema; + RETURN_NOT_OK( + FromParquetSchema(file_reader_->metadata()->schema(), column_indices, &schema)); + if (num_threads <= 1) { + result.reset(new FileTableDeserializer(std::move(nodes), schema, file_reader_)); + } else { + // When reading in parallel, we construct a FileArrayDeserializer + // for each node (aka column). The ParallelTableDeserializer + // simply invokes the array deserializer in a separate thread and + // holds onto its result + std::vector> array_deserializers; + for (auto&& node : nodes) { + array_deserializers.emplace_back( + new FileArrayDeserializer(std::move(node), file_reader_)); + } + result.reset(new ParallelTableDeserializer(schema, std::move(array_deserializers), + num_threads)); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildNodes( + std::vector const& indices, + std::vector>& result, bool in_index_order) { + auto node = std::const_pointer_cast( + file_reader_->metadata()->schema()->schema_root()); + if (!node->is_group()) { + std::ostringstream stream; + stream << "Expected a group node at the root of the schema, "; + stream << "but did not encounter one"; + return Status::Invalid(stream.str()); + } + auto group_node = std::static_pointer_cast(node); + using location = decltype(indices.begin() - indices.end()); + using reader_and_indices = std::tuple, location>; + std::vector deserializers; + for (int ii = 0; ii < group_node->field_count(); ++ii) { + std::vector indices_seen; + std::unique_ptr deserializer; + RETURN_NOT_OK(BuildNode(group_node->field(ii), indices, indices_seen, deserializer)); + if (!deserializer) { + continue; + } + auto earliest = indices.end(); + for (auto index_seen : indices_seen) { + auto current = std::find(indices.begin(), indices.end(), index_seen); + if (current < earliest) { + earliest = current; + } + } + if (earliest == indices.end()) { + std::ostringstream stream; + stream << "Could not determine the ordering of the fields"; + stream << " for " << group_node->field(ii)->path()->ToDotString(); + stream << " because the indices for the columns seen did not "; + stream << " match any that were given"; + return Status::Invalid(stream.str()); + } + deserializers.emplace_back( + std::make_tuple(std::move(deserializer), earliest - indices.begin())); + } + if (deserializers.empty()) { + std::ostringstream stream; + stream << "Unable to match any of the columns to the specified indices: ["; + bool first = true; + for (auto index : indices) { + if (!first) { + stream << ", "; + } else { + first = false; + } + stream << index; + } + stream << "]"; + return Status::Invalid(stream.str()); + } + if (in_index_order) { + std::sort(deserializers.begin(), deserializers.end(), + [](reader_and_indices const& first, reader_and_indices const& second) { + return std::get<1>(first) < std::get<1>(second); + }); + } + for (auto& current : deserializers) { + result.emplace_back(std::move(std::get<0>(current))); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildNode(std::shared_ptr const& node, + std::vector const& indices, + std::vector& indices_seen, + std::unique_ptr& result, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels) { + IncrementLevels(node, repetition_levels, definition_levels); + if (node->is_group()) { + return BuildGroupNode(node, repetition_levels, definition_levels, indices, + indices_seen, result); + } else { + return BuildPrimitiveNode(node, repetition_levels, definition_levels, indices, + indices_seen, result); + } +} + +Status DeserializerBuilder::Impl::BuildPrimitiveNode( + std::shared_ptr const& node, RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result) { + auto const schema = file_reader_->metadata()->schema(); + auto column_index = schema->ColumnIndex(*node); + auto found = std::find(indices.begin(), indices.end(), column_index); + if (found == indices.end()) { + // This was not specified in the list of indices, so it doesn't + // show up in the resulting deserializer. Parent's will check for + // null and possibly filter themselves out (e.g. - a struct with + // no fields that match nodes with columns in indices) + return Status::OK(); + } + indices_seen.emplace_back(column_index); + auto column_descriptor = schema->Column(column_index); + if (*definition_levels > 0) { + if (*repetition_levels > 0) { + // The path to the node either contains only repeated nodes or + // a mix of repeated and optional nodes. + return MakeTypedPrimitiveDeserializerNode( + node, column_descriptor, column_index, result); + } else { + // A field that is either itself optional or a descendant of + // an optional field (e.g. - member of a struct that is optional) + // but does not have any ancestors in the tree that are repeated. + // It therefore won't have any repetition levels. + return MakeTypedPrimitiveDeserializerNode( + node, column_descriptor, column_index, result); + } + } else if (*repetition_levels == 0) { + // A field that is always present (max_definition_level == + // 0 && max_repetition_level == 0) and therefore doesn't + // need repetition levels or definition levels + return MakeTypedPrimitiveDeserializerNode( + node, column_descriptor, column_index, result); + } else { + // This should never happen since a non-zero max repetition level + // should have a non-zero definition level + std::ostringstream stream; + stream << "Primitive node " << node->path()->ToDotString(); + stream << " has a max_repetition_level or "; + stream << *definition_levels; + stream << " and a max_definition_level of "; + stream << *repetition_levels; + return Status::Invalid(stream.str()); + } + if (node->is_repeated()) { + result.reset(new ListDeserializerNode(std::move(result), pool_, repetition_levels, + definition_levels)); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildGroupNode( + std::shared_ptr const& node, RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result) { + // TODO: implement a proper Visitor interface in Node + auto group_node = std::static_pointer_cast(node); + if (node->logical_type() == LogicalType::LIST) { + return BuildListNode(group_node, repetition_levels, definition_levels, indices, + indices_seen, result); + } + std::vector> children; + std::vector> fields; + for (int ii = 0; ii < group_node->field_count(); ++ii) { + auto child_node = group_node->field(static_cast(ii)); + std::unique_ptr child; + RETURN_NOT_OK(BuildNode(child_node, indices, indices_seen, child, repetition_levels, + definition_levels)); + // Check to see if the child is valid. If so, + // it or at least one of its descendants was in + // the list of indices. + if (child) { + // TODO: deal with metadata args + fields.emplace_back(std::make_shared<::arrow::Field>( + child_node->name(), child->GetArrayBuilder()->type(), + // If the child node is required, + // then it's not nullable + !child_node->is_required())); + children.emplace_back(std::move(child)); + } + } + if (children.empty()) { + // All of the descendants have been filtered out because + // of the indices + return Status::OK(); + } + if (node->is_optional()) { + result.reset(new NestedDeserializerNode(definition_levels, std::move(children), + fields, pool_)); + } else { + result.reset(new NestedDeserializerNode(definition_levels, std::move(children), + fields, pool_)); + } + if (node->is_repeated()) { + std::ostringstream stream; + stream << "Deserializing from Parquet repeated nodes to Arrow "; + stream << "is not supported without a LIST annotation"; + return Status::Invalid(stream.str()); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildListNode( + std::shared_ptr const& node, RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result) { + // + // A list node must have this schema: + // + // (required|optional) group outer { + // repeated group inner { + // (required|optiona) (group|primitive) element [{ children }] + // } + // } + // + // We construct a single node from outer and inner, using + // the definition level from outer and the repetition level + // from inner. This allows us to determine null lists when + // outer is optional, and empty lists in both cases. + if (node->is_repeated()) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " is repeated. List nodes can only be "; + stream << " optional or required, and must contain "; + stream << " a single child repeated group child node"; + return Status::Invalid(stream.str()); + } + if (node->field_count() != 1) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " has multiple children, it should have"; + stream << " a single repeated group child node"; + return Status::Invalid(stream.str()); + } + auto child_node = node->field(0); + if (!child_node->is_repeated()) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " should have a single repeated group child node, but "; + stream << " its child is not a repeated node"; + return Status::Invalid(stream.str()); + } + if (!child_node->is_group()) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " should have a single repeated group child node, but "; + stream << " its child is not a repeated primitive node"; + return Status::Invalid(stream.str()); + } + + // Since we've obtained the child node without calling + // BuildNode, we need to call IncrementLevels before + // building its children. We also save the definition level + // before we increment them, so that we can use it if this + // node is nullable. + auto current_max_definition_level = definition_levels; + IncrementLevels(child_node, repetition_levels, definition_levels); + auto group_node = std::static_pointer_cast(child_node); + if (group_node->field_count() > 1) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " should have a repeated group node as its child,"; + stream << " which has one child. However, the repeated group"; + stream << " node has " << group_node->field_count() << "children"; + return Status::Invalid(stream.str()); + } + std::unique_ptr child; + RETURN_NOT_OK(BuildNode(group_node->field(0), indices, indices_seen, child, + repetition_levels, definition_levels)); + if (!child) { + // The descendants have been filtered out based on indices + // so we retrun OK with a null result so that caller will + // know that this list has been filtered + return Status::OK(); + } + // If this node is required, then we defer the list building + // to the child node, since the list cannot be nullable and + // this node is effectively a pass through to the repeated + // child node. However, if this node is optional, we need to + // make a NullableListDeserializerNode, which will first check to + // see if there are any primitives in its tree that have + // a definition level equal to its max_definition_level. If + // there aren't, then it writes a null and skips writing + // any of its children. + if (node->is_optional()) { + result.reset(new NullableListDeserializerNode( + std::move(child), pool_, repetition_levels, current_max_definition_level)); + } else { + result.reset(new ListDeserializerNode(std::move(child), pool_, repetition_levels, + current_max_definition_level)); + } + return Status::OK(); +} + +DeserializerBuilder::DeserializerBuilder( + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, int64_t buffer_size) + : impl_(new Impl(file_reader, pool, buffer_size)) {} + +DeserializerBuilder::~DeserializerBuilder() {} + +::arrow::Status DeserializerBuilder::BuildSchemaNodeDeserializer( + int schema_index, std::unique_ptr& result) { + return impl_->BuildSchemaNodeDeserializer(schema_index, result); +} + +::arrow::Status DeserializerBuilder::BuildColumnDeserializer( + int column_index, std::unique_ptr& result) { + return impl_->BuildColumnDeserializer(column_index, result); +} + +::arrow::Status DeserializerBuilder::BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result) { + return impl_->BuildColumnChunkDeserializer(column_index, row_group_index, result); +} + +::arrow::Status DeserializerBuilder::BuildRowGroupDeserializer( + int row_group_index, const std::vector& indices, int num_threads, + std::unique_ptr& result) { + return impl_->BuildRowGroupDeserializer(row_group_index, indices, num_threads, result); +} + +::arrow::Status DeserializerBuilder::BuildFileDeserializer( + const std::vector& column_indices, int num_threads, + std::unique_ptr& result) { + return impl_->BuildFileDeserializer(column_indices, num_threads, result); +} + +::arrow::Status DeserializerBuilder::BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result) { + return impl_->BuildArrayBatchedDeserializer(column_index, result); +} + +} // namespace arrow +} // namespace parquet diff --git a/src/parquet/arrow/deserializer.h b/src/parquet/arrow/deserializer.h new file mode 100644 index 00000000..0f2c3711 --- /dev/null +++ b/src/parquet/arrow/deserializer.h @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#ifndef PARQUET_ARROW_DESERIALIZER_H +#define PARQUET_ARROW_DESERIALIZER_H + +#include +#include + +#include "arrow/status.h" +#include "parquet/util/visibility.h" + +namespace arrow { + +class Array; +class MemoryPool; +class Schema; +class Table; + +} // namespace arrow + +namespace parquet { + +class ParquetFileReader; +class RowGroupReader; + +namespace arrow { + +class DeserializerNode; + +// +// This header file contains classes that can deserialize from Parquet +// files into Arrow arrays and tables. It supports nested structures +// such as this (taken from the Dremel paper): +// +// message root { +// required int32 DocId; +// optional group Links { +// required group Forward_Outer (LIST) { +// repeated group Forward_Inner { +// required int32 Forward; +// } +// } +// required group Backward_Outer (LIST) { +// repeated group Backward_Inner { +// required int32 Backward; +// } +// } +// } +// required group Name_Outer (LIST) { +// repeated group Name_Inner { +// required group Name_Element { +// required group Language_Outer (LIST) { +// repeated group Language_Inner { +// required group Language_Element { +// required binary Code (UTF8); +// optional binary Country (UTF8); +// } +// } +// } +// optional binary Url (UTF8); +// } +// } +// } +// } +// +// There are two types of serializers, one for arrays and one for tables. +// To obtain a deserializer, first construct a DeserializerBuilder and +// then call one of its Build* methods. +// +// The contract of the ArrayDeserializer::DeserializeArray and +// TableDeserializer::DeserializeTable is that they should be called once +// and only once. + +// +// Interface for deserializing a Parquet node into an Arrow Array. A node can +// contain multiple columns in the case of a Parquet group node. +// +class PARQUET_EXPORT ArrayDeserializer { + public: + virtual ~ArrayDeserializer(); + + // \brief Deserializes the node for which this deserializer was constructed + // \param[out] array to deserialize into + // \return error status if there is a problem reading the data from the + // Parquet file or in constructing the array + virtual ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) = 0; + + protected: + ArrayDeserializer(std::unique_ptr&& node); + + std::unique_ptr node_; +}; + +// +// Interface for deserializing a Parquet node into Arrow Arrays in batches. A node +// can contain multiple columns in the case of a Parquet group node. +// +class PARQUET_EXPORT ArrayBatchedDeserializer { + public: + virtual ~ArrayBatchedDeserializer(); + + // \brief Deserializes a batch of Parquet rows + // \param[in] number of records to deserialize in this batch + // \param[out] array to deserialize into + // \return error status if there is a problem reading the data from the + // Parquet file or in constructing the array. The number of rows + // will be the minimum of num_records and the remaining records + // available for deserialization + virtual ::arrow::Status DeserializeBatch(int64_t num_records, + std::shared_ptr<::arrow::Array>& array) = 0; +}; + +// +// Interface for deserializing Parquet nodes in an Arrow table. The schema +// of the Table will depend on which method was invoked in DeserializerBuilder. +// For example, if column indices are specified in the opposite order of how +// they appear in the Parquet file, the resulting Arrow schema will have its +// fields in the order in which the indices are specified. +// +class PARQUET_EXPORT TableDeserializer { + public: + virtual ~TableDeserializer(); + + // \brief Deserializes Parquet nodes into an Arrow Table + // \param[out] table to deserialize into + // \return error status if there is a problem reading the data from the + // Parquet file or in constructing the Table. + virtual ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& result) = 0; + + protected: + TableDeserializer(std::shared_ptr<::arrow::Schema> const& schema); + + // Schema, which was most likely obtained from a call to + // FromParquetSchema + std::shared_ptr<::arrow::Schema> schema_; +}; + +// +// A builder of deserializers. This is the main entry point to obtaining +// Parquet deserializers. +// +class PARQUET_EXPORT DeserializerBuilder { + public: + // + // \brief Create a new deserializer builder + // \param[in] the reader that is the source of the data to deserialize + // \param[in] the memory pool to use for builders and buffers that + // are created for reading primitive values + // \param[in] the deserializers use buffers for reading Parquet definition + // and repetition levels, as well as values. This denotes how + // many values will be buffered when reading from primitive + // columns. + DeserializerBuilder(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, + // TODO: make this configurable from a reader + // properties object + int64_t buffer_size = 1024); + + // Defined so that we can hide the impl and use a unique_ptr + ~DeserializerBuilder(); + + // + // \brief builds an array deserializer that will deserialize a particular + // node in the Parquet schema. + // \param[in] the index which refers to one of the top + // level nodes in the Parquet schema. For exmaple, in the example schema + // defined at the top of this file, index 0 would be the DocId, index 1 + // would be the Links, and index 2 would be Name_Outer. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the schema_index + // is out of bounds + // + ::arrow::Status BuildSchemaNodeDeserializer(int schema_index, + std::unique_ptr& result); + + // + // \brief builds an array deserializer that will deserialize a particular + // node in the Parquet schema. + // \param[in] the index which refers to a primitive node in the Parquet + // schema. In the example schema at the top of this file, + // DocId would have index 0, Forward index 1, Backward index 2, + // Code index 3, and so forth. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the column_index + // is out of bounds. + // + ::arrow::Status BuildColumnDeserializer(int column_index, + std::unique_ptr& result); + + // + // \brief builds an array deserializer that cant deserialize a particular + // node in the Parquet schema in batches. + // \param[in] the index which refers to a primitive node in the Parquet + // schema. In the example schema at the top of this file, + // DocId would have index 0, Forward index 1, Backward index 2, + // Code index 3, and so forth. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the column_index + // is out of bounds. + // + ::arrow::Status BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result); + + // + // \brief builds an array deserializer that will deserialize a particular + // node in the Parquet schema for a particular row group. This + // is known as a ColumnChunk in the Parquet metadata. + // \param[in] the index which refers to a primitive node in the Parquet + // schema. In the example schema at the top of this file, + // DocId would have index 0, Forward index 1, Backward index 2, + // Code index 3, and so forth. + // \param[in] the row group to deserialize + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the column_index + // or row_group_index is out of bounds. + // + ::arrow::Status BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result); + + // + // \brief builds a table deserializer that will deserialize all of the + // nodes in the Parquet schema for a particular row group. + // \param[in] the row group to deserialize + // \param[in] the column indices to use for filtering the Parquet schema. + // For example, if column_index 2 is specified for the example schema + // at the top of this file, it will include Code and all of its + // ancestors in the resulting table. + // \param[in] the number of threads to use when deserializing. This + // will cause the top level columns to be read in parallel + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the row_group_index + // is out of bounds. + // + ::arrow::Status BuildRowGroupDeserializer(int row_group_index, + const std::vector& indices, + int num_threads, + std::unique_ptr& result); + + // \brief build a table deserializer that will only deserialize nodes that + // contain the specified column indices. + // \param[in] the column indices to use for filtering the Parquet schema. + // For example, if column_index 2 is specified for the example schema + // at the top of this file, it will include Code and all of its + // ancestors in the resulting table. + // \param[in] the number of threads to use when deserializing. This + // will cause the top level columns to be read in parallel + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if all of the + // column indices are out of bounds. + ::arrow::Status BuildFileDeserializer(const std::vector& column_indices, + int num_threads, + std::unique_ptr& result); + + private: + // Hiding implementation due to a large number of member template methods + // with SFINAE. + class PARQUET_NO_EXPORT Impl; + std::unique_ptr impl_; +}; + +} // namespace arrow +} // namespace parquet + +#endif // PARQUET_ARROW_DESERIALIZER_H diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 1f933e64..161eb99e 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "arrow/util/logging.h" #include "arrow/util/parallel.h" +#include "parquet/arrow/deserializer.h" #include "parquet/arrow/record_reader.h" #include "parquet/arrow/schema.h" #include "parquet/column_reader.h" @@ -66,93 +68,9 @@ namespace arrow { using ::arrow::BitUtil::BytesForBits; -constexpr int64_t kJulianToUnixEpochDays = 2440588LL; -constexpr int64_t kMillisecondsInADay = 86400000LL; -constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL; - -static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timestamp) { - int64_t days_since_epoch = impala_timestamp.value[2] - kJulianToUnixEpochDays; - int64_t nanoseconds = *(reinterpret_cast(&(impala_timestamp.value))); - return days_since_epoch * kNanosecondsInADay + nanoseconds; -} - template using ArrayType = typename ::arrow::TypeTraits::ArrayType; -// ---------------------------------------------------------------------- -// Iteration utilities - -// Abstraction to decouple row group iteration details from the ColumnReader, -// so we can read only a single row group if we want -class FileColumnIterator { - public: - explicit FileColumnIterator(int column_index, ParquetFileReader* reader) - : column_index_(column_index), - reader_(reader), - schema_(reader->metadata()->schema()) {} - - virtual ~FileColumnIterator() {} - - virtual std::unique_ptr<::parquet::PageReader> NextChunk() = 0; - - const SchemaDescriptor* schema() const { return schema_; } - - const ColumnDescriptor* descr() const { return schema_->Column(column_index_); } - - std::shared_ptr metadata() const { return reader_->metadata(); } - - int column_index() const { return column_index_; } - - protected: - int column_index_; - ParquetFileReader* reader_; - const SchemaDescriptor* schema_; -}; - -class AllRowGroupsIterator : public FileColumnIterator { - public: - explicit AllRowGroupsIterator(int column_index, ParquetFileReader* reader) - : FileColumnIterator(column_index, reader), next_row_group_(0) {} - - std::unique_ptr<::parquet::PageReader> NextChunk() override { - std::unique_ptr<::parquet::PageReader> result; - if (next_row_group_ < reader_->metadata()->num_row_groups()) { - result = reader_->RowGroup(next_row_group_)->GetColumnPageReader(column_index_); - next_row_group_++; - } else { - result = nullptr; - } - return result; - } - - private: - int next_row_group_; -}; - -class SingleRowGroupIterator : public FileColumnIterator { - public: - explicit SingleRowGroupIterator(int column_index, int row_group_number, - ParquetFileReader* reader) - : FileColumnIterator(column_index, reader), - row_group_number_(row_group_number), - done_(false) {} - - std::unique_ptr<::parquet::PageReader> NextChunk() override { - if (done_) { - return nullptr; - } - - auto result = - reader_->RowGroup(row_group_number_)->GetColumnPageReader(column_index_); - done_ = true; - return result; - } - - private: - int row_group_number_; - bool done_; -}; - class RowGroupRecordBatchReader : public ::arrow::RecordBatchReader { public: explicit RowGroupRecordBatchReader(const std::vector& row_group_indices, @@ -218,11 +136,6 @@ class FileReader::Impl { Status GetColumn(int i, std::unique_ptr* out); Status ReadSchemaField(int i, std::shared_ptr* out); - Status ReadSchemaField(int i, const std::vector& indices, - std::shared_ptr* out); - Status GetReaderForNode(int index, const Node* node, const std::vector& indices, - int16_t def_level, - std::unique_ptr* out); Status ReadColumn(int i, std::shared_ptr* out); Status ReadColumnChunk(int column_index, int row_group_index, std::shared_ptr* out); @@ -235,9 +148,6 @@ class FileReader::Impl { Status ReadTable(std::shared_ptr
* table); Status ReadRowGroup(int i, std::shared_ptr
* table); - bool CheckForFlatColumn(const ColumnDescriptor* descr); - bool CheckForFlatListColumn(const ColumnDescriptor* descr); - const ParquetFileReader* parquet_reader() const { return reader_.get(); } int num_row_groups() const { return reader_->metadata()->num_row_groups(); } @@ -250,192 +160,38 @@ class FileReader::Impl { private: MemoryPool* pool_; - std::unique_ptr reader_; + std::shared_ptr reader_; int num_threads_; }; -class ColumnReader::ColumnReaderImpl { - public: - virtual ~ColumnReaderImpl() {} - virtual Status NextBatch(int64_t records_to_read, std::shared_ptr* out) = 0; - virtual Status GetDefLevels(const int16_t** data, size_t* length) = 0; - virtual Status GetRepLevels(const int16_t** data, size_t* length) = 0; - virtual const std::shared_ptr field() = 0; -}; - -// Reader implementation for primitive arrays -class PARQUET_NO_EXPORT PrimitiveImpl : public ColumnReader::ColumnReaderImpl { - public: - PrimitiveImpl(MemoryPool* pool, std::unique_ptr input) - : pool_(pool), input_(std::move(input)), descr_(input_->descr()) { - record_reader_ = RecordReader::Make(descr_, pool_); - DCHECK(NodeToField(*input_->descr()->schema_node(), &field_).ok()); - NextRowGroup(); - } - - Status NextBatch(int64_t records_to_read, std::shared_ptr* out) override; - - template - Status WrapIntoListArray(std::shared_ptr* array); - - Status GetDefLevels(const int16_t** data, size_t* length) override; - Status GetRepLevels(const int16_t** data, size_t* length) override; - - const std::shared_ptr field() override { return field_; } - - private: - void NextRowGroup(); - - MemoryPool* pool_; - std::unique_ptr input_; - const ColumnDescriptor* descr_; - - std::shared_ptr record_reader_; - - std::shared_ptr field_; -}; - -// Reader implementation for struct array -class PARQUET_NO_EXPORT StructImpl : public ColumnReader::ColumnReaderImpl { - public: - explicit StructImpl(const std::vector>& children, - int16_t struct_def_level, MemoryPool* pool, const Node* node) - : children_(children), - struct_def_level_(struct_def_level), - pool_(pool), - def_levels_buffer_(pool) { - InitField(node, children); - } - - Status NextBatch(int64_t records_to_read, std::shared_ptr* out) override; - Status GetDefLevels(const int16_t** data, size_t* length) override; - Status GetRepLevels(const int16_t** data, size_t* length) override; - const std::shared_ptr field() override { return field_; } - - private: - std::vector> children_; - int16_t struct_def_level_; - MemoryPool* pool_; - std::shared_ptr field_; - PoolBuffer def_levels_buffer_; - - Status DefLevelsToNullArray(std::shared_ptr* null_bitmap, int64_t* null_count); - void InitField(const Node* node, - const std::vector>& children); -}; - FileReader::FileReader(MemoryPool* pool, std::unique_ptr reader) : impl_(new FileReader::Impl(pool, std::move(reader))) {} FileReader::~FileReader() {} Status FileReader::Impl::GetColumn(int i, std::unique_ptr* out) { - std::unique_ptr input(new AllRowGroupsIterator(i, reader_.get())); - - std::unique_ptr impl( - new PrimitiveImpl(pool_, std::move(input))); - *out = std::unique_ptr(new ColumnReader(std::move(impl))); - return Status::OK(); -} - -Status FileReader::Impl::GetReaderForNode( - int index, const Node* node, const std::vector& indices, int16_t def_level, - std::unique_ptr* out) { - *out = nullptr; - - if (IsSimpleStruct(node)) { - const schema::GroupNode* group = static_cast(node); - std::vector> children; - for (int i = 0; i < group->field_count(); i++) { - std::unique_ptr child_reader; - // TODO(itaiin): Remove the -1 index hack when all types of nested reads - // are supported. This currently just signals the lower level reader resolution - // to abort - RETURN_NOT_OK(GetReaderForNode(index, group->field(i).get(), indices, - static_cast(def_level + 1), &child_reader)); - if (child_reader != nullptr) { - children.push_back(std::move(child_reader)); - } - } - - if (children.size() > 0) { - *out = std::unique_ptr( - new StructImpl(children, def_level, pool_, node)); - } - } else { - // This should be a flat field case - translate the field index to - // the correct column index by walking down to the leaf node - const Node* walker = node; - while (!walker->is_primitive()) { - DCHECK(walker->is_group()); - auto group = static_cast(walker); - if (group->field_count() != 1) { - return Status::NotImplemented("lists with structs are not supported."); - } - walker = group->field(0).get(); - } - auto column_index = reader_->metadata()->schema()->ColumnIndex(*walker); - - // If the index of the column is found then a reader for the coliumn is needed. - // Otherwise *out keeps the nullptr value. - if (std::find(indices.begin(), indices.end(), column_index) != indices.end()) { - std::unique_ptr reader; - RETURN_NOT_OK(GetColumn(column_index, &reader)); - *out = std::move(reader->impl_); - } - } - + DeserializerBuilder builder(reader_, pool_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildArrayBatchedDeserializer(i, deserializer)); + (*out).reset(new ColumnReader(deserializer)); return Status::OK(); } Status FileReader::Impl::ReadSchemaField(int i, std::shared_ptr* out) { - std::vector indices(reader_->metadata()->num_columns()); - - for (size_t j = 0; j < indices.size(); ++j) { - indices[j] = static_cast(j); - } - - return ReadSchemaField(i, indices, out); -} - -Status FileReader::Impl::ReadSchemaField(int i, const std::vector& indices, - std::shared_ptr* out) { - auto parquet_schema = reader_->metadata()->schema(); - - auto node = parquet_schema->group_node()->field(i).get(); - std::unique_ptr reader_impl; - - RETURN_NOT_OK(GetReaderForNode(i, node, indices, 1, &reader_impl)); - if (reader_impl == nullptr) { - *out = nullptr; - return Status::OK(); - } - - std::unique_ptr reader(new ColumnReader(std::move(reader_impl))); - - // TODO(wesm): This calculation doesn't make much sense when we have repeated - // schema nodes - int64_t records_to_read = 0; - - const FileMetaData& metadata = *reader_->metadata(); - for (int j = 0; j < metadata.num_row_groups(); j++) { - records_to_read += metadata.RowGroup(j)->ColumnChunk(i)->num_values(); - } - - return reader->NextBatch(records_to_read, out); + DeserializerBuilder builder(reader_, pool_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildSchemaNodeDeserializer(i, deserializer)); + deserializer->DeserializeArray(*out); + return Status::OK(); } Status FileReader::Impl::ReadColumn(int i, std::shared_ptr* out) { - std::unique_ptr flat_column_reader; - RETURN_NOT_OK(GetColumn(i, &flat_column_reader)); - - int64_t records_to_read = 0; - for (int j = 0; j < reader_->metadata()->num_row_groups(); j++) { - records_to_read += reader_->metadata()->RowGroup(j)->ColumnChunk(i)->num_values(); - } - - return flat_column_reader->NextBatch(records_to_read, out); + DeserializerBuilder builder(reader_, pool_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildColumnDeserializer(i, deserializer)); + deserializer->DeserializeArray(*out); + return Status::OK(); } Status FileReader::Impl::GetSchema(const std::vector& indices, @@ -447,91 +203,31 @@ Status FileReader::Impl::GetSchema(const std::vector& indices, Status FileReader::Impl::ReadColumnChunk(int column_index, int row_group_index, std::shared_ptr* out) { - auto rg_metadata = reader_->metadata()->RowGroup(row_group_index); - int64_t records_to_read = rg_metadata->ColumnChunk(column_index)->num_values(); - - std::unique_ptr input( - new SingleRowGroupIterator(column_index, row_group_index, reader_.get())); - - std::unique_ptr impl( - new PrimitiveImpl(pool_, std::move(input))); - ColumnReader flat_column_reader(std::move(impl)); - - std::shared_ptr array; - RETURN_NOT_OK(flat_column_reader.NextBatch(records_to_read, &array)); - *out = array; + DeserializerBuilder builder(reader_, pool_); + std::unique_ptr deserializer; + RETURN_NOT_OK( + builder.BuildColumnChunkDeserializer(column_index, row_group_index, deserializer)); + deserializer->DeserializeArray(*out); return Status::OK(); } Status FileReader::Impl::ReadRowGroup(int row_group_index, const std::vector& indices, std::shared_ptr<::arrow::Table>* out) { - std::shared_ptr<::arrow::Schema> schema; - RETURN_NOT_OK(GetSchema(indices, &schema)); - - auto rg_metadata = reader_->metadata()->RowGroup(row_group_index); - - int num_columns = static_cast(indices.size()); - int nthreads = std::min(num_threads_, num_columns); - std::vector> columns(num_columns); - - // TODO(wesm): Refactor to share more code with ReadTable - - auto ReadColumnFunc = [&indices, &row_group_index, &schema, &columns, this](int i) { - int column_index = indices[i]; - - std::shared_ptr array; - RETURN_NOT_OK(ReadColumnChunk(column_index, row_group_index, &array)); - columns[i] = std::make_shared(schema->field(i), array); - return Status::OK(); - }; - - if (nthreads == 1) { - for (int i = 0; i < num_columns; i++) { - RETURN_NOT_OK(ReadColumnFunc(i)); - } - } else { - RETURN_NOT_OK(ParallelFor(nthreads, num_columns, ReadColumnFunc)); - } - - *out = Table::Make(schema, columns); + DeserializerBuilder builder(reader_, pool_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildRowGroupDeserializer(row_group_index, indices, num_threads_, + deserializer)); + deserializer->DeserializeTable(*out); return Status::OK(); } Status FileReader::Impl::ReadTable(const std::vector& indices, std::shared_ptr
* out) { - std::shared_ptr<::arrow::Schema> schema; - RETURN_NOT_OK(GetSchema(indices, &schema)); - - // We only need to read schema fields which have columns indicated - // in the indices vector - std::vector field_indices; - if (!ColumnIndicesToFieldIndices(*reader_->metadata()->schema(), indices, - &field_indices)) { - return Status::Invalid("Invalid column index"); - } - - std::vector> columns(field_indices.size()); - auto ReadColumnFunc = [&indices, &field_indices, &schema, &columns, this](int i) { - std::shared_ptr array; - RETURN_NOT_OK(ReadSchemaField(field_indices[i], indices, &array)); - columns[i] = std::make_shared(schema->field(i), array); - return Status::OK(); - }; - - int num_fields = static_cast(field_indices.size()); - int nthreads = std::min(num_threads_, num_fields); - if (nthreads == 1) { - for (int i = 0; i < num_fields; i++) { - RETURN_NOT_OK(ReadColumnFunc(i)); - } - } else { - RETURN_NOT_OK(ParallelFor(nthreads, num_fields, ReadColumnFunc)); - } - - std::shared_ptr
table = Table::Make(schema, columns); - RETURN_NOT_OK(table->Validate()); - *out = table; + DeserializerBuilder builder(reader_, pool_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildFileDeserializer(indices, num_threads_, deserializer)); + deserializer->DeserializeTable(*out); return Status::OK(); } @@ -688,773 +384,13 @@ const ParquetFileReader* FileReader::parquet_reader() const { return impl_->parquet_reader(); } -template -Status PrimitiveImpl::WrapIntoListArray(std::shared_ptr* array) { - const int16_t* def_levels = record_reader_->def_levels(); - const int16_t* rep_levels = record_reader_->rep_levels(); - const int64_t total_levels_read = record_reader_->levels_position(); - - std::shared_ptr<::arrow::Schema> arrow_schema; - RETURN_NOT_OK(FromParquetSchema(input_->schema(), {input_->column_index()}, - input_->metadata()->key_value_metadata(), - &arrow_schema)); - std::shared_ptr current_field = arrow_schema->field(0); - - if (descr_->max_repetition_level() > 0) { - // Walk downwards to extract nullability - std::vector nullable; - std::vector> offset_builders; - std::vector> valid_bits_builders; - nullable.push_back(current_field->nullable()); - while (current_field->type()->num_children() > 0) { - if (current_field->type()->num_children() > 1) { - return Status::NotImplemented( - "Fields with more than one child are not supported."); - } else { - if (current_field->type()->id() != ::arrow::Type::LIST) { - return Status::NotImplemented( - "Currently only nesting with Lists is supported."); - } - current_field = current_field->type()->child(0); - } - offset_builders.emplace_back( - std::make_shared<::arrow::Int32Builder>(::arrow::int32(), pool_)); - valid_bits_builders.emplace_back( - std::make_shared<::arrow::BooleanBuilder>(::arrow::boolean(), pool_)); - nullable.push_back(current_field->nullable()); - } - - int64_t list_depth = offset_builders.size(); - // This describes the minimal definition that describes a level that - // reflects a value in the primitive values array. - int16_t values_def_level = descr_->max_definition_level(); - if (nullable[nullable.size() - 1]) { - values_def_level--; - } - - // The definition levels that are needed so that a list is declared - // as empty and not null. - std::vector empty_def_level(list_depth); - int def_level = 0; - for (int i = 0; i < list_depth; i++) { - if (nullable[i]) { - def_level++; - } - empty_def_level[i] = static_cast(def_level); - def_level++; - } - - int32_t values_offset = 0; - std::vector null_counts(list_depth, 0); - for (int64_t i = 0; i < total_levels_read; i++) { - int16_t rep_level = rep_levels[i]; - if (rep_level < descr_->max_repetition_level()) { - for (int64_t j = rep_level; j < list_depth; j++) { - if (j == (list_depth - 1)) { - RETURN_NOT_OK(offset_builders[j]->Append(values_offset)); - } else { - RETURN_NOT_OK(offset_builders[j]->Append( - static_cast(offset_builders[j + 1]->length()))); - } - - if (((empty_def_level[j] - 1) == def_levels[i]) && (nullable[j])) { - RETURN_NOT_OK(valid_bits_builders[j]->Append(false)); - null_counts[j]++; - break; - } else { - RETURN_NOT_OK(valid_bits_builders[j]->Append(true)); - if (empty_def_level[j] == def_levels[i]) { - break; - } - } - } - } - if (def_levels[i] >= values_def_level) { - values_offset++; - } - } - // Add the final offset to all lists - for (int64_t j = 0; j < list_depth; j++) { - if (j == (list_depth - 1)) { - RETURN_NOT_OK(offset_builders[j]->Append(values_offset)); - } else { - RETURN_NOT_OK(offset_builders[j]->Append( - static_cast(offset_builders[j + 1]->length()))); - } - } - - std::vector> offsets; - std::vector> valid_bits; - std::vector list_lengths; - for (int64_t j = 0; j < list_depth; j++) { - list_lengths.push_back(offset_builders[j]->length() - 1); - std::shared_ptr array; - RETURN_NOT_OK(offset_builders[j]->Finish(&array)); - offsets.emplace_back(std::static_pointer_cast(array)->values()); - RETURN_NOT_OK(valid_bits_builders[j]->Finish(&array)); - valid_bits.emplace_back(std::static_pointer_cast(array)->values()); - } - - std::shared_ptr output(*array); - for (int64_t j = list_depth - 1; j >= 0; j--) { - auto list_type = - ::arrow::list(::arrow::field("item", output->type(), nullable[j + 1])); - output = std::make_shared<::arrow::ListArray>( - list_type, list_lengths[j], offsets[j], output, valid_bits[j], null_counts[j]); - } - *array = output; - } - return Status::OK(); -} - -template -struct supports_fast_path_impl { - using ArrowCType = typename ArrowType::c_type; - using ParquetCType = typename ParquetType::c_type; - static constexpr bool value = std::is_same::value; -}; - -template -struct supports_fast_path_impl { - static constexpr bool value = false; -}; - -template -struct supports_fast_path_impl { - static constexpr bool value = false; -}; - -template -using supports_fast_path = - typename std::enable_if::value>::type; - -template -struct TransferFunctor { - using ArrowCType = typename ArrowType::c_type; - using ParquetCType = typename ParquetType::c_type; - - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - static_assert(!std::is_same::value, - "The fast path transfer functor should be used " - "for primitive values"); - - int64_t length = reader->values_written(); - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(ArrowCType), &data)); - - auto values = reinterpret_cast(reader->values()); - auto out_ptr = reinterpret_cast(data->mutable_data()); - std::copy(values, values + length, out_ptr); - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared>(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared>(type, length, data); - } - return Status::OK(); - } -}; - -template -struct TransferFunctor> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - std::shared_ptr values = reader->ReleaseValues(); - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared>(type, length, values, is_valid, - reader->null_count()); - } else { - *out = std::make_shared>(type, length, values); - } - return Status::OK(); - } -}; - -template <> -struct TransferFunctor<::arrow::BooleanType, BooleanType> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - std::shared_ptr data; - - const int64_t buffer_size = BytesForBits(length); - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, buffer_size, &data)); - - // Transfer boolean values to packed bitmap - auto values = reinterpret_cast(reader->values()); - uint8_t* data_ptr = data->mutable_data(); - memset(data_ptr, 0, buffer_size); - - for (int64_t i = 0; i < length; i++) { - if (values[i]) { - ::arrow::BitUtil::SetBit(data_ptr, i); - } - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - RETURN_NOT_OK(is_valid->Resize(BytesForBits(length), false)); - *out = std::make_shared(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared(type, length, data); - } - return Status::OK(); - } -}; - -template <> -struct TransferFunctor<::arrow::TimestampType, Int96Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - auto values = reinterpret_cast(reader->values()); - - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(int64_t), &data)); - - auto data_ptr = reinterpret_cast(data->mutable_data()); - for (int64_t i = 0; i < length; i++) { - *data_ptr++ = impala_timestamp_to_nanoseconds(values[i]); - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared(type, length, data); - } - - return Status::OK(); - } -}; - -template <> -struct TransferFunctor<::arrow::Date64Type, Int32Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - auto values = reinterpret_cast(reader->values()); - - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(int64_t), &data)); - auto out_ptr = reinterpret_cast(data->mutable_data()); - - for (int64_t i = 0; i < length; i++) { - *out_ptr++ = static_cast(values[i]) * kMillisecondsInADay; - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared<::arrow::Date64Array>(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared<::arrow::Date64Array>(type, length, data); - } - return Status::OK(); - } -}; - -template -struct TransferFunctor< - ArrowType, ParquetType, - typename std::enable_if::value || - std::is_same::value>::type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - RETURN_NOT_OK(reader->builder()->Finish(out)); - - if (type->id() == ::arrow::Type::STRING) { - // Convert from BINARY type to STRING - auto new_data = (*out)->data()->Copy(); - new_data->type = type; - *out = ::arrow::MakeArray(new_data); - } - return Status::OK(); - } -}; - -static uint64_t BytesToInteger(const uint8_t* bytes, int32_t start, int32_t stop) { - using ::arrow::BitUtil::FromBigEndian; - - const int32_t length = stop - start; - - DCHECK_GE(length, 0); - DCHECK_LE(length, 8); - - switch (length) { - case 0: - return 0; - case 1: - return bytes[start]; - case 2: - return FromBigEndian(*reinterpret_cast(bytes + start)); - case 3: { - const uint64_t first_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_byte = bytes[stop - 1]; - return first_two_bytes << 8 | last_byte; - } - case 4: - return FromBigEndian(*reinterpret_cast(bytes + start)); - case 5: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_byte = bytes[stop - 1]; - return first_four_bytes << 8 | last_byte; - } - case 6: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start + 4)); - return first_four_bytes << 16 | last_two_bytes; - } - case 7: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t second_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start + 4)); - const uint64_t last_byte = bytes[stop - 1]; - return first_four_bytes << 24 | second_two_bytes << 8 | last_byte; - } - case 8: - return FromBigEndian(*reinterpret_cast(bytes + start)); - default: { - DCHECK(false); - return UINT64_MAX; - } - } -} - -static constexpr int32_t kMinDecimalBytes = 1; -static constexpr int32_t kMaxDecimalBytes = 16; - -/// \brief Convert a sequence of big-endian bytes to one int64_t (high bits) and one -/// uint64_t (low bits). -static void BytesToIntegerPair(const uint8_t* bytes, - const int32_t total_number_of_bytes_used, int64_t* high, - uint64_t* low) { - DCHECK_GE(total_number_of_bytes_used, kMinDecimalBytes); - DCHECK_LE(total_number_of_bytes_used, kMaxDecimalBytes); - - /// Bytes are coming in big-endian, so the first byte is the MSB and therefore holds the - /// sign bit. - const bool is_negative = static_cast(bytes[0]) < 0; - - /// Sign extend the low bits if necessary - *low = UINT64_MAX * (is_negative && total_number_of_bytes_used < 8); - *high = -1 * (is_negative && total_number_of_bytes_used < kMaxDecimalBytes); - - /// Stop byte of the high bytes - const int32_t high_bits_offset = std::max(0, total_number_of_bytes_used - 8); - - /// Shift left enough bits to make room for the incoming int64_t - *high <<= high_bits_offset * CHAR_BIT; - - /// Preserve the upper bits by inplace OR-ing the int64_t - *high |= BytesToInteger(bytes, 0, high_bits_offset); - - /// Stop byte of the low bytes - const int32_t low_bits_offset = std::min(total_number_of_bytes_used, 8); - - /// Shift left enough bits to make room for the incoming uint64_t - *low <<= low_bits_offset * CHAR_BIT; - - /// Preserve the upper bits by inplace OR-ing the uint64_t - *low |= BytesToInteger(bytes, high_bits_offset, total_number_of_bytes_used); -} - -static inline void RawBytesToDecimalBytes(const uint8_t* value, int32_t byte_width, - uint8_t* out_buf) { - // view the first 8 bytes as an unsigned 64-bit integer - auto low = reinterpret_cast(out_buf); - - // view the second 8 bytes as a signed 64-bit integer - auto high = reinterpret_cast(out_buf + sizeof(uint64_t)); - - // Convert the fixed size binary array bytes into a Decimal128 compatible layout - BytesToIntegerPair(value, byte_width, high, low); -} - -/// \brief Convert an array of FixedLenByteArrays to an arrow::Decimal128Array -/// We do this by: -/// 1. Creating a arrow::FixedSizeBinaryArray from the RecordReader's builder -/// 2. Allocating a buffer for the arrow::Decimal128Array -/// 3. Converting the big-endian bytes in the FixedSizeBinaryArray to two integers -/// representing the high and low bits of each decimal value. -template <> -struct TransferFunctor<::arrow::Decimal128Type, FLBAType> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL); - - // Finish the built data into a temporary array - std::shared_ptr array; - RETURN_NOT_OK(reader->builder()->Finish(&array)); - const auto& fixed_size_binary_array = - static_cast(*array); - - // Get the byte width of the values in the FixedSizeBinaryArray. Most of the time - // this will be different from the decimal array width because we write the minimum - // number of bytes necessary to represent a given precision - const int32_t byte_width = - static_cast(*fixed_size_binary_array.type()) - .byte_width(); - - // The byte width of each decimal value - const int32_t type_length = - static_cast(*type).byte_width(); - - // number of elements in the entire array - const int64_t length = fixed_size_binary_array.length(); - - // allocate memory for the decimal array - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * type_length, &data)); - - // raw bytes that we can write to - uint8_t* out_ptr = data->mutable_data(); - - // convert each FixedSizeBinary value to valid decimal bytes - const int64_t null_count = fixed_size_binary_array.null_count(); - if (null_count > 0) { - for (int64_t i = 0; i < length; ++i, out_ptr += type_length) { - if (!fixed_size_binary_array.IsNull(i)) { - RawBytesToDecimalBytes(fixed_size_binary_array.GetValue(i), byte_width, - out_ptr); - } - } - } else { - for (int64_t i = 0; i < length; ++i, out_ptr += type_length) { - RawBytesToDecimalBytes(fixed_size_binary_array.GetValue(i), byte_width, out_ptr); - } - } - - *out = std::make_shared<::arrow::Decimal128Array>( - type, length, data, fixed_size_binary_array.null_bitmap(), null_count); - return Status::OK(); - } -}; - -/// \brief Convert an Int32 or Int64 array into a Decimal128Array -/// The parquet spec allows systems to write decimals in int32, int64 if the values are -/// small enough to fit in less 4 bytes or less than 8 bytes, respectively. -/// This function implements the conversion from int32 and int64 arrays to decimal arrays. -template ::value || - std::is_same::value>::type> -static Status DecimalIntegerTransfer(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL); - - const int64_t length = reader->values_written(); - - using ElementType = typename ParquetIntegerType::c_type; - static_assert(std::is_same::value || - std::is_same::value, - "ElementType must be int32_t or int64_t"); - - const auto values = reinterpret_cast(reader->values()); - - const auto& decimal_type = static_cast(*type); - const int64_t type_length = decimal_type.byte_width(); - - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * type_length, &data)); - uint8_t* out_ptr = data->mutable_data(); - - using ::arrow::BitUtil::FromLittleEndian; - - for (int64_t i = 0; i < length; ++i, out_ptr += type_length) { - // sign/zero extend int32_t values, otherwise a no-op - const auto value = static_cast(values[i]); - - auto out_ptr_view = reinterpret_cast(out_ptr); - - // No-op on little endian machines, byteswap on big endian - out_ptr_view[0] = FromLittleEndian(static_cast(value)); - - // no need to byteswap here because we're sign/zero extending exactly 8 bytes - out_ptr_view[1] = static_cast(value < 0 ? -1 : 0); - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared<::arrow::Decimal128Array>(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared<::arrow::Decimal128Array>(type, length, data); - } - return Status::OK(); -} - -template <> -struct TransferFunctor<::arrow::Decimal128Type, Int32Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - return DecimalIntegerTransfer(reader, pool, type, out); - } -}; - -template <> -struct TransferFunctor<::arrow::Decimal128Type, Int64Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - return DecimalIntegerTransfer(reader, pool, type, out); - } -}; - -#define TRANSFER_DATA(ArrowType, ParquetType) \ - TransferFunctor func; \ - RETURN_NOT_OK(func(record_reader_.get(), pool_, field_->type(), out)); \ - RETURN_NOT_OK(WrapIntoListArray(out)) - -#define TRANSFER_CASE(ENUM, ArrowType, ParquetType) \ - case ::arrow::Type::ENUM: { \ - TRANSFER_DATA(ArrowType, ParquetType); \ - } break; - -Status PrimitiveImpl::NextBatch(int64_t records_to_read, std::shared_ptr* out) { - try { - // Pre-allocation gives much better performance for flat columns - record_reader_->Reserve(records_to_read); - - record_reader_->Reset(); - while (records_to_read > 0) { - if (!record_reader_->HasMoreData()) { - break; - } - int64_t records_read = record_reader_->ReadRecords(records_to_read); - records_to_read -= records_read; - if (records_read == 0) { - NextRowGroup(); - } - } - } catch (const ::parquet::ParquetException& e) { - return ::arrow::Status::IOError(e.what()); - } - - switch (field_->type()->id()) { - TRANSFER_CASE(BOOL, ::arrow::BooleanType, BooleanType) - TRANSFER_CASE(UINT8, ::arrow::UInt8Type, Int32Type) - TRANSFER_CASE(INT8, ::arrow::Int8Type, Int32Type) - TRANSFER_CASE(UINT16, ::arrow::UInt16Type, Int32Type) - TRANSFER_CASE(INT16, ::arrow::Int16Type, Int32Type) - TRANSFER_CASE(UINT32, ::arrow::UInt32Type, Int32Type) - TRANSFER_CASE(INT32, ::arrow::Int32Type, Int32Type) - TRANSFER_CASE(UINT64, ::arrow::UInt64Type, Int64Type) - TRANSFER_CASE(INT64, ::arrow::Int64Type, Int64Type) - TRANSFER_CASE(FLOAT, ::arrow::FloatType, FloatType) - TRANSFER_CASE(DOUBLE, ::arrow::DoubleType, DoubleType) - TRANSFER_CASE(STRING, ::arrow::StringType, ByteArrayType) - TRANSFER_CASE(BINARY, ::arrow::BinaryType, ByteArrayType) - TRANSFER_CASE(DATE32, ::arrow::Date32Type, Int32Type) - TRANSFER_CASE(DATE64, ::arrow::Date64Type, Int32Type) - TRANSFER_CASE(FIXED_SIZE_BINARY, ::arrow::FixedSizeBinaryType, FLBAType) - case ::arrow::Type::NA: { - *out = std::make_shared<::arrow::NullArray>(record_reader_->values_written()); - RETURN_NOT_OK(WrapIntoListArray(out)); - break; - } - case ::arrow::Type::DECIMAL: { - switch (descr_->physical_type()) { - case ::parquet::Type::INT32: { - TRANSFER_DATA(::arrow::Decimal128Type, Int32Type); - } break; - case ::parquet::Type::INT64: { - TRANSFER_DATA(::arrow::Decimal128Type, Int64Type); - } break; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - TRANSFER_DATA(::arrow::Decimal128Type, FLBAType); - } break; - default: - return Status::Invalid( - "Physical type for decimal must be int32, int64, or fixed length binary"); - } - } break; - case ::arrow::Type::TIMESTAMP: { - ::arrow::TimestampType* timestamp_type = - static_cast<::arrow::TimestampType*>(field_->type().get()); - switch (timestamp_type->unit()) { - case ::arrow::TimeUnit::MILLI: - case ::arrow::TimeUnit::MICRO: { - TRANSFER_DATA(::arrow::TimestampType, Int64Type); - } break; - case ::arrow::TimeUnit::NANO: { - TRANSFER_DATA(::arrow::TimestampType, Int96Type); - } break; - default: - return Status::NotImplemented("TimeUnit not supported"); - } - } break; - TRANSFER_CASE(TIME32, ::arrow::Time32Type, Int32Type) - TRANSFER_CASE(TIME64, ::arrow::Time64Type, Int64Type) - default: - std::stringstream ss; - ss << "No support for reading columns of type " << field_->type()->ToString(); - return Status::NotImplemented(ss.str()); - } - - return Status::OK(); -} - -void PrimitiveImpl::NextRowGroup() { - std::unique_ptr page_reader = input_->NextChunk(); - record_reader_->SetPageReader(std::move(page_reader)); -} - -Status PrimitiveImpl::GetDefLevels(const int16_t** data, size_t* length) { - *data = record_reader_->def_levels(); - *length = record_reader_->levels_written(); - return Status::OK(); -} - -Status PrimitiveImpl::GetRepLevels(const int16_t** data, size_t* length) { - *data = record_reader_->rep_levels(); - *length = record_reader_->levels_written(); - return Status::OK(); -} - -ColumnReader::ColumnReader(std::unique_ptr impl) - : impl_(std::move(impl)) {} +ColumnReader::ColumnReader(std::unique_ptr& deserializer) + : deserializer_(std::move(deserializer)) {} ColumnReader::~ColumnReader() {} Status ColumnReader::NextBatch(int64_t records_to_read, std::shared_ptr* out) { - return impl_->NextBatch(records_to_read, out); -} - -// StructImpl methods - -Status StructImpl::DefLevelsToNullArray(std::shared_ptr* null_bitmap_out, - int64_t* null_count_out) { - std::shared_ptr null_bitmap; - auto null_count = 0; - const int16_t* def_levels_data; - size_t def_levels_length; - RETURN_NOT_OK(GetDefLevels(&def_levels_data, &def_levels_length)); - RETURN_NOT_OK(GetEmptyBitmap(pool_, def_levels_length, &null_bitmap)); - uint8_t* null_bitmap_ptr = null_bitmap->mutable_data(); - for (size_t i = 0; i < def_levels_length; i++) { - if (def_levels_data[i] < struct_def_level_) { - // Mark null - null_count += 1; - } else { - DCHECK_EQ(def_levels_data[i], struct_def_level_); - ::arrow::BitUtil::SetBit(null_bitmap_ptr, i); - } - } - - *null_count_out = null_count; - *null_bitmap_out = (null_count == 0) ? nullptr : null_bitmap; - return Status::OK(); -} - -// TODO(itaiin): Consider caching the results of this calculation - -// note that this is only used once for each read for now -Status StructImpl::GetDefLevels(const int16_t** data, size_t* length) { - *data = nullptr; - if (children_.size() == 0) { - // Empty struct - *length = 0; - return Status::OK(); - } - - // We have at least one child - const int16_t* child_def_levels; - size_t child_length; - RETURN_NOT_OK(children_[0]->GetDefLevels(&child_def_levels, &child_length)); - auto size = child_length * sizeof(int16_t); - RETURN_NOT_OK(def_levels_buffer_.Resize(size)); - // Initialize with the minimal def level - std::memset(def_levels_buffer_.mutable_data(), -1, size); - auto result_levels = reinterpret_cast(def_levels_buffer_.mutable_data()); - - // When a struct is defined, all of its children def levels are at least at - // nesting level, and def level equals nesting level. - // When a struct is not defined, all of its children def levels are less than - // the nesting level, and the def level equals max(children def levels) - // All other possibilities are malformed definition data. - for (auto& child : children_) { - size_t current_child_length; - RETURN_NOT_OK(child->GetDefLevels(&child_def_levels, ¤t_child_length)); - DCHECK_EQ(child_length, current_child_length); - for (size_t i = 0; i < child_length; i++) { - // Check that value is either uninitialized, or current - // and previous children def levels agree on the struct level - DCHECK((result_levels[i] == -1) || ((result_levels[i] >= struct_def_level_) == - (child_def_levels[i] >= struct_def_level_))); - result_levels[i] = - std::max(result_levels[i], std::min(child_def_levels[i], struct_def_level_)); - } - } - *data = reinterpret_cast(def_levels_buffer_.data()); - *length = child_length; - return Status::OK(); -} - -void StructImpl::InitField( - const Node* node, const std::vector>& children) { - // Make a shallow node to field conversion from the children fields - std::vector> fields(children.size()); - for (size_t i = 0; i < children.size(); i++) { - fields[i] = children[i]->field(); - } - auto type = ::arrow::struct_(fields); - field_ = ::arrow::field(node->name(), type); -} - -Status StructImpl::GetRepLevels(const int16_t** data, size_t* length) { - return Status::NotImplemented("GetRepLevels is not implemented for struct"); -} - -Status StructImpl::NextBatch(int64_t records_to_read, std::shared_ptr* out) { - std::vector> children_arrays; - std::shared_ptr null_bitmap; - int64_t null_count; - - // Gather children arrays and def levels - for (auto& child : children_) { - std::shared_ptr child_array; - - RETURN_NOT_OK(child->NextBatch(records_to_read, &child_array)); - children_arrays.push_back(child_array); - } - - RETURN_NOT_OK(DefLevelsToNullArray(&null_bitmap, &null_count)); - - int64_t struct_length = children_arrays[0]->length(); - for (size_t i = 1; i < children_arrays.size(); ++i) { - if (children_arrays[i]->length() != struct_length) { - // TODO(wesm): This should really only occur if the Parquet file is - // malformed. Should this be a DCHECK? - return Status::Invalid("Struct children had different lengths"); - } - } - - *out = std::make_shared(field()->type(), struct_length, children_arrays, - null_bitmap, null_count); - return Status::OK(); + return deserializer_->DeserializeBatch(records_to_read, *out); } std::shared_ptr RowGroupReader::Column(int column_index) { diff --git a/src/parquet/arrow/reader.h b/src/parquet/arrow/reader.h index 4d68c610..612944a8 100644 --- a/src/parquet/arrow/reader.h +++ b/src/parquet/arrow/reader.h @@ -39,6 +39,7 @@ namespace parquet { namespace arrow { +class ArrayBatchedDeserializer; class ColumnChunkReader; class ColumnReader; class RowGroupReader; @@ -249,7 +250,6 @@ class PARQUET_EXPORT ColumnChunkReader { // might change in the future. class PARQUET_EXPORT ColumnReader { public: - class PARQUET_NO_EXPORT ColumnReaderImpl; virtual ~ColumnReader(); // Scan the next array of the indicated size. The actual size of the @@ -264,8 +264,8 @@ class PARQUET_EXPORT ColumnReader { ::arrow::Status NextBatch(int64_t batch_size, std::shared_ptr<::arrow::Array>* out); private: - std::unique_ptr impl_; - explicit ColumnReader(std::unique_ptr impl); + std::unique_ptr deserializer_; + explicit ColumnReader(std::unique_ptr& deserializer); friend class FileReader; friend class PrimitiveImpl; diff --git a/src/parquet/arrow/writer.cc b/src/parquet/arrow/writer.cc index 50b4649e..e0080d87 100644 --- a/src/parquet/arrow/writer.cc +++ b/src/parquet/arrow/writer.cc @@ -216,9 +216,11 @@ class LevelBuilder { if (level_null_count && level_valid_bitmap == nullptr) { // Special case: this is a null array (all elements are null) RETURN_NOT_OK(def_levels_.Append(static_cast(def_level + 1))); - } else if (nullable_level && ((level_null_count == 0) || - BitUtil::GetBit(level_valid_bitmap, - inner_offset + i + array_offsets_[recursion_level]))) { + } else if (nullable_level && + ((level_null_count == 0) || + BitUtil::GetBit( + level_valid_bitmap, + inner_offset + i + array_offsets_[recursion_level]))) { // Non-null element in a null level RETURN_NOT_OK(def_levels_.Append(static_cast(def_level + 2))); } else { diff --git a/src/parquet/statistics-test.cc b/src/parquet/statistics-test.cc index bf3d1968..674e972d 100644 --- a/src/parquet/statistics-test.cc +++ b/src/parquet/statistics-test.cc @@ -194,8 +194,9 @@ bool* TestRowGroupStatistics::GetValuesPointer(std::vector& v } template -typename std::vector TestRowGroupStatistics< - TestType>::GetDeepCopy(const std::vector& values) { +typename std::vector +TestRowGroupStatistics::GetDeepCopy( + const std::vector& values) { return values; }