From a576ebff24e1c42b65e45dcc6610b5473e4c7e36 Mon Sep 17 00:00:00 2001 From: Joshua Storck Date: Thu, 10 May 2018 15:43:40 -0400 Subject: [PATCH 1/6] Initial cut of implementing deserialization of arbitrary nested groups from Parquet to Arrow --- CMakeLists.txt | 1 + data/dremel.parquet | Bin 0 -> 1264 bytes src/parquet/arrow/arrow-reader-writer-test.cc | 149 +- src/parquet/arrow/arrow-schema-test.cc | 11 +- src/parquet/arrow/deserializer.cc | 1972 +++++++++++++++++ src/parquet/arrow/deserializer.h | 280 +++ src/parquet/arrow/reader.cc | 1055 +-------- src/parquet/arrow/reader.h | 6 +- src/parquet/arrow/writer.cc | 8 +- src/parquet/statistics-test.cc | 5 +- 10 files changed, 2446 insertions(+), 1041 deletions(-) create mode 100644 data/dremel.parquet create mode 100644 src/parquet/arrow/deserializer.cc create mode 100644 src/parquet/arrow/deserializer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e100a37..dc087966 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -658,6 +658,7 @@ add_custom_command( # Library config set(LIBPARQUET_SRCS + src/parquet/arrow/deserializer.cc src/parquet/arrow/reader.cc src/parquet/arrow/record_reader.cc src/parquet/arrow/schema.cc diff --git a/data/dremel.parquet b/data/dremel.parquet new file mode 100644 index 0000000000000000000000000000000000000000..93c8075fbd593ca98debc46160d04c1d28fb7a57 GIT binary patch literal 1264 zcmcIk!A{#i5FKwe!HNQks?N%h5Vu%8goGmH5^(_1q98>OMe4aY*b0cSPK_<#$|rE< zBl<<0dqR9b51sYKSqGY^mr9<^?0DXr*?Aie4u+o*u!_|Na&k&o0w@E(M>5*)<9-9} z>;=682Haw_0@zF?eInZ<8<0`n4(>FFWyZuQ$bnhvmsX~jw{+qeo;9$XCE6+h7*pKJ z=Zsm5tVn(!x04%HFp#tDctp59_LmT-Dy*R3v=SkJ#;33;PXBr$bv9j9)xU`2X{XgX z$oYpkzl*H)`$LaMVHnfIkk!E{ehJTyeFSJTr`W%gS2xI^4bJ*e7~Oi2|NS(Hg9sVr zx4DTPOR1uoV{MkJSisph^vuxvt zo*XCM_zl#AbMWds*HB=EM!~zGACN?JBPw;~taZag#?ifr6uw21bSHRivo>)=v)S^! z*lYO_^)hKrz34|0#KPXBLDx%An@?A8u`706(qbv7lYx3#hD8D9yYwJt;KCLoF>*UA zh@Y%C%tnb+TeD5MbApJH7>c3$WUXL-A@)jo*t?u|In5T-GJ@#Rb1Bl&@06J-BFf^y zqQ`w%)yXUSkDZvyK+Q-Y-B& table, int64_t row_group const std::shared_ptr& arrow_properties = default_arrow_writer_properties()) { std::shared_ptr result; - DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties); + ASSERT_NO_FATAL_FAILURE( + DoSimpleRoundtrip(table, 1, row_group_size, {}, &result, arrow_properties)); ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result, false)); } @@ -554,8 +556,8 @@ class TestParquetIO : public ::testing::Test { void ReadAndCheckSingleColumnTable(const std::shared_ptr& values) { std::shared_ptr<::arrow::Table> out; std::unique_ptr reader; - ReaderFromSink(&reader); - ReadTableFromFile(std::move(reader), &out); + ASSERT_NO_FATAL_FAILURE(ReaderFromSink(&reader)); + ASSERT_NO_FATAL_FAILURE(ReadTableFromFile(std::move(reader), &out)); ASSERT_EQ(1, out->num_columns()); ASSERT_EQ(values->length(), out->num_rows()); @@ -1043,8 +1045,8 @@ TEST_F(TestNullParquetIO, NullListColumn) { std::shared_ptr
out; std::unique_ptr reader; - this->ReaderFromSink(&reader); - this->ReadTableFromFile(std::move(reader), &out); + ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader)); + ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out)); ASSERT_EQ(1, out->num_columns()); ASSERT_EQ(offsets.size() - 1, out->num_rows()); @@ -1132,7 +1134,7 @@ class TestPrimitiveParquetIO : public TestParquetIO { ASSERT_NO_FATAL_FAILURE(MakeTestFile(values, num_chunks, &file_reader)); std::shared_ptr
out; - this->ReadTableFromFile(std::move(file_reader), &out); + ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(file_reader), &out)); ASSERT_EQ(1, out->num_columns()); ASSERT_EQ(SMALL_SIZE, out->num_rows()); @@ -1244,6 +1246,136 @@ TEST(TestArrowReadWrite, DateTimeTypes) { ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*table, *result)); } +TEST(TestArrowReadWrite, DremelExampleTest) { + std::string dir_string(test::get_data_dir()); + std::stringstream ss; + ss << dir_string << "/dremel.parquet"; + auto parquet_reader = ParquetFileReader::OpenFile(ss.str()); + parquet::arrow::FileReader reader(::arrow::default_memory_pool(), + std::move(parquet_reader)); + std::shared_ptr<::arrow::Table> out; + ASSERT_OK(reader.ReadTable(&out)); + + // Dremel paper uses int64, but the sample file + // stored them as int32 + auto forward = std::make_shared<::arrow::Int32Builder>(); + auto backward = std::make_shared<::arrow::Int32Builder>(); + auto docId = std::make_shared<::arrow::Int32Builder>(); + auto country = std::make_shared<::arrow::StringBuilder>(); + auto code = std::make_shared<::arrow::StringBuilder>(); + auto url = std::make_shared<::arrow::StringBuilder>(); + + std::vector> language_fields; + language_fields.emplace_back( + std::make_shared<::arrow::Field>("Code", ::arrow::utf8(), false)); + language_fields.emplace_back( + std::make_shared<::arrow::Field>("Country", ::arrow::utf8())); + + auto language_type = std::make_shared<::arrow::StructType>(language_fields); + std::vector> language_builders = {code, country}; + auto language = std::make_shared<::arrow::StructBuilder>( + language_type, ::arrow::default_memory_pool(), std::move(language_builders)); + auto language_list = + std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), language); + + auto backward_list = + std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), backward); + auto forward_list = + std::make_shared<::arrow::ListBuilder>(::arrow::default_memory_pool(), forward); + std::vector> links_fields; + links_fields.emplace_back( + std::make_shared<::arrow::Field>("Backward_Outer", backward_list->type(), false)); + links_fields.emplace_back( + std::make_shared<::arrow::Field>("Forward_Outer", backward_list->type(), false)); + auto links_type = std::make_shared<::arrow::StructType>(links_fields); + ::arrow::StructBuilder links(links_type, ::arrow::default_memory_pool(), + {forward_list, backward_list}); + + std::vector> name_fields; + name_fields.emplace_back( + std::make_shared<::arrow::Field>("Language_Outer", language_list->type(), false)); + name_fields.emplace_back(std::make_shared<::arrow::Field>("Url", url->type())); + + auto name_type = std::make_shared<::arrow::StructType>(name_fields); + std::vector> name_builders = {language_list, + url}; + auto name = std::make_shared<::arrow::StructBuilder>( + name_type, ::arrow::default_memory_pool(), std::move(name_builders)); + ::arrow::ListBuilder name_list(::arrow::default_memory_pool(), name); + + // Row 1 + docId->Append(10); + links.Append(); + name_list.Append(); + + // 3 forward links + forward_list->Append(); + forward->Append(20); + forward->Append(40); + forward->Append(60); + // No backward links + backward_list->Append(); + + // First name + name->Append(); + language_list->Append(); + language->Append(); + code->Append("en-us"); + country->Append("us"); + + language->Append(); + code->Append("en"); + country->AppendNull(); + + url->Append("http://A"); + + // Second name + name->Append(); + language_list->Append(); + url->Append("http://B"); + + // Third name + name->Append(); + language_list->Append(); + language->Append(); + code->Append("en-gb"); + country->Append("gb"); + url->AppendNull(); + + // Row 2 + docId->Append(20); + links.Append(); + name_list.Append(); + + backward_list->Append(); + backward->Append(10); + backward->Append(30); + forward_list->Append(); + forward->Append(80); + + name->Append(); + language_list->Append(); + url->Append("http://C"); + + std::shared_ptr<::arrow::Array> docIdArray, nameArray, linksArray; + + ASSERT_OK(docId->Finish(&docIdArray)); + ASSERT_OK(name_list.Finish(&nameArray)); + ASSERT_OK(links.Finish(&linksArray)); + + std::vector> fields; + fields.emplace_back( + std::make_shared<::arrow::Field>("DocId", docIdArray->type(), false)); + fields.emplace_back(std::make_shared<::arrow::Field>("Links", linksArray->type())); + fields.emplace_back(std::make_shared<::arrow::Field>("Name", nameArray->type(), false)); + + auto schema = std::make_shared<::arrow::Schema>(fields); + + auto expected = ::arrow::Table::Make(schema, {docIdArray, linksArray, nameArray}); + + ASSERT_NO_FATAL_FAILURE(AssertTablesEqual(*expected, *out)); +} + TEST(TestArrowReadWrite, CoerceTimestamps) { using ::arrow::ArrayFromVector; using ::arrow::field; @@ -1445,8 +1577,8 @@ TEST(TestArrowReadWrite, CoerceTimestampsAndSupportDeprecatedInt96) { using ::arrow::Schema; using ::arrow::Table; using ::arrow::TimeUnit; - using ::arrow::TimestampType; using ::arrow::TimestampBuilder; + using ::arrow::TimestampType; using ::arrow::default_memory_pool; auto timestamp_type = std::make_shared(TimeUnit::NANO); @@ -2161,6 +2293,7 @@ TEST_F(TestNestedSchemaRead, ReadIntoTableFull) { std::shared_ptr
table; ASSERT_OK_NO_THROW(reader_->ReadTable(&table)); + ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS); ASSERT_EQ(table->num_columns(), 2); ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 2); @@ -2228,9 +2361,11 @@ TEST_F(TestNestedSchemaRead, ReadTablePartial) { } TEST_F(TestNestedSchemaRead, StructAndListTogetherUnsupported) { +#if 0 ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::REPEATED)); std::shared_ptr
table; ASSERT_RAISES(NotImplemented, reader_->ReadTable(&table)); +#endif } TEST_P(TestNestedSchemaRead, DeepNestedSchemaRead) { diff --git a/src/parquet/arrow/arrow-schema-test.cc b/src/parquet/arrow/arrow-schema-test.cc index 5c16c044..cb2b8508 100644 --- a/src/parquet/arrow/arrow-schema-test.cc +++ b/src/parquet/arrow/arrow-schema-test.cc @@ -62,8 +62,8 @@ class TestConvertParquetSchema : public ::testing::Test { for (int i = 0; i < expected_schema->num_fields(); ++i) { auto lhs = result_schema_->field(i); auto rhs = expected_schema->field(i); - EXPECT_TRUE(lhs->Equals(rhs)) << i << " " << lhs->ToString() - << " != " << rhs->ToString(); + EXPECT_TRUE(lhs->Equals(rhs)) + << i << " " << lhs->ToString() << " != " << rhs->ToString(); } } @@ -607,9 +607,10 @@ TEST_F(TestConvertParquetSchema, ParquetRepeatedNestedSchema) { auto inner_group_type = std::make_shared<::arrow::StructType>(inner_group_fields); auto outer_group_fields = { std::make_shared("leaf2", INT32, true), - std::make_shared("innerGroup", ::arrow::list(std::make_shared( - "innerGroup", inner_group_type, false)), - false)}; + std::make_shared( + "innerGroup", + ::arrow::list(std::make_shared("innerGroup", inner_group_type, false)), + false)}; auto outer_group_type = std::make_shared<::arrow::StructType>(outer_group_fields); arrow_fields.push_back(std::make_shared("leaf1", INT32, true)); diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc new file mode 100644 index 00000000..88742ee9 --- /dev/null +++ b/src/parquet/arrow/deserializer.cc @@ -0,0 +1,1972 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/arrow/deserializer.h" + +#include +#include + +#include "arrow/builder.h" +#include "arrow/table.h" +#include "arrow/type.h" +#include "arrow/util/bit-util.h" +#include "arrow/util/decimal.h" +#include "arrow/util/logging.h" +#include "parquet/arrow/schema.h" +#include "parquet/file_reader.h" +#include "parquet/types.h" + +namespace arrow { + +class Array; +} + +using arrow::Array; +using arrow::Status; +using arrow::Table; + +using parquet::schema::GroupNode; +using parquet::schema::Node; +using parquet::schema::PrimitiveNode; + +namespace parquet { + +namespace arrow { + +// Start: copied from reader.cc +static uint64_t BytesToInteger(const uint8_t* bytes, int32_t start, int32_t stop) { + using ::arrow::BitUtil::FromBigEndian; + + const int32_t length = stop - start; + + DCHECK_GE(length, 0); + DCHECK_LE(length, 8); + + switch (length) { + case 0: + return 0; + case 1: + return bytes[start]; + case 2: + return FromBigEndian(*reinterpret_cast(bytes + start)); + case 3: { + const uint64_t first_two_bytes = + FromBigEndian(*reinterpret_cast(bytes + start)); + const uint64_t last_byte = bytes[stop - 1]; + return first_two_bytes << 8 | last_byte; + } + case 4: + return FromBigEndian(*reinterpret_cast(bytes + start)); + case 5: { + const uint64_t first_four_bytes = + FromBigEndian(*reinterpret_cast(bytes + start)); + const uint64_t last_byte = bytes[stop - 1]; + return first_four_bytes << 8 | last_byte; + } + case 6: { + const uint64_t first_four_bytes = + FromBigEndian(*reinterpret_cast(bytes + start)); + const uint64_t last_two_bytes = + FromBigEndian(*reinterpret_cast(bytes + start + 4)); + return first_four_bytes << 16 | last_two_bytes; + } + case 7: { + const uint64_t first_four_bytes = + FromBigEndian(*reinterpret_cast(bytes + start)); + const uint64_t second_two_bytes = + FromBigEndian(*reinterpret_cast(bytes + start + 4)); + const uint64_t last_byte = bytes[stop - 1]; + return first_four_bytes << 24 | second_two_bytes << 8 | last_byte; + } + case 8: + return FromBigEndian(*reinterpret_cast(bytes + start)); + default: { + DCHECK(false); + return UINT64_MAX; + } + } +} + +static constexpr int32_t kMinDecimalBytes = 1; +static constexpr int32_t kMaxDecimalBytes = 16; + +/// \brief Convert a sequence of big-endian bytes to one int64_t (high bits) and one +/// uint64_t (low bits). +static void BytesToIntegerPair(const uint8_t* bytes, + const int32_t total_number_of_bytes_used, int64_t* high, + uint64_t* low) { + DCHECK_GE(total_number_of_bytes_used, kMinDecimalBytes); + DCHECK_LE(total_number_of_bytes_used, kMaxDecimalBytes); + + /// Bytes are coming in big-endian, so the first byte is the MSB and therefore holds the + /// sign bit. + const bool is_negative = static_cast(bytes[0]) < 0; + + /// Sign extend the low bits if necessary + *low = UINT64_MAX * (is_negative && total_number_of_bytes_used < 8); + *high = -1 * (is_negative && total_number_of_bytes_used < kMaxDecimalBytes); + + /// Stop byte of the high bytes + const int32_t high_bits_offset = std::max(0, total_number_of_bytes_used - 8); + + /// Shift left enough bits to make room for the incoming int64_t + *high <<= high_bits_offset * CHAR_BIT; + + /// Preserve the upper bits by inplace OR-ing the int64_t + *high |= BytesToInteger(bytes, 0, high_bits_offset); + + /// Stop byte of the low bytes + const int32_t low_bits_offset = std::min(total_number_of_bytes_used, 8); + + /// Shift left enough bits to make room for the incoming uint64_t + *low <<= low_bits_offset * CHAR_BIT; + + /// Preserve the upper bits by inplace OR-ing the uint64_t + *low |= BytesToInteger(bytes, high_bits_offset, total_number_of_bytes_used); +} + +constexpr int64_t kJulianToUnixEpochDays = 2440588LL; +constexpr int64_t kMillisecondsInADay = 86400000LL; +constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL; + +static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timestamp) { + int64_t days_since_epoch = impala_timestamp.value[2] - kJulianToUnixEpochDays; + int64_t nanoseconds = *(reinterpret_cast(&(impala_timestamp.value))); + return days_since_epoch * kNanosecondsInADay + nanoseconds; +} +// End: copied from reader.cc + +// +// This is a utility class that is used by primitive node deserializer +// nodes. It's similar to a std::list, but stores its data as a contiguous +// array of memory because it's passed to calls to ReadBatch. +template +class PrimitiveDeserializerArray { + public: + PrimitiveDeserializerArray(int64_t capacity) + : // TODO: use memory pools and buffers + data_(new T[capacity]), + capacity_(capacity), + size_(0), + index_(0) {} + + ~PrimitiveDeserializerArray() { delete[] data_; } + + bool const Empty() const { return index_ >= size_; } + + T Front() const { return data_[index_]; } + + void PopFront() { ++index_; } + + void Reset() { + index_ = 0; + size_ = 0; + } + + T* data_; + + int64_t capacity_; + + int64_t size_; + + int64_t index_; +}; + +// +// Interface for implementations that are able to deserialize +// Parquet primitive and group nodes. Successive calls to +// Deserialize will cause this node to append values or nulls +// to its builder. +// +class DeserializerNode { + public: + using LevelSet = std::set; + + // + // \brief Deserializes the next set of values. + // \param[in] this is used to find out what the next set of + // repetitions will be. This is used by list + // (aka - repeated) nodes in their Deserialize method + // to determine whether or not they need to repeat + // and call Deserialize for their child nodes + // again before popping back to their parent node. + // \param[in] the current repeition level. When this is first called + // when deserializing a row, it should be zero. It may be + // non-zero when a list node is calling a child node + // during a repetition. In those cases, a child node may + // do nothing because it's next value doesn't repeat at + // the current repeition level. + virtual void Deserialize(LevelSet& next_repetitions, + int16_t current_repetition_level) = 0; + + // \brief Skips deserialization. This is used by group and list + // nodes when they are nullable, in which they call this + // method on their child nodes so that they do not append + // any values to their builders. + virtual void Skip(LevelSet& next_repetitions) = 0; + + // \brief returns the maximum definition level of all of this + // node's descendants. In the case of a primitive node + // it will be the next definition level in the file (if + // one exists) or zero otherwise. In the case of a group + // node, it will simply take the maximum of all of its + // child nodes. + // \return the maximum definition level for this nodes and its + // descendants + virtual int16_t CurrentMaxDefinitionLevel() = 0; + + // \brief returns the array buidler that was used by this node + // to append values and null in calls to Deserialize + // \return the array builder + virtual std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() = 0; + + // \brief initializes this node for a new row group. This is used + // by primitive node deserializers to obtain a column reader + // for that row group + virtual void InitializeRowGroup( + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) = 0; +}; + +template +class PrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, BuilderType& builder, + ParquetValueType value) { + using value_type = typename BuilderType::value_type; + builder.Append(static_cast(value)); + } +}; + +template +class PrimitiveAppender<::parquet::ByteArray, BuilderType> { + public: + template + void Append(const ColumnDescriptor* descriptor, BuilderType& builder, + ParquetValueType value) { + builder.Append(value.ptr, value.len); + } +}; + +template +class PrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, BuilderType& builder, + ParquetValueType value) { + builder.Append(value.ptr); + } +}; + +template <> +class PrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + ParquetValueType value) { + int64_t high; + uint64_t low; + BytesToIntegerPair(value.ptr, value.len, &high, &low); + builder.Append(::arrow::Decimal128(high, low)); + } +}; + +template <> +class PrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + ParquetValueType value) { + int64_t high; + uint64_t low; + BytesToIntegerPair(value.ptr, descriptor->type_length(), &high, &low); + builder.Append(::arrow::Decimal128(high, low)); + } +}; + +template +class PrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + ParquetValueType value) { + builder.Append(::arrow::Decimal128(value)); + } +}; + +template <> +class PrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, ::arrow::TimestampBuilder& builder, + ParquetValueType value) { + builder.Append(impala_timestamp_to_nanoseconds(value)); + } +}; + +class NullPrimitiveAppender { + public: + template + void Append(const ColumnDescriptor* descriptor, ::arrow::NullBuilder& builder, + ParquetValueType value) { + // This should never be called. + // TODO: add error values and have this return + // Status::Invalid. + } +}; + +template +class PrimitiveAppender + : public NullPrimitiveAppender {}; + +// These specializations avoid ambiguity with other ones where FLBA is +// specified but the builder type isn't (ambiguous with the class right above) +template <> +class PrimitiveAppender : public NullPrimitiveAppender {}; + +template <> +class PrimitiveAppender : public NullPrimitiveAppender { +}; + +template +class PrimitiveDeserializerNode : public DeserializerNode { + public: + PrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, + int64_t buffer_size, + std::shared_ptr const& builder) + : descriptor_(descriptor), + column_index_(column_index), + values_(buffer_size), + builder_(builder) {} + + std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() { return builder_; } + + void InitializeRowGroup(std::shared_ptr const& row_group_reader) final { + column_reader_ = std::static_pointer_cast( + row_group_reader->Column(column_index_)); + } + + protected: + void Append() { + appender_.Append(descriptor_, *builder_, values_.Front()); + values_.PopFront(); + } + + const ColumnDescriptor* const descriptor_; + + int const column_index_; + + PrimitiveDeserializerArray values_; + + std::shared_ptr builder_; + + PrimitiveAppender appender_; + + std::shared_ptr column_reader_; +}; + +template +class TypedPrimitiveDeserializerNode; + +template +class TypedPrimitiveDeserializerNode + : public PrimitiveDeserializerNode { + public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, + int const column_index, std::size_t buffer_size, + std::shared_ptr const& builder) + : PrimitiveDeserializerNode( + descriptor, column_index, buffer_size, builder), + definition_levels_(buffer_size) {} + + void Skip(DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + return; + } + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + this->values_.PopFront(); + } + definition_levels_.PopFront(); + } + + int16_t CurrentMaxDefinitionLevel() final { + if (!Fill()) { + // TODO: change the return type to bool when there is + // better error handling for not filling values + return 0; + } + return definition_levels_.Front(); + } + + void Deserialize(DeserializerNode::LevelSet& next_repetitions, + int16_t current_repetition_level) final { + if (!Fill()) { + // TODO: check the current_repetition_level against + // the max repetition level and throw an exception + // if necessary? + return; + } + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + // The value is not null + this->Append(); + } else { + this->builder_->AppendNull(); + } + definition_levels_.PopFront(); + } + + private: + bool Fill() { + if (!definition_levels_.Empty()) { + return true; + } + definition_levels_.Reset(); + this->values_.Reset(); + definition_levels_.size_ = this->column_reader_->ReadBatch( + definition_levels_.capacity_, definition_levels_.data_, 0, this->values_.data_, + &this->values_.size_); + return !definition_levels_.Empty(); + } + + PrimitiveDeserializerArray definition_levels_; +}; + +template +class TypedPrimitiveDeserializerNode + : public PrimitiveDeserializerNode { + public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, + int column_index, std::size_t buffer_size, + std::shared_ptr const& builder) + : PrimitiveDeserializerNode( + descriptor, column_index, buffer_size, builder), + definition_levels_(buffer_size), + repetition_levels_(buffer_size) {} + + int16_t CurrentMaxDefinitionLevel() final { + if (!Fill()) { + return false; + } + return definition_levels_.Front(); + } + + void Skip(DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) { + return; + } + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + this->values_.PopFront(); + } + definition_levels_.PopFront(); + AddRepetitions(next_repetitions); + } + + void Deserialize(DeserializerNode::LevelSet& next_repetitions, + int16_t current_repetition_level) final { + if (!Fill()) { + // TODO: check the current_repetition_level against + // the max repetition level and throw an exception + // if necessary? + return; + } + // See Dremel paper: second entry for Links.Backward would + // apply in this case and return + if (repetition_levels_.Front() != current_repetition_level) { + return; + } + + if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { + // The value is not null + this->Append(); + } else { + // Insert a null + this->builder_->AppendNull(); + } + definition_levels_.PopFront(); + AddRepetitions(next_repetitions); + } + + private: + void AddRepetitions(DeserializerNode::LevelSet& next_repetitions) { + repetition_levels_.PopFront(); + if (Fill()) { + next_repetitions.insert(repetition_levels_.Front()); + } + } + + bool Fill() { + if (!repetition_levels_.Empty()) { + return true; + } + definition_levels_.Reset(); + repetition_levels_.Reset(); + this->values_.Reset(); + auto levels_read = this->column_reader_->ReadBatch( + definition_levels_.capacity_, definition_levels_.data_, repetition_levels_.data_, + this->values_.data_, &this->values_.size_); + definition_levels_.size_ = levels_read; + repetition_levels_.size_ = levels_read; + + return !definition_levels_.Empty(); + } + + PrimitiveDeserializerArray definition_levels_; + + PrimitiveDeserializerArray repetition_levels_; +}; + +template +class TypedPrimitiveDeserializerNode + : public PrimitiveDeserializerNode { + public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, + int column_index, std::size_t buffer_size, + std::shared_ptr const& builder) + : PrimitiveDeserializerNode( + descriptor, column_index, buffer_size, builder) {} + + int16_t CurrentMaxDefinitionLevel() final { + return this->descriptor_->max_definition_level(); + } + + void Skip(DeserializerNode::LevelSet& next_repetitions) final { + // This should only happen if there is some problem + // with the reading of the data. + } + + void Deserialize(DeserializerNode::LevelSet& next_repetitions, + int16_t current_repetition_level) final { + if (!Fill()) { + // TODO: check the current_repetition_level against + // the max repetition level and throw an exception + // if necessary? + return; + } + this->Append(); + } + + private: + bool Fill() { + if (!this->values_.Empty()) { + return true; + } + this->values_.Reset(); + // Ignore the return value here + this->column_reader_->ReadBatch(this->values_.capacity_, 0, 0, this->values_.data_, + &this->values_.size_); + return !this->values_.Empty(); + } +}; + +class ListDeserializerNode : public DeserializerNode { + public: + ListDeserializerNode(std::unique_ptr&& node_reader, + int16_t max_repetition_level, int16_t max_definition_level) + : node_reader_(std::move(node_reader)), + max_repetition_level_(max_repetition_level), + max_definition_level_(max_definition_level), + builder_(std::make_shared<::arrow::ListBuilder>( + ::arrow::default_memory_pool(), node_reader_->GetArrayBuilder())) {} + + int16_t CurrentMaxDefinitionLevel() final { + return node_reader_->CurrentMaxDefinitionLevel(); + } + + void Skip(LevelSet& next_repetitions) final { node_reader_->Skip(next_repetitions); } + + void Deserialize(LevelSet& next_repetitions, int16_t current_repetition_level) { + builder_->Append(); + if (node_reader_->CurrentMaxDefinitionLevel() == max_definition_level_) { + // This is just an empty list + node_reader_->Skip(next_repetitions); + } else { + // Deserialize the first one + node_reader_->Deserialize(next_repetitions, current_repetition_level); + // Deserialize the repetitions + DeserializeRepetitions(next_repetitions); + } + } + + std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } + + void InitializeRowGroup(std::shared_ptr const& row_group_reader) { + node_reader_->InitializeRowGroup(row_group_reader); + } + + protected: + void DeserializeRepetitions(LevelSet& next_repetitions) { + bool repetition_at_this_level = true; + while (!next_repetitions.empty() && repetition_at_this_level) { + // next_repetitions is a sorted set, so end + // is the highest value. If it's lower than our + // max_repetition_level_, we just return from this + // function. Otherwise, we erase highest, descend again, + // and then re-check if we repeated at this level + auto highest = next_repetitions.end(); + --highest; + repetition_at_this_level = (*highest == max_repetition_level_); + if (repetition_at_this_level) { + next_repetitions.erase(highest); + node_reader_->Deserialize(next_repetitions, max_repetition_level_); + } + } + } + + std::unique_ptr node_reader_; + + int16_t max_repetition_level_; + + int16_t max_definition_level_; + + std::shared_ptr<::arrow::ListBuilder> builder_; +}; + +class NullableListDeserializerNode : public ListDeserializerNode { + public: + NullableListDeserializerNode(std::unique_ptr&& node_reader, + int16_t max_repetition_level, int16_t max_definition_level) + : ListDeserializerNode(std::forward>(node_reader), + max_repetition_level, max_definition_level) {} + + void Deserialize(LevelSet& next_repetitions, int16_t current_repetition_level) final { + auto max_definition_level = node_reader_->CurrentMaxDefinitionLevel(); + if (max_definition_level < max_definition_level_) { + // It's null + builder_->AppendNull(); + node_reader_->Skip(next_repetitions); + } else if (max_definition_level == max_definition_level_) { + // Empty list + builder_->Append(); + node_reader_->Skip(next_repetitions); + } else { + builder_->Append(); + node_reader_->Deserialize(next_repetitions, current_repetition_level); + } + // TODO: should this be in the else block above? + DeserializeRepetitions(next_repetitions); + } +}; + +template +class NestedDeserializerNode : public DeserializerNode { + public: + NestedDeserializerNode(std::string const& name, int16_t max_definition_level, + std::vector>&& children, + std::vector>& fields) + : name_(name), + children_(std::move(children)), + max_definition_level_(max_definition_level) { + std::vector> child_builders; + for (auto const& child : children_) { + child_builders.emplace_back(child->GetArrayBuilder()); + } + auto arrow_type = std::make_shared<::arrow::StructType>(fields); + builder_ = std::make_shared<::arrow::StructBuilder>( + arrow_type, ::arrow::default_memory_pool(), std::move(child_builders)); + } + + void Skip(LevelSet& next_repetitions) { + for (auto& child : children_) { + child->Skip(next_repetitions); + } + } + + int16_t CurrentMaxDefinitionLevel() final { + int16_t max = 0; + for (auto& child : children_) { + auto current = child->CurrentMaxDefinitionLevel(); + if (current > max) { + max = current; + } + } + return max; + } + + void Deserialize(LevelSet& next_repetitions, int16_t current_repetition_level) final { + // TODO: detect this at tree construction time. If all of + // the fields inside this nested node are required, then + // there is no need to do this check. + auto has_data = !Optional; + if (Optional) { + has_data = CurrentMaxDefinitionLevel() >= max_definition_level_; + } + if (has_data) { + builder_->Append(); + } else { + builder_->AppendNull(); + } + // For structs, we call Deserialize even when they don't have data. + // This ensures that the length of the child arrays match + // the length of the struct bitmap. + for (auto& child : children_) { + child->Deserialize(next_repetitions, current_repetition_level); + } + } + + std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } + + void InitializeRowGroup(std::shared_ptr const& row_group_reader) final { + for (auto& child : children_) { + child->InitializeRowGroup(row_group_reader); + } + } + + private: + std::string name_; + + std::vector> children_; + + int16_t max_definition_level_; + + std::shared_ptr<::arrow::StructBuilder> builder_; +}; + +ArrayDeserializer::ArrayDeserializer(std::unique_ptr&& node) + : node_(std::move(node)) {} + +ArrayDeserializer::~ArrayDeserializer() {} + +class FileArrayDeserializer : public ArrayDeserializer { + public: + FileArrayDeserializer(std::unique_ptr&& node, + std::shared_ptr const& file_reader) + : ArrayDeserializer(std::forward>(node)), + file_reader_(file_reader) {} + + ~FileArrayDeserializer() {} + + ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final { + DeserializerNode::LevelSet repetitions; + auto const num_row_groups = file_reader_->metadata()->num_row_groups(); + for (int ii = 0; ii < num_row_groups; ++ii) { + auto row_group_reader = file_reader_->RowGroup(ii); + node_->InitializeRowGroup(row_group_reader); + auto const num_rows = row_group_reader_->metadata()->num_rows(); + for (int jj = 0; jj < num_rows; ++jj) { + node_->Deserialize(repetitions, /* repetition level */ 0); + repetitions.clear(); + } + } + return node_->GetArrayBuilder()->Finish(&array); + } + + private: + std::shared_ptr file_reader_; + + std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; +}; + +ArrayBatchedDeserializer::~ArrayBatchedDeserializer() {} + +class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { + public: + FileArrayBatchedDeserializer(std::unique_ptr&& node, + std::shared_ptr const& file_reader) + : node_(std::move(node)), + file_reader_(file_reader), + current_row_group_(file_reader->RowGroup(0)), + current_row_group_index_(0), + current_row_(0) { + node_->InitializeRowGroup(current_row_group_); + } + + ~FileArrayBatchedDeserializer() {} + + ::arrow::Status DeserializeBatch(int64_t num_records, + std::shared_ptr<::arrow::Array>& array) final { + DeserializerNode::LevelSet repetitions; + int64_t records_read = 0; + auto const num_row_groups = file_reader_->metadata()->num_row_groups(); + while (records_read < num_records && current_row_group_index_ < num_row_groups) { + auto const num_rows = current_row_group_->metadata()->num_rows(); + for (; current_row_ < num_rows && records_read < num_records; + ++current_row_, ++records_read) { + node_->Deserialize(repetitions, /* repetition_level */ 0); + repetitions.clear(); + } + if (current_row_ >= num_rows) { + ++current_row_group_index_; + if (current_row_group_index_ < num_row_groups) { + current_row_group_ = file_reader_->RowGroup(current_row_group_index_); + node_->InitializeRowGroup(current_row_group_); + current_row_ = 0; + } + } + } + return node_->GetArrayBuilder()->Finish(&array); + } + + private: + std::unique_ptr node_; + + std::shared_ptr<::parquet::ParquetFileReader> file_reader_; + + std::shared_ptr<::parquet::RowGroupReader> current_row_group_; + + int current_row_group_index_; + + int current_row_; +}; + +class ColumnChunkDeserializer : public ArrayDeserializer { + public: + ColumnChunkDeserializer( + std::unique_ptr&& node, + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) + : ArrayDeserializer(std::move(node)), row_group_reader_(row_group_reader) { + node_->InitializeRowGroup(row_group_reader_); + } + + ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final { + DeserializerNode::LevelSet repetitions; + auto const num_rows = row_group_reader_->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + node_->Deserialize(repetitions, /* repetition level */ 0); + repetitions.clear(); + } + return node_->GetArrayBuilder()->Finish(&array); + } + + private: + std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; +}; + +TableDeserializer::TableDeserializer( + std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema) + : nodes_(std::move(nodes)), schema_(schema) {} + +TableDeserializer::~TableDeserializer() {} + +::arrow::Status TableDeserializer::MakeTable(std::shared_ptr<::arrow::Table>& result) { + std::vector> arrays; + for (auto const& node : nodes_) { + std::shared_ptr<::arrow::Array> array; + RETURN_NOT_OK(node->GetArrayBuilder()->Finish(&array)); + arrays.emplace_back(std::move(array)); + } + result = Table::Make(schema_, arrays); + return Status::OK(); +} + +class RowGroupDeserializer : public TableDeserializer { + public: + RowGroupDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema, + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) + : TableDeserializer( + std::forward>>(nodes), schema), + row_group_reader_(row_group_reader) { + for (auto& node : nodes_) { + node->InitializeRowGroup(row_group_reader); + } + } + + ~RowGroupDeserializer() {} + + ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final { + DeserializerNode::LevelSet repetitions; + auto const num_rows = row_group_reader_->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + for (auto& node : nodes_) { + node->Deserialize(repetitions, /* repetition level */ 0); + repetitions.clear(); + } + } + return MakeTable(table); + } + + private: + std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; +}; + +class FileTableDeserializer : public TableDeserializer { + public: + FileTableDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema, + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader) + : TableDeserializer( + std::forward>>(nodes), schema), + file_reader_(file_reader) {} + + ~FileTableDeserializer() {} + + ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) { + DeserializerNode::LevelSet repetitions; + auto num_row_groups = file_reader_->metadata()->num_row_groups(); + for (int ii = 0; ii < num_row_groups; ++ii) { + auto row_group_reader = file_reader_->RowGroup(ii); + for (auto& node : nodes_) { + node->InitializeRowGroup(row_group_reader); + } + auto const num_rows = row_group_reader->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + for (auto& node : nodes_) { + node->Deserialize(repetitions, /* repetition level */ 0); + repetitions.clear(); + } + } + } + return MakeTable(table); + } + + private: + std::shared_ptr<::parquet::ParquetFileReader> file_reader_; +}; + +class DeserializerBuilder::Impl { + public: + Impl(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + int64_t buffer_size); + + ::arrow::Status BuildSchemaNodeDeserializer(int schema_index, + std::unique_ptr& result); + + ::arrow::Status BuildColumnDeserializer(int column_index, + std::unique_ptr& result); + + ::arrow::Status BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result); + + ::arrow::Status BuildRowGroupDeserializer(int row_group_index, + const std::vector& column_indices, + std::unique_ptr& result); + + ::arrow::Status BuildFileDeserializer(const std::vector& column_indices, + std::unique_ptr& result); + + ::arrow::Status BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result); + + private: + std::vector const FillColumnIndices() const { + auto const num_columns = file_reader_->metadata()->num_columns(); + std::vector indices(num_columns); + for (int ii = 0; ii < num_columns; ++ii) { + indices[ii] = ii; + } + return indices; + } + + Status BuildNodes(std::vector const& column_indices, + std::vector>& nodes, + bool in_index_order = true); + + // TODO: make repetition_levels and definition_levels + // tagged/typed integers: + // + // struct RepetitionTag; + // + // template + // TaggedType; + // + // using Repetition = TaggedType + Status BuildNode(std::shared_ptr const& node, + std::vector const& indices, std::vector& indices_seen, + std::unique_ptr& result, + int16_t repetition_levels = 0, int16_t definition_levels = 0); + + Status BuildPrimitiveNode(std::shared_ptr const& node, + int16_t repetition_levels, int16_t definition_levels, + std::vector const& indices, + std::vector& indices_seen, + std::unique_ptr& result); + + Status BuildGroupNode(std::shared_ptr const& node, + int16_t repetition_levels, int16_t definition_levels, + std::vector const& indices, std::vector& indices_seen, + std::unique_ptr& result); + + Status BuildListNode(std::shared_ptr const& node, + int16_t repetition_levels, int16_t definition_levels, + std::vector const& indices, std::vector& indices_seen, + std::unique_ptr& result); + + template + Status MakeTypedPrimitiveDeserializerNode( + std::shared_ptr const& node, + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) { + auto primitive_node = std::static_pointer_cast(node); + switch (primitive_node->physical_type()) { + case Type::BOOLEAN: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::INT32: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::INT64: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::INT96: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::FLOAT: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::DOUBLE: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::BYTE_ARRAY: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + case Type::FIXED_LEN_BYTE_ARRAY: + return MakeLogicalPrimitiveDeserializerNode( + primitive_node, column_descriptor, column_index, result); + default: + std::ostringstream stream; + stream << "Unrecognized type when creating a primitive node reader"; + return Status::Invalid(stream.str()); + } + } + + template + Status MakeLogicalPrimitiveDeserializerNode( + std::shared_ptr const& node, + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) { + switch (node->logical_type()) { + case parquet::LogicalType::DATE: + return MakeDateDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::TIME_MILLIS: + return MakeTimeMillisDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::TIME_MICROS: + return MakeTimeMicrosDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::TIMESTAMP_MILLIS: + return MakeTimestampDeserializerNode( + column_descriptor, column_index, ::arrow::TimeUnit::MILLI, result); + case parquet::LogicalType::TIMESTAMP_MICROS: + return MakeTimestampDeserializerNode( + column_descriptor, column_index, ::arrow::TimeUnit::MICRO, result); + case parquet::LogicalType::DECIMAL: + return MakeDecimalDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::UINT_8: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::UINT_16: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::UINT_32: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::UINT_64: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::INT_8: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::INT_16: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::INT_32: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::INT_64: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::UTF8: + return MakeUTF8DeserializerNode( + column_descriptor, column_index, result); + case parquet::LogicalType::NA: + return MakeNullDeserializerNode( + column_descriptor, column_index, result); + default: + switch (node->physical_type()) { + case Type::BOOLEAN: + return MakeBooleanDeserializerNode( + column_descriptor, column_index, result); + case Type::INT32: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case Type::INT64: + return MakeIntegerDeserializerNode( + column_descriptor, column_index, result); + case Type::FLOAT: + return MakeFloatDeserializerNode( + column_descriptor, column_index, result); + case Type::DOUBLE: + return MakeDoubleDeserializerNode( + column_descriptor, column_index, result); + case Type::BYTE_ARRAY: + return MakeByteArrayDeserializerNode(column_descriptor, + column_index, result); + case Type::FIXED_LEN_BYTE_ARRAY: + return MakeFixedLengthByteArrayDeserializerNode( + column_descriptor, column_index, result); + case Type::INT96: + return MakeImpalaTimestampDeserializerNode( + column_descriptor, column_index, result); + default: + std::ostringstream stream; + stream << "Unsupported physical type " << node->physical_type() << std::endl; + return Status::Invalid(stream.str()); + } + } + } + + template + Status MakeNullDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::TimestampType>(::arrow::TimeUnit::NANO); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, + Status>::type + MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a nano timestamp field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int96"; + return Status::Invalid(stream.str()); + } + + template + struct SupportsDecimal { + static constexpr auto value = + (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value); + }; + + template + typename std::enable_if::value, Status>::type + MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::Decimal128Type>( + column_descriptor->type_precision(), column_descriptor->type_scale()); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, Status>::type + MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a decimal field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int32, int64, or fixed length byte array "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a boolean field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an boolean "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIME_MILLIS field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int32 "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::Time32Type>(::arrow::TimeUnit::MILLI); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIME_MILLIS field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int32 "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::Time64Type>(::arrow::TimeUnit::MICRO); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIME_MICROS field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int64"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, ::arrow::TimeUnit::type time_unit, + std::unique_ptr& result) { + auto arrow_type = std::make_shared<::arrow::TimestampType>(time_unit); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, + Status>::type + MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, ::arrow::TimeUnit::type time_unit, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a TIMESTAMP field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an int64"; + return Status::Invalid(stream.str()); + } + + template + struct CanCastInteger { + static constexpr auto parquet_is_int = + std::is_integral::value; + + static constexpr auto arrow_is_int = + std::is_integral::value; + + static constexpr auto parquet_size = sizeof(typename ParquetType::c_type); + + static constexpr auto arrow_size = sizeof(typename ArrayBuilderType::value_type); + + static constexpr auto both_ints = parquet_is_int && arrow_is_int; + + static constexpr auto value = (both_ints && parquet_size >= arrow_size); + }; + + template + typename std::enable_if::value, + Status>::type + MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, + column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make an integer field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not an integer or not the correct size "; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, ::arrow::utf8(), + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, + Status>::type + MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make an UTF8 field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a byte array"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a byte array field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a byte array"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeFixedLengthByteArrayDeserializerNode( + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) + + { + auto arrow_type = + std::make_shared<::arrow::FixedSizeBinaryType>(column_descriptor->type_length()); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, arrow_type, + ::arrow::default_memory_pool()); + } + + template + typename std::enable_if::value, + Status>::type + MakeFixedLengthByteArrayDeserializerNode( + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a byte array field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a fixed length byte array"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, + std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a double field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a double type"; + return Status::Invalid(stream.str()); + } + + template + typename std::enable_if::value, + Status>::type + MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) + + { + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result); + } + + template + typename std::enable_if::value, + Status>::type + MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, + int column_index, std::unique_ptr& result) { + std::ostringstream stream; + stream << "Type mismatch: attempt to make a float field for " + << column_descriptor->path()->ToDotString(); + stream << ", but it was not a float type"; + return Status::Invalid(stream.str()); + } + + template + Status MakeTypedPrimitiveNodeWithBuilder( + const ColumnDescriptor* const column_descriptor, int column_index, + std::unique_ptr& result, Args&&... args) { + auto builder = std::make_shared(std::forward(args)...); + using ReturnType = + TypedPrimitiveDeserializerNode, ArrayBuilderType, + HasDefinition, HasRepetition>; + result = std::unique_ptr( + new ReturnType(column_descriptor, column_index, buffer_size_, builder)); + return Status::OK(); + } + + void IncrementLevels(std::shared_ptr const& node, + int16_t& repetition_levels, int16_t& definition_levels) { + if (!node->is_required()) { + ++definition_levels; + } + if (node->is_repeated()) { + ++repetition_levels; + } + } + + std::shared_ptr<::parquet::ParquetFileReader> file_reader_; + + int64_t buffer_size_; +}; + +DeserializerBuilder::Impl::Impl( + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, int64_t buffer_size) + : file_reader_(file_reader), buffer_size_(buffer_size) {} + +::arrow::Status DeserializerBuilder::Impl::BuildSchemaNodeDeserializer( + int schema_index, std::unique_ptr& result) { + std::vector> nodes; + // Pass false to in_index_order so that the nodes are created + // in schema order so we can look it up that way + auto const column_indices = FillColumnIndices(); + RETURN_NOT_OK(BuildNodes(column_indices, nodes, false /* in_index_order */)); + if (static_cast(schema_index) >= nodes.size()) { + std::ostringstream stream; + stream << "Could not find schema node with index "; + stream << schema_index << ", there were only " << nodes.size(); + stream << "nodes"; + return Status::Invalid(stream.str()); + } + // nodes is guaranteed to be non-empty after calling BuildNodes + result.reset(new FileArrayDeserializer(std::move(nodes[schema_index]), file_reader_)); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildColumnDeserializer( + int column_index, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes({column_index}, nodes)); + // nodes is guaranteed to be non-empty after calling BuildNodes + if (nodes.size() > 1) { + std::ostringstream stream; + stream << "Unable to build a deserializer for column index "; + stream << column_index << ", multiple deserializer nodes "; + stream << "were created"; + return Status::Invalid(stream.str()); + } + result.reset(new FileArrayDeserializer(std::move(nodes[0]), file_reader_)); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes({column_index}, nodes)); + // nodes is guaranteed to be non-empty after calling BuildNodes + if (nodes.size() > 1) { + std::ostringstream stream; + stream << "Unable to build an array batched deserializer for column index "; + stream << column_index << ", multiple deserializer nodes "; + stream << "were created"; + return Status::Invalid(stream.str()); + } + result.reset(new FileArrayBatchedDeserializer(std::move(nodes[0]), file_reader_)); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes({column_index}, nodes)); + // nodes is guaranteed to be non-empty after calling BuildNodes + if (nodes.size() > 1) { + std::ostringstream stream; + stream << "Unable to build a deserializer for column chunk "; + stream << "with column index " << column_index << " and "; + stream << "row group " << row_group_index << ", multiple deserializer nodes "; + stream << "were created"; + return Status::Invalid(stream.str()); + } + result.reset(new ColumnChunkDeserializer(std::move(nodes[0]), + file_reader_->RowGroup(row_group_index))); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildRowGroupDeserializer( + int row_group_index, const std::vector& indices, + std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes(indices, nodes)); + std::shared_ptr<::arrow::Schema> schema; + RETURN_NOT_OK(FromParquetSchema(file_reader_->metadata()->schema(), indices, &schema)); + result.reset(new RowGroupDeserializer(std::move(nodes), schema, + file_reader_->RowGroup(row_group_index))); + return Status::OK(); +} + +::arrow::Status DeserializerBuilder::Impl::BuildFileDeserializer( + const std::vector& column_indices, std::unique_ptr& result) { + std::vector> nodes; + RETURN_NOT_OK(BuildNodes(column_indices, nodes)); + std::shared_ptr<::arrow::Schema> schema; + RETURN_NOT_OK( + FromParquetSchema(file_reader_->metadata()->schema(), column_indices, &schema)); + result.reset(new FileTableDeserializer(std::move(nodes), schema, file_reader_)); + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildNodes( + std::vector const& indices, + std::vector>& result, bool in_index_order) { + auto node = std::const_pointer_cast( + file_reader_->metadata()->schema()->schema_root()); + if (!node->is_group()) { + std::ostringstream stream; + stream << "Expected a group node at the root of the schema, "; + stream << "but did not encounter one"; + return Status::Invalid(stream.str()); + } + auto group_node = std::static_pointer_cast(node); + using location = decltype(indices.begin() - indices.end()); + using reader_and_indices = std::tuple, location>; + std::vector deserializers; + for (int ii = 0; ii < group_node->field_count(); ++ii) { + std::vector indices_seen; + std::unique_ptr deserializer; + RETURN_NOT_OK(BuildNode(group_node->field(ii), indices, indices_seen, deserializer)); + if (!deserializer) { + continue; + } + auto earliest = indices.end(); + for (auto index_seen : indices_seen) { + auto current = std::find(indices.begin(), indices.end(), index_seen); + if (current < earliest) { + earliest = current; + } + } + if (earliest == indices.end()) { + std::ostringstream stream; + stream << "Could not determine the ordering of the fields"; + stream << " for " << group_node->field(ii)->path()->ToDotString(); + stream << " because the indices for the columns seen did not "; + stream << " match any that were given"; + return Status::Invalid(stream.str()); + } + deserializers.emplace_back( + std::make_tuple(std::move(deserializer), earliest - indices.begin())); + } + if (deserializers.empty()) { + std::ostringstream stream; + stream << "Unable to match any of the columns to the specified indices: ["; + bool first = true; + for (auto index : indices) { + if (!first) { + stream << ", "; + } else { + first = false; + } + stream << index; + } + stream << "]"; + return Status::Invalid(stream.str()); + } + if (in_index_order) { + std::sort(deserializers.begin(), deserializers.end(), + [](reader_and_indices const& first, reader_and_indices const& second) { + return std::get<1>(first) < std::get<1>(second); + }); + } + for (auto& current : deserializers) { + result.emplace_back(std::move(std::get<0>(current))); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildNode(std::shared_ptr const& node, + std::vector const& indices, + std::vector& indices_seen, + std::unique_ptr& result, + int16_t repetition_levels, + int16_t definition_levels) { + IncrementLevels(node, repetition_levels, definition_levels); + if (node->is_group()) { + return BuildGroupNode(node, repetition_levels, definition_levels, indices, + indices_seen, result); + } else { + return BuildPrimitiveNode(node, repetition_levels, definition_levels, indices, + indices_seen, result); + } +} + +Status DeserializerBuilder::Impl::BuildPrimitiveNode( + std::shared_ptr const& node, int16_t repetition_levels, + int16_t definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result) { + auto const schema = file_reader_->metadata()->schema(); + auto column_index = schema->ColumnIndex(*node); + auto found = std::find(indices.begin(), indices.end(), column_index); + if (found == indices.end()) { + // This was not specified in the list of indices, so it doesn't + // show up in the resulting deserializer. Parent's will check for + // null and possibly filter themselves out (e.g. - a struct with + // no fields that match nodes with columns in indices) + return Status::OK(); + } + indices_seen.emplace_back(column_index); + auto column_descriptor = schema->Column(column_index); + if (definition_levels > 0) { + if (repetition_levels > 0) { + // The path to the node either contains only repeated nodes or + // a mix of repeated and optional nodes. + return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, + column_index, result); + } else { + // A field that is either itself optional or a descendant of + // an optional field (e.g. - member of a struct that is optional) + // but does not have any ancestors in the tree that are repeated. + // It therefore won't have any repetition levels. + return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, + column_index, result); + } + } else if (repetition_levels == 0) { + // A field that is always present (max_definition_level == + // 0 && max_repetition_level == 0) and therefore doesn't + // need repetition levels or definition levels + return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, + column_index, result); + } else { + // This should never happen since a non-zero max repetition level + // should have a non-zero definition level + std::ostringstream stream; + stream << "Primitive node " << node->path()->ToDotString(); + stream << " has a max_repetition_level or "; + stream << definition_levels; + stream << " and a max_definition_level of "; + stream << repetition_levels; + return Status::Invalid(stream.str()); + } + if (node->is_repeated()) { + result.reset(new ListDeserializerNode(std::move(result), repetition_levels, + definition_levels)); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildGroupNode( + std::shared_ptr const& node, int16_t repetition_levels, + int16_t definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result) { + // TODO: implement a proper Visitor interface in Node + auto group_node = std::static_pointer_cast(node); + if (node->logical_type() == LogicalType::LIST) { + return BuildListNode(group_node, repetition_levels, definition_levels, indices, + indices_seen, result); + } + std::vector> children; + std::vector> fields; + for (int ii = 0; ii < group_node->field_count(); ++ii) { + auto child_node = group_node->field(static_cast(ii)); + std::unique_ptr child; + RETURN_NOT_OK(BuildNode(child_node, indices, indices_seen, child, repetition_levels, + definition_levels)); + // Check to see if the child is valid. If so, + // it or at least one of its descendants was in + // the list of indices. + if (child) { + // TODO: deal with metadata args + fields.emplace_back(std::make_shared<::arrow::Field>( + child_node->name(), child->GetArrayBuilder()->type(), + // If the child node is required, + // then it's not nullable + !child_node->is_required())); + children.emplace_back(std::move(child)); + } + } + if (children.empty()) { + // All of the descendants have been filtered out because + // of the indices + return Status::OK(); + } + if (node->is_optional()) { + result.reset(new NestedDeserializerNode(node->name(), definition_levels, + std::move(children), fields)); + } else { + result.reset(new NestedDeserializerNode(node->name(), definition_levels, + std::move(children), fields)); + } + if (node->is_repeated()) { + // TODO: do we even test this? It may actually be implemented + // incorrectly because it doesn't contain the nesting that + // exists in LIST nodes. For example, this code path would + // be run for a node like this: + // + // repeated int32 element + // + // But that's the same code which is used for this: + // + // required group (LIST) { + // repated group list_group { + // required int32 element + // } + // } + // + // The danger of using the same implementation is that in this + // path we can't differentiate between an empty list and a list + // with a null + result.reset(new ListDeserializerNode(std::move(result), repetition_levels, + definition_levels)); + } + return Status::OK(); +} + +Status DeserializerBuilder::Impl::BuildListNode( + std::shared_ptr const& node, int16_t repetition_levels, + int16_t definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result) { + // + // A list node must have this schema: + // + // (required|optional) group outer { + // repeated group inner { + // (required|optiona) (group|primitive) element [{ children }] + // } + // } + // + // We construct a single node from outer and inner, using + // the definition level from outer and the repetition level + // from inner. This allows us to determine null lists when + // outer is optional, and empty lists in both cases. + if (node->is_repeated()) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " is repeated. List nodes can only be "; + stream << " optional or required, and must contain "; + stream << " a single child repeated group child node"; + return Status::Invalid(stream.str()); + } + if (node->field_count() != 1) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " has multiple children, it should have"; + stream << " a single repeated group child node"; + return Status::Invalid(stream.str()); + } + auto child_node = node->field(0); + if (!child_node->is_repeated()) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " should have a single repeated group child node, but "; + stream << " its child is not a repeated node"; + return Status::Invalid(stream.str()); + } + if (!child_node->is_group()) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " should have a single repeated group child node, but "; + stream << " its child is not a repeated primitive node"; + return Status::Invalid(stream.str()); + } + + // Since we've obtained the child node without calling + // BuildNode, we need to call IncrementLevels before + // building its children. We also save the definition level + // before we increment them, so that we can use it if this + // node is nullable. + auto current_max_definition_level = definition_levels; + IncrementLevels(child_node, repetition_levels, definition_levels); + auto group_node = std::static_pointer_cast(child_node); + if (group_node->field_count() > 1) { + std::ostringstream stream; + stream << "List node " << node->path()->ToDotString(); + stream << " should have a repeated group node as its child,"; + stream << " which has one child. However, the repeated group"; + stream << " node has " << group_node->field_count() << "children"; + return Status::Invalid(stream.str()); + } + std::unique_ptr child; + RETURN_NOT_OK(BuildNode(group_node->field(0), indices, indices_seen, child, + repetition_levels, definition_levels)); + if (!child) { + // The descendants have been filtered out based on indices + // so we retrun OK with a null result so that caller will + // know that this list has been filtered + return Status::OK(); + } + // If this node is required, then we defer the list building + // to the child node, since the list cannot be nullable and + // this node is effectively a pass through to the repeated + // child node. However, if this node is optional, we need to + // make a NullableListDeserializerNode, which will first check to + // see if there are any primitives in its tree that have + // a definition level equal to its max_definition_level. If + // there aren't, then it writes a null and skips writing + // any of its children. + if (node->is_optional()) { + result.reset(new NullableListDeserializerNode(std::move(child), repetition_levels, + current_max_definition_level)); + } else { + result.reset(new ListDeserializerNode(std::move(child), repetition_levels, + current_max_definition_level)); + } + return Status::OK(); +} + +DeserializerBuilder::DeserializerBuilder( + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, int64_t buffer_size) + : impl_(new Impl(file_reader, buffer_size)) {} + +DeserializerBuilder::~DeserializerBuilder() {} + +::arrow::Status DeserializerBuilder::BuildSchemaNodeDeserializer( + int schema_index, std::unique_ptr& result) { + return impl_->BuildSchemaNodeDeserializer(schema_index, result); +} + +::arrow::Status DeserializerBuilder::BuildColumnDeserializer( + int column_index, std::unique_ptr& result) { + return impl_->BuildColumnDeserializer(column_index, result); +} + +::arrow::Status DeserializerBuilder::BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result) { + return impl_->BuildColumnChunkDeserializer(column_index, row_group_index, result); +} + +::arrow::Status DeserializerBuilder::BuildRowGroupDeserializer( + int row_group_index, const std::vector& indices, + std::unique_ptr& result) { + return impl_->BuildRowGroupDeserializer(row_group_index, indices, result); +} + +::arrow::Status DeserializerBuilder::BuildFileDeserializer( + const std::vector& column_indices, std::unique_ptr& result) { + return impl_->BuildFileDeserializer(column_indices, result); +} + +::arrow::Status DeserializerBuilder::BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result) { + return impl_->BuildArrayBatchedDeserializer(column_index, result); +} + +} // namespace arrow +} // namespace parquet diff --git a/src/parquet/arrow/deserializer.h b/src/parquet/arrow/deserializer.h new file mode 100644 index 00000000..80b93fe7 --- /dev/null +++ b/src/parquet/arrow/deserializer.h @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#ifndef PARQUET_ARROW_DESERIALIZER_H +#define PARQUET_ARROW_DESERIALIZER_H + +#include +#include + +#include "arrow/status.h" +#include "parquet/util/visibility.h" + +namespace arrow { + +class Array; +class Schema; +class Table; + +} // namespace arrow + +namespace parquet { + +class ParquetFileReader; +class RowGroupReader; + +namespace arrow { + +class DeserializerNode; + +// +// This header file contains classes that can deserialize from Parquet +// files into Arrow arrays and tables. It supports nested structures +// such as this (taken from the Dremel paper): +// +// message root { +// required int32 DocId; +// optional group Links { +// required group Forward_Outer (LIST) { +// repeated group Forward_Inner { +// required int32 Forward; +// } +// } +// required group Backward_Outer (LIST) { +// repeated group Backward_Inner { +// required int32 Backward; +// } +// } +// } +// required group Name_Outer (LIST) { +// repeated group Name_Inner { +// required group Name_Element { +// required group Language_Outer (LIST) { +// repeated group Language_Inner { +// required group Language_Element { +// required binary Code (UTF8); +// optional binary Country (UTF8); +// } +// } +// } +// optional binary Url (UTF8); +// } +// } +// } +// } +// +// There are two types of serializers, one for arrays and one for tables. +// To obtain a deserializer, first construct a DeserializerBuilder and +// then call one of its Build* methods. +// +// The contract of the ArrayDeserializer::DeserializeArray and +// TableDeserializer::DeserializeTable is that they should be called once +// and only once. + +// +// Interface for deserializing a Parquet node into an Arrow Array. A node can +// contain multiple columns in the case of a Parquet group node. +// +class PARQUET_EXPORT ArrayDeserializer { + public: + virtual ~ArrayDeserializer(); + + // \brief Deserializes the node for which this deserializer was constructed + // \param[out] array to deserialize into + // \return error status if there is a problem reading the data from the + // Parquet file or in constructing the array + virtual ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) = 0; + + protected: + ArrayDeserializer(std::unique_ptr&& node); + + std::unique_ptr node_; +}; + +// +// Interface for deserializing a Parquet node into Arrow Arrays in batches. A node +// can contain multiple columns in the case of a Parquet group node. +// +class PARQUET_EXPORT ArrayBatchedDeserializer { + public: + virtual ~ArrayBatchedDeserializer(); + + // \brief Deserializes a batch of Parquet rows + // \param[in] number of records to deserialize in this batch + // \param[out] array to deserialize into + // \return error status if there is a problem reading the data from the + // Parquet file or in constructing the array. The number of rows + // will be the minimum of num_records and the remaining records + // available for deserialization + virtual ::arrow::Status DeserializeBatch(int64_t num_records, + std::shared_ptr<::arrow::Array>& array) = 0; +}; + +// +// Interface for deserializing Parquet nodes in an Arrow table. The schema +// of the Table will depend on which method was invoked in DeserializerBuilder. +// For example, if column indices are specified in the opposite order of how +// they appear in the Parquet file, the resulting Arrow schema will have its +// fields in the order in which the indices are specified. +// +class PARQUET_EXPORT TableDeserializer { + public: + virtual ~TableDeserializer(); + + // \brief Deserializes Parquet nodes into an Arrow Table + // \param[out] table to deserialize into + // \return error status if there is a problem reading the data from the + // Parquet file or in constructing the Table. + virtual ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& result) = 0; + + protected: + TableDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema); + + // \brief Convenience method which iterates over the nodes in this + // deserializer and constructs a table from them, using + // the schema passed to this object's constructor + // \return error status if there is an issue constructing the table + ::arrow::Status MakeTable(std::shared_ptr<::arrow::Table>& result); + + std::vector> nodes_; + + private: + // Schema, which was most likely obtained from a call to + // FromParquetSchema + std::shared_ptr<::arrow::Schema> schema_; +}; + +// +// A builder of deserializers. This is the main entry point to obtaining +// Parquet deserializers. +// +class PARQUET_EXPORT DeserializerBuilder { + public: + // + // \brief Create a new deserializer builder + // \param[in] the reader that is the source of the data to deserialize + // \param[in] the deserializers use buffers for reading Parquet definition + // and repetition levels, as well as values. This denotes how + // many values will be buffered when reading from primitive + // columns. + DeserializerBuilder(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + // TODO: make this configurable from a reader + // properties object + int64_t buffer_size = 1024); + + // Defined so that we can hide the impl and use a unique_ptr + ~DeserializerBuilder(); + + // + // \brief builds an array deserializer that will deserialize a particular + // node in the Parquet schema. + // \param[in] the index which refers to one of the top + // level nodes in the Parquet schema. For exmaple, in the example schema + // defined at the top of this file, index 0 would be the DocId, index 1 + // would be the Links, and index 2 would be Name_Outer. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the schema_index + // is out of bounds + // + ::arrow::Status BuildSchemaNodeDeserializer(int schema_index, + std::unique_ptr& result); + + // + // \brief builds an array deserializer that will deserialize a particular + // node in the Parquet schema. + // \param[in] the index which refers to a primitive node in the Parquet + // schema. In the example schema at the top of this file, + // DocId would have index 0, Forward index 1, Backward index 2, + // Code index 3, and so forth. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the column_index + // is out of bounds. + // + ::arrow::Status BuildColumnDeserializer(int column_index, + std::unique_ptr& result); + + // + // \brief builds an array deserializer that cant deserialize a particular + // node in the Parquet schema in batches. + // \param[in] the index which refers to a primitive node in the Parquet + // schema. In the example schema at the top of this file, + // DocId would have index 0, Forward index 1, Backward index 2, + // Code index 3, and so forth. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the column_index + // is out of bounds. + // + ::arrow::Status BuildArrayBatchedDeserializer( + int column_index, std::unique_ptr& result); + + // + // \brief builds an array deserializer that will deserialize a particular + // node in the Parquet schema for a particular row group. This + // is known as a ColumnChunk in the Parquet metadata. + // \param[in] the index which refers to a primitive node in the Parquet + // schema. In the example schema at the top of this file, + // DocId would have index 0, Forward index 1, Backward index 2, + // Code index 3, and so forth. + // \param[in] the row group to deserialize + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the column_index + // or row_group_index is out of bounds. + // + ::arrow::Status BuildColumnChunkDeserializer( + int column_index, int row_group_index, std::unique_ptr& result); + + // + // \brief builds a table deserializer that will deserialize all of the + // nodes in the Parquet schema for a particular row group. + // \param[in] the row group to deserialize + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if the row_group_index + // is out of bounds. + // + ::arrow::Status BuildRowGroupDeserializer(int row_group_index, + const std::vector& indices, + std::unique_ptr& result); + + // \brief build a table deserializer that will only deserialize nodes that + // contain the specified column indices. + // \param[in] the column indices to use for filtering the Parquet schema. + // For example, if column_index 2 is specified for the example schema + // at the top of this file, it will include Code and all of its + // ancestors in the resulting table. + // \param[out] the deserializer + // \return error status if there is a problem building the appropriate + // deserializers for various column types or if all of the + // column indices are out of bounds. + ::arrow::Status BuildFileDeserializer(const std::vector& column_indices, + std::unique_ptr& result); + + private: + // Hiding implementation due to a large number of member template methods + // with SFINAE. + class PARQUET_NO_EXPORT Impl; + std::unique_ptr impl_; +}; + +} // namespace arrow +} // namespace parquet + +#endif // PARQUET_ARROW_DESERIALIZER_H diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 1f933e64..21de2003 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "arrow/util/logging.h" #include "arrow/util/parallel.h" +#include "parquet/arrow/deserializer.h" #include "parquet/arrow/record_reader.h" #include "parquet/arrow/schema.h" #include "parquet/column_reader.h" @@ -66,16 +68,6 @@ namespace arrow { using ::arrow::BitUtil::BytesForBits; -constexpr int64_t kJulianToUnixEpochDays = 2440588LL; -constexpr int64_t kMillisecondsInADay = 86400000LL; -constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL; - -static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timestamp) { - int64_t days_since_epoch = impala_timestamp.value[2] - kJulianToUnixEpochDays; - int64_t nanoseconds = *(reinterpret_cast(&(impala_timestamp.value))); - return days_since_epoch * kNanosecondsInADay + nanoseconds; -} - template using ArrayType = typename ::arrow::TypeTraits::ArrayType; @@ -218,11 +210,6 @@ class FileReader::Impl { Status GetColumn(int i, std::unique_ptr* out); Status ReadSchemaField(int i, std::shared_ptr* out); - Status ReadSchemaField(int i, const std::vector& indices, - std::shared_ptr* out); - Status GetReaderForNode(int index, const Node* node, const std::vector& indices, - int16_t def_level, - std::unique_ptr* out); Status ReadColumn(int i, std::shared_ptr* out); Status ReadColumnChunk(int column_index, int row_group_index, std::shared_ptr* out); @@ -250,192 +237,38 @@ class FileReader::Impl { private: MemoryPool* pool_; - std::unique_ptr reader_; + std::shared_ptr reader_; int num_threads_; }; -class ColumnReader::ColumnReaderImpl { - public: - virtual ~ColumnReaderImpl() {} - virtual Status NextBatch(int64_t records_to_read, std::shared_ptr* out) = 0; - virtual Status GetDefLevels(const int16_t** data, size_t* length) = 0; - virtual Status GetRepLevels(const int16_t** data, size_t* length) = 0; - virtual const std::shared_ptr field() = 0; -}; - -// Reader implementation for primitive arrays -class PARQUET_NO_EXPORT PrimitiveImpl : public ColumnReader::ColumnReaderImpl { - public: - PrimitiveImpl(MemoryPool* pool, std::unique_ptr input) - : pool_(pool), input_(std::move(input)), descr_(input_->descr()) { - record_reader_ = RecordReader::Make(descr_, pool_); - DCHECK(NodeToField(*input_->descr()->schema_node(), &field_).ok()); - NextRowGroup(); - } - - Status NextBatch(int64_t records_to_read, std::shared_ptr* out) override; - - template - Status WrapIntoListArray(std::shared_ptr* array); - - Status GetDefLevels(const int16_t** data, size_t* length) override; - Status GetRepLevels(const int16_t** data, size_t* length) override; - - const std::shared_ptr field() override { return field_; } - - private: - void NextRowGroup(); - - MemoryPool* pool_; - std::unique_ptr input_; - const ColumnDescriptor* descr_; - - std::shared_ptr record_reader_; - - std::shared_ptr field_; -}; - -// Reader implementation for struct array -class PARQUET_NO_EXPORT StructImpl : public ColumnReader::ColumnReaderImpl { - public: - explicit StructImpl(const std::vector>& children, - int16_t struct_def_level, MemoryPool* pool, const Node* node) - : children_(children), - struct_def_level_(struct_def_level), - pool_(pool), - def_levels_buffer_(pool) { - InitField(node, children); - } - - Status NextBatch(int64_t records_to_read, std::shared_ptr* out) override; - Status GetDefLevels(const int16_t** data, size_t* length) override; - Status GetRepLevels(const int16_t** data, size_t* length) override; - const std::shared_ptr field() override { return field_; } - - private: - std::vector> children_; - int16_t struct_def_level_; - MemoryPool* pool_; - std::shared_ptr field_; - PoolBuffer def_levels_buffer_; - - Status DefLevelsToNullArray(std::shared_ptr* null_bitmap, int64_t* null_count); - void InitField(const Node* node, - const std::vector>& children); -}; - FileReader::FileReader(MemoryPool* pool, std::unique_ptr reader) : impl_(new FileReader::Impl(pool, std::move(reader))) {} FileReader::~FileReader() {} Status FileReader::Impl::GetColumn(int i, std::unique_ptr* out) { - std::unique_ptr input(new AllRowGroupsIterator(i, reader_.get())); - - std::unique_ptr impl( - new PrimitiveImpl(pool_, std::move(input))); - *out = std::unique_ptr(new ColumnReader(std::move(impl))); - return Status::OK(); -} - -Status FileReader::Impl::GetReaderForNode( - int index, const Node* node, const std::vector& indices, int16_t def_level, - std::unique_ptr* out) { - *out = nullptr; - - if (IsSimpleStruct(node)) { - const schema::GroupNode* group = static_cast(node); - std::vector> children; - for (int i = 0; i < group->field_count(); i++) { - std::unique_ptr child_reader; - // TODO(itaiin): Remove the -1 index hack when all types of nested reads - // are supported. This currently just signals the lower level reader resolution - // to abort - RETURN_NOT_OK(GetReaderForNode(index, group->field(i).get(), indices, - static_cast(def_level + 1), &child_reader)); - if (child_reader != nullptr) { - children.push_back(std::move(child_reader)); - } - } - - if (children.size() > 0) { - *out = std::unique_ptr( - new StructImpl(children, def_level, pool_, node)); - } - } else { - // This should be a flat field case - translate the field index to - // the correct column index by walking down to the leaf node - const Node* walker = node; - while (!walker->is_primitive()) { - DCHECK(walker->is_group()); - auto group = static_cast(walker); - if (group->field_count() != 1) { - return Status::NotImplemented("lists with structs are not supported."); - } - walker = group->field(0).get(); - } - auto column_index = reader_->metadata()->schema()->ColumnIndex(*walker); - - // If the index of the column is found then a reader for the coliumn is needed. - // Otherwise *out keeps the nullptr value. - if (std::find(indices.begin(), indices.end(), column_index) != indices.end()) { - std::unique_ptr reader; - RETURN_NOT_OK(GetColumn(column_index, &reader)); - *out = std::move(reader->impl_); - } - } - + DeserializerBuilder builder(reader_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildArrayBatchedDeserializer(i, deserializer)); + (*out).reset(new ColumnReader(deserializer)); return Status::OK(); } Status FileReader::Impl::ReadSchemaField(int i, std::shared_ptr* out) { - std::vector indices(reader_->metadata()->num_columns()); - - for (size_t j = 0; j < indices.size(); ++j) { - indices[j] = static_cast(j); - } - - return ReadSchemaField(i, indices, out); -} - -Status FileReader::Impl::ReadSchemaField(int i, const std::vector& indices, - std::shared_ptr* out) { - auto parquet_schema = reader_->metadata()->schema(); - - auto node = parquet_schema->group_node()->field(i).get(); - std::unique_ptr reader_impl; - - RETURN_NOT_OK(GetReaderForNode(i, node, indices, 1, &reader_impl)); - if (reader_impl == nullptr) { - *out = nullptr; - return Status::OK(); - } - - std::unique_ptr reader(new ColumnReader(std::move(reader_impl))); - - // TODO(wesm): This calculation doesn't make much sense when we have repeated - // schema nodes - int64_t records_to_read = 0; - - const FileMetaData& metadata = *reader_->metadata(); - for (int j = 0; j < metadata.num_row_groups(); j++) { - records_to_read += metadata.RowGroup(j)->ColumnChunk(i)->num_values(); - } - - return reader->NextBatch(records_to_read, out); + DeserializerBuilder builder(reader_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildSchemaNodeDeserializer(i, deserializer)); + deserializer->DeserializeArray(*out); + return Status::OK(); } Status FileReader::Impl::ReadColumn(int i, std::shared_ptr* out) { - std::unique_ptr flat_column_reader; - RETURN_NOT_OK(GetColumn(i, &flat_column_reader)); - - int64_t records_to_read = 0; - for (int j = 0; j < reader_->metadata()->num_row_groups(); j++) { - records_to_read += reader_->metadata()->RowGroup(j)->ColumnChunk(i)->num_values(); - } - - return flat_column_reader->NextBatch(records_to_read, out); + DeserializerBuilder builder(reader_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildColumnDeserializer(i, deserializer)); + deserializer->DeserializeArray(*out); + return Status::OK(); } Status FileReader::Impl::GetSchema(const std::vector& indices, @@ -447,91 +280,31 @@ Status FileReader::Impl::GetSchema(const std::vector& indices, Status FileReader::Impl::ReadColumnChunk(int column_index, int row_group_index, std::shared_ptr* out) { - auto rg_metadata = reader_->metadata()->RowGroup(row_group_index); - int64_t records_to_read = rg_metadata->ColumnChunk(column_index)->num_values(); - - std::unique_ptr input( - new SingleRowGroupIterator(column_index, row_group_index, reader_.get())); - - std::unique_ptr impl( - new PrimitiveImpl(pool_, std::move(input))); - ColumnReader flat_column_reader(std::move(impl)); - - std::shared_ptr array; - RETURN_NOT_OK(flat_column_reader.NextBatch(records_to_read, &array)); - *out = array; + DeserializerBuilder builder(reader_); + std::unique_ptr deserializer; + RETURN_NOT_OK( + builder.BuildColumnChunkDeserializer(column_index, row_group_index, deserializer)); + deserializer->DeserializeArray(*out); return Status::OK(); } Status FileReader::Impl::ReadRowGroup(int row_group_index, const std::vector& indices, std::shared_ptr<::arrow::Table>* out) { - std::shared_ptr<::arrow::Schema> schema; - RETURN_NOT_OK(GetSchema(indices, &schema)); - - auto rg_metadata = reader_->metadata()->RowGroup(row_group_index); - - int num_columns = static_cast(indices.size()); - int nthreads = std::min(num_threads_, num_columns); - std::vector> columns(num_columns); - - // TODO(wesm): Refactor to share more code with ReadTable - - auto ReadColumnFunc = [&indices, &row_group_index, &schema, &columns, this](int i) { - int column_index = indices[i]; - - std::shared_ptr array; - RETURN_NOT_OK(ReadColumnChunk(column_index, row_group_index, &array)); - columns[i] = std::make_shared(schema->field(i), array); - return Status::OK(); - }; - - if (nthreads == 1) { - for (int i = 0; i < num_columns; i++) { - RETURN_NOT_OK(ReadColumnFunc(i)); - } - } else { - RETURN_NOT_OK(ParallelFor(nthreads, num_columns, ReadColumnFunc)); - } - - *out = Table::Make(schema, columns); + DeserializerBuilder builder(reader_); + std::unique_ptr deserializer; + RETURN_NOT_OK( + builder.BuildRowGroupDeserializer(row_group_index, indices, deserializer)); + deserializer->DeserializeTable(*out); return Status::OK(); } Status FileReader::Impl::ReadTable(const std::vector& indices, std::shared_ptr
* out) { - std::shared_ptr<::arrow::Schema> schema; - RETURN_NOT_OK(GetSchema(indices, &schema)); - - // We only need to read schema fields which have columns indicated - // in the indices vector - std::vector field_indices; - if (!ColumnIndicesToFieldIndices(*reader_->metadata()->schema(), indices, - &field_indices)) { - return Status::Invalid("Invalid column index"); - } - - std::vector> columns(field_indices.size()); - auto ReadColumnFunc = [&indices, &field_indices, &schema, &columns, this](int i) { - std::shared_ptr array; - RETURN_NOT_OK(ReadSchemaField(field_indices[i], indices, &array)); - columns[i] = std::make_shared(schema->field(i), array); - return Status::OK(); - }; - - int num_fields = static_cast(field_indices.size()); - int nthreads = std::min(num_threads_, num_fields); - if (nthreads == 1) { - for (int i = 0; i < num_fields; i++) { - RETURN_NOT_OK(ReadColumnFunc(i)); - } - } else { - RETURN_NOT_OK(ParallelFor(nthreads, num_fields, ReadColumnFunc)); - } - - std::shared_ptr
table = Table::Make(schema, columns); - RETURN_NOT_OK(table->Validate()); - *out = table; + DeserializerBuilder builder(reader_); + std::unique_ptr deserializer; + RETURN_NOT_OK(builder.BuildFileDeserializer(indices, deserializer)); + deserializer->DeserializeTable(*out); return Status::OK(); } @@ -688,773 +461,13 @@ const ParquetFileReader* FileReader::parquet_reader() const { return impl_->parquet_reader(); } -template -Status PrimitiveImpl::WrapIntoListArray(std::shared_ptr* array) { - const int16_t* def_levels = record_reader_->def_levels(); - const int16_t* rep_levels = record_reader_->rep_levels(); - const int64_t total_levels_read = record_reader_->levels_position(); - - std::shared_ptr<::arrow::Schema> arrow_schema; - RETURN_NOT_OK(FromParquetSchema(input_->schema(), {input_->column_index()}, - input_->metadata()->key_value_metadata(), - &arrow_schema)); - std::shared_ptr current_field = arrow_schema->field(0); - - if (descr_->max_repetition_level() > 0) { - // Walk downwards to extract nullability - std::vector nullable; - std::vector> offset_builders; - std::vector> valid_bits_builders; - nullable.push_back(current_field->nullable()); - while (current_field->type()->num_children() > 0) { - if (current_field->type()->num_children() > 1) { - return Status::NotImplemented( - "Fields with more than one child are not supported."); - } else { - if (current_field->type()->id() != ::arrow::Type::LIST) { - return Status::NotImplemented( - "Currently only nesting with Lists is supported."); - } - current_field = current_field->type()->child(0); - } - offset_builders.emplace_back( - std::make_shared<::arrow::Int32Builder>(::arrow::int32(), pool_)); - valid_bits_builders.emplace_back( - std::make_shared<::arrow::BooleanBuilder>(::arrow::boolean(), pool_)); - nullable.push_back(current_field->nullable()); - } - - int64_t list_depth = offset_builders.size(); - // This describes the minimal definition that describes a level that - // reflects a value in the primitive values array. - int16_t values_def_level = descr_->max_definition_level(); - if (nullable[nullable.size() - 1]) { - values_def_level--; - } - - // The definition levels that are needed so that a list is declared - // as empty and not null. - std::vector empty_def_level(list_depth); - int def_level = 0; - for (int i = 0; i < list_depth; i++) { - if (nullable[i]) { - def_level++; - } - empty_def_level[i] = static_cast(def_level); - def_level++; - } - - int32_t values_offset = 0; - std::vector null_counts(list_depth, 0); - for (int64_t i = 0; i < total_levels_read; i++) { - int16_t rep_level = rep_levels[i]; - if (rep_level < descr_->max_repetition_level()) { - for (int64_t j = rep_level; j < list_depth; j++) { - if (j == (list_depth - 1)) { - RETURN_NOT_OK(offset_builders[j]->Append(values_offset)); - } else { - RETURN_NOT_OK(offset_builders[j]->Append( - static_cast(offset_builders[j + 1]->length()))); - } - - if (((empty_def_level[j] - 1) == def_levels[i]) && (nullable[j])) { - RETURN_NOT_OK(valid_bits_builders[j]->Append(false)); - null_counts[j]++; - break; - } else { - RETURN_NOT_OK(valid_bits_builders[j]->Append(true)); - if (empty_def_level[j] == def_levels[i]) { - break; - } - } - } - } - if (def_levels[i] >= values_def_level) { - values_offset++; - } - } - // Add the final offset to all lists - for (int64_t j = 0; j < list_depth; j++) { - if (j == (list_depth - 1)) { - RETURN_NOT_OK(offset_builders[j]->Append(values_offset)); - } else { - RETURN_NOT_OK(offset_builders[j]->Append( - static_cast(offset_builders[j + 1]->length()))); - } - } - - std::vector> offsets; - std::vector> valid_bits; - std::vector list_lengths; - for (int64_t j = 0; j < list_depth; j++) { - list_lengths.push_back(offset_builders[j]->length() - 1); - std::shared_ptr array; - RETURN_NOT_OK(offset_builders[j]->Finish(&array)); - offsets.emplace_back(std::static_pointer_cast(array)->values()); - RETURN_NOT_OK(valid_bits_builders[j]->Finish(&array)); - valid_bits.emplace_back(std::static_pointer_cast(array)->values()); - } - - std::shared_ptr output(*array); - for (int64_t j = list_depth - 1; j >= 0; j--) { - auto list_type = - ::arrow::list(::arrow::field("item", output->type(), nullable[j + 1])); - output = std::make_shared<::arrow::ListArray>( - list_type, list_lengths[j], offsets[j], output, valid_bits[j], null_counts[j]); - } - *array = output; - } - return Status::OK(); -} - -template -struct supports_fast_path_impl { - using ArrowCType = typename ArrowType::c_type; - using ParquetCType = typename ParquetType::c_type; - static constexpr bool value = std::is_same::value; -}; - -template -struct supports_fast_path_impl { - static constexpr bool value = false; -}; - -template -struct supports_fast_path_impl { - static constexpr bool value = false; -}; - -template -using supports_fast_path = - typename std::enable_if::value>::type; - -template -struct TransferFunctor { - using ArrowCType = typename ArrowType::c_type; - using ParquetCType = typename ParquetType::c_type; - - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - static_assert(!std::is_same::value, - "The fast path transfer functor should be used " - "for primitive values"); - - int64_t length = reader->values_written(); - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(ArrowCType), &data)); - - auto values = reinterpret_cast(reader->values()); - auto out_ptr = reinterpret_cast(data->mutable_data()); - std::copy(values, values + length, out_ptr); - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared>(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared>(type, length, data); - } - return Status::OK(); - } -}; - -template -struct TransferFunctor> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - std::shared_ptr values = reader->ReleaseValues(); - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared>(type, length, values, is_valid, - reader->null_count()); - } else { - *out = std::make_shared>(type, length, values); - } - return Status::OK(); - } -}; - -template <> -struct TransferFunctor<::arrow::BooleanType, BooleanType> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - std::shared_ptr data; - - const int64_t buffer_size = BytesForBits(length); - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, buffer_size, &data)); - - // Transfer boolean values to packed bitmap - auto values = reinterpret_cast(reader->values()); - uint8_t* data_ptr = data->mutable_data(); - memset(data_ptr, 0, buffer_size); - - for (int64_t i = 0; i < length; i++) { - if (values[i]) { - ::arrow::BitUtil::SetBit(data_ptr, i); - } - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - RETURN_NOT_OK(is_valid->Resize(BytesForBits(length), false)); - *out = std::make_shared(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared(type, length, data); - } - return Status::OK(); - } -}; - -template <> -struct TransferFunctor<::arrow::TimestampType, Int96Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - auto values = reinterpret_cast(reader->values()); - - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(int64_t), &data)); - - auto data_ptr = reinterpret_cast(data->mutable_data()); - for (int64_t i = 0; i < length; i++) { - *data_ptr++ = impala_timestamp_to_nanoseconds(values[i]); - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared(type, length, data); - } - - return Status::OK(); - } -}; - -template <> -struct TransferFunctor<::arrow::Date64Type, Int32Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - int64_t length = reader->values_written(); - auto values = reinterpret_cast(reader->values()); - - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(int64_t), &data)); - auto out_ptr = reinterpret_cast(data->mutable_data()); - - for (int64_t i = 0; i < length; i++) { - *out_ptr++ = static_cast(values[i]) * kMillisecondsInADay; - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared<::arrow::Date64Array>(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared<::arrow::Date64Array>(type, length, data); - } - return Status::OK(); - } -}; - -template -struct TransferFunctor< - ArrowType, ParquetType, - typename std::enable_if::value || - std::is_same::value>::type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - RETURN_NOT_OK(reader->builder()->Finish(out)); - - if (type->id() == ::arrow::Type::STRING) { - // Convert from BINARY type to STRING - auto new_data = (*out)->data()->Copy(); - new_data->type = type; - *out = ::arrow::MakeArray(new_data); - } - return Status::OK(); - } -}; - -static uint64_t BytesToInteger(const uint8_t* bytes, int32_t start, int32_t stop) { - using ::arrow::BitUtil::FromBigEndian; - - const int32_t length = stop - start; - - DCHECK_GE(length, 0); - DCHECK_LE(length, 8); - - switch (length) { - case 0: - return 0; - case 1: - return bytes[start]; - case 2: - return FromBigEndian(*reinterpret_cast(bytes + start)); - case 3: { - const uint64_t first_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_byte = bytes[stop - 1]; - return first_two_bytes << 8 | last_byte; - } - case 4: - return FromBigEndian(*reinterpret_cast(bytes + start)); - case 5: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_byte = bytes[stop - 1]; - return first_four_bytes << 8 | last_byte; - } - case 6: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start + 4)); - return first_four_bytes << 16 | last_two_bytes; - } - case 7: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t second_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start + 4)); - const uint64_t last_byte = bytes[stop - 1]; - return first_four_bytes << 24 | second_two_bytes << 8 | last_byte; - } - case 8: - return FromBigEndian(*reinterpret_cast(bytes + start)); - default: { - DCHECK(false); - return UINT64_MAX; - } - } -} - -static constexpr int32_t kMinDecimalBytes = 1; -static constexpr int32_t kMaxDecimalBytes = 16; - -/// \brief Convert a sequence of big-endian bytes to one int64_t (high bits) and one -/// uint64_t (low bits). -static void BytesToIntegerPair(const uint8_t* bytes, - const int32_t total_number_of_bytes_used, int64_t* high, - uint64_t* low) { - DCHECK_GE(total_number_of_bytes_used, kMinDecimalBytes); - DCHECK_LE(total_number_of_bytes_used, kMaxDecimalBytes); - - /// Bytes are coming in big-endian, so the first byte is the MSB and therefore holds the - /// sign bit. - const bool is_negative = static_cast(bytes[0]) < 0; - - /// Sign extend the low bits if necessary - *low = UINT64_MAX * (is_negative && total_number_of_bytes_used < 8); - *high = -1 * (is_negative && total_number_of_bytes_used < kMaxDecimalBytes); - - /// Stop byte of the high bytes - const int32_t high_bits_offset = std::max(0, total_number_of_bytes_used - 8); - - /// Shift left enough bits to make room for the incoming int64_t - *high <<= high_bits_offset * CHAR_BIT; - - /// Preserve the upper bits by inplace OR-ing the int64_t - *high |= BytesToInteger(bytes, 0, high_bits_offset); - - /// Stop byte of the low bytes - const int32_t low_bits_offset = std::min(total_number_of_bytes_used, 8); - - /// Shift left enough bits to make room for the incoming uint64_t - *low <<= low_bits_offset * CHAR_BIT; - - /// Preserve the upper bits by inplace OR-ing the uint64_t - *low |= BytesToInteger(bytes, high_bits_offset, total_number_of_bytes_used); -} - -static inline void RawBytesToDecimalBytes(const uint8_t* value, int32_t byte_width, - uint8_t* out_buf) { - // view the first 8 bytes as an unsigned 64-bit integer - auto low = reinterpret_cast(out_buf); - - // view the second 8 bytes as a signed 64-bit integer - auto high = reinterpret_cast(out_buf + sizeof(uint64_t)); - - // Convert the fixed size binary array bytes into a Decimal128 compatible layout - BytesToIntegerPair(value, byte_width, high, low); -} - -/// \brief Convert an array of FixedLenByteArrays to an arrow::Decimal128Array -/// We do this by: -/// 1. Creating a arrow::FixedSizeBinaryArray from the RecordReader's builder -/// 2. Allocating a buffer for the arrow::Decimal128Array -/// 3. Converting the big-endian bytes in the FixedSizeBinaryArray to two integers -/// representing the high and low bits of each decimal value. -template <> -struct TransferFunctor<::arrow::Decimal128Type, FLBAType> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL); - - // Finish the built data into a temporary array - std::shared_ptr array; - RETURN_NOT_OK(reader->builder()->Finish(&array)); - const auto& fixed_size_binary_array = - static_cast(*array); - - // Get the byte width of the values in the FixedSizeBinaryArray. Most of the time - // this will be different from the decimal array width because we write the minimum - // number of bytes necessary to represent a given precision - const int32_t byte_width = - static_cast(*fixed_size_binary_array.type()) - .byte_width(); - - // The byte width of each decimal value - const int32_t type_length = - static_cast(*type).byte_width(); - - // number of elements in the entire array - const int64_t length = fixed_size_binary_array.length(); - - // allocate memory for the decimal array - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * type_length, &data)); - - // raw bytes that we can write to - uint8_t* out_ptr = data->mutable_data(); - - // convert each FixedSizeBinary value to valid decimal bytes - const int64_t null_count = fixed_size_binary_array.null_count(); - if (null_count > 0) { - for (int64_t i = 0; i < length; ++i, out_ptr += type_length) { - if (!fixed_size_binary_array.IsNull(i)) { - RawBytesToDecimalBytes(fixed_size_binary_array.GetValue(i), byte_width, - out_ptr); - } - } - } else { - for (int64_t i = 0; i < length; ++i, out_ptr += type_length) { - RawBytesToDecimalBytes(fixed_size_binary_array.GetValue(i), byte_width, out_ptr); - } - } - - *out = std::make_shared<::arrow::Decimal128Array>( - type, length, data, fixed_size_binary_array.null_bitmap(), null_count); - return Status::OK(); - } -}; - -/// \brief Convert an Int32 or Int64 array into a Decimal128Array -/// The parquet spec allows systems to write decimals in int32, int64 if the values are -/// small enough to fit in less 4 bytes or less than 8 bytes, respectively. -/// This function implements the conversion from int32 and int64 arrays to decimal arrays. -template ::value || - std::is_same::value>::type> -static Status DecimalIntegerTransfer(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL); - - const int64_t length = reader->values_written(); - - using ElementType = typename ParquetIntegerType::c_type; - static_assert(std::is_same::value || - std::is_same::value, - "ElementType must be int32_t or int64_t"); - - const auto values = reinterpret_cast(reader->values()); - - const auto& decimal_type = static_cast(*type); - const int64_t type_length = decimal_type.byte_width(); - - std::shared_ptr data; - RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * type_length, &data)); - uint8_t* out_ptr = data->mutable_data(); - - using ::arrow::BitUtil::FromLittleEndian; - - for (int64_t i = 0; i < length; ++i, out_ptr += type_length) { - // sign/zero extend int32_t values, otherwise a no-op - const auto value = static_cast(values[i]); - - auto out_ptr_view = reinterpret_cast(out_ptr); - - // No-op on little endian machines, byteswap on big endian - out_ptr_view[0] = FromLittleEndian(static_cast(value)); - - // no need to byteswap here because we're sign/zero extending exactly 8 bytes - out_ptr_view[1] = static_cast(value < 0 ? -1 : 0); - } - - if (reader->nullable_values()) { - std::shared_ptr is_valid = reader->ReleaseIsValid(); - *out = std::make_shared<::arrow::Decimal128Array>(type, length, data, is_valid, - reader->null_count()); - } else { - *out = std::make_shared<::arrow::Decimal128Array>(type, length, data); - } - return Status::OK(); -} - -template <> -struct TransferFunctor<::arrow::Decimal128Type, Int32Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - return DecimalIntegerTransfer(reader, pool, type, out); - } -}; - -template <> -struct TransferFunctor<::arrow::Decimal128Type, Int64Type> { - Status operator()(RecordReader* reader, MemoryPool* pool, - const std::shared_ptr<::arrow::DataType>& type, - std::shared_ptr* out) { - return DecimalIntegerTransfer(reader, pool, type, out); - } -}; - -#define TRANSFER_DATA(ArrowType, ParquetType) \ - TransferFunctor func; \ - RETURN_NOT_OK(func(record_reader_.get(), pool_, field_->type(), out)); \ - RETURN_NOT_OK(WrapIntoListArray(out)) - -#define TRANSFER_CASE(ENUM, ArrowType, ParquetType) \ - case ::arrow::Type::ENUM: { \ - TRANSFER_DATA(ArrowType, ParquetType); \ - } break; - -Status PrimitiveImpl::NextBatch(int64_t records_to_read, std::shared_ptr* out) { - try { - // Pre-allocation gives much better performance for flat columns - record_reader_->Reserve(records_to_read); - - record_reader_->Reset(); - while (records_to_read > 0) { - if (!record_reader_->HasMoreData()) { - break; - } - int64_t records_read = record_reader_->ReadRecords(records_to_read); - records_to_read -= records_read; - if (records_read == 0) { - NextRowGroup(); - } - } - } catch (const ::parquet::ParquetException& e) { - return ::arrow::Status::IOError(e.what()); - } - - switch (field_->type()->id()) { - TRANSFER_CASE(BOOL, ::arrow::BooleanType, BooleanType) - TRANSFER_CASE(UINT8, ::arrow::UInt8Type, Int32Type) - TRANSFER_CASE(INT8, ::arrow::Int8Type, Int32Type) - TRANSFER_CASE(UINT16, ::arrow::UInt16Type, Int32Type) - TRANSFER_CASE(INT16, ::arrow::Int16Type, Int32Type) - TRANSFER_CASE(UINT32, ::arrow::UInt32Type, Int32Type) - TRANSFER_CASE(INT32, ::arrow::Int32Type, Int32Type) - TRANSFER_CASE(UINT64, ::arrow::UInt64Type, Int64Type) - TRANSFER_CASE(INT64, ::arrow::Int64Type, Int64Type) - TRANSFER_CASE(FLOAT, ::arrow::FloatType, FloatType) - TRANSFER_CASE(DOUBLE, ::arrow::DoubleType, DoubleType) - TRANSFER_CASE(STRING, ::arrow::StringType, ByteArrayType) - TRANSFER_CASE(BINARY, ::arrow::BinaryType, ByteArrayType) - TRANSFER_CASE(DATE32, ::arrow::Date32Type, Int32Type) - TRANSFER_CASE(DATE64, ::arrow::Date64Type, Int32Type) - TRANSFER_CASE(FIXED_SIZE_BINARY, ::arrow::FixedSizeBinaryType, FLBAType) - case ::arrow::Type::NA: { - *out = std::make_shared<::arrow::NullArray>(record_reader_->values_written()); - RETURN_NOT_OK(WrapIntoListArray(out)); - break; - } - case ::arrow::Type::DECIMAL: { - switch (descr_->physical_type()) { - case ::parquet::Type::INT32: { - TRANSFER_DATA(::arrow::Decimal128Type, Int32Type); - } break; - case ::parquet::Type::INT64: { - TRANSFER_DATA(::arrow::Decimal128Type, Int64Type); - } break; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - TRANSFER_DATA(::arrow::Decimal128Type, FLBAType); - } break; - default: - return Status::Invalid( - "Physical type for decimal must be int32, int64, or fixed length binary"); - } - } break; - case ::arrow::Type::TIMESTAMP: { - ::arrow::TimestampType* timestamp_type = - static_cast<::arrow::TimestampType*>(field_->type().get()); - switch (timestamp_type->unit()) { - case ::arrow::TimeUnit::MILLI: - case ::arrow::TimeUnit::MICRO: { - TRANSFER_DATA(::arrow::TimestampType, Int64Type); - } break; - case ::arrow::TimeUnit::NANO: { - TRANSFER_DATA(::arrow::TimestampType, Int96Type); - } break; - default: - return Status::NotImplemented("TimeUnit not supported"); - } - } break; - TRANSFER_CASE(TIME32, ::arrow::Time32Type, Int32Type) - TRANSFER_CASE(TIME64, ::arrow::Time64Type, Int64Type) - default: - std::stringstream ss; - ss << "No support for reading columns of type " << field_->type()->ToString(); - return Status::NotImplemented(ss.str()); - } - - return Status::OK(); -} - -void PrimitiveImpl::NextRowGroup() { - std::unique_ptr page_reader = input_->NextChunk(); - record_reader_->SetPageReader(std::move(page_reader)); -} - -Status PrimitiveImpl::GetDefLevels(const int16_t** data, size_t* length) { - *data = record_reader_->def_levels(); - *length = record_reader_->levels_written(); - return Status::OK(); -} - -Status PrimitiveImpl::GetRepLevels(const int16_t** data, size_t* length) { - *data = record_reader_->rep_levels(); - *length = record_reader_->levels_written(); - return Status::OK(); -} - -ColumnReader::ColumnReader(std::unique_ptr impl) - : impl_(std::move(impl)) {} +ColumnReader::ColumnReader(std::unique_ptr& deserializer) + : deserializer_(std::move(deserializer)) {} ColumnReader::~ColumnReader() {} Status ColumnReader::NextBatch(int64_t records_to_read, std::shared_ptr* out) { - return impl_->NextBatch(records_to_read, out); -} - -// StructImpl methods - -Status StructImpl::DefLevelsToNullArray(std::shared_ptr* null_bitmap_out, - int64_t* null_count_out) { - std::shared_ptr null_bitmap; - auto null_count = 0; - const int16_t* def_levels_data; - size_t def_levels_length; - RETURN_NOT_OK(GetDefLevels(&def_levels_data, &def_levels_length)); - RETURN_NOT_OK(GetEmptyBitmap(pool_, def_levels_length, &null_bitmap)); - uint8_t* null_bitmap_ptr = null_bitmap->mutable_data(); - for (size_t i = 0; i < def_levels_length; i++) { - if (def_levels_data[i] < struct_def_level_) { - // Mark null - null_count += 1; - } else { - DCHECK_EQ(def_levels_data[i], struct_def_level_); - ::arrow::BitUtil::SetBit(null_bitmap_ptr, i); - } - } - - *null_count_out = null_count; - *null_bitmap_out = (null_count == 0) ? nullptr : null_bitmap; - return Status::OK(); -} - -// TODO(itaiin): Consider caching the results of this calculation - -// note that this is only used once for each read for now -Status StructImpl::GetDefLevels(const int16_t** data, size_t* length) { - *data = nullptr; - if (children_.size() == 0) { - // Empty struct - *length = 0; - return Status::OK(); - } - - // We have at least one child - const int16_t* child_def_levels; - size_t child_length; - RETURN_NOT_OK(children_[0]->GetDefLevels(&child_def_levels, &child_length)); - auto size = child_length * sizeof(int16_t); - RETURN_NOT_OK(def_levels_buffer_.Resize(size)); - // Initialize with the minimal def level - std::memset(def_levels_buffer_.mutable_data(), -1, size); - auto result_levels = reinterpret_cast(def_levels_buffer_.mutable_data()); - - // When a struct is defined, all of its children def levels are at least at - // nesting level, and def level equals nesting level. - // When a struct is not defined, all of its children def levels are less than - // the nesting level, and the def level equals max(children def levels) - // All other possibilities are malformed definition data. - for (auto& child : children_) { - size_t current_child_length; - RETURN_NOT_OK(child->GetDefLevels(&child_def_levels, ¤t_child_length)); - DCHECK_EQ(child_length, current_child_length); - for (size_t i = 0; i < child_length; i++) { - // Check that value is either uninitialized, or current - // and previous children def levels agree on the struct level - DCHECK((result_levels[i] == -1) || ((result_levels[i] >= struct_def_level_) == - (child_def_levels[i] >= struct_def_level_))); - result_levels[i] = - std::max(result_levels[i], std::min(child_def_levels[i], struct_def_level_)); - } - } - *data = reinterpret_cast(def_levels_buffer_.data()); - *length = child_length; - return Status::OK(); -} - -void StructImpl::InitField( - const Node* node, const std::vector>& children) { - // Make a shallow node to field conversion from the children fields - std::vector> fields(children.size()); - for (size_t i = 0; i < children.size(); i++) { - fields[i] = children[i]->field(); - } - auto type = ::arrow::struct_(fields); - field_ = ::arrow::field(node->name(), type); -} - -Status StructImpl::GetRepLevels(const int16_t** data, size_t* length) { - return Status::NotImplemented("GetRepLevels is not implemented for struct"); -} - -Status StructImpl::NextBatch(int64_t records_to_read, std::shared_ptr* out) { - std::vector> children_arrays; - std::shared_ptr null_bitmap; - int64_t null_count; - - // Gather children arrays and def levels - for (auto& child : children_) { - std::shared_ptr child_array; - - RETURN_NOT_OK(child->NextBatch(records_to_read, &child_array)); - children_arrays.push_back(child_array); - } - - RETURN_NOT_OK(DefLevelsToNullArray(&null_bitmap, &null_count)); - - int64_t struct_length = children_arrays[0]->length(); - for (size_t i = 1; i < children_arrays.size(); ++i) { - if (children_arrays[i]->length() != struct_length) { - // TODO(wesm): This should really only occur if the Parquet file is - // malformed. Should this be a DCHECK? - return Status::Invalid("Struct children had different lengths"); - } - } - - *out = std::make_shared(field()->type(), struct_length, children_arrays, - null_bitmap, null_count); - return Status::OK(); + return deserializer_->DeserializeBatch(records_to_read, *out); } std::shared_ptr RowGroupReader::Column(int column_index) { diff --git a/src/parquet/arrow/reader.h b/src/parquet/arrow/reader.h index 4d68c610..612944a8 100644 --- a/src/parquet/arrow/reader.h +++ b/src/parquet/arrow/reader.h @@ -39,6 +39,7 @@ namespace parquet { namespace arrow { +class ArrayBatchedDeserializer; class ColumnChunkReader; class ColumnReader; class RowGroupReader; @@ -249,7 +250,6 @@ class PARQUET_EXPORT ColumnChunkReader { // might change in the future. class PARQUET_EXPORT ColumnReader { public: - class PARQUET_NO_EXPORT ColumnReaderImpl; virtual ~ColumnReader(); // Scan the next array of the indicated size. The actual size of the @@ -264,8 +264,8 @@ class PARQUET_EXPORT ColumnReader { ::arrow::Status NextBatch(int64_t batch_size, std::shared_ptr<::arrow::Array>* out); private: - std::unique_ptr impl_; - explicit ColumnReader(std::unique_ptr impl); + std::unique_ptr deserializer_; + explicit ColumnReader(std::unique_ptr& deserializer); friend class FileReader; friend class PrimitiveImpl; diff --git a/src/parquet/arrow/writer.cc b/src/parquet/arrow/writer.cc index 50b4649e..e0080d87 100644 --- a/src/parquet/arrow/writer.cc +++ b/src/parquet/arrow/writer.cc @@ -216,9 +216,11 @@ class LevelBuilder { if (level_null_count && level_valid_bitmap == nullptr) { // Special case: this is a null array (all elements are null) RETURN_NOT_OK(def_levels_.Append(static_cast(def_level + 1))); - } else if (nullable_level && ((level_null_count == 0) || - BitUtil::GetBit(level_valid_bitmap, - inner_offset + i + array_offsets_[recursion_level]))) { + } else if (nullable_level && + ((level_null_count == 0) || + BitUtil::GetBit( + level_valid_bitmap, + inner_offset + i + array_offsets_[recursion_level]))) { // Non-null element in a null level RETURN_NOT_OK(def_levels_.Append(static_cast(def_level + 2))); } else { diff --git a/src/parquet/statistics-test.cc b/src/parquet/statistics-test.cc index bf3d1968..674e972d 100644 --- a/src/parquet/statistics-test.cc +++ b/src/parquet/statistics-test.cc @@ -194,8 +194,9 @@ bool* TestRowGroupStatistics::GetValuesPointer(std::vector& v } template -typename std::vector TestRowGroupStatistics< - TestType>::GetDeepCopy(const std::vector& values) { +typename std::vector +TestRowGroupStatistics::GetDeepCopy( + const std::vector& values) { return values; } From c653510b038547ce6039578c7594147c097a5a47 Mon Sep 17 00:00:00 2001 From: Joshua Storck Date: Fri, 11 May 2018 16:51:05 -0400 Subject: [PATCH 2/6] Adding documentation, use of memory pools, more error checking, stronger types for definition/repetition levels, and strong type for primitive serializer type --- src/parquet/arrow/arrow-reader-writer-test.cc | 4 +- src/parquet/arrow/deserializer.cc | 1186 ++++++++++++----- src/parquet/arrow/deserializer.h | 12 +- src/parquet/arrow/reader.cc | 12 +- 4 files changed, 836 insertions(+), 378 deletions(-) diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc index 96a8c554..a4eeca67 100644 --- a/src/parquet/arrow/arrow-reader-writer-test.cc +++ b/src/parquet/arrow/arrow-reader-writer-test.cc @@ -510,8 +510,8 @@ class TestParquetIO : public ::testing::Test { std::shared_ptr out; std::unique_ptr reader; - ReaderFromSink(&reader); - ReadSingleColumnFile(std::move(reader), &out); + ASSERT_NO_FATAL_FAILURE(ReaderFromSink(&reader)); + ASSERT_NO_FATAL_FAILURE(ReadSingleColumnFile(std::move(reader), &out)); internal::AssertArraysEqual(values, *out); } diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc index 88742ee9..4bb278d5 100644 --- a/src/parquet/arrow/deserializer.cc +++ b/src/parquet/arrow/deserializer.cc @@ -21,6 +21,7 @@ #include #include "arrow/builder.h" +#include "arrow/memory_pool.h" #include "arrow/table.h" #include "arrow/type.h" #include "arrow/util/bit-util.h" @@ -150,6 +151,43 @@ static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timest } // End: copied from reader.cc +// This is a "tagged" integer type for definition/repetition levels. +// The tag gives us strong typing so that we can differentiate between +// definition and repeition level arguments in the DeserializerBuilder +// build function calls. +template +class Level +{ + +public: + + Level() : value_(0) + { + } + + Level& operator++() + { + ++value_; + return *this; + } + + int16_t operator*() const + { + return value_; + } + +private: + + int16_t value_; + +}; + +struct Repetition; +struct Definition; + +using RepetitionLevel = Level; +using DefinitionLevel = Level; + // // This is a utility class that is used by primitive node deserializer // nodes. It's similar to a std::list, but stores its data as a contiguous @@ -157,15 +195,25 @@ static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timest template class PrimitiveDeserializerArray { public: - PrimitiveDeserializerArray(int64_t capacity) - : // TODO: use memory pools and buffers - data_(new T[capacity]), - capacity_(capacity), - size_(0), - index_(0) {} + PrimitiveDeserializerArray(::arrow::MemoryPool* pool, int64_t capacity) + : + pool_(pool), + capacity_(capacity), + size_(0), + index_(0) {} - ~PrimitiveDeserializerArray() { delete[] data_; } + ~PrimitiveDeserializerArray() { + pool_->Free(reinterpret_cast(data_), capacity_); + } + Status Initialize() + { + uint8_t* out; + RETURN_NOT_OK(pool_->Allocate(sizeof(T) * capacity_, &out)); + data_ = reinterpret_cast(out); + return Status::OK(); + } + bool const Empty() const { return index_ >= size_; } T Front() const { return data_[index_]; } @@ -177,6 +225,8 @@ class PrimitiveDeserializerArray { size_ = 0; } + ::arrow::MemoryPool* pool_; + T* data_; int64_t capacity_; @@ -198,26 +248,34 @@ class DeserializerNode { // // \brief Deserializes the next set of values. - // \param[in] this is used to find out what the next set of - // repetitions will be. This is used by list - // (aka - repeated) nodes in their Deserialize method - // to determine whether or not they need to repeat - // and call Deserialize for their child nodes - // again before popping back to their parent node. // \param[in] the current repeition level. When this is first called // when deserializing a row, it should be zero. It may be // non-zero when a list node is calling a child node // during a repetition. In those cases, a child node may // do nothing because it's next value doesn't repeat at // the current repeition level. - virtual void Deserialize(LevelSet& next_repetitions, - int16_t current_repetition_level) = 0; + // \param[out] this is used to find out what the next set of + // repetitions will be. This is used by list + // (aka - repeated) nodes in their Deserialize method + // to determine whether or not they need to repeat + // and call Deserialize for their child nodes + // again before popping back to their parent node. + // \return error status if there is no more data for this node + virtual Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) = 0; // \brief Skips deserialization. This is used by group and list // nodes when they are nullable, in which they call this // method on their child nodes so that they do not append // any values to their builders. - virtual void Skip(LevelSet& next_repetitions) = 0; + // \param[out] this is used to find out what the next set of + // repetitions will be. This is used by list + // (aka - repeated) nodes in their Deserialize method + // to determine whether or not they need to repeat + // and call Deserialize for their child nodes + // again before popping back to their parent node. + // \return error status if there is no more data for this node + virtual Status Skip(LevelSet& next_repetitions) = 0; // \brief returns the maximum definition level of all of this // node's descendants. In the case of a primitive node @@ -225,9 +283,10 @@ class DeserializerNode { // one exists) or zero otherwise. In the case of a group // node, it will simply take the maximum of all of its // child nodes. - // \return the maximum definition level for this nodes and its - // descendants - virtual int16_t CurrentMaxDefinitionLevel() = 0; + // \param[out] the maximum definition level for this nodes and its + // descendants + // \return error status if there is no more data for this node + virtual Status CurrentMaxDefinitionLevel(int16_t& level) = 0; // \brief returns the array buidler that was used by this node // to append values and null in calls to Deserialize @@ -237,18 +296,30 @@ class DeserializerNode { // \brief initializes this node for a new row group. This is used // by primitive node deserializers to obtain a column reader // for that row group - virtual void InitializeRowGroup( + // \return status error if this node is a primitive node and the + // column index is out of bounds + virtual Status InitializeRowGroup( std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) = 0; }; +// +// The PrimitiveAppender class is used adapt the reading of parquet +// values from a ColumnReader to calls to a builder's append method. +// The reason this is needed is that the signature of the builder's +// Append method varies by type. For example, byte arrays take +// a pointer and a length whereas primitive types take just a value. +// By using partial specialization for various types, the deserializer +// code for primitive types does not need specialization but instead +// delegates to these set of classes to handle it. +// template class PrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, BuilderType& builder, + Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, ParquetValueType value) { using value_type = typename BuilderType::value_type; - builder.Append(static_cast(value)); + return builder.Append(static_cast(value)); } }; @@ -256,9 +327,9 @@ template class PrimitiveAppender<::parquet::ByteArray, BuilderType> { public: template - void Append(const ColumnDescriptor* descriptor, BuilderType& builder, + Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, ParquetValueType value) { - builder.Append(value.ptr, value.len); + return builder.Append(value.ptr, value.len); } }; @@ -266,9 +337,9 @@ template class PrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, BuilderType& builder, + Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, ParquetValueType value) { - builder.Append(value.ptr); + return builder.Append(value.ptr); } }; @@ -276,12 +347,12 @@ template <> class PrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, ParquetValueType value) { int64_t high; uint64_t low; BytesToIntegerPair(value.ptr, value.len, &high, &low); - builder.Append(::arrow::Decimal128(high, low)); + return builder.Append(::arrow::Decimal128(high, low)); } }; @@ -289,12 +360,12 @@ template <> class PrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, ParquetValueType value) { int64_t high; uint64_t low; BytesToIntegerPair(value.ptr, descriptor->type_length(), &high, &low); - builder.Append(::arrow::Decimal128(high, low)); + return builder.Append(::arrow::Decimal128(high, low)); } }; @@ -302,9 +373,9 @@ template class PrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, + Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, ParquetValueType value) { - builder.Append(::arrow::Decimal128(value)); + return builder.Append(::arrow::Decimal128(value)); } }; @@ -312,20 +383,20 @@ template <> class PrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, ::arrow::TimestampBuilder& builder, + Status Append(const ColumnDescriptor* descriptor, ::arrow::TimestampBuilder& builder, ParquetValueType value) { - builder.Append(impala_timestamp_to_nanoseconds(value)); + return builder.Append(impala_timestamp_to_nanoseconds(value)); } }; class NullPrimitiveAppender { public: template - void Append(const ColumnDescriptor* descriptor, ::arrow::NullBuilder& builder, + Status Append(const ColumnDescriptor* descriptor, ::arrow::NullBuilder& builder, ParquetValueType value) { - // This should never be called. - // TODO: add error values and have this return - // Status::Invalid. + std::ostringstream stream; + stream << "Unexpected call to NullPrimitiveAppender::Append"; + return Status::Invalid(stream.str()); } }; @@ -342,28 +413,66 @@ template <> class PrimitiveAppender : public NullPrimitiveAppender { }; +// +// This is the base class for primitive types deserializers. +// The ColumnReaderType should be an instantiation of +// TypeColumnReader (see column_reader.h). The ArrayBuilderType +// should be an instantiation of one of the Arrow builders +// (e.g. - Int32Builder). This class is used for initializing +// the column reader and appending values, whereas the sub-classes +// handle reading of repetition and definition levels and implement +// the DeserializerNode virtual functions. +// template class PrimitiveDeserializerNode : public DeserializerNode { - public: + +public: + PrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, int64_t buffer_size, - std::shared_ptr const& builder) + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) : descriptor_(descriptor), column_index_(column_index), - values_(buffer_size), + values_(pool, buffer_size), builder_(builder) {} std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() { return builder_; } - void InitializeRowGroup(std::shared_ptr const& row_group_reader) final { - column_reader_ = std::static_pointer_cast( - row_group_reader->Column(column_index_)); + // \brief initializes the column reader for this deserializer + // \param[in] the reader for the row group from which the column reader + // is obtained + // \return error status if the column index is out of bounds + Status InitializeRowGroup(std::shared_ptr const& row_group_reader) final { + if (column_index_ < row_group_reader->metadata()->num_columns()) + { + column_reader_ = std::static_pointer_cast( + row_group_reader->Column(column_index_)); + } + else + { + std::ostringstream stream; + stream << "Invalid column index " << column_index_ << " for row group "; + return Status::Invalid(stream.str()); + } + return Status::OK(); } + virtual Status Initialize() + { + return values_.Initialize(); + } + protected: - void Append() { - appender_.Append(descriptor_, *builder_, values_.Front()); + + // \brief appends the next value in this node's values array. It is + // assumed that the caller has checked to make sure that the + // values buffer is not empty + // \return error status if the call to the builder's Append function fails + Status Append() { + auto status = appender_.Append(descriptor_, *builder_, values_.Front()); values_.PopFront(); + return status; } const ColumnDescriptor* const descriptor_; @@ -379,58 +488,120 @@ class PrimitiveDeserializerNode : public DeserializerNode { std::shared_ptr column_reader_; }; -template +// This enum is used to differentiate between the three +// different types of primitive deserializers +enum PrimitiveSerializerType +{ + // This means that the node in the tree has no ancestors (including + // itself) that are repeated or optional + REQUIRED, + // This means that the node in the tree has only ancestors + // (including itself) that are required or optional, and therefore + // doesn't need to store repetition levels but does need to store + // definition levels + OPTIONAL, + // This means that the node in the tree has at least one ancestor + // (including itself) + REPEATED +}; + +// +// The TypedPrimitiveDeserializerNode does the heavy lifting of +// reading repetition and definition levels and implementing +// the DeserializerNode interface for primitive types. There are +// three specializations based on whether or not the field +// has definition levels or repetition levels (e.g. - a required +// field at depth 1 doesn't have repeition or definition levels +// and they are not stored in the parquet file). By using +// specialization in this manner, we save space and also don't +// have to do things like check if the max repetition or +// max definition level is zero. Unfortunately, this pattern is not +// yet being used in the column reader classes, so those checks +// still happen in calls to ReadBatch. +// +template class TypedPrimitiveDeserializerNode; +// Specialization for a field that has definition levels but no +// repetition levels. For example, the following schemas would +// create this type of node: +// +// message root { +// optional int32 field; +// } +// +// message root { +// optional group contact { +// required bytearray name (UTF8); +// optional int32 phone_number; +// } +// +// Both name and phone_number would be serialized with this type of +// node because they have a max definition level of 1 and 2 respectively. +// template -class TypedPrimitiveDeserializerNode +class TypedPrimitiveDeserializerNode : public PrimitiveDeserializerNode { public: + TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int const column_index, std::size_t buffer_size, - std::shared_ptr const& builder) + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) : PrimitiveDeserializerNode( - descriptor, column_index, buffer_size, builder), - definition_levels_(buffer_size) {} + descriptor, column_index, buffer_size, builder, pool), + definition_levels_(pool, buffer_size) {} - void Skip(DeserializerNode::LevelSet& next_repetitions) final { + Status Initialize() final + { + RETURN_NOT_OK(definition_levels_.Initialize()); + return PrimitiveDeserializerNode::Initialize(); + } + + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { if (!Fill()) { - return; + std::ostringstream stream; + stream << "Unable to skip data"; + return Status::IOError(stream.str()); } if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { this->values_.PopFront(); } definition_levels_.PopFront(); + return Status::OK(); } - int16_t CurrentMaxDefinitionLevel() final { + Status CurrentMaxDefinitionLevel(int16_t& level) final { if (!Fill()) { - // TODO: change the return type to bool when there is - // better error handling for not filling values - return 0; + std::ostringstream stream; + stream << "Unable to read current max definition level"; + return Status::IOError(stream.str()); } - return definition_levels_.Front(); + level = definition_levels_.Front(); + return Status::OK(); } - void Deserialize(DeserializerNode::LevelSet& next_repetitions, - int16_t current_repetition_level) final { + Status Deserialize(RepetitionLevel current_repetition_level, + DeserializerNode::LevelSet& next_repetitions) final { if (!Fill()) { - // TODO: check the current_repetition_level against - // the max repetition level and throw an exception - // if necessary? - return; + std::ostringstream stream; + stream << "Unable to deserialize value"; + return Status::IOError(stream.str()); } if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { // The value is not null - this->Append(); + RETURN_NOT_OK(this->Append()); } else { - this->builder_->AppendNull(); + RETURN_NOT_OK(this->builder_->AppendNull()); } definition_levels_.PopFront(); + return Status::OK(); } private: + bool Fill() { if (!definition_levels_.Empty()) { return true; @@ -444,64 +615,95 @@ class TypedPrimitiveDeserializerNode definition_levels_; + }; +// +// Specialization for a field that has both definition and repetition levels. +// For example, the following schema would create this type of node: +// +// message root { +// required group list_outer (LIST) { +// repeated group list_inner { +// required int32 value; +// } +// } +// } template -class TypedPrimitiveDeserializerNode +class TypedPrimitiveDeserializerNode : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, std::size_t buffer_size, - std::shared_ptr const& builder) + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) : PrimitiveDeserializerNode( - descriptor, column_index, buffer_size, builder), - definition_levels_(buffer_size), - repetition_levels_(buffer_size) {} + descriptor, column_index, buffer_size, builder, pool), + definition_levels_(pool, buffer_size), + repetition_levels_(pool, buffer_size) {} - int16_t CurrentMaxDefinitionLevel() final { + Status Initialize() final + { + RETURN_NOT_OK(definition_levels_.Initialize()); + RETURN_NOT_OK(repetition_levels_.Initialize()); + return PrimitiveDeserializerNode::Initialize(); + } + + Status CurrentMaxDefinitionLevel(int16_t& level) final { if (!Fill()) { - return false; + std::ostringstream stream; + stream << "Unable to determine current max definition level"; + return Status::IOError(stream.str()); } - return definition_levels_.Front(); + level = definition_levels_.Front(); + return Status::OK(); } - void Skip(DeserializerNode::LevelSet& next_repetitions) final { + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { if (!Fill()) { - return; + std::ostringstream stream; + stream << "Unable to determine skip data"; + return Status::IOError(stream.str()); } if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { this->values_.PopFront(); } definition_levels_.PopFront(); AddRepetitions(next_repetitions); + return Status::OK(); } - void Deserialize(DeserializerNode::LevelSet& next_repetitions, - int16_t current_repetition_level) final { - if (!Fill()) { - // TODO: check the current_repetition_level against - // the max repetition level and throw an exception - // if necessary? - return; + Status Deserialize(RepetitionLevel current_repetition_level, + DeserializerNode::LevelSet& next_repetitions) final { + if (!Fill()) + { + std::ostringstream stream; + stream << "Unable to fill values when deserializing"; + return Status::IOError(stream.str()); } // See Dremel paper: second entry for Links.Backward would // apply in this case and return - if (repetition_levels_.Front() != current_repetition_level) { - return; + if (repetition_levels_.Front() != *current_repetition_level) { + return Status::OK(); } if (definition_levels_.Front() == this->descriptor_->max_definition_level()) { // The value is not null - this->Append(); + RETURN_NOT_OK(this->Append()); } else { // Insert a null - this->builder_->AppendNull(); + RETURN_NOT_OK(this->builder_->AppendNull()); } definition_levels_.PopFront(); AddRepetitions(next_repetitions); + return Status::OK(); } private: + + // This is factored into its own method because it's used in both + // Skip and Deserialize void AddRepetitions(DeserializerNode::LevelSet& next_repetitions) { repetition_levels_.PopFront(); if (Fill()) { @@ -530,37 +732,57 @@ class TypedPrimitiveDeserializerNode repetition_levels_; }; +// +// Specialization for a field that has neither definition nor repetition levels. +// For example, all of the primitive fields in the following schema +// would create this type of node: +// +// message root { +// required group contact { +// required bytearray name (UTF8); +// required group address { +// required bytearray street (UTF8); +// required int32 number; +// required bytearray city (UTF8); +// required int8 zip; +// } +// } +// } template -class TypedPrimitiveDeserializerNode +class TypedPrimitiveDeserializerNode : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, std::size_t buffer_size, - std::shared_ptr const& builder) + std::shared_ptr const& builder, + ::arrow::MemoryPool* pool) : PrimitiveDeserializerNode( - descriptor, column_index, buffer_size, builder) {} + descriptor, column_index, buffer_size, builder, pool) {} - int16_t CurrentMaxDefinitionLevel() final { - return this->descriptor_->max_definition_level(); + Status CurrentMaxDefinitionLevel(int16_t& level) final { + level = this->descriptor_->max_definition_level(); + return Status::OK(); } - void Skip(DeserializerNode::LevelSet& next_repetitions) final { - // This should only happen if there is some problem - // with the reading of the data. + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { + std::ostringstream stream; + stream << "Call to Skip made when it shouldn't have"; + return Status::Invalid(stream.str()); } - void Deserialize(DeserializerNode::LevelSet& next_repetitions, - int16_t current_repetition_level) final { + Status Deserialize(RepetitionLevel current_repetition_level, + DeserializerNode::LevelSet& next_repetitions) final { if (!Fill()) { - // TODO: check the current_repetition_level against - // the max repetition level and throw an exception - // if necessary? - return; + std::ostringstream stream; + stream << "Unable to load more values in call to Deserialize"; + return Status::IOError(stream.str()); } - this->Append(); + return this->Append(); } private: + bool Fill() { if (!this->values_.Empty()) { return true; @@ -571,45 +793,83 @@ class TypedPrimitiveDeserializerNodevalues_.size_); return !this->values_.Empty(); } + }; +// +// Deserializer for lists that are required. The Parquet spec on list +// types requires that the schema for a required list looks likes this: +// +// required group list_outer (LIST) { +// repeated group list_inner { +// (required|optional) element ... +// } +// } +// +// The reason for the outer level of nesting is that without it +// you wouldn't be able to differentiate a list of a single null +// from an empty list. By having this extra level, an empty list +// is indicated when the max definition level of the list_outer's +// descendants are equal to list_outer's max definition level, and +// a list of null would be indicated by a definition level of +// list_inner's descendants that is equal to list_inner's max +// definition level. +// +// The constructor of this class takes the max_definition_level of +// list_outer and the max_repetition_level of list_inner class ListDeserializerNode : public DeserializerNode { - public: + +public: + ListDeserializerNode(std::unique_ptr&& node_reader, - int16_t max_repetition_level, int16_t max_definition_level) + ::arrow::MemoryPool* pool, + RepetitionLevel max_repetition_level, + DefinitionLevel max_definition_level) : node_reader_(std::move(node_reader)), max_repetition_level_(max_repetition_level), max_definition_level_(max_definition_level), builder_(std::make_shared<::arrow::ListBuilder>( - ::arrow::default_memory_pool(), node_reader_->GetArrayBuilder())) {} + pool, node_reader_->GetArrayBuilder())) {} - int16_t CurrentMaxDefinitionLevel() final { - return node_reader_->CurrentMaxDefinitionLevel(); + Status CurrentMaxDefinitionLevel(int16_t& level) final { + return node_reader_->CurrentMaxDefinitionLevel(level); } - void Skip(LevelSet& next_repetitions) final { node_reader_->Skip(next_repetitions); } + Status Skip(LevelSet& next_repetitions) final { + return node_reader_->Skip(next_repetitions); + } - void Deserialize(LevelSet& next_repetitions, int16_t current_repetition_level) { - builder_->Append(); - if (node_reader_->CurrentMaxDefinitionLevel() == max_definition_level_) { + Status Deserialize(RepetitionLevel current_repetition_level, LevelSet& next_repetitions) { + RETURN_NOT_OK(builder_->Append()); + int16_t current_max_definition_level; + RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level)); + if (current_max_definition_level == *max_definition_level_) { // This is just an empty list - node_reader_->Skip(next_repetitions); + return node_reader_->Skip(next_repetitions); } else { // Deserialize the first one - node_reader_->Deserialize(next_repetitions, current_repetition_level); + RETURN_NOT_OK(node_reader_->Deserialize(current_repetition_level, next_repetitions)); // Deserialize the repetitions - DeserializeRepetitions(next_repetitions); + return DeserializeRepetitions(next_repetitions); } } std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } - void InitializeRowGroup(std::shared_ptr const& row_group_reader) { - node_reader_->InitializeRowGroup(row_group_reader); + Status InitializeRowGroup(std::shared_ptr const& row_group_reader) { + return node_reader_->InitializeRowGroup(row_group_reader); } protected: - void DeserializeRepetitions(LevelSet& next_repetitions) { + + // \brief keeps appending elements to the current list as long as + // the descendants of this node have repetitions at this list's + // repetition level + // \param[in] the set of next repetitions from the last call to + // children's Deserialize functions. + // \return status error if calls to child node Deserialize functions + // produce an error + Status DeserializeRepetitions(LevelSet& next_repetitions) { bool repetition_at_this_level = true; while (!next_repetitions.empty() && repetition_at_this_level) { // next_repetitions is a sorted set, so end @@ -619,57 +879,92 @@ class ListDeserializerNode : public DeserializerNode { // and then re-check if we repeated at this level auto highest = next_repetitions.end(); --highest; - repetition_at_this_level = (*highest == max_repetition_level_); + repetition_at_this_level = (*highest == *max_repetition_level_); if (repetition_at_this_level) { next_repetitions.erase(highest); - node_reader_->Deserialize(next_repetitions, max_repetition_level_); + RETURN_NOT_OK(node_reader_->Deserialize(max_repetition_level_, next_repetitions)); } } + return Status::OK(); } std::unique_ptr node_reader_; - int16_t max_repetition_level_; + RepetitionLevel max_repetition_level_; - int16_t max_definition_level_; + DefinitionLevel max_definition_level_; std::shared_ptr<::arrow::ListBuilder> builder_; }; +// +// Deserializer for lists that are optional. The Parquet spec on list +// types requires that the schema for an optional list looks likes this: +// +// optional group list_outer (LIST) { +// repeated group list_inner { +// (required|optional) element ... +// } +// } +// +// When deserializing, if the max definition level of this node's +// children is less than list_outer's max definition level, then +// we append a null. If it's equal to it, then we create an empty +// list. Otherwise, we create a non-empty list. +// +// The constructor of this class takes the max_definition_level of +// list_outer and the max_repetition_level of list_inner class NullableListDeserializerNode : public ListDeserializerNode { - public: + +public: + NullableListDeserializerNode(std::unique_ptr&& node_reader, - int16_t max_repetition_level, int16_t max_definition_level) + ::arrow::MemoryPool* pool, + RepetitionLevel max_repetition_level, + DefinitionLevel max_definition_level) : ListDeserializerNode(std::forward>(node_reader), - max_repetition_level, max_definition_level) {} + pool, max_repetition_level, max_definition_level) {} - void Deserialize(LevelSet& next_repetitions, int16_t current_repetition_level) final { - auto max_definition_level = node_reader_->CurrentMaxDefinitionLevel(); - if (max_definition_level < max_definition_level_) { + Status Deserialize(RepetitionLevel current_repetition_level, LevelSet& next_repetitions) final { + int16_t current_max_definition_level; + RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level)); + if (current_max_definition_level < *max_definition_level_) { // It's null - builder_->AppendNull(); - node_reader_->Skip(next_repetitions); - } else if (max_definition_level == max_definition_level_) { + RETURN_NOT_OK(builder_->AppendNull()); + RETURN_NOT_OK(node_reader_->Skip(next_repetitions)); + } else if (current_max_definition_level == *max_definition_level_) { // Empty list - builder_->Append(); - node_reader_->Skip(next_repetitions); + RETURN_NOT_OK(builder_->Append()); + RETURN_NOT_OK(node_reader_->Skip(next_repetitions)); } else { - builder_->Append(); - node_reader_->Deserialize(next_repetitions, current_repetition_level); + RETURN_NOT_OK(builder_->Append()); + RETURN_NOT_OK(node_reader_->Deserialize(current_repetition_level, next_repetitions)); + RETURN_NOT_OK(DeserializeRepetitions(next_repetitions)); } - // TODO: should this be in the else block above? - DeserializeRepetitions(next_repetitions); + return Status::OK(); } }; +// +// Serializer for a group (aka struct) node. The schema for such a node +// would look like this; +// +// (repeated|required) group pair { +// required int32 key; +// required bytearray value (UTF8); +// } +// +// This has an Optional template parameter so that we can remove unnecessary checks +// for required nodes at compile time. +// template class NestedDeserializerNode : public DeserializerNode { public: - NestedDeserializerNode(std::string const& name, int16_t max_definition_level, + NestedDeserializerNode(DefinitionLevel max_definition_level, std::vector>&& children, - std::vector>& fields) - : name_(name), - children_(std::move(children)), + std::vector>& fields, + ::arrow::MemoryPool* pool) + : children_(std::move(children)), max_definition_level_(max_definition_level) { std::vector> child_builders; for (auto const& child : children_) { @@ -677,70 +972,84 @@ class NestedDeserializerNode : public DeserializerNode { } auto arrow_type = std::make_shared<::arrow::StructType>(fields); builder_ = std::make_shared<::arrow::StructBuilder>( - arrow_type, ::arrow::default_memory_pool(), std::move(child_builders)); + arrow_type, pool, std::move(child_builders)); } - void Skip(LevelSet& next_repetitions) { + Status Skip(LevelSet& next_repetitions) { for (auto& child : children_) { - child->Skip(next_repetitions); + RETURN_NOT_OK(child->Skip(next_repetitions)); } + return Status::OK(); } - int16_t CurrentMaxDefinitionLevel() final { - int16_t max = 0; + Status CurrentMaxDefinitionLevel(int16_t& max) final { + max = 0; for (auto& child : children_) { - auto current = child->CurrentMaxDefinitionLevel(); + int16_t current; + RETURN_NOT_OK(child->CurrentMaxDefinitionLevel(current)); if (current > max) { max = current; } } - return max; + return Status::OK(); } - void Deserialize(LevelSet& next_repetitions, int16_t current_repetition_level) final { + Status Deserialize(RepetitionLevel current_repetition_level, LevelSet& next_repetitions) final { // TODO: detect this at tree construction time. If all of // the fields inside this nested node are required, then // there is no need to do this check. auto has_data = !Optional; if (Optional) { - has_data = CurrentMaxDefinitionLevel() >= max_definition_level_; + int16_t current_max_definition_level; + RETURN_NOT_OK(CurrentMaxDefinitionLevel(current_max_definition_level)); + has_data = current_max_definition_level >= *max_definition_level_; } if (has_data) { - builder_->Append(); + RETURN_NOT_OK(builder_->Append()); } else { - builder_->AppendNull(); + RETURN_NOT_OK(builder_->AppendNull()); } // For structs, we call Deserialize even when they don't have data. // This ensures that the length of the child arrays match // the length of the struct bitmap. for (auto& child : children_) { - child->Deserialize(next_repetitions, current_repetition_level); + RETURN_NOT_OK(child->Deserialize(current_repetition_level, next_repetitions)); } + return Status::OK(); } std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } - void InitializeRowGroup(std::shared_ptr const& row_group_reader) final { + Status InitializeRowGroup(std::shared_ptr const& row_group_reader) final { for (auto& child : children_) { - child->InitializeRowGroup(row_group_reader); + RETURN_NOT_OK(child->InitializeRowGroup(row_group_reader)); } + return Status::OK(); } private: - std::string name_; std::vector> children_; - int16_t max_definition_level_; + DefinitionLevel max_definition_level_; std::shared_ptr<::arrow::StructBuilder> builder_; }; +// The following set of classes are used to drive the +// iteration over the input data in a particular manner. +// +// TODO: except for the batched array serializer, it might +// be possible to collapse these classes through the +// use of iterators. ArrayDeserializer::ArrayDeserializer(std::unique_ptr&& node) : node_(std::move(node)) {} ArrayDeserializer::~ArrayDeserializer() {} +// This class will deserialize a single array from the entire +// file. The node that is passed to the constructor may +// contain multiple primitive nodes as its descendants. class FileArrayDeserializer : public ArrayDeserializer { public: FileArrayDeserializer(std::unique_ptr&& node, @@ -755,10 +1064,10 @@ class FileArrayDeserializer : public ArrayDeserializer { auto const num_row_groups = file_reader_->metadata()->num_row_groups(); for (int ii = 0; ii < num_row_groups; ++ii) { auto row_group_reader = file_reader_->RowGroup(ii); - node_->InitializeRowGroup(row_group_reader); + RETURN_NOT_OK(node_->InitializeRowGroup(row_group_reader)); auto const num_rows = row_group_reader_->metadata()->num_rows(); for (int jj = 0; jj < num_rows; ++jj) { - node_->Deserialize(repetitions, /* repetition level */ 0); + RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); repetitions.clear(); } } @@ -773,6 +1082,9 @@ class FileArrayDeserializer : public ArrayDeserializer { ArrayBatchedDeserializer::~ArrayBatchedDeserializer() {} +// This class will deserialize a single array from the entire +// file in batches. The node that is passed to the constructor may +// contain multiple primitive nodes as its descendants. class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { public: FileArrayBatchedDeserializer(std::unique_ptr&& node, @@ -782,11 +1094,15 @@ class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { current_row_group_(file_reader->RowGroup(0)), current_row_group_index_(0), current_row_(0) { - node_->InitializeRowGroup(current_row_group_); } ~FileArrayBatchedDeserializer() {} + Status Initialize() + { + return node_->InitializeRowGroup(current_row_group_); + } + ::arrow::Status DeserializeBatch(int64_t num_records, std::shared_ptr<::arrow::Array>& array) final { DeserializerNode::LevelSet repetitions; @@ -796,14 +1112,14 @@ class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { auto const num_rows = current_row_group_->metadata()->num_rows(); for (; current_row_ < num_rows && records_read < num_records; ++current_row_, ++records_read) { - node_->Deserialize(repetitions, /* repetition_level */ 0); + RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); repetitions.clear(); } if (current_row_ >= num_rows) { ++current_row_group_index_; if (current_row_group_index_ < num_row_groups) { current_row_group_ = file_reader_->RowGroup(current_row_group_index_); - node_->InitializeRowGroup(current_row_group_); + RETURN_NOT_OK(node_->InitializeRowGroup(current_row_group_)); current_row_ = 0; } } @@ -823,20 +1139,29 @@ class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { int current_row_; }; +// This class will deserialize a single array from a single +// row group for a single node. The DeserializeBuilder only +// constructs this for a node that has at most one primitive +// descendant, but it could just as well be used for nodes that +// contain multiple primitive descendants. class ColumnChunkDeserializer : public ArrayDeserializer { public: ColumnChunkDeserializer( std::unique_ptr&& node, std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) : ArrayDeserializer(std::move(node)), row_group_reader_(row_group_reader) { - node_->InitializeRowGroup(row_group_reader_); } + Status Initialize() + { + return node_->InitializeRowGroup(row_group_reader_); + } + ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final { DeserializerNode::LevelSet repetitions; auto const num_rows = row_group_reader_->metadata()->num_rows(); for (int ii = 0; ii < num_rows; ++ii) { - node_->Deserialize(repetitions, /* repetition level */ 0); + RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); repetitions.clear(); } return node_->GetArrayBuilder()->Finish(&array); @@ -864,6 +1189,23 @@ ::arrow::Status TableDeserializer::MakeTable(std::shared_ptr<::arrow::Table>& re return Status::OK(); } +Status TableDeserializer::DeserializeRowGroup(std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) +{ + for (auto& node : nodes_) { + RETURN_NOT_OK(node->InitializeRowGroup(row_group_reader)); + } + DeserializerNode::LevelSet repetitions; + auto const num_rows = row_group_reader->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + for (auto& node : nodes_) { + RETURN_NOT_OK(node->Deserialize(RepetitionLevel(), repetitions)); + repetitions.clear(); + } + } + return Status::OK(); +} + +// This class will deserialize all of the specified nodes from a single row group. class RowGroupDeserializer : public TableDeserializer { public: RowGroupDeserializer(std::vector>&& nodes, @@ -871,23 +1213,14 @@ class RowGroupDeserializer : public TableDeserializer { std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) : TableDeserializer( std::forward>>(nodes), schema), - row_group_reader_(row_group_reader) { - for (auto& node : nodes_) { - node->InitializeRowGroup(row_group_reader); - } + row_group_reader_(row_group_reader) + { } ~RowGroupDeserializer() {} ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final { - DeserializerNode::LevelSet repetitions; - auto const num_rows = row_group_reader_->metadata()->num_rows(); - for (int ii = 0; ii < num_rows; ++ii) { - for (auto& node : nodes_) { - node->Deserialize(repetitions, /* repetition level */ 0); - repetitions.clear(); - } - } + DeserializeRowGroup(row_group_reader_); return MakeTable(table); } @@ -895,6 +1228,7 @@ class RowGroupDeserializer : public TableDeserializer { std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; }; +// This class wil deserialize all of the specified nodes from a Parquet file class FileTableDeserializer : public TableDeserializer { public: FileTableDeserializer(std::vector>&& nodes, @@ -907,20 +1241,9 @@ class FileTableDeserializer : public TableDeserializer { ~FileTableDeserializer() {} ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) { - DeserializerNode::LevelSet repetitions; auto num_row_groups = file_reader_->metadata()->num_row_groups(); for (int ii = 0; ii < num_row_groups; ++ii) { - auto row_group_reader = file_reader_->RowGroup(ii); - for (auto& node : nodes_) { - node->InitializeRowGroup(row_group_reader); - } - auto const num_rows = row_group_reader->metadata()->num_rows(); - for (int ii = 0; ii < num_rows; ++ii) { - for (auto& node : nodes_) { - node->Deserialize(repetitions, /* repetition level */ 0); - repetitions.clear(); - } - } + DeserializeRowGroup(file_reader_->RowGroup(ii)); } return MakeTable(table); } @@ -929,9 +1252,12 @@ class FileTableDeserializer : public TableDeserializer { std::shared_ptr<::parquet::ParquetFileReader> file_reader_; }; +// This class does the main work of constructing deserializer +// nodes from Parquet schema nodes. class DeserializerBuilder::Impl { public: Impl(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, int64_t buffer_size); ::arrow::Status BuildSchemaNodeDeserializer(int schema_index, @@ -963,41 +1289,149 @@ class DeserializerBuilder::Impl { return indices; } + + // \brief calls BuildNode for each node in the parquet file's + // schema + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[out] the deserializer nodes + // \param[in] if specified, the nodes will appear in the result + // list based on the order in which they appeared in + // column_indices. For example, if the Parquet schema has two + // primitive nodes and the column_indices given is {1, 0}, + // then the resulting table will have the fields reversed + // in the Arrow schema. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported Status BuildNodes(std::vector const& column_indices, std::vector>& nodes, bool in_index_order = true); - // TODO: make repetition_levels and definition_levels - // tagged/typed integers: - // - // struct RepetitionTag; - // - // template - // TaggedType; - // - // using Repetition = TaggedType + // \brief builds a deserializer node based on the Parquet node. + // \param[in] the parquet node + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported Status BuildNode(std::shared_ptr const& node, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result, - int16_t repetition_levels = 0, int16_t definition_levels = 0); - + RepetitionLevel repetition_levels = RepetitionLevel(), + DefinitionLevel definition_levels = DefinitionLevel()); + + // \brief builds a primitive deserializer node based on the Parquet node. + // \param[in] the parquet node, which must point to a PrimitiveNode + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported Status BuildPrimitiveNode(std::shared_ptr const& node, - int16_t repetition_levels, int16_t definition_levels, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result); + // \brief builds a group deserializer node based on the Parquet node. + // \param[in] the parquet node, which must point to a GroupNode + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported Status BuildGroupNode(std::shared_ptr const& node, - int16_t repetition_levels, int16_t definition_levels, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result); + // \brief builds a list deserializer node based on the Parquet node. + // \param[in] the parquet node, which must point to a GroupNode which + // has a LIST annotation. + // \param[in] indices that are used to filter indicate which + // primitive columns to include. The indices correspond + // to the index you would obtain from calling + // SchemaDescriptor::Column + // \param[in] these indicate which indices were used in building + // this node. This is used to re-order the arrays in + // the schema of the deserialized Table for that it + // matches the order in which the indices were specified. + // \param[out] the deserializer node + // \param[in] repetition levels seen so far. Since this function is + // called recursively for group nodes, the value is + // incremented as the tree is traversed. Not specifying + // this value in the call should only be done when the + // node is the root node. + // \param[in] definition levels seen so far. Used in the same manner + // as the repetition levels. + // \return error status if all of the column indices were out of bounds + // or any of the converted/physical type specifications in the + // Parquet schema are unsupported Status BuildListNode(std::shared_ptr const& node, - int16_t repetition_levels, int16_t definition_levels, + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result); - template + // \brief builds a typed primitive node. This is the main entry point + // for a number of SFINAE based calls. + // \param[in] the parquet node, which must point to a PrimitiveNode + // \param[in] the column descriptor, which is used in error + // messages to report what node had an unsupported type + // \parma[in] the index of the column, which is stored in the node + // and then later used to lookup the column reader from + // the row group reader + // \param[out] the deserializer node + // \return error status if any of the converted/physical type + // specifications in the Parquet schema are unsupported + template Status MakeTypedPrimitiveDeserializerNode( std::shared_ptr const& node, const ColumnDescriptor* const column_descriptor, int column_index, @@ -1005,36 +1439,28 @@ class DeserializerBuilder::Impl { auto primitive_node = std::static_pointer_cast(node); switch (primitive_node->physical_type()) { case Type::BOOLEAN: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::INT32: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::INT64: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::INT96: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::FLOAT: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::DOUBLE: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::BYTE_ARRAY: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::FIXED_LEN_BYTE_ARRAY: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); default: std::ostringstream stream; @@ -1043,98 +1469,106 @@ class DeserializerBuilder::Impl { } } - template + // \brief builds a typed primitive node, taking into account it's + // logical (aka converted) type. + // \param[in] the parquet node, which must point to a PrimitiveNode + // \param[in] the column descriptor, which is used in error + // messages to report what node had an unsupported type + // \parma[in] the index of the column, which is stored in the node + // and then later used to lookup the column reader from + // the row group reader + // \param[out] the deserializer node + // \return error status if any of the converted/physical type + // specifications in the Parquet schema are unsupported + template Status MakeLogicalPrimitiveDeserializerNode( std::shared_ptr const& node, const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { switch (node->logical_type()) { case parquet::LogicalType::DATE: - return MakeDateDeserializerNode( + return MakeDateDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::TIME_MILLIS: - return MakeTimeMillisDeserializerNode( + return MakeTimeMillisDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::TIME_MICROS: - return MakeTimeMicrosDeserializerNode( + return MakeTimeMicrosDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::TIMESTAMP_MILLIS: - return MakeTimestampDeserializerNode( + return MakeTimestampDeserializerNode( column_descriptor, column_index, ::arrow::TimeUnit::MILLI, result); case parquet::LogicalType::TIMESTAMP_MICROS: - return MakeTimestampDeserializerNode( + return MakeTimestampDeserializerNode( column_descriptor, column_index, ::arrow::TimeUnit::MICRO, result); case parquet::LogicalType::DECIMAL: - return MakeDecimalDeserializerNode( + return MakeDecimalDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::UINT_8: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::UINT_16: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::UINT_32: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::UINT_64: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::INT_8: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::INT_16: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::INT_32: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::INT_64: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case parquet::LogicalType::UTF8: - return MakeUTF8DeserializerNode( + return MakeUTF8DeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::NA: - return MakeNullDeserializerNode( + return MakeNullDeserializerNode( column_descriptor, column_index, result); default: switch (node->physical_type()) { case Type::BOOLEAN: - return MakeBooleanDeserializerNode( + return MakeBooleanDeserializerNode( column_descriptor, column_index, result); case Type::INT32: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case Type::INT64: return MakeIntegerDeserializerNode( + SerializerType>( column_descriptor, column_index, result); case Type::FLOAT: - return MakeFloatDeserializerNode( + return MakeFloatDeserializerNode( column_descriptor, column_index, result); case Type::DOUBLE: - return MakeDoubleDeserializerNode( + return MakeDoubleDeserializerNode( column_descriptor, column_index, result); case Type::BYTE_ARRAY: - return MakeByteArrayDeserializerNode(column_descriptor, + return MakeByteArrayDeserializerNode(column_descriptor, column_index, result); case Type::FIXED_LEN_BYTE_ARRAY: - return MakeFixedLengthByteArrayDeserializerNode( + return MakeFixedLengthByteArrayDeserializerNode( column_descriptor, column_index, result); case Type::INT96: - return MakeImpalaTimestampDeserializerNode( + return MakeImpalaTimestampDeserializerNode( column_descriptor, column_index, result); default: std::ostringstream stream; @@ -1144,16 +1578,26 @@ class DeserializerBuilder::Impl { } } - template + // + // The functions below create the various logical types of primitive nodes. + // Since the ParquetType template parameter is determined at run-time and we + // bind that earlier in the call stack, we can use SFINAE to limit the + // allowable types for a particular field (e.g. - Timestamp types can only + // be constructed from an int64 physical type). + // + // Most of the code is boiler-plate, except for the fact that there are + // different builders for the various types. + + template Status MakeNullDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result); + SerializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1161,12 +1605,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::TimestampType>(::arrow::TimeUnit::NANO); return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, arrow_type, - ::arrow::default_memory_pool()); + SerializerType>( + column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1188,7 +1631,7 @@ class DeserializerBuilder::Impl { std::is_same::value); }; - template + template typename std::enable_if::value, Status>::type MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, @@ -1196,12 +1639,11 @@ class DeserializerBuilder::Impl { auto arrow_type = std::make_shared<::arrow::Decimal128Type>( column_descriptor->type_precision(), column_descriptor->type_scale()); return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, arrow_type, - ::arrow::default_memory_pool()); + SerializerType>( + column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, @@ -1213,18 +1655,18 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result); + SerializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1237,17 +1679,17 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result); + SerializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1259,7 +1701,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1267,12 +1709,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::Time32Type>(::arrow::TimeUnit::MILLI); return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, arrow_type, - ::arrow::default_memory_pool()); + SerializerType>( + column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1285,7 +1726,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1293,12 +1734,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::Time64Type>(::arrow::TimeUnit::MICRO); return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, arrow_type, - ::arrow::default_memory_pool()); + SerializerType>( + column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1311,7 +1751,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1319,12 +1759,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::TimestampType>(time_unit); return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, arrow_type, - ::arrow::default_memory_pool()); + SerializerType>( + column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1354,21 +1793,19 @@ class DeserializerBuilder::Impl { static constexpr auto value = (both_ints && parquet_size >= arrow_size); }; - template + template typename std::enable_if::value, Status>::type MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { - return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result); + return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, + column_index, result, + pool_); } - template + template typename std::enable_if::value, Status>::type MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1381,20 +1818,19 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { - return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, ::arrow::utf8(), - ::arrow::default_memory_pool()); + return MakeTypedPrimitiveNodeWithBuilder( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1406,7 +1842,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1415,11 +1851,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result); + SerializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1432,7 +1868,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeFixedLengthByteArrayDeserializerNode( @@ -1443,12 +1879,11 @@ class DeserializerBuilder::Impl { auto arrow_type = std::make_shared<::arrow::FixedSizeBinaryType>(column_descriptor->type_length()); return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, arrow_type, - ::arrow::default_memory_pool()); + SerializerType>( + column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeFixedLengthByteArrayDeserializerNode( @@ -1461,7 +1896,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1469,11 +1904,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result); + SerializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1486,7 +1921,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1494,11 +1929,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result); + SerializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1510,22 +1945,39 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + // \brief The primary function for creating a primitive node. This will + // create an array builder of the specified type, passing + // args to its constructor, and finally pass that builder + // to the TypedPrimitiveDeserializerNode constructor + // \param[in] the column descriptor, which is used in error + // messages to report what node had an unsupported type + // \parma[in] the index of the column, which is stored in the node + // and then later used to lookup the column reader from + // the row group reader + // \param[in] arguments that are passed to the BuilderType's + // constructor + // \param[out] the deserializer node + // \return error status if any of the converted/physical type + // specifications in the Parquet schema are unsupported + template Status MakeTypedPrimitiveNodeWithBuilder( const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result, Args&&... args) { auto builder = std::make_shared(std::forward(args)...); using ReturnType = TypedPrimitiveDeserializerNode, ArrayBuilderType, - HasDefinition, HasRepetition>; - result = std::unique_ptr( - new ReturnType(column_descriptor, column_index, buffer_size_, builder)); + SerializerType>; + auto node = std::unique_ptr(new ReturnType(column_descriptor, column_index, buffer_size_, builder, pool_)); + RETURN_NOT_OK(node->Initialize()); + result = std::move(node); return Status::OK(); } + // This method is pulled out here instead of inside BuildNode because + // we skip the "middle" node when constructing List nodes. void IncrementLevels(std::shared_ptr const& node, - int16_t& repetition_levels, int16_t& definition_levels) { + RepetitionLevel& repetition_levels, + DefinitionLevel& definition_levels) { if (!node->is_required()) { ++definition_levels; } @@ -1536,12 +1988,16 @@ class DeserializerBuilder::Impl { std::shared_ptr<::parquet::ParquetFileReader> file_reader_; + ::arrow::MemoryPool* pool_; + int64_t buffer_size_; }; DeserializerBuilder::Impl::Impl( - std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, int64_t buffer_size) - : file_reader_(file_reader), buffer_size_(buffer_size) {} + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, + int64_t buffer_size) + : file_reader_(file_reader), pool_(pool), buffer_size_(buffer_size) {} ::arrow::Status DeserializerBuilder::Impl::BuildSchemaNodeDeserializer( int schema_index, std::unique_ptr& result) { @@ -1590,7 +2046,9 @@ ::arrow::Status DeserializerBuilder::Impl::BuildArrayBatchedDeserializer( stream << "were created"; return Status::Invalid(stream.str()); } - result.reset(new FileArrayBatchedDeserializer(std::move(nodes[0]), file_reader_)); + std::unique_ptr deserializer(new FileArrayBatchedDeserializer(std::move(nodes[0]), file_reader_)); + RETURN_NOT_OK(deserializer->Initialize()); + result = std::move(deserializer); return Status::OK(); } @@ -1607,8 +2065,10 @@ ::arrow::Status DeserializerBuilder::Impl::BuildColumnChunkDeserializer( stream << "were created"; return Status::Invalid(stream.str()); } - result.reset(new ColumnChunkDeserializer(std::move(nodes[0]), - file_reader_->RowGroup(row_group_index))); + std::unique_ptr deserializer(new ColumnChunkDeserializer(std::move(nodes[0]), + file_reader_->RowGroup(row_group_index))); + RETURN_NOT_OK(deserializer->Initialize()); + result = std::move(deserializer); return Status::OK(); } @@ -1706,8 +2166,8 @@ Status DeserializerBuilder::Impl::BuildNode(std::shared_ptr const& n std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result, - int16_t repetition_levels, - int16_t definition_levels) { + RepetitionLevel repetition_levels, + DefinitionLevel definition_levels) { IncrementLevels(node, repetition_levels, definition_levels); if (node->is_group()) { return BuildGroupNode(node, repetition_levels, definition_levels, indices, @@ -1719,8 +2179,8 @@ Status DeserializerBuilder::Impl::BuildNode(std::shared_ptr const& n } Status DeserializerBuilder::Impl::BuildPrimitiveNode( - std::shared_ptr const& node, int16_t repetition_levels, - int16_t definition_levels, std::vector const& indices, + std::shared_ptr const& node, RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result) { auto const schema = file_reader_->metadata()->schema(); auto column_index = schema->ColumnIndex(*node); @@ -1734,25 +2194,25 @@ Status DeserializerBuilder::Impl::BuildPrimitiveNode( } indices_seen.emplace_back(column_index); auto column_descriptor = schema->Column(column_index); - if (definition_levels > 0) { - if (repetition_levels > 0) { + if (*definition_levels > 0) { + if (*repetition_levels > 0) { // The path to the node either contains only repeated nodes or // a mix of repeated and optional nodes. - return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, + return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, column_index, result); } else { // A field that is either itself optional or a descendant of // an optional field (e.g. - member of a struct that is optional) // but does not have any ancestors in the tree that are repeated. // It therefore won't have any repetition levels. - return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, + return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, column_index, result); } - } else if (repetition_levels == 0) { + } else if (*repetition_levels == 0) { // A field that is always present (max_definition_level == // 0 && max_repetition_level == 0) and therefore doesn't // need repetition levels or definition levels - return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, + return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, column_index, result); } else { // This should never happen since a non-zero max repetition level @@ -1760,21 +2220,21 @@ Status DeserializerBuilder::Impl::BuildPrimitiveNode( std::ostringstream stream; stream << "Primitive node " << node->path()->ToDotString(); stream << " has a max_repetition_level or "; - stream << definition_levels; + stream << *definition_levels; stream << " and a max_definition_level of "; - stream << repetition_levels; + stream << *repetition_levels; return Status::Invalid(stream.str()); } if (node->is_repeated()) { - result.reset(new ListDeserializerNode(std::move(result), repetition_levels, + result.reset(new ListDeserializerNode(std::move(result), pool_, repetition_levels, definition_levels)); } return Status::OK(); } Status DeserializerBuilder::Impl::BuildGroupNode( - std::shared_ptr const& node, int16_t repetition_levels, - int16_t definition_levels, std::vector const& indices, + std::shared_ptr const& node, RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result) { // TODO: implement a proper Visitor interface in Node auto group_node = std::static_pointer_cast(node); @@ -1808,40 +2268,26 @@ Status DeserializerBuilder::Impl::BuildGroupNode( return Status::OK(); } if (node->is_optional()) { - result.reset(new NestedDeserializerNode(node->name(), definition_levels, - std::move(children), fields)); + result.reset(new NestedDeserializerNode(definition_levels, + std::move(children), fields, + pool_)); } else { - result.reset(new NestedDeserializerNode(node->name(), definition_levels, - std::move(children), fields)); + result.reset(new NestedDeserializerNode(definition_levels, + std::move(children), fields, + pool_)); } if (node->is_repeated()) { - // TODO: do we even test this? It may actually be implemented - // incorrectly because it doesn't contain the nesting that - // exists in LIST nodes. For example, this code path would - // be run for a node like this: - // - // repeated int32 element - // - // But that's the same code which is used for this: - // - // required group (LIST) { - // repated group list_group { - // required int32 element - // } - // } - // - // The danger of using the same implementation is that in this - // path we can't differentiate between an empty list and a list - // with a null - result.reset(new ListDeserializerNode(std::move(result), repetition_levels, - definition_levels)); + std::ostringstream stream; + stream << "Deserializing from Parquet repeated nodes to Arrow "; + stream << "is not supported without a LIST annotation"; + return Status::Invalid(stream.str()); } return Status::OK(); } Status DeserializerBuilder::Impl::BuildListNode( - std::shared_ptr const& node, int16_t repetition_levels, - int16_t definition_levels, std::vector const& indices, + std::shared_ptr const& node, RepetitionLevel repetition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result) { // // A list node must have this schema: @@ -1922,18 +2368,20 @@ Status DeserializerBuilder::Impl::BuildListNode( // there aren't, then it writes a null and skips writing // any of its children. if (node->is_optional()) { - result.reset(new NullableListDeserializerNode(std::move(child), repetition_levels, + result.reset(new NullableListDeserializerNode(std::move(child), pool_, repetition_levels, current_max_definition_level)); } else { - result.reset(new ListDeserializerNode(std::move(child), repetition_levels, + result.reset(new ListDeserializerNode(std::move(child), pool_, repetition_levels, current_max_definition_level)); } return Status::OK(); } DeserializerBuilder::DeserializerBuilder( - std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, int64_t buffer_size) - : impl_(new Impl(file_reader, buffer_size)) {} + std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, + int64_t buffer_size) + : impl_(new Impl(file_reader, pool, buffer_size)) {} DeserializerBuilder::~DeserializerBuilder() {} diff --git a/src/parquet/arrow/deserializer.h b/src/parquet/arrow/deserializer.h index 80b93fe7..77806a4b 100644 --- a/src/parquet/arrow/deserializer.h +++ b/src/parquet/arrow/deserializer.h @@ -26,6 +26,7 @@ namespace arrow { class Array; +class MemoryPool; class Schema; class Table; @@ -39,7 +40,7 @@ class RowGroupReader; namespace arrow { class DeserializerNode; - + // // This header file contains classes that can deserialize from Parquet // files into Arrow arrays and tables. It supports nested structures @@ -152,6 +153,12 @@ class PARQUET_EXPORT TableDeserializer { std::vector> nodes_; + // \brief Convenience method which iterates over all rows in the rows + // group and deserializes each node + // \param[in] the row group reader + // \return status error if there is a problem deserializing the row group + ::arrow::Status DeserializeRowGroup(std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader); + private: // Schema, which was most likely obtained from a call to // FromParquetSchema @@ -167,11 +174,14 @@ class PARQUET_EXPORT DeserializerBuilder { // // \brief Create a new deserializer builder // \param[in] the reader that is the source of the data to deserialize + // \param[in] the memory pool to use for builders and buffers that + // are created for reading primitive values // \param[in] the deserializers use buffers for reading Parquet definition // and repetition levels, as well as values. This denotes how // many values will be buffered when reading from primitive // columns. DeserializerBuilder(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, + ::arrow::MemoryPool* pool, // TODO: make this configurable from a reader // properties object int64_t buffer_size = 1024); diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 21de2003..7e1f15b6 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -248,7 +248,7 @@ FileReader::FileReader(MemoryPool* pool, std::unique_ptr read FileReader::~FileReader() {} Status FileReader::Impl::GetColumn(int i, std::unique_ptr* out) { - DeserializerBuilder builder(reader_); + DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; RETURN_NOT_OK(builder.BuildArrayBatchedDeserializer(i, deserializer)); (*out).reset(new ColumnReader(deserializer)); @@ -256,7 +256,7 @@ Status FileReader::Impl::GetColumn(int i, std::unique_ptr* out) { } Status FileReader::Impl::ReadSchemaField(int i, std::shared_ptr* out) { - DeserializerBuilder builder(reader_); + DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; RETURN_NOT_OK(builder.BuildSchemaNodeDeserializer(i, deserializer)); deserializer->DeserializeArray(*out); @@ -264,7 +264,7 @@ Status FileReader::Impl::ReadSchemaField(int i, std::shared_ptr* out) { } Status FileReader::Impl::ReadColumn(int i, std::shared_ptr* out) { - DeserializerBuilder builder(reader_); + DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; RETURN_NOT_OK(builder.BuildColumnDeserializer(i, deserializer)); deserializer->DeserializeArray(*out); @@ -280,7 +280,7 @@ Status FileReader::Impl::GetSchema(const std::vector& indices, Status FileReader::Impl::ReadColumnChunk(int column_index, int row_group_index, std::shared_ptr* out) { - DeserializerBuilder builder(reader_); + DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; RETURN_NOT_OK( builder.BuildColumnChunkDeserializer(column_index, row_group_index, deserializer)); @@ -291,7 +291,7 @@ Status FileReader::Impl::ReadColumnChunk(int column_index, int row_group_index, Status FileReader::Impl::ReadRowGroup(int row_group_index, const std::vector& indices, std::shared_ptr<::arrow::Table>* out) { - DeserializerBuilder builder(reader_); + DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; RETURN_NOT_OK( builder.BuildRowGroupDeserializer(row_group_index, indices, deserializer)); @@ -301,7 +301,7 @@ Status FileReader::Impl::ReadRowGroup(int row_group_index, Status FileReader::Impl::ReadTable(const std::vector& indices, std::shared_ptr
* out) { - DeserializerBuilder builder(reader_); + DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; RETURN_NOT_OK(builder.BuildFileDeserializer(indices, deserializer)); deserializer->DeserializeTable(*out); From a5bdee51e40aefd0d0d45e17429eef9f3219af46 Mon Sep 17 00:00:00 2001 From: Joshua Storck Date: Sun, 13 May 2018 00:52:53 -0400 Subject: [PATCH 3/6] Moving conversion of Decimal128 from big endian bytes into static method in Decimal128 class --- src/parquet/arrow/deserializer.cc | 542 +++++++++++------------------- 1 file changed, 201 insertions(+), 341 deletions(-) diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc index 4bb278d5..3e4fcedf 100644 --- a/src/parquet/arrow/deserializer.cc +++ b/src/parquet/arrow/deserializer.cc @@ -48,98 +48,6 @@ namespace parquet { namespace arrow { -// Start: copied from reader.cc -static uint64_t BytesToInteger(const uint8_t* bytes, int32_t start, int32_t stop) { - using ::arrow::BitUtil::FromBigEndian; - - const int32_t length = stop - start; - - DCHECK_GE(length, 0); - DCHECK_LE(length, 8); - - switch (length) { - case 0: - return 0; - case 1: - return bytes[start]; - case 2: - return FromBigEndian(*reinterpret_cast(bytes + start)); - case 3: { - const uint64_t first_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_byte = bytes[stop - 1]; - return first_two_bytes << 8 | last_byte; - } - case 4: - return FromBigEndian(*reinterpret_cast(bytes + start)); - case 5: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_byte = bytes[stop - 1]; - return first_four_bytes << 8 | last_byte; - } - case 6: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t last_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start + 4)); - return first_four_bytes << 16 | last_two_bytes; - } - case 7: { - const uint64_t first_four_bytes = - FromBigEndian(*reinterpret_cast(bytes + start)); - const uint64_t second_two_bytes = - FromBigEndian(*reinterpret_cast(bytes + start + 4)); - const uint64_t last_byte = bytes[stop - 1]; - return first_four_bytes << 24 | second_two_bytes << 8 | last_byte; - } - case 8: - return FromBigEndian(*reinterpret_cast(bytes + start)); - default: { - DCHECK(false); - return UINT64_MAX; - } - } -} - -static constexpr int32_t kMinDecimalBytes = 1; -static constexpr int32_t kMaxDecimalBytes = 16; - -/// \brief Convert a sequence of big-endian bytes to one int64_t (high bits) and one -/// uint64_t (low bits). -static void BytesToIntegerPair(const uint8_t* bytes, - const int32_t total_number_of_bytes_used, int64_t* high, - uint64_t* low) { - DCHECK_GE(total_number_of_bytes_used, kMinDecimalBytes); - DCHECK_LE(total_number_of_bytes_used, kMaxDecimalBytes); - - /// Bytes are coming in big-endian, so the first byte is the MSB and therefore holds the - /// sign bit. - const bool is_negative = static_cast(bytes[0]) < 0; - - /// Sign extend the low bits if necessary - *low = UINT64_MAX * (is_negative && total_number_of_bytes_used < 8); - *high = -1 * (is_negative && total_number_of_bytes_used < kMaxDecimalBytes); - - /// Stop byte of the high bytes - const int32_t high_bits_offset = std::max(0, total_number_of_bytes_used - 8); - - /// Shift left enough bits to make room for the incoming int64_t - *high <<= high_bits_offset * CHAR_BIT; - - /// Preserve the upper bits by inplace OR-ing the int64_t - *high |= BytesToInteger(bytes, 0, high_bits_offset); - - /// Stop byte of the low bytes - const int32_t low_bits_offset = std::min(total_number_of_bytes_used, 8); - - /// Shift left enough bits to make room for the incoming uint64_t - *low <<= low_bits_offset * CHAR_BIT; - - /// Preserve the upper bits by inplace OR-ing the uint64_t - *low |= BytesToInteger(bytes, high_bits_offset, total_number_of_bytes_used); -} - constexpr int64_t kJulianToUnixEpochDays = 2440588LL; constexpr int64_t kMillisecondsInADay = 86400000LL; constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL; @@ -156,30 +64,19 @@ static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timest // definition and repeition level arguments in the DeserializerBuilder // build function calls. template -class Level -{ - -public: - - Level() : value_(0) - { - } +class Level { + public: + Level() : value_(0) {} - Level& operator++() - { + Level& operator++() { ++value_; return *this; } - int16_t operator*() const - { - return value_; - } - -private: + int16_t operator*() const { return value_; } + private: int16_t value_; - }; struct Repetition; @@ -187,7 +84,7 @@ struct Definition; using RepetitionLevel = Level; using DefinitionLevel = Level; - + // // This is a utility class that is used by primitive node deserializer // nodes. It's similar to a std::list, but stores its data as a contiguous @@ -196,24 +93,19 @@ template class PrimitiveDeserializerArray { public: PrimitiveDeserializerArray(::arrow::MemoryPool* pool, int64_t capacity) - : - pool_(pool), - capacity_(capacity), - size_(0), - index_(0) {} + : pool_(pool), capacity_(capacity), size_(0), index_(0) {} ~PrimitiveDeserializerArray() { pool_->Free(reinterpret_cast(data_), capacity_); } - Status Initialize() - { + Status Initialize() { uint8_t* out; RETURN_NOT_OK(pool_->Allocate(sizeof(T) * capacity_, &out)); data_ = reinterpret_cast(out); return Status::OK(); } - + bool const Empty() const { return index_ >= size_; } T Front() const { return data_[index_]; } @@ -226,7 +118,7 @@ class PrimitiveDeserializerArray { } ::arrow::MemoryPool* pool_; - + T* data_; int64_t capacity_; @@ -262,7 +154,7 @@ class DeserializerNode { // again before popping back to their parent node. // \return error status if there is no more data for this node virtual Status Deserialize(RepetitionLevel current_repetition_level, - LevelSet& next_repetitions) = 0; + LevelSet& next_repetitions) = 0; // \brief Skips deserialization. This is used by group and list // nodes when they are nullable, in which they call this @@ -274,7 +166,7 @@ class DeserializerNode { // to determine whether or not they need to repeat // and call Deserialize for their child nodes // again before popping back to their parent node. - // \return error status if there is no more data for this node + // \return error status if there is no more data for this node virtual Status Skip(LevelSet& next_repetitions) = 0; // \brief returns the maximum definition level of all of this @@ -317,7 +209,7 @@ class PrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, - ParquetValueType value) { + ParquetValueType value) { using value_type = typename BuilderType::value_type; return builder.Append(static_cast(value)); } @@ -328,7 +220,7 @@ class PrimitiveAppender<::parquet::ByteArray, BuilderType> { public: template Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, - ParquetValueType value) { + ParquetValueType value) { return builder.Append(value.ptr, value.len); } }; @@ -338,7 +230,7 @@ class PrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, BuilderType& builder, - ParquetValueType value) { + ParquetValueType value) { return builder.Append(value.ptr); } }; @@ -348,11 +240,10 @@ class PrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, - ParquetValueType value) { - int64_t high; - uint64_t low; - BytesToIntegerPair(value.ptr, value.len, &high, &low); - return builder.Append(::arrow::Decimal128(high, low)); + ParquetValueType value) { + ::arrow::Decimal128 converted; + RETURN_NOT_OK(::arrow::Decimal128::FromBigEndian(value.ptr, value.len, &converted)); + return builder.Append(converted); } }; @@ -361,11 +252,11 @@ class PrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, - ParquetValueType value) { - int64_t high; - uint64_t low; - BytesToIntegerPair(value.ptr, descriptor->type_length(), &high, &low); - return builder.Append(::arrow::Decimal128(high, low)); + ParquetValueType value) { + ::arrow::Decimal128 converted; + RETURN_NOT_OK(::arrow::Decimal128::FromBigEndian(value.ptr, descriptor->type_length(), + &converted)); + return builder.Append(converted); } }; @@ -374,7 +265,7 @@ class PrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, ::arrow::DecimalBuilder& builder, - ParquetValueType value) { + ParquetValueType value) { return builder.Append(::arrow::Decimal128(value)); } }; @@ -384,7 +275,7 @@ class PrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, ::arrow::TimestampBuilder& builder, - ParquetValueType value) { + ParquetValueType value) { return builder.Append(impala_timestamp_to_nanoseconds(value)); } }; @@ -393,7 +284,7 @@ class NullPrimitiveAppender { public: template Status Append(const ColumnDescriptor* descriptor, ::arrow::NullBuilder& builder, - ParquetValueType value) { + ParquetValueType value) { std::ostringstream stream; stream << "Unexpected call to NullPrimitiveAppender::Append"; return Status::Invalid(stream.str()); @@ -422,16 +313,14 @@ class PrimitiveAppender : public NullPrimitiveA // the column reader and appending values, whereas the sub-classes // handle reading of repetition and definition levels and implement // the DeserializerNode virtual functions. -// +// template class PrimitiveDeserializerNode : public DeserializerNode { - -public: - + public: PrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, int64_t buffer_size, std::shared_ptr const& builder, - ::arrow::MemoryPool* pool) + ::arrow::MemoryPool* pool) : descriptor_(descriptor), column_index_(column_index), values_(pool, buffer_size), @@ -443,14 +332,12 @@ class PrimitiveDeserializerNode : public DeserializerNode { // \param[in] the reader for the row group from which the column reader // is obtained // \return error status if the column index is out of bounds - Status InitializeRowGroup(std::shared_ptr const& row_group_reader) final { - if (column_index_ < row_group_reader->metadata()->num_columns()) - { + Status InitializeRowGroup( + std::shared_ptr const& row_group_reader) final { + if (column_index_ < row_group_reader->metadata()->num_columns()) { column_reader_ = std::static_pointer_cast( - row_group_reader->Column(column_index_)); - } - else - { + row_group_reader->Column(column_index_)); + } else { std::ostringstream stream; stream << "Invalid column index " << column_index_ << " for row group "; return Status::Invalid(stream.str()); @@ -458,13 +345,9 @@ class PrimitiveDeserializerNode : public DeserializerNode { return Status::OK(); } - virtual Status Initialize() - { - return values_.Initialize(); - } - - protected: + virtual Status Initialize() { return values_.Initialize(); } + protected: // \brief appends the next value in this node's values array. It is // assumed that the caller has checked to make sure that the // values buffer is not empty @@ -490,8 +373,7 @@ class PrimitiveDeserializerNode : public DeserializerNode { // This enum is used to differentiate between the three // different types of primitive deserializers -enum PrimitiveSerializerType -{ +enum PrimitiveSerializerType { // This means that the node in the tree has no ancestors (including // itself) that are repeated or optional REQUIRED, @@ -504,7 +386,7 @@ enum PrimitiveSerializerType // (including itself) REPEATED }; - + // // The TypedPrimitiveDeserializerNode does the heavy lifting of // reading repetition and definition levels and implementing @@ -520,7 +402,7 @@ enum PrimitiveSerializerType // still happen in calls to ReadBatch. // template + PrimitiveSerializerType Serializertype> class TypedPrimitiveDeserializerNode; // Specialization for a field that has definition levels but no @@ -535,31 +417,29 @@ class TypedPrimitiveDeserializerNode; // optional group contact { // required bytearray name (UTF8); // optional int32 phone_number; -// } +// } // // Both name and phone_number would be serialized with this type of -// node because they have a max definition level of 1 and 2 respectively. -// +// node because they have a max definition level of 1 and 2 respectively. +// template class TypedPrimitiveDeserializerNode + PrimitiveSerializerType::OPTIONAL> : public PrimitiveDeserializerNode { public: - TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int const column_index, std::size_t buffer_size, std::shared_ptr const& builder, - ::arrow::MemoryPool* pool) + ::arrow::MemoryPool* pool) : PrimitiveDeserializerNode( - descriptor, column_index, buffer_size, builder, pool), - definition_levels_(pool, buffer_size) {} + descriptor, column_index, buffer_size, builder, pool), + definition_levels_(pool, buffer_size) {} - Status Initialize() final - { + Status Initialize() final { RETURN_NOT_OK(definition_levels_.Initialize()); - return PrimitiveDeserializerNode::Initialize(); + return PrimitiveDeserializerNode::Initialize(); } - + Status Skip(DeserializerNode::LevelSet& next_repetitions) final { if (!Fill()) { std::ostringstream stream; @@ -584,7 +464,7 @@ class TypedPrimitiveDeserializerNode definition_levels_; - }; // @@ -625,31 +503,30 @@ class TypedPrimitiveDeserializerNode class TypedPrimitiveDeserializerNode + PrimitiveSerializerType::REPEATED> : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, std::size_t buffer_size, std::shared_ptr const& builder, - ::arrow::MemoryPool* pool) + ::arrow::MemoryPool* pool) : PrimitiveDeserializerNode( - descriptor, column_index, buffer_size, builder, pool), - definition_levels_(pool, buffer_size), - repetition_levels_(pool, buffer_size) {} + descriptor, column_index, buffer_size, builder, pool), + definition_levels_(pool, buffer_size), + repetition_levels_(pool, buffer_size) {} - Status Initialize() final - { + Status Initialize() final { RETURN_NOT_OK(definition_levels_.Initialize()); RETURN_NOT_OK(repetition_levels_.Initialize()); return PrimitiveDeserializerNode::Initialize(); } - + Status CurrentMaxDefinitionLevel(int16_t& level) final { if (!Fill()) { std::ostringstream stream; @@ -675,9 +552,8 @@ class TypedPrimitiveDeserializerNode class TypedPrimitiveDeserializerNode + PrimitiveSerializerType::REQUIRED> : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, int column_index, std::size_t buffer_size, std::shared_ptr const& builder, - ::arrow::MemoryPool* pool) + ::arrow::MemoryPool* pool) : PrimitiveDeserializerNode( - descriptor, column_index, buffer_size, builder, pool) {} + descriptor, column_index, buffer_size, builder, pool) {} Status CurrentMaxDefinitionLevel(int16_t& level) final { level = this->descriptor_->max_definition_level(); @@ -772,7 +647,7 @@ class TypedPrimitiveDeserializerNodevalues_.Empty()) { return true; @@ -793,7 +667,6 @@ class TypedPrimitiveDeserializerNodevalues_.size_); return !this->values_.Empty(); } - }; // @@ -814,17 +687,14 @@ class TypedPrimitiveDeserializerNode&& node_reader, - ::arrow::MemoryPool* pool, - RepetitionLevel max_repetition_level, - DefinitionLevel max_definition_level) + ::arrow::MemoryPool* pool, RepetitionLevel max_repetition_level, + DefinitionLevel max_definition_level) : node_reader_(std::move(node_reader)), max_repetition_level_(max_repetition_level), max_definition_level_(max_definition_level), @@ -839,7 +709,8 @@ class ListDeserializerNode : public DeserializerNode { return node_reader_->Skip(next_repetitions); } - Status Deserialize(RepetitionLevel current_repetition_level, LevelSet& next_repetitions) { + Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) { RETURN_NOT_OK(builder_->Append()); int16_t current_max_definition_level; RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level)); @@ -848,7 +719,8 @@ class ListDeserializerNode : public DeserializerNode { return node_reader_->Skip(next_repetitions); } else { // Deserialize the first one - RETURN_NOT_OK(node_reader_->Deserialize(current_repetition_level, next_repetitions)); + RETURN_NOT_OK( + node_reader_->Deserialize(current_repetition_level, next_repetitions)); // Deserialize the repetitions return DeserializeRepetitions(next_repetitions); } @@ -861,7 +733,6 @@ class ListDeserializerNode : public DeserializerNode { } protected: - // \brief keeps appending elements to the current list as long as // the descendants of this node have repetitions at this list's // repetition level @@ -885,7 +756,7 @@ class ListDeserializerNode : public DeserializerNode { RETURN_NOT_OK(node_reader_->Deserialize(max_repetition_level_, next_repetitions)); } } - return Status::OK(); + return Status::OK(); } std::unique_ptr node_reader_; @@ -915,17 +786,16 @@ class ListDeserializerNode : public DeserializerNode { // The constructor of this class takes the max_definition_level of // list_outer and the max_repetition_level of list_inner class NullableListDeserializerNode : public ListDeserializerNode { - -public: - + public: NullableListDeserializerNode(std::unique_ptr&& node_reader, - ::arrow::MemoryPool* pool, + ::arrow::MemoryPool* pool, RepetitionLevel max_repetition_level, - DefinitionLevel max_definition_level) + DefinitionLevel max_definition_level) : ListDeserializerNode(std::forward>(node_reader), pool, max_repetition_level, max_definition_level) {} - Status Deserialize(RepetitionLevel current_repetition_level, LevelSet& next_repetitions) final { + Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) final { int16_t current_max_definition_level; RETURN_NOT_OK(node_reader_->CurrentMaxDefinitionLevel(current_max_definition_level)); if (current_max_definition_level < *max_definition_level_) { @@ -938,8 +808,9 @@ class NullableListDeserializerNode : public ListDeserializerNode { RETURN_NOT_OK(node_reader_->Skip(next_repetitions)); } else { RETURN_NOT_OK(builder_->Append()); - RETURN_NOT_OK(node_reader_->Deserialize(current_repetition_level, next_repetitions)); - RETURN_NOT_OK(DeserializeRepetitions(next_repetitions)); + RETURN_NOT_OK( + node_reader_->Deserialize(current_repetition_level, next_repetitions)); + RETURN_NOT_OK(DeserializeRepetitions(next_repetitions)); } return Status::OK(); } @@ -956,23 +827,22 @@ class NullableListDeserializerNode : public ListDeserializerNode { // // This has an Optional template parameter so that we can remove unnecessary checks // for required nodes at compile time. -// +// template class NestedDeserializerNode : public DeserializerNode { public: NestedDeserializerNode(DefinitionLevel max_definition_level, std::vector>&& children, std::vector>& fields, - ::arrow::MemoryPool* pool) - : children_(std::move(children)), - max_definition_level_(max_definition_level) { + ::arrow::MemoryPool* pool) + : children_(std::move(children)), max_definition_level_(max_definition_level) { std::vector> child_builders; for (auto const& child : children_) { child_builders.emplace_back(child->GetArrayBuilder()); } auto arrow_type = std::make_shared<::arrow::StructType>(fields); - builder_ = std::make_shared<::arrow::StructBuilder>( - arrow_type, pool, std::move(child_builders)); + builder_ = std::make_shared<::arrow::StructBuilder>(arrow_type, pool, + std::move(child_builders)); } Status Skip(LevelSet& next_repetitions) { @@ -994,7 +864,8 @@ class NestedDeserializerNode : public DeserializerNode { return Status::OK(); } - Status Deserialize(RepetitionLevel current_repetition_level, LevelSet& next_repetitions) final { + Status Deserialize(RepetitionLevel current_repetition_level, + LevelSet& next_repetitions) final { // TODO: detect this at tree construction time. If all of // the fields inside this nested node are required, then // there is no need to do this check. @@ -1020,7 +891,8 @@ class NestedDeserializerNode : public DeserializerNode { std::shared_ptr<::arrow::ArrayBuilder> GetArrayBuilder() final { return builder_; } - Status InitializeRowGroup(std::shared_ptr const& row_group_reader) final { + Status InitializeRowGroup( + std::shared_ptr const& row_group_reader) final { for (auto& child : children_) { RETURN_NOT_OK(child->InitializeRowGroup(row_group_reader)); } @@ -1028,7 +900,6 @@ class NestedDeserializerNode : public DeserializerNode { } private: - std::vector> children_; DefinitionLevel max_definition_level_; @@ -1041,7 +912,7 @@ class NestedDeserializerNode : public DeserializerNode { // // TODO: except for the batched array serializer, it might // be possible to collapse these classes through the -// use of iterators. +// use of iterators. ArrayDeserializer::ArrayDeserializer(std::unique_ptr&& node) : node_(std::move(node)) {} @@ -1049,7 +920,7 @@ ArrayDeserializer::~ArrayDeserializer() {} // This class will deserialize a single array from the entire // file. The node that is passed to the constructor may -// contain multiple primitive nodes as its descendants. +// contain multiple primitive nodes as its descendants. class FileArrayDeserializer : public ArrayDeserializer { public: FileArrayDeserializer(std::unique_ptr&& node, @@ -1084,7 +955,7 @@ ArrayBatchedDeserializer::~ArrayBatchedDeserializer() {} // This class will deserialize a single array from the entire // file in batches. The node that is passed to the constructor may -// contain multiple primitive nodes as its descendants. +// contain multiple primitive nodes as its descendants. class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { public: FileArrayBatchedDeserializer(std::unique_ptr&& node, @@ -1093,16 +964,12 @@ class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { file_reader_(file_reader), current_row_group_(file_reader->RowGroup(0)), current_row_group_index_(0), - current_row_(0) { - } + current_row_(0) {} ~FileArrayBatchedDeserializer() {} - Status Initialize() - { - return node_->InitializeRowGroup(current_row_group_); - } - + Status Initialize() { return node_->InitializeRowGroup(current_row_group_); } + ::arrow::Status DeserializeBatch(int64_t num_records, std::shared_ptr<::arrow::Array>& array) final { DeserializerNode::LevelSet repetitions; @@ -1143,20 +1010,16 @@ class FileArrayBatchedDeserializer : public ArrayBatchedDeserializer { // row group for a single node. The DeserializeBuilder only // constructs this for a node that has at most one primitive // descendant, but it could just as well be used for nodes that -// contain multiple primitive descendants. +// contain multiple primitive descendants. class ColumnChunkDeserializer : public ArrayDeserializer { public: ColumnChunkDeserializer( std::unique_ptr&& node, std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) - : ArrayDeserializer(std::move(node)), row_group_reader_(row_group_reader) { - } + : ArrayDeserializer(std::move(node)), row_group_reader_(row_group_reader) {} + + Status Initialize() { return node_->InitializeRowGroup(row_group_reader_); } - Status Initialize() - { - return node_->InitializeRowGroup(row_group_reader_); - } - ::arrow::Status DeserializeArray(std::shared_ptr<::arrow::Array>& array) final { DeserializerNode::LevelSet repetitions; auto const num_rows = row_group_reader_->metadata()->num_rows(); @@ -1189,12 +1052,12 @@ ::arrow::Status TableDeserializer::MakeTable(std::shared_ptr<::arrow::Table>& re return Status::OK(); } -Status TableDeserializer::DeserializeRowGroup(std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) -{ +Status TableDeserializer::DeserializeRowGroup( + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) { for (auto& node : nodes_) { RETURN_NOT_OK(node->InitializeRowGroup(row_group_reader)); } - DeserializerNode::LevelSet repetitions; + DeserializerNode::LevelSet repetitions; auto const num_rows = row_group_reader->metadata()->num_rows(); for (int ii = 0; ii < num_rows; ++ii) { for (auto& node : nodes_) { @@ -1204,7 +1067,7 @@ Status TableDeserializer::DeserializeRowGroup(std::shared_ptr<::parquet::RowGrou } return Status::OK(); } - + // This class will deserialize all of the specified nodes from a single row group. class RowGroupDeserializer : public TableDeserializer { public: @@ -1213,9 +1076,7 @@ class RowGroupDeserializer : public TableDeserializer { std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) : TableDeserializer( std::forward>>(nodes), schema), - row_group_reader_(row_group_reader) - { - } + row_group_reader_(row_group_reader) {} ~RowGroupDeserializer() {} @@ -1228,7 +1089,7 @@ class RowGroupDeserializer : public TableDeserializer { std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; }; -// This class wil deserialize all of the specified nodes from a Parquet file +// This class wil deserialize all of the specified nodes from a Parquet file class FileTableDeserializer : public TableDeserializer { public: FileTableDeserializer(std::vector>&& nodes, @@ -1257,8 +1118,7 @@ class FileTableDeserializer : public TableDeserializer { class DeserializerBuilder::Impl { public: Impl(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, - ::arrow::MemoryPool* pool, - int64_t buffer_size); + ::arrow::MemoryPool* pool, int64_t buffer_size); ::arrow::Status BuildSchemaNodeDeserializer(int schema_index, std::unique_ptr& result); @@ -1289,10 +1149,9 @@ class DeserializerBuilder::Impl { return indices; } - // \brief calls BuildNode for each node in the parquet file's // schema - // \param[in] indices that are used to filter indicate which + // \param[in] indices that are used to filter indicate which // primitive columns to include. The indices correspond // to the index you would obtain from calling // SchemaDescriptor::Column @@ -1312,7 +1171,7 @@ class DeserializerBuilder::Impl { // \brief builds a deserializer node based on the Parquet node. // \param[in] the parquet node - // \param[in] indices that are used to filter indicate which + // \param[in] indices that are used to filter indicate which // primitive columns to include. The indices correspond // to the index you would obtain from calling // SchemaDescriptor::Column @@ -1335,11 +1194,11 @@ class DeserializerBuilder::Impl { std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result, RepetitionLevel repetition_levels = RepetitionLevel(), - DefinitionLevel definition_levels = DefinitionLevel()); + DefinitionLevel definition_levels = DefinitionLevel()); // \brief builds a primitive deserializer node based on the Parquet node. // \param[in] the parquet node, which must point to a PrimitiveNode - // \param[in] indices that are used to filter indicate which + // \param[in] indices that are used to filter indicate which // primitive columns to include. The indices correspond // to the index you would obtain from calling // SchemaDescriptor::Column @@ -1360,14 +1219,14 @@ class DeserializerBuilder::Impl { // Parquet schema are unsupported Status BuildPrimitiveNode(std::shared_ptr const& node, RepetitionLevel repetition_levels, - DefinitionLevel definition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result); // \brief builds a group deserializer node based on the Parquet node. // \param[in] the parquet node, which must point to a GroupNode - // \param[in] indices that are used to filter indicate which + // \param[in] indices that are used to filter indicate which // primitive columns to include. The indices correspond // to the index you would obtain from calling // SchemaDescriptor::Column @@ -1388,14 +1247,14 @@ class DeserializerBuilder::Impl { // Parquet schema are unsupported Status BuildGroupNode(std::shared_ptr const& node, RepetitionLevel repetition_levels, - DefinitionLevel definition_levels, + DefinitionLevel definition_levels, std::vector const& indices, std::vector& indices_seen, std::unique_ptr& result); // \brief builds a list deserializer node based on the Parquet node. // \param[in] the parquet node, which must point to a GroupNode which // has a LIST annotation. - // \param[in] indices that are used to filter indicate which + // \param[in] indices that are used to filter indicate which // primitive columns to include. The indices correspond // to the index you would obtain from calling // SchemaDescriptor::Column @@ -1416,12 +1275,12 @@ class DeserializerBuilder::Impl { // Parquet schema are unsupported Status BuildListNode(std::shared_ptr const& node, RepetitionLevel repetition_levels, - DefinitionLevel definition_levels, - std::vector const& indices, std::vector& indices_seen, + DefinitionLevel definition_levels, std::vector const& indices, + std::vector& indices_seen, std::unique_ptr& result); // \brief builds a typed primitive node. This is the main entry point - // for a number of SFINAE based calls. + // for a number of SFINAE based calls. // \param[in] the parquet node, which must point to a PrimitiveNode // \param[in] the column descriptor, which is used in error // messages to report what node had an unsupported type @@ -1430,7 +1289,7 @@ class DeserializerBuilder::Impl { // the row group reader // \param[out] the deserializer node // \return error status if any of the converted/physical type - // specifications in the Parquet schema are unsupported + // specifications in the Parquet schema are unsupported template Status MakeTypedPrimitiveDeserializerNode( std::shared_ptr const& node, @@ -1457,7 +1316,8 @@ class DeserializerBuilder::Impl { return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::BYTE_ARRAY: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::FIXED_LEN_BYTE_ARRAY: return MakeLogicalPrimitiveDeserializerNode( @@ -1479,7 +1339,7 @@ class DeserializerBuilder::Impl { // the row group reader // \param[out] the deserializer node // \return error status if any of the converted/physical type - // specifications in the Parquet schema are unsupported + // specifications in the Parquet schema are unsupported template Status MakeLogicalPrimitiveDeserializerNode( std::shared_ptr const& node, @@ -1506,36 +1366,36 @@ class DeserializerBuilder::Impl { column_descriptor, column_index, result); case parquet::LogicalType::UINT_8: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UINT_16: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UINT_32: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UINT_64: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_8: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_16: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_32: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_64: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UTF8: return MakeUTF8DeserializerNode( column_descriptor, column_index, result); @@ -1549,12 +1409,12 @@ class DeserializerBuilder::Impl { column_descriptor, column_index, result); case Type::INT32: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case Type::INT64: return MakeIntegerDeserializerNode( - column_descriptor, column_index, result); + SerializerType>(column_descriptor, + column_index, result); case Type::FLOAT: return MakeFloatDeserializerNode( column_descriptor, column_index, result); @@ -1562,8 +1422,8 @@ class DeserializerBuilder::Impl { return MakeDoubleDeserializerNode( column_descriptor, column_index, result); case Type::BYTE_ARRAY: - return MakeByteArrayDeserializerNode(column_descriptor, - column_index, result); + return MakeByteArrayDeserializerNode( + column_descriptor, column_index, result); case Type::FIXED_LEN_BYTE_ARRAY: return MakeFixedLengthByteArrayDeserializerNode( column_descriptor, column_index, result); @@ -1578,7 +1438,7 @@ class DeserializerBuilder::Impl { } } - // + // // The functions below create the various logical types of primitive nodes. // Since the ParquetType template parameter is determined at run-time and we // bind that earlier in the call stack, we can use SFINAE to limit the @@ -1587,14 +1447,14 @@ class DeserializerBuilder::Impl { // // Most of the code is boiler-plate, except for the fact that there are // different builders for the various types. - + template Status MakeNullDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1662,8 +1522,8 @@ class DeserializerBuilder::Impl { int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1685,8 +1545,8 @@ class DeserializerBuilder::Impl { MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1793,19 +1653,21 @@ class DeserializerBuilder::Impl { static constexpr auto value = (both_ints && parquet_size >= arrow_size); }; - template + template typename std::enable_if::value, Status>::type MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { - return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, - pool_); + return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, + column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1826,8 +1688,8 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1851,8 +1713,8 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1904,8 +1766,8 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1929,8 +1791,8 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder( - column_descriptor, column_index, result, pool_); + SerializerType>(column_descriptor, + column_index, result, pool_); } template @@ -1958,16 +1820,17 @@ class DeserializerBuilder::Impl { // constructor // \param[out] the deserializer node // \return error status if any of the converted/physical type - // specifications in the Parquet schema are unsupported - template + // specifications in the Parquet schema are unsupported + template Status MakeTypedPrimitiveNodeWithBuilder( const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result, Args&&... args) { auto builder = std::make_shared(std::forward(args)...); - using ReturnType = - TypedPrimitiveDeserializerNode, ArrayBuilderType, - SerializerType>; - auto node = std::unique_ptr(new ReturnType(column_descriptor, column_index, buffer_size_, builder, pool_)); + using ReturnType = TypedPrimitiveDeserializerNode, + ArrayBuilderType, SerializerType>; + auto node = std::unique_ptr( + new ReturnType(column_descriptor, column_index, buffer_size_, builder, pool_)); RETURN_NOT_OK(node->Initialize()); result = std::move(node); return Status::OK(); @@ -1977,7 +1840,7 @@ class DeserializerBuilder::Impl { // we skip the "middle" node when constructing List nodes. void IncrementLevels(std::shared_ptr const& node, RepetitionLevel& repetition_levels, - DefinitionLevel& definition_levels) { + DefinitionLevel& definition_levels) { if (!node->is_required()) { ++definition_levels; } @@ -1989,15 +1852,14 @@ class DeserializerBuilder::Impl { std::shared_ptr<::parquet::ParquetFileReader> file_reader_; ::arrow::MemoryPool* pool_; - + int64_t buffer_size_; }; DeserializerBuilder::Impl::Impl( std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, - ::arrow::MemoryPool* pool, - int64_t buffer_size) - : file_reader_(file_reader), pool_(pool), buffer_size_(buffer_size) {} + ::arrow::MemoryPool* pool, int64_t buffer_size) + : file_reader_(file_reader), pool_(pool), buffer_size_(buffer_size) {} ::arrow::Status DeserializerBuilder::Impl::BuildSchemaNodeDeserializer( int schema_index, std::unique_ptr& result) { @@ -2046,7 +1908,8 @@ ::arrow::Status DeserializerBuilder::Impl::BuildArrayBatchedDeserializer( stream << "were created"; return Status::Invalid(stream.str()); } - std::unique_ptr deserializer(new FileArrayBatchedDeserializer(std::move(nodes[0]), file_reader_)); + std::unique_ptr deserializer( + new FileArrayBatchedDeserializer(std::move(nodes[0]), file_reader_)); RETURN_NOT_OK(deserializer->Initialize()); result = std::move(deserializer); return Status::OK(); @@ -2065,8 +1928,8 @@ ::arrow::Status DeserializerBuilder::Impl::BuildColumnChunkDeserializer( stream << "were created"; return Status::Invalid(stream.str()); } - std::unique_ptr deserializer(new ColumnChunkDeserializer(std::move(nodes[0]), - file_reader_->RowGroup(row_group_index))); + std::unique_ptr deserializer(new ColumnChunkDeserializer( + std::move(nodes[0]), file_reader_->RowGroup(row_group_index))); RETURN_NOT_OK(deserializer->Initialize()); result = std::move(deserializer); return Status::OK(); @@ -2198,22 +2061,22 @@ Status DeserializerBuilder::Impl::BuildPrimitiveNode( if (*repetition_levels > 0) { // The path to the node either contains only repeated nodes or // a mix of repeated and optional nodes. - return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, - column_index, result); + return MakeTypedPrimitiveDeserializerNode( + node, column_descriptor, column_index, result); } else { // A field that is either itself optional or a descendant of // an optional field (e.g. - member of a struct that is optional) // but does not have any ancestors in the tree that are repeated. // It therefore won't have any repetition levels. - return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, - column_index, result); + return MakeTypedPrimitiveDeserializerNode( + node, column_descriptor, column_index, result); } } else if (*repetition_levels == 0) { // A field that is always present (max_definition_level == // 0 && max_repetition_level == 0) and therefore doesn't // need repetition levels or definition levels - return MakeTypedPrimitiveDeserializerNode(node, column_descriptor, - column_index, result); + return MakeTypedPrimitiveDeserializerNode( + node, column_descriptor, column_index, result); } else { // This should never happen since a non-zero max repetition level // should have a non-zero definition level @@ -2268,13 +2131,11 @@ Status DeserializerBuilder::Impl::BuildGroupNode( return Status::OK(); } if (node->is_optional()) { - result.reset(new NestedDeserializerNode(definition_levels, - std::move(children), fields, - pool_)); + result.reset(new NestedDeserializerNode(definition_levels, std::move(children), + fields, pool_)); } else { - result.reset(new NestedDeserializerNode(definition_levels, - std::move(children), fields, - pool_)); + result.reset(new NestedDeserializerNode(definition_levels, std::move(children), + fields, pool_)); } if (node->is_repeated()) { std::ostringstream stream; @@ -2368,8 +2229,8 @@ Status DeserializerBuilder::Impl::BuildListNode( // there aren't, then it writes a null and skips writing // any of its children. if (node->is_optional()) { - result.reset(new NullableListDeserializerNode(std::move(child), pool_, repetition_levels, - current_max_definition_level)); + result.reset(new NullableListDeserializerNode( + std::move(child), pool_, repetition_levels, current_max_definition_level)); } else { result.reset(new ListDeserializerNode(std::move(child), pool_, repetition_levels, current_max_definition_level)); @@ -2379,9 +2240,8 @@ Status DeserializerBuilder::Impl::BuildListNode( DeserializerBuilder::DeserializerBuilder( std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, - ::arrow::MemoryPool* pool, - int64_t buffer_size) - : impl_(new Impl(file_reader, pool, buffer_size)) {} + ::arrow::MemoryPool* pool, int64_t buffer_size) + : impl_(new Impl(file_reader, pool, buffer_size)) {} DeserializerBuilder::~DeserializerBuilder() {} From 8345f9fb54f8f02a50cf6783366234cf59ef5a18 Mon Sep 17 00:00:00 2001 From: Joshua Storck Date: Tue, 15 May 2018 18:34:49 -0400 Subject: [PATCH 4/6] Removing unused code and fixing formatting --- src/parquet/arrow/deserializer.cc | 1 - src/parquet/arrow/deserializer.h | 11 +++-- src/parquet/arrow/reader.cc | 77 ------------------------------- 3 files changed, 6 insertions(+), 83 deletions(-) diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc index 3e4fcedf..d627b8f3 100644 --- a/src/parquet/arrow/deserializer.cc +++ b/src/parquet/arrow/deserializer.cc @@ -57,7 +57,6 @@ static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timest int64_t nanoseconds = *(reinterpret_cast(&(impala_timestamp.value))); return days_since_epoch * kNanosecondsInADay + nanoseconds; } -// End: copied from reader.cc // This is a "tagged" integer type for definition/repetition levels. // The tag gives us strong typing so that we can differentiate between diff --git a/src/parquet/arrow/deserializer.h b/src/parquet/arrow/deserializer.h index 77806a4b..f790a904 100644 --- a/src/parquet/arrow/deserializer.h +++ b/src/parquet/arrow/deserializer.h @@ -26,7 +26,7 @@ namespace arrow { class Array; -class MemoryPool; +class MemoryPool; class Schema; class Table; @@ -40,7 +40,7 @@ class RowGroupReader; namespace arrow { class DeserializerNode; - + // // This header file contains classes that can deserialize from Parquet // files into Arrow arrays and tables. It supports nested structures @@ -157,8 +157,9 @@ class PARQUET_EXPORT TableDeserializer { // group and deserializes each node // \param[in] the row group reader // \return status error if there is a problem deserializing the row group - ::arrow::Status DeserializeRowGroup(std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader); - + ::arrow::Status DeserializeRowGroup( + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader); + private: // Schema, which was most likely obtained from a call to // FromParquetSchema @@ -181,7 +182,7 @@ class PARQUET_EXPORT DeserializerBuilder { // many values will be buffered when reading from primitive // columns. DeserializerBuilder(std::shared_ptr<::parquet::ParquetFileReader> const& file_reader, - ::arrow::MemoryPool* pool, + ::arrow::MemoryPool* pool, // TODO: make this configurable from a reader // properties object int64_t buffer_size = 1024); diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 7e1f15b6..8a6af040 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -71,80 +71,6 @@ using ::arrow::BitUtil::BytesForBits; template using ArrayType = typename ::arrow::TypeTraits::ArrayType; -// ---------------------------------------------------------------------- -// Iteration utilities - -// Abstraction to decouple row group iteration details from the ColumnReader, -// so we can read only a single row group if we want -class FileColumnIterator { - public: - explicit FileColumnIterator(int column_index, ParquetFileReader* reader) - : column_index_(column_index), - reader_(reader), - schema_(reader->metadata()->schema()) {} - - virtual ~FileColumnIterator() {} - - virtual std::unique_ptr<::parquet::PageReader> NextChunk() = 0; - - const SchemaDescriptor* schema() const { return schema_; } - - const ColumnDescriptor* descr() const { return schema_->Column(column_index_); } - - std::shared_ptr metadata() const { return reader_->metadata(); } - - int column_index() const { return column_index_; } - - protected: - int column_index_; - ParquetFileReader* reader_; - const SchemaDescriptor* schema_; -}; - -class AllRowGroupsIterator : public FileColumnIterator { - public: - explicit AllRowGroupsIterator(int column_index, ParquetFileReader* reader) - : FileColumnIterator(column_index, reader), next_row_group_(0) {} - - std::unique_ptr<::parquet::PageReader> NextChunk() override { - std::unique_ptr<::parquet::PageReader> result; - if (next_row_group_ < reader_->metadata()->num_row_groups()) { - result = reader_->RowGroup(next_row_group_)->GetColumnPageReader(column_index_); - next_row_group_++; - } else { - result = nullptr; - } - return result; - } - - private: - int next_row_group_; -}; - -class SingleRowGroupIterator : public FileColumnIterator { - public: - explicit SingleRowGroupIterator(int column_index, int row_group_number, - ParquetFileReader* reader) - : FileColumnIterator(column_index, reader), - row_group_number_(row_group_number), - done_(false) {} - - std::unique_ptr<::parquet::PageReader> NextChunk() override { - if (done_) { - return nullptr; - } - - auto result = - reader_->RowGroup(row_group_number_)->GetColumnPageReader(column_index_); - done_ = true; - return result; - } - - private: - int row_group_number_; - bool done_; -}; - class RowGroupRecordBatchReader : public ::arrow::RecordBatchReader { public: explicit RowGroupRecordBatchReader(const std::vector& row_group_indices, @@ -222,9 +148,6 @@ class FileReader::Impl { Status ReadTable(std::shared_ptr
* table); Status ReadRowGroup(int i, std::shared_ptr
* table); - bool CheckForFlatColumn(const ColumnDescriptor* descr); - bool CheckForFlatListColumn(const ColumnDescriptor* descr); - const ParquetFileReader* parquet_reader() const { return reader_.get(); } int num_row_groups() const { return reader_->metadata()->num_row_groups(); } From 720192f8e3e2230f43dcc8d80503307eb44027e9 Mon Sep 17 00:00:00 2001 From: Joshua Storck Date: Wed, 16 May 2018 11:10:03 -0400 Subject: [PATCH 5/6] Renaming PrimitiveSerializerType to PrimitiveDeserializerType --- src/parquet/arrow/deserializer.cc | 209 +++++++++++++++--------------- 1 file changed, 106 insertions(+), 103 deletions(-) diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc index d627b8f3..a2ac5328 100644 --- a/src/parquet/arrow/deserializer.cc +++ b/src/parquet/arrow/deserializer.cc @@ -372,7 +372,7 @@ class PrimitiveDeserializerNode : public DeserializerNode { // This enum is used to differentiate between the three // different types of primitive deserializers -enum PrimitiveSerializerType { +enum PrimitiveDeserializerType { // This means that the node in the tree has no ancestors (including // itself) that are repeated or optional REQUIRED, @@ -401,7 +401,7 @@ enum PrimitiveSerializerType { // still happen in calls to ReadBatch. // template + PrimitiveDeserializerType DeserializerType> class TypedPrimitiveDeserializerNode; // Specialization for a field that has definition levels but no @@ -423,7 +423,7 @@ class TypedPrimitiveDeserializerNode; // template class TypedPrimitiveDeserializerNode + PrimitiveDeserializerType::OPTIONAL> : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, @@ -508,7 +508,7 @@ class TypedPrimitiveDeserializerNode class TypedPrimitiveDeserializerNode + PrimitiveDeserializerType::REPEATED> : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, @@ -624,7 +624,7 @@ class TypedPrimitiveDeserializerNode class TypedPrimitiveDeserializerNode + PrimitiveDeserializerType::REQUIRED> : public PrimitiveDeserializerNode { public: TypedPrimitiveDeserializerNode(const ColumnDescriptor* const descriptor, @@ -1289,7 +1289,7 @@ class DeserializerBuilder::Impl { // \param[out] the deserializer node // \return error status if any of the converted/physical type // specifications in the Parquet schema are unsupported - template + template Status MakeTypedPrimitiveDeserializerNode( std::shared_ptr const& node, const ColumnDescriptor* const column_descriptor, int column_index, @@ -1297,29 +1297,31 @@ class DeserializerBuilder::Impl { auto primitive_node = std::static_pointer_cast(node); switch (primitive_node->physical_type()) { case Type::BOOLEAN: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::INT32: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::INT64: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::INT96: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::FLOAT: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::DOUBLE: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); case Type::BYTE_ARRAY: return MakeLogicalPrimitiveDeserializerNode( + DeserializerType>( primitive_node, column_descriptor, column_index, result); case Type::FIXED_LEN_BYTE_ARRAY: - return MakeLogicalPrimitiveDeserializerNode( + return MakeLogicalPrimitiveDeserializerNode( primitive_node, column_descriptor, column_index, result); default: std::ostringstream stream; @@ -1339,95 +1341,96 @@ class DeserializerBuilder::Impl { // \param[out] the deserializer node // \return error status if any of the converted/physical type // specifications in the Parquet schema are unsupported - template + template Status MakeLogicalPrimitiveDeserializerNode( std::shared_ptr const& node, const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { switch (node->logical_type()) { case parquet::LogicalType::DATE: - return MakeDateDeserializerNode( + return MakeDateDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::TIME_MILLIS: - return MakeTimeMillisDeserializerNode( + return MakeTimeMillisDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::TIME_MICROS: - return MakeTimeMicrosDeserializerNode( + return MakeTimeMicrosDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::TIMESTAMP_MILLIS: - return MakeTimestampDeserializerNode( + return MakeTimestampDeserializerNode( column_descriptor, column_index, ::arrow::TimeUnit::MILLI, result); case parquet::LogicalType::TIMESTAMP_MICROS: - return MakeTimestampDeserializerNode( + return MakeTimestampDeserializerNode( column_descriptor, column_index, ::arrow::TimeUnit::MICRO, result); case parquet::LogicalType::DECIMAL: - return MakeDecimalDeserializerNode( + return MakeDecimalDeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::UINT_8: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UINT_16: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UINT_32: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UINT_64: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_8: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_16: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_32: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::INT_64: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case parquet::LogicalType::UTF8: - return MakeUTF8DeserializerNode( + return MakeUTF8DeserializerNode( column_descriptor, column_index, result); case parquet::LogicalType::NA: - return MakeNullDeserializerNode( + return MakeNullDeserializerNode( column_descriptor, column_index, result); default: switch (node->physical_type()) { case Type::BOOLEAN: - return MakeBooleanDeserializerNode( + return MakeBooleanDeserializerNode( column_descriptor, column_index, result); case Type::INT32: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case Type::INT64: return MakeIntegerDeserializerNode(column_descriptor, - column_index, result); + DeserializerType>(column_descriptor, + column_index, result); case Type::FLOAT: - return MakeFloatDeserializerNode( + return MakeFloatDeserializerNode( column_descriptor, column_index, result); case Type::DOUBLE: - return MakeDoubleDeserializerNode( + return MakeDoubleDeserializerNode( column_descriptor, column_index, result); case Type::BYTE_ARRAY: - return MakeByteArrayDeserializerNode( + return MakeByteArrayDeserializerNode( column_descriptor, column_index, result); case Type::FIXED_LEN_BYTE_ARRAY: - return MakeFixedLengthByteArrayDeserializerNode( + return MakeFixedLengthByteArrayDeserializerNode( column_descriptor, column_index, result); case Type::INT96: - return MakeImpalaTimestampDeserializerNode( + return MakeImpalaTimestampDeserializerNode( column_descriptor, column_index, result); default: std::ostringstream stream; @@ -1447,16 +1450,16 @@ class DeserializerBuilder::Impl { // Most of the code is boiler-plate, except for the fact that there are // different builders for the various types. - template + template Status MakeNullDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1464,11 +1467,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::TimestampType>(::arrow::TimeUnit::NANO); return MakeTypedPrimitiveNodeWithBuilder( + DeserializerType>( column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeImpalaTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1490,7 +1493,7 @@ class DeserializerBuilder::Impl { std::is_same::value); }; - template + template typename std::enable_if::value, Status>::type MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, @@ -1498,11 +1501,11 @@ class DeserializerBuilder::Impl { auto arrow_type = std::make_shared<::arrow::Decimal128Type>( column_descriptor->type_precision(), column_descriptor->type_scale()); return MakeTypedPrimitiveNodeWithBuilder( + DeserializerType>( column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeDecimalDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, @@ -1514,18 +1517,18 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeBooleanDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1538,17 +1541,17 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result) { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeDateDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1560,7 +1563,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1568,11 +1571,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::Time32Type>(::arrow::TimeUnit::MILLI); return MakeTypedPrimitiveNodeWithBuilder( + DeserializerType>( column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeTimeMillisDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1585,7 +1588,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1593,11 +1596,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::Time64Type>(::arrow::TimeUnit::MICRO); return MakeTypedPrimitiveNodeWithBuilder( + DeserializerType>( column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeTimeMicrosDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1610,7 +1613,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1618,11 +1621,11 @@ class DeserializerBuilder::Impl { std::unique_ptr& result) { auto arrow_type = std::make_shared<::arrow::TimestampType>(time_unit); return MakeTypedPrimitiveNodeWithBuilder( + DeserializerType>( column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeTimestampDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1653,7 +1656,7 @@ class DeserializerBuilder::Impl { }; template + PrimitiveDeserializerType DeserializerType> typename std::enable_if::value, Status>::type MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1661,12 +1664,12 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } template + PrimitiveDeserializerType DeserializerType> typename std::enable_if::value, Status>::type MakeIntegerDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1679,7 +1682,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1687,11 +1690,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeUTF8DeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1703,7 +1706,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1712,11 +1715,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeByteArrayDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1729,7 +1732,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeFixedLengthByteArrayDeserializerNode( @@ -1740,11 +1743,11 @@ class DeserializerBuilder::Impl { auto arrow_type = std::make_shared<::arrow::FixedSizeBinaryType>(column_descriptor->type_length()); return MakeTypedPrimitiveNodeWithBuilder( + DeserializerType>( column_descriptor, column_index, result, arrow_type, pool_); } - template + template typename std::enable_if::value, Status>::type MakeFixedLengthByteArrayDeserializerNode( @@ -1757,7 +1760,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1765,11 +1768,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeDoubleDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1782,7 +1785,7 @@ class DeserializerBuilder::Impl { return Status::Invalid(stream.str()); } - template + template typename std::enable_if::value, Status>::type MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1790,11 +1793,11 @@ class DeserializerBuilder::Impl { { return MakeTypedPrimitiveNodeWithBuilder(column_descriptor, - column_index, result, pool_); + DeserializerType>( + column_descriptor, column_index, result, pool_); } - template + template typename std::enable_if::value, Status>::type MakeFloatDeserializerNode(const ColumnDescriptor* const column_descriptor, @@ -1821,13 +1824,13 @@ class DeserializerBuilder::Impl { // \return error status if any of the converted/physical type // specifications in the Parquet schema are unsupported template + PrimitiveDeserializerType DeserializerType, typename... Args> Status MakeTypedPrimitiveNodeWithBuilder( const ColumnDescriptor* const column_descriptor, int column_index, std::unique_ptr& result, Args&&... args) { auto builder = std::make_shared(std::forward(args)...); using ReturnType = TypedPrimitiveDeserializerNode, - ArrayBuilderType, SerializerType>; + ArrayBuilderType, DeserializerType>; auto node = std::unique_ptr( new ReturnType(column_descriptor, column_index, buffer_size_, builder, pool_)); RETURN_NOT_OK(node->Initialize()); @@ -2060,21 +2063,21 @@ Status DeserializerBuilder::Impl::BuildPrimitiveNode( if (*repetition_levels > 0) { // The path to the node either contains only repeated nodes or // a mix of repeated and optional nodes. - return MakeTypedPrimitiveDeserializerNode( + return MakeTypedPrimitiveDeserializerNode( node, column_descriptor, column_index, result); } else { // A field that is either itself optional or a descendant of // an optional field (e.g. - member of a struct that is optional) // but does not have any ancestors in the tree that are repeated. // It therefore won't have any repetition levels. - return MakeTypedPrimitiveDeserializerNode( + return MakeTypedPrimitiveDeserializerNode( node, column_descriptor, column_index, result); } } else if (*repetition_levels == 0) { // A field that is always present (max_definition_level == // 0 && max_repetition_level == 0) and therefore doesn't // need repetition levels or definition levels - return MakeTypedPrimitiveDeserializerNode( + return MakeTypedPrimitiveDeserializerNode( node, column_descriptor, column_index, result); } else { // This should never happen since a non-zero max repetition level From 50acb98c87055a702314700bf7d2ed0faf1c131d Mon Sep 17 00:00:00 2001 From: Joshua Storck Date: Thu, 5 Jul 2018 08:56:14 -0400 Subject: [PATCH 6/6] Adding ability to deserialize from Parquet to Arrow with multiple threads. Refactored parts of TableDeserializer into a sub-class so that a new ParallelTableDeserializer could be created. Fixed a bug in FileArrayDeserializer that was using an unnecessary and uninitialized member variable. --- src/parquet/arrow/deserializer.cc | 183 +++++++++++++++++++++++------- src/parquet/arrow/deserializer.h | 29 ++--- src/parquet/arrow/reader.cc | 6 +- 3 files changed, 156 insertions(+), 62 deletions(-) diff --git a/src/parquet/arrow/deserializer.cc b/src/parquet/arrow/deserializer.cc index a2ac5328..d8dce5eb 100644 --- a/src/parquet/arrow/deserializer.cc +++ b/src/parquet/arrow/deserializer.cc @@ -27,6 +27,7 @@ #include "arrow/util/bit-util.h" #include "arrow/util/decimal.h" #include "arrow/util/logging.h" +#include "arrow/util/parallel.h" #include "parquet/arrow/schema.h" #include "parquet/file_reader.h" #include "parquet/types.h" @@ -935,7 +936,7 @@ class FileArrayDeserializer : public ArrayDeserializer { for (int ii = 0; ii < num_row_groups; ++ii) { auto row_group_reader = file_reader_->RowGroup(ii); RETURN_NOT_OK(node_->InitializeRowGroup(row_group_reader)); - auto const num_rows = row_group_reader_->metadata()->num_rows(); + auto const num_rows = row_group_reader->metadata()->num_rows(); for (int jj = 0; jj < num_rows; ++jj) { RETURN_NOT_OK(node_->Deserialize(RepetitionLevel(), repetitions)); repetitions.clear(); @@ -946,8 +947,6 @@ class FileArrayDeserializer : public ArrayDeserializer { private: std::shared_ptr file_reader_; - - std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; }; ArrayBatchedDeserializer::~ArrayBatchedDeserializer() {} @@ -1033,47 +1032,112 @@ class ColumnChunkDeserializer : public ArrayDeserializer { std::shared_ptr<::parquet::RowGroupReader> row_group_reader_; }; -TableDeserializer::TableDeserializer( - std::vector>&& nodes, - std::shared_ptr<::arrow::Schema> const& schema) - : nodes_(std::move(nodes)), schema_(schema) {} +TableDeserializer::TableDeserializer(std::shared_ptr<::arrow::Schema> const& schema) + : schema_(schema) {} TableDeserializer::~TableDeserializer() {} -::arrow::Status TableDeserializer::MakeTable(std::shared_ptr<::arrow::Table>& result) { - std::vector> arrays; - for (auto const& node : nodes_) { - std::shared_ptr<::arrow::Array> array; - RETURN_NOT_OK(node->GetArrayBuilder()->Finish(&array)); - arrays.emplace_back(std::move(array)); +// +// A table deserializer that does it's deserialization +// in a single thread. This is in contrast to +// ParallelTableDeserializer which deserializes each +// top level array in the schema in a separate thread. +class SerialTableDeserializer : public TableDeserializer { + protected: + SerialTableDeserializer(std::vector>&& nodes, + std::shared_ptr<::arrow::Schema> const& schema) + : TableDeserializer(schema), nodes_(std::move(nodes)) {} + + // \brief Convenience method which iterates over the nodes in this + // deserializer and constructs a table from them, using + // the schema passed to this object's constructor + // \return error status if there is an issue constructing the table + ::arrow::Status MakeTable(std::shared_ptr<::arrow::Table>& result) { + std::vector> arrays; + for (auto const& node : nodes_) { + std::shared_ptr<::arrow::Array> array; + RETURN_NOT_OK(node->GetArrayBuilder()->Finish(&array)); + arrays.emplace_back(std::move(array)); + } + result = Table::Make(schema_, arrays); + return Status::OK(); } - result = Table::Make(schema_, arrays); - return Status::OK(); -} -Status TableDeserializer::DeserializeRowGroup( - std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) { - for (auto& node : nodes_) { - RETURN_NOT_OK(node->InitializeRowGroup(row_group_reader)); - } - DeserializerNode::LevelSet repetitions; - auto const num_rows = row_group_reader->metadata()->num_rows(); - for (int ii = 0; ii < num_rows; ++ii) { + // \brief Convenience method which iterates over all rows in the rows + // group and deserializes each node + // \param[in] the row group reader + // \return status error if there is a problem deserializing the row group + Status DeserializeRowGroup( + std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) { for (auto& node : nodes_) { - RETURN_NOT_OK(node->Deserialize(RepetitionLevel(), repetitions)); - repetitions.clear(); + RETURN_NOT_OK(node->InitializeRowGroup(row_group_reader)); + } + DeserializerNode::LevelSet repetitions; + auto const num_rows = row_group_reader->metadata()->num_rows(); + for (int ii = 0; ii < num_rows; ++ii) { + for (auto& node : nodes_) { + RETURN_NOT_OK(node->Deserialize(RepetitionLevel(), repetitions)); + repetitions.clear(); + } } + return Status::OK(); } - return Status::OK(); -} + + private: + std::vector> nodes_; +}; + +// +// This is a deserializer that can deserialize columns in +// parallel using multiple threads. It takes a vector of +// ArrayDeserializer's and calls DeserializeArray separately +// in each thread. Each of the ArrayDeserializer's will either +// have a separate row group reader, or will create row group +// readers as it iterates through the file. This ensures that +// there is no sharing of data across the threads. +// +class ParallelTableDeserializer : public TableDeserializer { + public: + ParallelTableDeserializer( + std::shared_ptr<::arrow::Schema> const& schema, + std::vector>&& array_deserializers, + int num_threads) + : TableDeserializer(schema), + num_threads_(num_threads), + array_deserializers_(std::move(array_deserializers)), + arrays_(array_deserializers_.size()) {} + + ~ParallelTableDeserializer() {} + + ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& table) final { + // For now, the parallelism is done a column basis. Other possible + // approaches are deserializing row groups or column chunks in parallel. + RETURN_NOT_OK(::arrow::ParallelFor( + num_threads_, static_cast(array_deserializers_.size()), [this](int index) { + auto const array_index = static_cast(index); + return array_deserializers_[array_index]->DeserializeArray( + arrays_[array_index]); + })); + table = Table::Make(schema_, arrays_); + return Status::OK(); + } + + private: + int num_threads_; + + std::vector> array_deserializers_; + + // Used to hold the output. + std::vector> arrays_; +}; // This class will deserialize all of the specified nodes from a single row group. -class RowGroupDeserializer : public TableDeserializer { +class RowGroupDeserializer : public SerialTableDeserializer { public: RowGroupDeserializer(std::vector>&& nodes, std::shared_ptr<::arrow::Schema> const& schema, std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader) - : TableDeserializer( + : SerialTableDeserializer( std::forward>>(nodes), schema), row_group_reader_(row_group_reader) {} @@ -1089,12 +1153,12 @@ class RowGroupDeserializer : public TableDeserializer { }; // This class wil deserialize all of the specified nodes from a Parquet file -class FileTableDeserializer : public TableDeserializer { +class FileTableDeserializer : public SerialTableDeserializer { public: FileTableDeserializer(std::vector>&& nodes, std::shared_ptr<::arrow::Schema> const& schema, std::shared_ptr<::parquet::ParquetFileReader> const& file_reader) - : TableDeserializer( + : SerialTableDeserializer( std::forward>>(nodes), schema), file_reader_(file_reader) {} @@ -1130,9 +1194,11 @@ class DeserializerBuilder::Impl { ::arrow::Status BuildRowGroupDeserializer(int row_group_index, const std::vector& column_indices, + int num_threads, std::unique_ptr& result); ::arrow::Status BuildFileDeserializer(const std::vector& column_indices, + int num_threads, std::unique_ptr& result); ::arrow::Status BuildArrayBatchedDeserializer( @@ -1938,25 +2004,59 @@ ::arrow::Status DeserializerBuilder::Impl::BuildColumnChunkDeserializer( } ::arrow::Status DeserializerBuilder::Impl::BuildRowGroupDeserializer( - int row_group_index, const std::vector& indices, + int row_group_index, const std::vector& indices, int num_threads, std::unique_ptr& result) { std::vector> nodes; RETURN_NOT_OK(BuildNodes(indices, nodes)); std::shared_ptr<::arrow::Schema> schema; RETURN_NOT_OK(FromParquetSchema(file_reader_->metadata()->schema(), indices, &schema)); - result.reset(new RowGroupDeserializer(std::move(nodes), schema, - file_reader_->RowGroup(row_group_index))); + if (num_threads <= 1) { + result.reset(new RowGroupDeserializer(std::move(nodes), schema, + file_reader_->RowGroup(row_group_index))); + } else { + // When reading in parallel, we construct a + // ColumnChunkDeserializer for each node (aka column). The + // ParallelTableDeserializer simply invokes the array deserializer + // in a separate thread and holds onto its result + std::vector> array_deserializers; + for (auto&& node : nodes) { + // Note: the call to RowGroup should be done for each node, + // so it has its own RowGroupReader. Otherwise, it would be + // shared across threads and would induce a race. + std::unique_ptr current(new ColumnChunkDeserializer( + std::move(node), file_reader_->RowGroup(row_group_index))); + RETURN_NOT_OK(current->Initialize()); + array_deserializers.push_back(std::move(current)); + } + result.reset(new ParallelTableDeserializer(schema, std::move(array_deserializers), + num_threads)); + } return Status::OK(); } ::arrow::Status DeserializerBuilder::Impl::BuildFileDeserializer( - const std::vector& column_indices, std::unique_ptr& result) { + const std::vector& column_indices, int num_threads, + std::unique_ptr& result) { std::vector> nodes; RETURN_NOT_OK(BuildNodes(column_indices, nodes)); std::shared_ptr<::arrow::Schema> schema; RETURN_NOT_OK( FromParquetSchema(file_reader_->metadata()->schema(), column_indices, &schema)); - result.reset(new FileTableDeserializer(std::move(nodes), schema, file_reader_)); + if (num_threads <= 1) { + result.reset(new FileTableDeserializer(std::move(nodes), schema, file_reader_)); + } else { + // When reading in parallel, we construct a FileArrayDeserializer + // for each node (aka column). The ParallelTableDeserializer + // simply invokes the array deserializer in a separate thread and + // holds onto its result + std::vector> array_deserializers; + for (auto&& node : nodes) { + array_deserializers.emplace_back( + new FileArrayDeserializer(std::move(node), file_reader_)); + } + result.reset(new ParallelTableDeserializer(schema, std::move(array_deserializers), + num_threads)); + } return Status::OK(); } @@ -2263,14 +2363,15 @@ ::arrow::Status DeserializerBuilder::BuildColumnChunkDeserializer( } ::arrow::Status DeserializerBuilder::BuildRowGroupDeserializer( - int row_group_index, const std::vector& indices, + int row_group_index, const std::vector& indices, int num_threads, std::unique_ptr& result) { - return impl_->BuildRowGroupDeserializer(row_group_index, indices, result); + return impl_->BuildRowGroupDeserializer(row_group_index, indices, num_threads, result); } ::arrow::Status DeserializerBuilder::BuildFileDeserializer( - const std::vector& column_indices, std::unique_ptr& result) { - return impl_->BuildFileDeserializer(column_indices, result); + const std::vector& column_indices, int num_threads, + std::unique_ptr& result) { + return impl_->BuildFileDeserializer(column_indices, num_threads, result); } ::arrow::Status DeserializerBuilder::BuildArrayBatchedDeserializer( diff --git a/src/parquet/arrow/deserializer.h b/src/parquet/arrow/deserializer.h index f790a904..0f2c3711 100644 --- a/src/parquet/arrow/deserializer.h +++ b/src/parquet/arrow/deserializer.h @@ -142,25 +142,8 @@ class PARQUET_EXPORT TableDeserializer { virtual ::arrow::Status DeserializeTable(std::shared_ptr<::arrow::Table>& result) = 0; protected: - TableDeserializer(std::vector>&& nodes, - std::shared_ptr<::arrow::Schema> const& schema); + TableDeserializer(std::shared_ptr<::arrow::Schema> const& schema); - // \brief Convenience method which iterates over the nodes in this - // deserializer and constructs a table from them, using - // the schema passed to this object's constructor - // \return error status if there is an issue constructing the table - ::arrow::Status MakeTable(std::shared_ptr<::arrow::Table>& result); - - std::vector> nodes_; - - // \brief Convenience method which iterates over all rows in the rows - // group and deserializes each node - // \param[in] the row group reader - // \return status error if there is a problem deserializing the row group - ::arrow::Status DeserializeRowGroup( - std::shared_ptr<::parquet::RowGroupReader> const& row_group_reader); - - private: // Schema, which was most likely obtained from a call to // FromParquetSchema std::shared_ptr<::arrow::Schema> schema_; @@ -256,6 +239,12 @@ class PARQUET_EXPORT DeserializerBuilder { // \brief builds a table deserializer that will deserialize all of the // nodes in the Parquet schema for a particular row group. // \param[in] the row group to deserialize + // \param[in] the column indices to use for filtering the Parquet schema. + // For example, if column_index 2 is specified for the example schema + // at the top of this file, it will include Code and all of its + // ancestors in the resulting table. + // \param[in] the number of threads to use when deserializing. This + // will cause the top level columns to be read in parallel // \param[out] the deserializer // \return error status if there is a problem building the appropriate // deserializers for various column types or if the row_group_index @@ -263,6 +252,7 @@ class PARQUET_EXPORT DeserializerBuilder { // ::arrow::Status BuildRowGroupDeserializer(int row_group_index, const std::vector& indices, + int num_threads, std::unique_ptr& result); // \brief build a table deserializer that will only deserialize nodes that @@ -271,11 +261,14 @@ class PARQUET_EXPORT DeserializerBuilder { // For example, if column_index 2 is specified for the example schema // at the top of this file, it will include Code and all of its // ancestors in the resulting table. + // \param[in] the number of threads to use when deserializing. This + // will cause the top level columns to be read in parallel // \param[out] the deserializer // \return error status if there is a problem building the appropriate // deserializers for various column types or if all of the // column indices are out of bounds. ::arrow::Status BuildFileDeserializer(const std::vector& column_indices, + int num_threads, std::unique_ptr& result); private: diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 8a6af040..161eb99e 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -216,8 +216,8 @@ Status FileReader::Impl::ReadRowGroup(int row_group_index, std::shared_ptr<::arrow::Table>* out) { DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; - RETURN_NOT_OK( - builder.BuildRowGroupDeserializer(row_group_index, indices, deserializer)); + RETURN_NOT_OK(builder.BuildRowGroupDeserializer(row_group_index, indices, num_threads_, + deserializer)); deserializer->DeserializeTable(*out); return Status::OK(); } @@ -226,7 +226,7 @@ Status FileReader::Impl::ReadTable(const std::vector& indices, std::shared_ptr
* out) { DeserializerBuilder builder(reader_, pool_); std::unique_ptr deserializer; - RETURN_NOT_OK(builder.BuildFileDeserializer(indices, deserializer)); + RETURN_NOT_OK(builder.BuildFileDeserializer(indices, num_threads_, deserializer)); deserializer->DeserializeTable(*out); return Status::OK(); }