From 43068032c78d8d1a61054c4b8e34543cc156eda3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 6 Nov 2019 10:07:47 -0600 Subject: [PATCH 1/4] Add "split blocks" and "self destruct" options to Table.to_pandas, with zero-copy operations for improved memory use when converting from Arrow to pandas --- cpp/src/arrow/python/arrow_to_pandas.cc | 2323 +++++++++++------------ cpp/src/arrow/python/arrow_to_pandas.h | 60 +- cpp/src/arrow/python/type_traits.h | 29 +- cpp/src/arrow/table.cc | 16 + cpp/src/arrow/table.h | 10 +- cpp/src/arrow/table_test.cc | 23 + cpp/src/arrow/type_traits.h | 13 + python/pyarrow/array.pxi | 108 +- python/pyarrow/includes/libarrow.pxd | 18 +- python/pyarrow/pandas_compat.py | 6 +- python/pyarrow/table.pxi | 60 +- python/pyarrow/tests/test_pandas.py | 60 +- 12 files changed, 1343 insertions(+), 1383 deletions(-) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 715de777a429..c64c1189f9fb 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -23,9 +23,12 @@ #include #include +#include #include +#include #include #include +#include #include #include "arrow/array.h" @@ -62,16 +65,36 @@ class MemoryPool; using internal::checked_cast; using internal::ParallelFor; -namespace py { +// ---------------------------------------------------------------------- +// PyCapsule code for setting ndarray base to reference C++ object + +struct ArrayCapsule { + std::shared_ptr array; +}; + +struct BufferCapsule { + std::shared_ptr buffer; +}; + +namespace { + +void ArrayCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Array")); +} -using internal::kNanosecondsInDay; -using internal::kPandasTimestampNull; +void BufferCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Buffer")); +} -using compute::Datum; -using compute::FunctionContext; +} // namespace // ---------------------------------------------------------------------- -// Utility code +// pandas 0.x DataFrame conversion internals + +namespace py { + +using internal::arrow_traits; +using internal::npy_traits; template struct WrapBytes {}; @@ -145,28 +168,6 @@ static inline bool ListTypeSupported(const DataType& type) { } return false; } -// ---------------------------------------------------------------------- -// PyCapsule code for setting ndarray base to reference C++ object - -struct ArrayCapsule { - std::shared_ptr array; -}; - -struct BufferCapsule { - std::shared_ptr buffer; -}; - -namespace { - -void ArrayCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Array")); -} - -void BufferCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Buffer")); -} - -} // namespace Status CapsulizeArray(const std::shared_ptr& arr, PyObject** out) { auto capsule = new ArrayCapsule{{arr}}; @@ -205,68 +206,28 @@ Status SetBufferBase(PyArrayObject* arr, const std::shared_ptr& buffer) return SetNdarrayBase(arr, base); } -// ---------------------------------------------------------------------- -// pandas 0.x DataFrame conversion internals - inline void set_numpy_metadata(int type, const DataType* datatype, PyArray_Descr* out) { + auto metadata = reinterpret_cast(out->c_metadata); if (type == NPY_DATETIME) { - auto date_dtype = reinterpret_cast(out->c_metadata); if (datatype->id() == Type::TIMESTAMP) { const auto& timestamp_type = checked_cast(*datatype); - - switch (timestamp_type.unit()) { - case TimestampType::Unit::SECOND: - date_dtype->meta.base = NPY_FR_s; - break; - case TimestampType::Unit::MILLI: - date_dtype->meta.base = NPY_FR_ms; - break; - case TimestampType::Unit::MICRO: - date_dtype->meta.base = NPY_FR_us; - break; - case TimestampType::Unit::NANO: - date_dtype->meta.base = NPY_FR_ns; - break; - } + metadata->meta.base = internal::NumPyFrequency(timestamp_type.unit()); } else { - // datatype->type == Type::DATE64 - date_dtype->meta.base = NPY_FR_D; + DCHECK(false) << "NPY_DATETIME views only supported for Arrow TIMESTAMP types"; } } else if (type == NPY_TIMEDELTA) { DCHECK_EQ(datatype->id(), Type::DURATION); - auto timedelta_dtype = - reinterpret_cast(out->c_metadata); const auto& duration_type = checked_cast(*datatype); - - switch (duration_type.unit()) { - case DurationType::Unit::SECOND: - timedelta_dtype->meta.base = NPY_FR_s; - break; - case DurationType::Unit::MILLI: - timedelta_dtype->meta.base = NPY_FR_ms; - break; - case DurationType::Unit::MICRO: - timedelta_dtype->meta.base = NPY_FR_us; - break; - case DurationType::Unit::NANO: - timedelta_dtype->meta.base = NPY_FR_ns; - break; - } + metadata->meta.base = internal::NumPyFrequency(duration_type.unit()); } } -namespace { - -Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, - const DataType* arrow_type, MemoryPool* pool, PyObject** out) { +Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, MemoryPool* pool, + PyObject** out) { // ARROW-6570: Allocate memory from MemoryPool for a couple reasons // // * Track allocations // * Get better performance through custom allocators - if (arrow_type != nullptr) { - set_numpy_metadata(descr->type_num, arrow_type, descr); - } - int64_t total_size = descr->elsize; for (int i = 0; i < nd; ++i) { total_size *= dims[i]; @@ -286,9 +247,49 @@ Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, return SetBufferBase(reinterpret_cast(*out), buffer); } -} // namespace +template +inline const T* GetPrimitiveValues(const Array& arr) { + if (arr.length() == 0) { + return nullptr; + } + int elsize = checked_cast(*arr.type()).bit_width() / 8; + const auto& prim_arr = checked_cast(arr); + return reinterpret_cast(prim_arr.values()->data() + arr.offset() * elsize); +} + +Status MakeNumPyView(std::shared_ptr arr, PyObject* py_ref, int npy_type, int ndim, + npy_intp* dims, PyObject** out) { + PyAcquireGIL lock; + + PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); + set_numpy_metadata(npy_type, arr->type().get(), descr); + PyObject* result = PyArray_NewFromDescr( + &PyArray_Type, descr, ndim, dims, /*strides=*/nullptr, + const_cast(GetPrimitiveValues(*arr)), /*flags=*/0, nullptr); + PyArrayObject* np_arr = reinterpret_cast(result); + if (np_arr == nullptr) { + // Error occurred, trust that error set + return Status::OK(); + } + + PyObject* base; + if (py_ref == nullptr) { + // Capsule will be owned by the ndarray, no incref necessary. See + // ARROW-1973 + RETURN_NOT_OK(CapsulizeArray(arr, &base)); + } else { + Py_INCREF(py_ref); + base = py_ref; + } + RETURN_NOT_OK(SetNdarrayBase(np_arr, base)); + + // Do not allow Arrow data to be mutated + PyArray_CLEARFLAGS(np_arr, NPY_ARRAY_WRITEABLE); + *out = result; + return Status::OK(); +} -class PandasBlock { +class PandasWriter { public: enum type { OBJECT, @@ -304,36 +305,143 @@ class PandasBlock { FLOAT, DOUBLE, BOOL, - DATETIME, - DATETIME_WITH_TZ, - TIMEDELTA, + DATETIME_DAY, + DATETIME_SECOND, + DATETIME_MILLI, + DATETIME_MICRO, + DATETIME_NANO, + DATETIME_NANO_TZ, + TIMEDELTA_SECOND, + TIMEDELTA_MILLI, + TIMEDELTA_MICRO, + TIMEDELTA_NANO, CATEGORICAL, EXTENSION }; - PandasBlock(const PandasOptions& options, int64_t num_rows, int num_columns) - : num_rows_(num_rows), num_columns_(num_columns), options_(options) {} - virtual ~PandasBlock() {} + PandasWriter(const PandasOptions& options, int64_t num_rows, int num_columns) + : options_(options), num_rows_(num_rows), num_columns_(num_columns) {} + virtual ~PandasWriter() {} - virtual Status Allocate() = 0; - virtual Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) = 0; + void SetBlockData(PyObject* arr) { + block_arr_.reset(arr); + block_data_ = + reinterpret_cast(PyArray_DATA(reinterpret_cast(arr))); + } + + /// \brief Either copy or wrap single array to create pandas-compatible array + /// for Series or DataFrame. num_columns_ can only be 1. Will try to zero + /// copy if possible (or error if not possible and zero_copy_only=True) + virtual Status TransferSingle(std::shared_ptr data, PyObject* py_ref) = 0; + + /// \brief Copy ChunkedArray into a multi-column block + virtual Status CopyInto(std::shared_ptr data, int64_t rel_placement) = 0; + + Status EnsurePlacementAllocated() { + std::lock_guard guard(allocation_lock_); + if (placement_data_ != nullptr) { + return Status::OK(); + } + PyAcquireGIL lock; + + npy_intp placement_dims[1] = {num_columns_}; + PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); + RETURN_IF_PYERROR(); + placement_arr_.reset(placement_arr); + placement_data_ = reinterpret_cast( + PyArray_DATA(reinterpret_cast(placement_arr))); + return Status::OK(); + } + + Status EnsureAllocated() { + std::lock_guard guard(allocation_lock_); + if (block_data_ != nullptr) { + return Status::OK(); + } + RETURN_NOT_OK(Allocate()); + return Status::OK(); + } - PyObject* block_arr() const { return block_arr_.obj(); } + virtual bool CanZeroCopy(const ChunkedArray& data) const { return false; } + + virtual Status Write(std::shared_ptr data, int64_t abs_placement, + int64_t rel_placement) { + RETURN_NOT_OK(EnsurePlacementAllocated()); + if (num_columns_ == 1) { + RETURN_NOT_OK(TransferSingle(data, /*py_ref=*/nullptr)); + } else { + RETURN_NOT_OK( + CheckNoZeroCopy("Cannot do zero copy conversion into " + "multi-column DataFrame block")); + RETURN_NOT_OK(EnsureAllocated()); + RETURN_NOT_OK(CopyInto(data, rel_placement)); + } + placement_data_[rel_placement] = abs_placement; + return Status::OK(); + } - virtual Status GetPyResult(PyObject** output) { + virtual Status GetDataFrameResult(PyObject** out) { PyObject* result = PyDict_New(); RETURN_IF_PYERROR(); - PyDict_SetItemString(result, "block", block_arr_.obj()); - PyDict_SetItemString(result, "placement", placement_arr_.obj()); + PyObject* block; + RETURN_NOT_OK(GetResultBlock(&block)); - *output = result; + PyDict_SetItemString(result, "block", block); + PyDict_SetItemString(result, "placement", placement_arr_.obj()); + RETURN_NOT_OK(AddResultMetadata(result)); + *out = result; return Status::OK(); } + virtual Status GetSeriesResult(PyObject** out) { return GetBlock1D(out); } + protected: + virtual Status AddResultMetadata(PyObject* result) { return Status::OK(); } + + Status GetBlock1D(PyObject** out) { + // For Series or for certain DataFrame block types, we need to shape to a + // 1D array when there is only one column + PyAcquireGIL lock; + + DCHECK_EQ(1, num_columns_); + + npy_intp new_dims[1] = {num_rows_}; + PyArray_Dims dims; + dims.ptr = new_dims; + dims.len = 1; + + *out = PyArray_Newshape(reinterpret_cast(block_arr_.obj()), &dims, + NPY_ANYORDER); + RETURN_IF_PYERROR(); + return Status::OK(); + } + + virtual Status GetResultBlock(PyObject** out) { + *out = block_arr_.obj(); + return Status::OK(); + } + + Status CheckNoZeroCopy(const std::string& message) { + if (options_.zero_copy_only) { + return Status::Invalid(message); + } + return Status::OK(); + } + + Status CheckNotZeroCopyOnly(const ChunkedArray& data) { + if (options_.zero_copy_only) { + return Status::Invalid("Needed to copy ", data.num_chunks(), " chunks with ", + data.null_count(), " nulls, but zero_copy_only was True"); + } + return Status::OK(); + } + + virtual Status Allocate() { + return Status::NotImplemented("Override Allocate in subclasses"); + } + Status AllocateNDArray(int npy_type, int ndim = 2) { PyAcquireGIL lock; @@ -353,60 +461,45 @@ class PandasBlock { block_arr = PyArray_SimpleNewFromDescr(ndim, block_dims, descr); RETURN_IF_PYERROR(); } else { - RETURN_NOT_OK(PyArray_NewFromPool(ndim, block_dims, descr, - /*arrow_type=*/nullptr, options_.pool, - &block_arr)); + RETURN_NOT_OK( + PyArray_NewFromPool(ndim, block_dims, descr, options_.pool, &block_arr)); } - npy_intp placement_dims[1] = {num_columns_}; - PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - - RETURN_IF_PYERROR(); - - block_arr_.reset(block_arr); - placement_arr_.reset(placement_arr); + SetBlockData(block_arr); + return Status::OK(); + } - block_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(block_arr))); + void SetDatetimeUnit(NPY_DATETIMEUNIT unit) { + PyAcquireGIL lock; + auto date_dtype = reinterpret_cast( + PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); + date_dtype->meta.base = unit; + } - placement_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(placement_arr))); + PandasOptions options_; - return Status::OK(); - } + std::mutex allocation_lock_; int64_t num_rows_; int num_columns_; OwnedRefNoGIL block_arr_; - uint8_t* block_data_; - - PandasOptions options_; + uint8_t* block_data_ = nullptr; // ndarray OwnedRefNoGIL placement_arr_; - int64_t* placement_data_; + int64_t* placement_data_ = nullptr; private: - ARROW_DISALLOW_COPY_AND_ASSIGN(PandasBlock); + ARROW_DISALLOW_COPY_AND_ASSIGN(PandasWriter); }; -template -inline const T* GetPrimitiveValues(const Array& arr) { - if (arr.length() == 0) { - return nullptr; - } - const auto& prim_arr = checked_cast(arr); - const T* raw_values = reinterpret_cast(prim_arr.values()->data()); - return raw_values + arr.offset(); -} - -template +template inline void ConvertIntegerWithNulls(const PandasOptions& options, - const ChunkedArray& data, double* out_values) { + const ChunkedArray& data, OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = *data.chunk(c); - const T* in_values = GetPrimitiveValues(arr); + const InType* in_values = GetPrimitiveValues(arr); // Upcast to double, set NaN as appropriate for (int i = 0; i < arr.length(); ++i) { @@ -440,40 +533,6 @@ inline void ConvertIntegerNoNullsCast(const PandasOptions& options, } } -static Status ConvertBooleanWithNulls(const PandasOptions& options, - const ChunkedArray& data, PyObject** out_values) { - PyAcquireGIL lock; - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); - - for (int64_t i = 0; i < arr.length(); ++i) { - if (arr.IsNull(i)) { - Py_INCREF(Py_None); - *out_values++ = Py_None; - } else if (arr.Value(i)) { - // True - Py_INCREF(Py_True); - *out_values++ = Py_True; - } else { - // False - Py_INCREF(Py_False); - *out_values++ = Py_False; - } - } - } - return Status::OK(); -} - -static void ConvertBooleanNoNulls(const PandasOptions& options, const ChunkedArray& data, - uint8_t* out_values) { - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = static_cast(arr.Value(i)); - } - } -} - // Generic Array -> PyObject** converter that handles object deduplication, if // requested template @@ -510,7 +569,6 @@ inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArra using ArrayType = typename TypeTraits::ArrayType; using Scalar = typename MemoizationTraits::Scalar; - PyAcquireGIL lock; // TODO(fsaintjacques): propagate memory pool. ::arrow::internal::ScalarMemoTable memo_table(default_memory_pool()); std::vector unique_values; @@ -547,52 +605,8 @@ inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArra return Status::OK(); } -template -static Status ConvertIntegerObjects(const PandasOptions& options, - const ChunkedArray& data, PyObject** out_values) { - using T = typename Type::c_type; - auto WrapValue = [](T value, PyObject** out) { - *out = std::is_signed::value ? PyLong_FromLongLong(value) - : PyLong_FromUnsignedLongLong(value); - RETURN_IF_PYERROR(); - return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} - -template -inline Status ConvertBinaryLike(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - auto WrapValue = [](const util::string_view& view, PyObject** out) { - *out = WrapBytes::Wrap(view.data(), view.length()); - if (*out == nullptr) { - PyErr_Clear(); - return Status::UnknownError("Wrapping ", view, " failed"); - } - return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} - -inline Status ConvertNulls(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - PyAcquireGIL lock; - for (int c = 0; c < data.num_chunks(); c++) { - std::shared_ptr arr = data.chunk(c); - - for (int64_t i = 0; i < arr->length(); ++i) { - // All values are null - Py_INCREF(Py_None); - *out_values = Py_None; - ++out_values; - } - } - return Status::OK(); -} - inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& data, PyObject** out_values) { - PyAcquireGIL lock; if (data.num_chunks() == 0) { return Status::OK(); } @@ -665,16 +679,22 @@ inline Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& auto value_type = checked_cast(*data.type()).value_type(); auto flat_column = std::make_shared(value_arrays, value_type); // TODO(ARROW-489): Currently we don't have a Python reference for single columns. - // Storing a reference to the whole Array would be to expensive. + // Storing a reference to the whole Array would be too expensive. + + // ARROW-3789(wesm): During refactoring I found that unit tests assumed that + // timestamp units would be preserved on list conversions in + // Table.to_pandas. So we set the option here to not coerce things to + // nanoseconds. Bit of a hack but this seemed the simplest thing to satisfy + // the existing unit tests + PandasOptions modified_options = options; + modified_options.coerce_temporal_nanoseconds = false; OwnedRefNoGIL owned_numpy_array; - RETURN_NOT_OK(ConvertChunkedArrayToPandas(options, flat_column, nullptr, + RETURN_NOT_OK(ConvertChunkedArrayToPandas(modified_options, flat_column, nullptr, owned_numpy_array.ref())); PyObject* numpy_array = owned_numpy_array.obj(); - PyAcquireGIL lock; - int64_t chunk_offset = 0; for (int c = 0; c < data.num_chunks(); c++) { auto arr = std::static_pointer_cast(data.chunk(c)); @@ -710,300 +730,300 @@ inline Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& return Status::OK(); } -template -inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) { +template +inline void ConvertNumericNullable(const ChunkedArray& data, InType na_value, + OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = *data.chunk(c); - const T* in_values = GetPrimitiveValues(arr); + const InType* in_values = GetPrimitiveValues(arr); if (arr.null_count() > 0) { for (int64_t i = 0; i < arr.length(); ++i) { *out_values++ = arr.IsNull(i) ? na_value : in_values[i]; } } else { - memcpy(out_values, in_values, sizeof(T) * arr.length()); + memcpy(out_values, in_values, sizeof(InType) * arr.length()); out_values += arr.length(); } } } template -inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value, +inline void ConvertNumericNullableCast(const ChunkedArray& data, InType na_value, OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = *data.chunk(c); const InType* in_values = GetPrimitiveValues(arr); for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]); + *out_values++ = arr.IsNull(i) ? static_cast(na_value) + : static_cast(in_values[i]); } } } -template -inline void ConvertDatetimeLikeNanos(const ChunkedArray& data, int64_t* out_values) { - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = *data.chunk(c); - const T* in_values = GetPrimitiveValues(arr); +template +class TypedPandasWriter : public PandasWriter { + public: + using T = typename npy_traits::value_type; - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? kPandasTimestampNull - : (static_cast(in_values[i]) * SHIFT); + using PandasWriter::PandasWriter; + + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + if (CanZeroCopy(*data)) { + PyObject* wrapped; + npy_intp dims[2] = {num_columns_, num_rows_}; + RETURN_NOT_OK( + MakeNumPyView(data->chunk(0), py_ref, NPY_TYPE, /*ndim=*/2, dims, &wrapped)); + SetBlockData(wrapped); + return Status::OK(); + } else { + RETURN_NOT_OK(CheckNotZeroCopyOnly(*data)); + RETURN_NOT_OK(EnsureAllocated()); + return CopyInto(data, /*rel_placement=*/0); } } -} -template -static Status ConvertDates(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - auto WrapValue = [](typename Type::c_type value, PyObject** out) { - RETURN_NOT_OK(internal::PyDate_from_int(value, Type::UNIT, out)); - RETURN_IF_PYERROR(); + Status CheckTypeExact(const DataType& type, Type::type expected) { + if (type.id() != expected) { + // TODO(wesm): stringify NumPy / pandas type + return Status::NotImplemented("Cannot write Arrow data of type ", type.ToString()); + } return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} + } -template -static Status ConvertTimes(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - const TimeUnit::type unit = checked_cast(*data.type()).unit(); + T* GetBlockColumnStart(int64_t rel_placement) { + return reinterpret_cast(block_data_) + rel_placement * num_rows_; + } - auto WrapValue = [unit](typename Type::c_type value, PyObject** out) { - RETURN_NOT_OK(internal::PyTime_from_int(value, unit, out)); - RETURN_IF_PYERROR(); - return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} + protected: + Status Allocate() override { return AllocateNDArray(NPY_TYPE); } +}; -static Status ConvertDecimals(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - PyAcquireGIL lock; - OwnedRef decimal; - OwnedRef Decimal; - RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); - RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal)); - PyObject* decimal_constructor = Decimal.obj(); +struct ObjectWriterVisitor { + const PandasOptions& options; + const ChunkedArray& data; + PyObject** out_values; - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); + Status Visit(const NullType& type) { + for (int c = 0; c < data.num_chunks(); c++) { + std::shared_ptr arr = data.chunk(c); - for (int64_t i = 0; i < arr.length(); ++i) { - if (arr.IsNull(i)) { + for (int64_t i = 0; i < arr->length(); ++i) { + // All values are null Py_INCREF(Py_None); - *out_values++ = Py_None; - } else { - *out_values++ = - internal::DecimalFromString(decimal_constructor, arr.FormatValue(i)); - RETURN_IF_PYERROR(); + *out_values = Py_None; + ++out_values; } } + return Status::OK(); } - return Status::OK(); -} - -#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum) \ - case Type::ArrowEnum: \ - RETURN_NOT_OK((ConvertListsLike(options_, *data, out_buffer))); \ - break; - -class ObjectBlock : public PandasBlock { - public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_OBJECT); } - - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + Status Visit(const BooleanType& type) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); - PyObject** out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; - - if (type == Type::BOOL) { - RETURN_NOT_OK(ConvertBooleanWithNulls(options_, *data, out_buffer)); - } else if (type == Type::UINT8) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT8) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::UINT16) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT16) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::UINT32) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT32) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::UINT64) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT64) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::BINARY) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::LARGE_BINARY) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::STRING) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::LARGE_STRING) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::FIXED_SIZE_BINARY) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::DATE32) { - RETURN_NOT_OK(ConvertDates(options_, *data, out_buffer)); - } else if (type == Type::DATE64) { - RETURN_NOT_OK(ConvertDates(options_, *data, out_buffer)); - } else if (type == Type::TIME32) { - RETURN_NOT_OK(ConvertTimes(options_, *data, out_buffer)); - } else if (type == Type::TIME64) { - RETURN_NOT_OK(ConvertTimes(options_, *data, out_buffer)); - } else if (type == Type::DECIMAL) { - RETURN_NOT_OK(ConvertDecimals(options_, *data, out_buffer)); - } else if (type == Type::NA) { - RETURN_NOT_OK(ConvertNulls(options_, *data, out_buffer)); - } else if (type == Type::LIST) { - auto list_type = std::static_pointer_cast(data->type()); - switch (list_type->value_type()->id()) { - CONVERTLISTSLIKE_CASE(BooleanType, BOOL) - CONVERTLISTSLIKE_CASE(UInt8Type, UINT8) - CONVERTLISTSLIKE_CASE(Int8Type, INT8) - CONVERTLISTSLIKE_CASE(UInt16Type, UINT16) - CONVERTLISTSLIKE_CASE(Int16Type, INT16) - CONVERTLISTSLIKE_CASE(UInt32Type, UINT32) - CONVERTLISTSLIKE_CASE(Int32Type, INT32) - CONVERTLISTSLIKE_CASE(UInt64Type, UINT64) - CONVERTLISTSLIKE_CASE(Int64Type, INT64) - CONVERTLISTSLIKE_CASE(Date32Type, DATE32) - CONVERTLISTSLIKE_CASE(Date64Type, DATE64) - CONVERTLISTSLIKE_CASE(Time32Type, TIME32) - CONVERTLISTSLIKE_CASE(Time64Type, TIME64) - CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP) - CONVERTLISTSLIKE_CASE(DurationType, DURATION) - CONVERTLISTSLIKE_CASE(FloatType, FLOAT) - CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE) - CONVERTLISTSLIKE_CASE(DecimalType, DECIMAL) - CONVERTLISTSLIKE_CASE(BinaryType, BINARY) - CONVERTLISTSLIKE_CASE(StringType, STRING) - CONVERTLISTSLIKE_CASE(ListType, LIST) - CONVERTLISTSLIKE_CASE(NullType, NA) - default: { - return Status::NotImplemented( - "Not implemented type for conversion from List to Pandas ObjectBlock: ", - list_type->value_type()->ToString()); + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values++ = Py_None; + } else if (arr.Value(i)) { + // True + Py_INCREF(Py_True); + *out_values++ = Py_True; + } else { + // False + Py_INCREF(Py_False); + *out_values++ = Py_False; } } - } else if (type == Type::STRUCT) { - RETURN_NOT_OK(ConvertStruct(options_, *data, out_buffer)); - } else { - return Status::NotImplemented("Unsupported type for object array output: ", - data->type()->ToString()); } - - placement_data_[rel_placement] = abs_placement; return Status::OK(); } -}; -template -class IntBlock : public PandasBlock { - public: - using PandasBlock::PandasBlock; - Status Allocate() override { - return AllocateNDArray(internal::arrow_traits::npy_type); + template + enable_if_integer Visit(const Type& type) { + using T = typename Type::c_type; + auto WrapValue = [](T value, PyObject** out) { + *out = std::is_signed::value ? PyLong_FromLongLong(value) + : PyLong_FromUnsignedLongLong(value); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); } - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); - - C_TYPE* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + template + enable_if_t::value || is_fixed_size_binary_type::value, + Status> + Visit(const Type& type) { + auto WrapValue = [](const util::string_view& view, PyObject** out) { + *out = WrapBytes::Wrap(view.data(), view.length()); + if (*out == nullptr) { + PyErr_Clear(); + return Status::UnknownError("Wrapping ", view, " failed"); + } + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } - if (type != ARROW_TYPE) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), " to a Pandas int", - sizeof(C_TYPE), " block"); - } + template + enable_if_date Visit(const Type& type) { + auto WrapValue = [](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyDate_from_int(value, Type::UNIT, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } - ConvertIntegerNoNullsSameType(options_, *data, out_buffer); - placement_data_[rel_placement] = abs_placement; - return Status::OK(); + template + enable_if_time Visit(const Type& type) { + const TimeUnit::type unit = type.unit(); + auto WrapValue = [unit](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyTime_from_int(value, unit, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); } -}; -using UInt8Block = IntBlock; -using Int8Block = IntBlock; -using UInt16Block = IntBlock; -using Int16Block = IntBlock; -using UInt32Block = IntBlock; -using Int32Block = IntBlock; -using UInt64Block = IntBlock; -using Int64Block = IntBlock; + Status Visit(const Decimal128Type& type) { + OwnedRef decimal; + OwnedRef Decimal; + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); + RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal)); + PyObject* decimal_constructor = Decimal.obj(); -class Float16Block : public PandasBlock { - public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_FLOAT16); } + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values++ = Py_None; + } else { + *out_values++ = + internal::DecimalFromString(decimal_constructor, arr.FormatValue(i)); + RETURN_IF_PYERROR(); + } + } + } - if (type != Type::HALF_FLOAT) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), - " to a Pandas float16 block"); + return Status::OK(); + } + + Status Visit(const ListType& type) { +#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum) \ + case Type::ArrowEnum: \ + return ConvertListsLike(options, data, out_values); + + switch (type.value_type()->id()) { + CONVERTLISTSLIKE_CASE(BooleanType, BOOL) + CONVERTLISTSLIKE_CASE(UInt8Type, UINT8) + CONVERTLISTSLIKE_CASE(Int8Type, INT8) + CONVERTLISTSLIKE_CASE(UInt16Type, UINT16) + CONVERTLISTSLIKE_CASE(Int16Type, INT16) + CONVERTLISTSLIKE_CASE(UInt32Type, UINT32) + CONVERTLISTSLIKE_CASE(Int32Type, INT32) + CONVERTLISTSLIKE_CASE(UInt64Type, UINT64) + CONVERTLISTSLIKE_CASE(Int64Type, INT64) + CONVERTLISTSLIKE_CASE(Date32Type, DATE32) + CONVERTLISTSLIKE_CASE(Date64Type, DATE64) + CONVERTLISTSLIKE_CASE(Time32Type, TIME32) + CONVERTLISTSLIKE_CASE(Time64Type, TIME64) + CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP) + CONVERTLISTSLIKE_CASE(DurationType, DURATION) + CONVERTLISTSLIKE_CASE(FloatType, FLOAT) + CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE) + CONVERTLISTSLIKE_CASE(DecimalType, DECIMAL) + CONVERTLISTSLIKE_CASE(BinaryType, BINARY) + CONVERTLISTSLIKE_CASE(StringType, STRING) + CONVERTLISTSLIKE_CASE(ListType, LIST) + CONVERTLISTSLIKE_CASE(NullType, NA) + default: { + return Status::NotImplemented( + "Not implemented type for conversion from List to Pandas: ", + type.value_type()->ToString()); + } } + } - npy_half* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + Status Visit(const StructType& type) { + return ConvertStruct(options, data, out_values); + } - ConvertNumericNullable(*data, NPY_HALF_NAN, out_buffer); - placement_data_[rel_placement] = abs_placement; - return Status::OK(); + template + enable_if_t::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_base_of::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + Status> + Visit(const Type& type) { + return Status::NotImplemented("No implemented conversion to object dtype: ", + type.ToString()); } }; -class Float32Block : public PandasBlock { +class ObjectWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); } + using TypedPandasWriter::TypedPandasWriter; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + PyAcquireGIL lock; + ObjectWriterVisitor visitor{this->options_, *data, + this->GetBlockColumnStart(rel_placement)}; + return VisitTypeInline(*data->type(), &visitor); + } +}; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); +static inline bool IsNonNullContiguous(const ChunkedArray& data) { + return data.num_chunks() == 1 && data.null_count() == 0; +} - if (type != Type::FLOAT) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), - " to a Pandas float32 block"); - } +template +class IntWriter : public TypedPandasWriter { + public: + using ArrowType = typename npy_traits::TypeClass; + using TypedPandasWriter::TypedPandasWriter; - float* out_buffer = reinterpret_cast(block_data_) + rel_placement * num_rows_; + bool CanZeroCopy(const ChunkedArray& data) const override { + return IsNonNullContiguous(data); + } - ConvertNumericNullable(*data, NAN, out_buffer); - placement_data_[rel_placement] = abs_placement; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + RETURN_NOT_OK(this->CheckTypeExact(*data->type(), ArrowType::type_id)); + ConvertIntegerNoNullsSameType( + this->options_, *data, this->GetBlockColumnStart(rel_placement)); return Status::OK(); } }; -class Float64Block : public PandasBlock { +template +class FloatWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); } + using ArrowType = typename npy_traits::TypeClass; + using TypedPandasWriter::TypedPandasWriter; + using T = typename ArrowType::c_type; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + bool CanZeroCopy(const ChunkedArray& data) const override { + return IsNonNullContiguous(data) && data.type()->id() == ArrowType::type_id; + } - double* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type in_type = data->type()->id(); + auto out_values = this->GetBlockColumnStart(rel_placement); -#define INTEGER_CASE(IN_TYPE) \ - ConvertIntegerWithNulls(options_, *data, out_buffer); \ +#define INTEGER_CASE(IN_TYPE) \ + ConvertIntegerWithNulls(this->options_, *data, out_values); \ break; - switch (type) { + switch (in_type) { case Type::UINT8: INTEGER_CASE(uint8_t); case Type::INT8: @@ -1020,87 +1040,178 @@ class Float64Block : public PandasBlock { INTEGER_CASE(uint64_t); case Type::INT64: INTEGER_CASE(int64_t); + case Type::HALF_FLOAT: + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); case Type::FLOAT: - ConvertNumericNullableCast(*data, NAN, out_buffer); + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); break; case Type::DOUBLE: - ConvertNumericNullable(*data, NAN, out_buffer); + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); break; default: return Status::NotImplemented("Cannot write Arrow data of type ", data->type()->ToString(), - " to a Pandas float64 block"); + " to a Pandas floating point block"); } #undef INTEGER_CASE - placement_data_[rel_placement] = abs_placement; return Status::OK(); } }; -class BoolBlock : public PandasBlock { +using UInt8Writer = IntWriter; +using Int8Writer = IntWriter; +using UInt16Writer = IntWriter; +using Int16Writer = IntWriter; +using UInt32Writer = IntWriter; +using Int32Writer = IntWriter; +using UInt64Writer = IntWriter; +using Int64Writer = IntWriter; +using Float16Writer = FloatWriter; +using Float32Writer = FloatWriter; +using Float64Writer = FloatWriter; + +class BoolWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_BOOL); } + using TypedPandasWriter::TypedPandasWriter; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - if (data->type()->id() != Type::BOOL) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), - " to a Pandas boolean block"); + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + RETURN_NOT_OK( + CheckNoZeroCopy("Zero copy conversions not possible with " + "boolean types")); + RETURN_NOT_OK(EnsureAllocated()); + return CopyInto(data, /*rel_placement=*/0); + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + RETURN_NOT_OK(this->CheckTypeExact(*data->type(), Type::BOOL)); + auto out_values = this->GetBlockColumnStart(rel_placement); + for (int c = 0; c < data->num_chunks(); c++) { + const auto& arr = checked_cast(*data->chunk(c)); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = static_cast(arr.Value(i)); + } } + return Status::OK(); + } +}; - uint8_t* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; +// ---------------------------------------------------------------------- +// Date / timestamp types - ConvertBooleanNoNulls(options_, *data, out_buffer); - placement_data_[rel_placement] = abs_placement; +template +inline void ConvertDatetimeLikeNanos(const ChunkedArray& data, int64_t* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); + + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? kPandasTimestampNull + : (static_cast(in_values[i]) * SHIFT); + } + } +} + +template +void ConvertDatesShift(const ChunkedArray& data, int64_t* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? kPandasTimestampNull + : static_cast(in_values[i]) / SHIFT; + } + } +} + +class DatetimeDayWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + int64_t* out_values = this->GetBlockColumnStart(rel_placement); + const auto& type = checked_cast(*data->type()); + switch (type.unit()) { + case DateUnit::DAY: + ConvertDatesShift(*data, out_values); + break; + case DateUnit::MILLI: + ConvertDatesShift(*data, out_values); + break; + } + return Status::OK(); + } + + protected: + Status Allocate() override { + RETURN_NOT_OK(this->AllocateNDArray(NPY_DATETIME)); + SetDatetimeUnit(NPY_FR_D); return Status::OK(); } }; -class DatetimeBlock : public PandasBlock { +template +class DatetimeWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status AllocateDatetime(int ndim) { - RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME, ndim)); + using TypedPandasWriter::TypedPandasWriter; - PyAcquireGIL lock; - auto date_dtype = reinterpret_cast( - PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); - date_dtype->meta.base = NPY_FR_ns; + bool CanZeroCopy(const ChunkedArray& data) const override { + if (data.type()->id() == Type::TIMESTAMP) { + const auto& type = checked_cast(*data.type()); + return IsNonNullContiguous(data) && type.unit() == UNIT; + } else { + return false; + } + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + const auto& ts_type = checked_cast(*data->type()); + DCHECK_EQ(UNIT, ts_type.unit()) << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, + this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } + + protected: + Status Allocate() override { + RETURN_NOT_OK(this->AllocateNDArray(NPY_DATETIME)); + SetDatetimeUnit(internal::NumPyFrequency(UNIT)); return Status::OK(); } +}; - Status Allocate() override { return AllocateDatetime(2); } +using DatetimeSecondWriter = DatetimeWriter; +using DatetimeMilliWriter = DatetimeWriter; +using DatetimeMicroWriter = DatetimeWriter; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); +class DatetimeNanoWriter : public DatetimeWriter { + public: + using DatetimeWriter::DatetimeWriter; - int64_t* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); if (type == Type::DATE32) { // Convert from days since epoch to datetime64[ns] - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (type == Type::DATE64) { // Date64Type is millisecond timestamp stored as int64_t // TODO(wesm): Do we want to make sure to zero out the milliseconds? - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (type == Type::TIMESTAMP) { const auto& ts_type = checked_cast(*data->type()); if (ts_type.unit() == TimeUnit::NANO) { - ConvertNumericNullable(*data, kPandasTimestampNull, out_buffer); + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); } else if (ts_type.unit() == TimeUnit::MICRO) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::MILLI) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::SECOND) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else { return Status::NotImplemented("Unsupported time unit"); } @@ -1109,45 +1220,81 @@ class DatetimeBlock : public PandasBlock { data->type()->ToString(), " to a Pandas datetime block."); } + return Status::OK(); + } +}; - placement_data_[rel_placement] = abs_placement; +class DatetimeTZWriter : public DatetimeNanoWriter { + public: + DatetimeTZWriter(const PandasOptions& options, const std::string& timezone, + int64_t num_rows) + : DatetimeNanoWriter(options, num_rows, 1), timezone_(timezone) {} + + protected: + Status GetResultBlock(PyObject** out) override { return GetBlock1D(out); } + + Status AddResultMetadata(PyObject* result) override { + PyObject* py_tz = PyUnicode_FromStringAndSize( + timezone_.c_str(), static_cast(timezone_.size())); + RETURN_IF_PYERROR(); + PyDict_SetItemString(result, "timezone", py_tz); return Status::OK(); } + + private: + std::string timezone_; }; -class TimedeltaBlock : public PandasBlock { +template +class TimedeltaWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status AllocateDatetime(int ndim) { - RETURN_NOT_OK(AllocateNDArray(NPY_TIMEDELTA, ndim)); + using TypedPandasWriter::TypedPandasWriter; - PyAcquireGIL lock; - auto date_dtype = reinterpret_cast( - PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); - date_dtype->meta.base = NPY_FR_ns; + Status AllocateTimedelta(int ndim) { + RETURN_NOT_OK(this->AllocateNDArray(NPY_TIMEDELTA, ndim)); + SetDatetimeUnit(internal::NumPyFrequency(UNIT)); return Status::OK(); } - Status Allocate() override { return AllocateDatetime(2); } + bool CanZeroCopy(const ChunkedArray& data) const override { + const auto& type = checked_cast(*data.type()); + return IsNonNullContiguous(data) && type.unit() == UNIT; + } - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + const auto& type = checked_cast(*data->type()); + DCHECK_EQ(UNIT, type.unit()) << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, + this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } + + protected: + Status Allocate() override { return AllocateTimedelta(2); } +}; - int64_t* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; +using TimedeltaSecondWriter = TimedeltaWriter; +using TimedeltaMilliWriter = TimedeltaWriter; +using TimedeltaMicroWriter = TimedeltaWriter; + +class TimedeltaNanoWriter : public TimedeltaWriter { + public: + using TimedeltaWriter::TimedeltaWriter; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); if (type == Type::DURATION) { const auto& ts_type = checked_cast(*data->type()); - if (ts_type.unit() == TimeUnit::NANO) { - ConvertNumericNullable(*data, kPandasTimestampNull, out_buffer); + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); } else if (ts_type.unit() == TimeUnit::MICRO) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::MILLI) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::SECOND) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else { return Status::NotImplemented("Unsupported time unit"); } @@ -1156,42 +1303,10 @@ class TimedeltaBlock : public PandasBlock { data->type()->ToString(), " to a Pandas timedelta block."); } - - placement_data_[rel_placement] = abs_placement; return Status::OK(); } }; -class DatetimeTZBlock : public DatetimeBlock { - public: - DatetimeTZBlock(const PandasOptions& options, const std::string& timezone, - int64_t num_rows) - : DatetimeBlock(options, num_rows, 1), timezone_(timezone) {} - - // Like Categorical, the internal ndarray is 1-dimensional - Status Allocate() override { return AllocateDatetime(1); } - - Status GetPyResult(PyObject** output) override { - PyObject* result = PyDict_New(); - RETURN_IF_PYERROR(); - - PyObject* py_tz = PyUnicode_FromStringAndSize( - timezone_.c_str(), static_cast(timezone_.size())); - RETURN_IF_PYERROR(); - - PyDict_SetItemString(result, "block", block_arr_.obj()); - PyDict_SetItemString(result, "timezone", py_tz); - PyDict_SetItemString(result, "placement", placement_arr_.obj()); - - *output = result; - - return Status::OK(); - } - - private: - std::string timezone_; -}; - Status MakeZeroLengthArray(const std::shared_ptr& type, std::shared_ptr* out) { std::unique_ptr builder; @@ -1228,24 +1343,77 @@ Status CheckDictionaryIndices(const Array& arr, int64_t dict_length) { return Status::OK(); } -class CategoricalBlock : public PandasBlock { +template +class CategoricalWriter + : public TypedPandasWriter::npy_type> { public: - explicit CategoricalBlock(const PandasOptions& options, int64_t num_rows) - : PandasBlock(options, num_rows, 1), ordered_(false), needs_copy_(false) {} + using TRAITS = arrow_traits; + using ArrayType = typename TypeTraits::ArrayType; + using T = typename TRAITS::T; - Status Allocate() override { - return Status::NotImplemented( - "CategoricalBlock allocation happens when calling Write"); + explicit CategoricalWriter(const PandasOptions& options, int64_t num_rows) + : TypedPandasWriter(options, num_rows, 1), + ordered_(false), + needs_copy_(false) {} + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + return Status::NotImplemented("categorical type"); } - template - Status WriteIndicesUniform(const ChunkedArray& data) { - using ArrayType = typename TypeTraits::ArrayType; - using TRAITS = internal::arrow_traits; - using T = typename TRAITS::T; + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + const auto& dict_type = checked_cast(*data->type()); + std::shared_ptr dict; + if (data->num_chunks() == 0) { + // no dictionary values => create empty array + RETURN_NOT_OK(this->AllocateNDArray(TRAITS::npy_type, 1)); + RETURN_NOT_OK(MakeZeroLengthArray(dict_type.value_type(), &dict)); + } else { + DCHECK_EQ(IndexType::type_id, dict_type.index_type()->id()); + RETURN_NOT_OK(WriteIndices(*data, &dict)); + } - RETURN_NOT_OK(AllocateNDArray(TRAITS::npy_type, 1)); - T* out_values = reinterpret_cast(block_data_); + PyObject* pydict; + RETURN_NOT_OK(ConvertArrayToPandas(this->options_, dict, nullptr, &pydict)); + dictionary_.reset(pydict); + ordered_ = dict_type.ordered(); + return Status::OK(); + } + + Status Write(std::shared_ptr data, int64_t abs_placement, + int64_t rel_placement) override { + RETURN_NOT_OK(this->EnsurePlacementAllocated()); + RETURN_NOT_OK(TransferSingle(data, /*py_ref=*/nullptr)); + this->placement_data_[rel_placement] = abs_placement; + return Status::OK(); + } + + Status GetSeriesResult(PyObject** out) override { + PyAcquireGIL lock; + + PyObject* result = PyDict_New(); + RETURN_IF_PYERROR(); + + // Expected single array dictionary layout + PyDict_SetItemString(result, "indices", this->block_arr_.obj()); + RETURN_IF_PYERROR(); + RETURN_NOT_OK(AddResultMetadata(result)); + + *out = result; + return Status::OK(); + } + + protected: + Status AddResultMetadata(PyObject* result) override { + PyDict_SetItemString(result, "dictionary", dictionary_.obj()); + PyObject* py_ordered = ordered_ ? Py_True : Py_False; + Py_INCREF(py_ordered); + PyDict_SetItemString(result, "ordered", py_ordered); + return Status::OK(); + } + + Status WriteIndicesUniform(const ChunkedArray& data) { + RETURN_NOT_OK(this->AllocateNDArray(TRAITS::npy_type, 1)); + T* out_values = reinterpret_cast(this->block_data_); for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = checked_cast(*data.chunk(c)); @@ -1269,22 +1437,17 @@ class CategoricalBlock : public PandasBlock { return Status::OK(); } - template Status WriteIndicesVarying(const ChunkedArray& data, std::shared_ptr* out_dict) { - using ArrayType = typename TypeTraits::ArrayType; - using TRAITS = internal::arrow_traits; - using T = typename TRAITS::T; - // Yield int32 indices to allow for dictionary outgrowing the current index // type - RETURN_NOT_OK(AllocateNDArray(NPY_INT32, 1)); - auto out_values = reinterpret_cast(block_data_); + RETURN_NOT_OK(this->AllocateNDArray(NPY_INT32, 1)); + auto out_values = reinterpret_cast(this->block_data_); const auto& dict_type = checked_cast(*data.type()); std::unique_ptr unifier; RETURN_NOT_OK( - DictionaryUnifier::Make(options_.pool, dict_type.value_type(), &unifier)); + DictionaryUnifier::Make(this->options_.pool, dict_type.value_type(), &unifier)); for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = checked_cast(*data.chunk(c)); const auto& indices = checked_cast(*arr.indices()); @@ -1314,203 +1477,70 @@ class CategoricalBlock : public PandasBlock { return unifier->GetResult(&unused_type, out_dict); } - template Status WriteIndices(const ChunkedArray& data, std::shared_ptr* out_dict) { - using ArrayType = typename TypeTraits::ArrayType; - using TRAITS = internal::arrow_traits; - using T = typename TRAITS::T; - DCHECK_GT(data.num_chunks(), 0); // Sniff the first chunk const auto& arr_first = checked_cast(*data.chunk(0)); const auto indices_first = std::static_pointer_cast(arr_first.indices()); - if (!needs_copy_ && data.num_chunks() == 1 && indices_first->null_count() == 0) { + if (data.num_chunks() == 1 && indices_first->null_count() == 0) { RETURN_NOT_OK(CheckDictionaryIndices(*indices_first, arr_first.dictionary()->length())); - RETURN_NOT_OK(WrapIndicesZeroCopy(TRAITS::npy_type, indices_first)); + + PyObject* wrapped; + npy_intp dims[1] = {this->num_rows_}; + RETURN_NOT_OK(MakeNumPyView(indices_first, /*py_ref=*/nullptr, TRAITS::npy_type, + /*ndim=*/1, dims, &wrapped)); + this->SetBlockData(wrapped); *out_dict = arr_first.dictionary(); } else { - if (options_.zero_copy_only) { - if (needs_copy_) { - return Status::Invalid("Need to allocate categorical memory, but ", - "only zero-copy conversions " - "allowed"); - } - - return Status::Invalid("Needed to copy ", data.num_chunks(), " chunks with ", - indices_first->null_count(), - " indices nulls, but zero_copy_only was True"); - } - + RETURN_NOT_OK(this->CheckNotZeroCopyOnly(data)); if (NeedDictionaryUnification(data)) { - RETURN_NOT_OK(WriteIndicesVarying(data, out_dict)); + RETURN_NOT_OK(WriteIndicesVarying(data, out_dict)); } else { - RETURN_NOT_OK(WriteIndicesUniform(data)); + RETURN_NOT_OK(WriteIndicesUniform(data)); *out_dict = arr_first.dictionary(); } } return Status::OK(); } - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - if (options_.strings_to_categorical && - (data->type()->id() == Type::STRING || data->type()->id() == Type::BINARY)) { - needs_copy_ = true; - compute::FunctionContext ctx(options_.pool); - - Datum out; - RETURN_NOT_OK(compute::DictionaryEncode(&ctx, data, &out)); - DCHECK_EQ(out.kind(), Datum::CHUNKED_ARRAY); - data = out.chunked_array(); - } - - const auto& dict_type = checked_cast(*data->type()); - std::shared_ptr dict; - if (data->num_chunks() == 0) { - // no dictionary values => create empty array - RETURN_NOT_OK(AllocateNDArray(/* any type */ NPY_INT32, 1)); - RETURN_NOT_OK(MakeZeroLengthArray(dict_type.value_type(), &dict)); - } else { - switch (dict_type.index_type()->id()) { - case Type::INT8: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - case Type::INT16: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - case Type::INT32: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - case Type::INT64: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - default: { - return Status::NotImplemented("Categorical index type not supported: ", - dict_type.index_type()->ToString()); - } - } - } - - placement_data_[rel_placement] = abs_placement; - PyObject* pydict; - RETURN_NOT_OK(ConvertArrayToPandas(options_, dict, nullptr, &pydict)); - dictionary_.reset(pydict); - ordered_ = dict_type.ordered(); - - return Status::OK(); - } - - Status GetPyResult(PyObject** output) override { - PyObject* result = PyDict_New(); - RETURN_IF_PYERROR(); - - PyDict_SetItemString(result, "block", block_arr_.obj()); - PyDict_SetItemString(result, "dictionary", dictionary_.obj()); - PyDict_SetItemString(result, "placement", placement_arr_.obj()); - - PyObject* py_ordered = ordered_ ? Py_True : Py_False; - Py_INCREF(py_ordered); - PyDict_SetItemString(result, "ordered", py_ordered); - - *output = result; - - return Status::OK(); - } - - PyObject* dictionary() const { return dictionary_.obj(); } - - protected: - template - Status WrapIndicesZeroCopy(int npy_type, - const std::shared_ptr& indices) { - const T* in_values = GetPrimitiveValues(*indices); - void* data = const_cast(in_values); - - PyAcquireGIL lock; - - PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); - if (descr == nullptr) { - // Error occurred, trust error state is set - return Status::OK(); - } - - npy_intp block_dims[1] = {num_rows_}; - PyObject* block_arr = PyArray_NewFromDescr(&PyArray_Type, descr, 1, block_dims, - nullptr, data, NPY_ARRAY_CARRAY, nullptr); - RETURN_IF_PYERROR(); - - // Add a reference to the underlying Array. Otherwise the array may be - // deleted once we leave the block conversion. - PyObject* base; - RETURN_NOT_OK(CapsulizeArray(indices, &base)); - RETURN_NOT_OK(SetNdarrayBase(reinterpret_cast(block_arr), base)); - - npy_intp placement_dims[1] = {num_columns_}; - PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - RETURN_IF_PYERROR(); - - block_arr_.reset(block_arr); - placement_arr_.reset(placement_arr); - - block_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(block_arr))); - - placement_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(placement_arr))); - - return Status::OK(); - } - OwnedRefNoGIL dictionary_; bool ordered_; bool needs_copy_; }; -class ExtensionBlock : public PandasBlock { +class ExtensionWriter : public PandasWriter { public: - using PandasBlock::PandasBlock; + using PandasWriter::PandasWriter; - // Don't create a block array here, only the placement array - Status Allocate() override { + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { PyAcquireGIL lock; - - npy_intp placement_dims[1] = {num_columns_}; - PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - - RETURN_IF_PYERROR(); - - placement_arr_.reset(placement_arr); - - placement_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(placement_arr))); - - return Status::OK(); - } - - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - PyAcquireGIL lock; - PyObject* py_array; py_array = wrap_chunked_array(data); py_array_.reset(py_array); - placement_data_[rel_placement] = abs_placement; return Status::OK(); } - Status GetPyResult(PyObject** output) override { + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + return TransferSingle(data, nullptr); + } + + Status GetDataFrameResult(PyObject** out) override { + PyAcquireGIL lock; PyObject* result = PyDict_New(); RETURN_IF_PYERROR(); PyDict_SetItemString(result, "py_array", py_array_.obj()); PyDict_SetItemString(result, "placement", placement_arr_.obj()); + *out = result; + return Status::OK(); + } - *output = result; - + Status GetSeriesResult(PyObject** out) override { + *out = py_array_.detach(); return Status::OK(); } @@ -1518,52 +1548,85 @@ class ExtensionBlock : public PandasBlock { OwnedRefNoGIL py_array_; }; -Status MakeBlock(const PandasOptions& options, PandasBlock::type type, int64_t num_rows, - int num_columns, std::shared_ptr* block) { -#define BLOCK_CASE(NAME, TYPE) \ - case PandasBlock::NAME: \ - *block = std::make_shared(options, num_rows, num_columns); \ +Status MakeWriter(const PandasOptions& options, PandasWriter::type writer_type, + const DataType& type, int64_t num_rows, int num_columns, + std::shared_ptr* writer) { +#define BLOCK_CASE(NAME, TYPE) \ + case PandasWriter::NAME: \ + *writer = std::make_shared(options, num_rows, num_columns); \ break; - switch (type) { - BLOCK_CASE(OBJECT, ObjectBlock); - BLOCK_CASE(UINT8, UInt8Block); - BLOCK_CASE(INT8, Int8Block); - BLOCK_CASE(UINT16, UInt16Block); - BLOCK_CASE(INT16, Int16Block); - BLOCK_CASE(UINT32, UInt32Block); - BLOCK_CASE(INT32, Int32Block); - BLOCK_CASE(UINT64, UInt64Block); - BLOCK_CASE(INT64, Int64Block); - BLOCK_CASE(HALF_FLOAT, Float16Block); - BLOCK_CASE(FLOAT, Float32Block); - BLOCK_CASE(DOUBLE, Float64Block); - BLOCK_CASE(BOOL, BoolBlock); - BLOCK_CASE(DATETIME, DatetimeBlock); - BLOCK_CASE(TIMEDELTA, TimedeltaBlock); + switch (writer_type) { + case PandasWriter::CATEGORICAL: { + const auto& index_type = *checked_cast(type).index_type(); + switch (index_type.id()) { + case Type::INT8: + *writer = std::make_shared>(options, num_rows); + break; + case Type::INT16: + *writer = std::make_shared>(options, num_rows); + break; + case Type::INT32: + *writer = std::make_shared>(options, num_rows); + break; + case Type::INT64: + *writer = std::make_shared>(options, num_rows); + break; + default: + return Status::NotImplemented("Unsupported categorical index type:", + index_type.ToString()); + } + } break; + case PandasWriter::EXTENSION: + *writer = std::make_shared(options, num_rows, num_columns); + break; + BLOCK_CASE(OBJECT, ObjectWriter); + BLOCK_CASE(UINT8, UInt8Writer); + BLOCK_CASE(INT8, Int8Writer); + BLOCK_CASE(UINT16, UInt16Writer); + BLOCK_CASE(INT16, Int16Writer); + BLOCK_CASE(UINT32, UInt32Writer); + BLOCK_CASE(INT32, Int32Writer); + BLOCK_CASE(UINT64, UInt64Writer); + BLOCK_CASE(INT64, Int64Writer); + BLOCK_CASE(HALF_FLOAT, Float16Writer); + BLOCK_CASE(FLOAT, Float32Writer); + BLOCK_CASE(DOUBLE, Float64Writer); + BLOCK_CASE(BOOL, BoolWriter); + BLOCK_CASE(DATETIME_DAY, DatetimeDayWriter); + BLOCK_CASE(DATETIME_SECOND, DatetimeSecondWriter); + BLOCK_CASE(DATETIME_MILLI, DatetimeMilliWriter); + BLOCK_CASE(DATETIME_MICRO, DatetimeMicroWriter); + BLOCK_CASE(DATETIME_NANO, DatetimeNanoWriter); + BLOCK_CASE(TIMEDELTA_SECOND, TimedeltaSecondWriter); + BLOCK_CASE(TIMEDELTA_MILLI, TimedeltaMilliWriter); + BLOCK_CASE(TIMEDELTA_MICRO, TimedeltaMicroWriter); + BLOCK_CASE(TIMEDELTA_NANO, TimedeltaNanoWriter); + case PandasWriter::DATETIME_NANO_TZ: { + const auto& ts_type = checked_cast(type); + *writer = std::make_shared(options, ts_type.timezone(), num_rows); + } break; default: return Status::NotImplemented("Unsupported block type"); } #undef BLOCK_CASE - return (*block)->Allocate(); + return Status::OK(); } -using BlockMap = std::unordered_map>; - -static Status GetPandasBlockType(const ChunkedArray& data, const PandasOptions& options, - PandasBlock::type* output_type) { -#define INTEGER_CASE(NAME) \ - *output_type = \ - data.null_count() > 0 \ - ? options.integer_object_nulls ? PandasBlock::OBJECT : PandasBlock::DOUBLE \ - : PandasBlock::NAME; \ +static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions& options, + PandasWriter::type* output_type) { +#define INTEGER_CASE(NAME) \ + *output_type = \ + data.null_count() > 0 \ + ? options.integer_object_nulls ? PandasWriter::OBJECT : PandasWriter::DOUBLE \ + : PandasWriter::NAME; \ break; switch (data.type()->id()) { case Type::BOOL: - *output_type = data.null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL; + *output_type = data.null_count() > 0 ? PandasWriter::OBJECT : PandasWriter::BOOL; break; case Type::UINT8: INTEGER_CASE(UINT8); @@ -1582,55 +1645,92 @@ static Status GetPandasBlockType(const ChunkedArray& data, const PandasOptions& case Type::INT64: INTEGER_CASE(INT64); case Type::HALF_FLOAT: - *output_type = PandasBlock::HALF_FLOAT; + *output_type = PandasWriter::HALF_FLOAT; break; case Type::FLOAT: - *output_type = PandasBlock::FLOAT; + *output_type = PandasWriter::FLOAT; break; case Type::DOUBLE: - *output_type = PandasBlock::DOUBLE; + *output_type = PandasWriter::DOUBLE; break; case Type::STRING: // fall through case Type::LARGE_STRING: // fall through case Type::BINARY: // fall through case Type::LARGE_BINARY: - if (options.strings_to_categorical) { - *output_type = PandasBlock::CATEGORICAL; - break; - } // fall through case Type::NA: // fall through case Type::FIXED_SIZE_BINARY: // fall through case Type::STRUCT: // fall through case Type::TIME32: // fall through case Type::TIME64: // fall through case Type::DECIMAL: // fall through - *output_type = PandasBlock::OBJECT; + *output_type = PandasWriter::OBJECT; break; case Type::DATE32: // fall through case Type::DATE64: - *output_type = options.date_as_object ? PandasBlock::OBJECT : PandasBlock::DATETIME; + if (options.date_as_object) { + *output_type = PandasWriter::OBJECT; + } else { + *output_type = options.coerce_temporal_nanoseconds ? PandasWriter::DATETIME_NANO + : PandasWriter::DATETIME_DAY; + } break; case Type::TIMESTAMP: { const auto& ts_type = checked_cast(*data.type()); if (ts_type.timezone() != "") { - *output_type = PandasBlock::DATETIME_WITH_TZ; + *output_type = PandasWriter::DATETIME_NANO_TZ; + } else if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::DATETIME_NANO; } else { - *output_type = PandasBlock::DATETIME; + switch (ts_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::DATETIME_SECOND; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::DATETIME_MILLI; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::DATETIME_MICRO; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::DATETIME_NANO; + break; + } + } + } break; + case Type::DURATION: { + const auto& dur_type = checked_cast(*data.type()); + if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::TIMEDELTA_NANO; + } else { + switch (dur_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::TIMEDELTA_SECOND; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::TIMEDELTA_MILLI; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::TIMEDELTA_MICRO; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::TIMEDELTA_NANO; + break; + } } } break; - case Type::DURATION: - *output_type = PandasBlock::TIMEDELTA; - break; case Type::LIST: { auto list_type = std::static_pointer_cast(data.type()); if (!ListTypeSupported(*list_type->value_type())) { return Status::NotImplemented("Not implemented type for list in DataFrameBlock: ", list_type->value_type()->ToString()); } - *output_type = PandasBlock::OBJECT; + *output_type = PandasWriter::OBJECT; } break; case Type::DICTIONARY: - *output_type = PandasBlock::CATEGORICAL; + *output_type = PandasWriter::CATEGORICAL; + break; + case Type::EXTENSION: + *output_type = PandasWriter::EXTENSION; break; default: return Status::NotImplemented( @@ -1640,66 +1740,108 @@ static Status GetPandasBlockType(const ChunkedArray& data, const PandasOptions& return Status::OK(); } -// Construct the exact pandas 0.x "BlockManager" memory layout +// Construct the exact pandas "BlockManager" memory layout // // * For each column determine the correct output pandas type // * Allocate 2D blocks (ncols x nrows) for each distinct data type in output // * Allocate block placement arrays // * Write Arrow columns out into each slice of memory; populate block // * placement arrays as we go -class DataFrameBlockCreator { +class PandasBlockCreator { public: - explicit DataFrameBlockCreator(const PandasOptions& options, - const std::unordered_set& extension_columns, - const std::shared_ptr& table) - : table_(table), options_(options), extension_columns_(extension_columns) {} + using WriterMap = std::unordered_map>; + + explicit PandasBlockCreator(const PandasOptions& options, + std::vector> fields, + std::vector> arrays) + : options_(options), fields_(std::move(fields)), arrays_(std::move(arrays)) { + num_columns_ = static_cast(arrays_.size()); + if (num_columns_ > 0) { + num_rows_ = arrays_[0]->length(); + } + column_block_placement_.resize(num_columns_); + } + + virtual Status Convert(PyObject** out) = 0; - Status Convert(PyObject** output) { - column_types_.resize(table_->num_columns()); - column_block_placement_.resize(table_->num_columns()); - type_counts_.clear(); - blocks_.clear(); + Status AppendBlocks(const WriterMap& blocks, PyObject* list) { + for (const auto& it : blocks) { + PyObject* item; + RETURN_NOT_OK(it.second->GetDataFrameResult(&item)); + if (PyList_Append(list, item) < 0) { + RETURN_IF_PYERROR(); + } + + // ARROW-1017; PyList_Append increments object refcount + Py_DECREF(item); + } + return Status::OK(); + } + + protected: + PandasOptions options_; + + std::vector> fields_; + std::vector> arrays_; + int64_t num_columns_; + int64_t num_rows_; + + // column num -> relative placement within internal block + std::vector column_block_placement_; +}; + +class ConsolidatedBlockCreator : public PandasBlockCreator { + public: + using PandasBlockCreator::PandasBlockCreator; + Status Convert(PyObject** out) override { + column_types_.resize(num_columns_); RETURN_NOT_OK(CreateBlocks()); RETURN_NOT_OK(WriteTableToBlocks()); + PyAcquireGIL lock; + + PyObject* result = PyList_New(0); + RETURN_IF_PYERROR(); + + RETURN_NOT_OK(AppendBlocks(blocks_, result)); + RETURN_NOT_OK(AppendBlocks(singleton_blocks_, result)); - return GetResultList(output); + *out = result; + return Status::OK(); + } + + Status GetBlockType(int column_index, PandasWriter::type* out) { + if (options_.extension_columns.count(fields_[column_index]->name())) { + *out = PandasWriter::EXTENSION; + return Status::OK(); + } else { + return GetPandasWriterType(*arrays_[column_index], options_, out); + } } Status CreateBlocks() { - for (int i = 0; i < table_->num_columns(); ++i) { - std::shared_ptr col = table_->column(i); - PandasBlock::type output_type = PandasBlock::OBJECT; - if (extension_columns_.count(table_->field(i)->name())) { - output_type = PandasBlock::EXTENSION; - } else { - RETURN_NOT_OK(GetPandasBlockType(*col, options_, &output_type)); - } + for (int i = 0; i < num_columns_; ++i) { + const DataType& type = *arrays_[i]->type(); + PandasWriter::type output_type; + RETURN_NOT_OK(GetBlockType(i, &output_type)); int block_placement = 0; - std::shared_ptr block; - if (output_type == PandasBlock::CATEGORICAL) { - block = std::make_shared(options_, table_->num_rows()); - categorical_blocks_[i] = block; - } else if (output_type == PandasBlock::DATETIME_WITH_TZ) { - const auto& ts_type = checked_cast(*col->type()); - block = std::make_shared(options_, ts_type.timezone(), - table_->num_rows()); - RETURN_NOT_OK(block->Allocate()); - datetimetz_blocks_[i] = block; - } else if (output_type == PandasBlock::EXTENSION) { - block = std::make_shared(options_, table_->num_rows(), 1); - RETURN_NOT_OK(block->Allocate()); - extension_blocks_[i] = block; + std::shared_ptr writer; + if (output_type == PandasWriter::CATEGORICAL || + output_type == PandasWriter::DATETIME_NANO_TZ || + output_type == PandasWriter::EXTENSION) { + RETURN_NOT_OK(MakeWriter(options_, output_type, type, num_rows_, + /*num_columns=*/1, &writer)); + singleton_blocks_[i] = writer; } else { - auto it = type_counts_.find(output_type); - if (it != type_counts_.end()) { + auto it = block_sizes_.find(output_type); + if (it != block_sizes_.end()) { block_placement = it->second; // Increment count - it->second += 1; + ++it->second; } else { // Add key to map - type_counts_[output_type] = 1; + block_sizes_[output_type] = 1; } } column_types_[i] = output_type; @@ -1707,559 +1849,220 @@ class DataFrameBlockCreator { } // Create normal non-categorical blocks - for (const auto& it : this->type_counts_) { - PandasBlock::type type = static_cast(it.first); - std::shared_ptr block; - RETURN_NOT_OK( - MakeBlock(this->options_, type, this->table_->num_rows(), it.second, &block)); - this->blocks_[type] = block; + for (const auto& it : this->block_sizes_) { + PandasWriter::type output_type = static_cast(it.first); + std::shared_ptr block; + RETURN_NOT_OK(MakeWriter(this->options_, output_type, /*unused*/ *null(), num_rows_, + it.second, &block)); + this->blocks_[output_type] = block; } return Status::OK(); } - Status GetBlock(int i, std::shared_ptr* block) { - PandasBlock::type output_type = this->column_types_[i]; - - if (output_type == PandasBlock::CATEGORICAL) { - auto it = this->categorical_blocks_.find(i); - if (it == this->blocks_.end()) { - return Status::KeyError("No categorical block allocated"); - } - *block = it->second; - } else if (output_type == PandasBlock::DATETIME_WITH_TZ) { - auto it = this->datetimetz_blocks_.find(i); - if (it == this->datetimetz_blocks_.end()) { - return Status::KeyError("No datetimetz block allocated"); - } - *block = it->second; - } else if (output_type == PandasBlock::EXTENSION) { - auto it = this->extension_blocks_.find(i); - if (it == this->extension_blocks_.end()) { - return Status::KeyError("No extension block allocated"); - } - *block = it->second; - } else { - auto it = this->blocks_.find(output_type); - if (it == this->blocks_.end()) { - return Status::KeyError("No block allocated"); - } - *block = it->second; + Status GetWriter(int i, std::shared_ptr* block) { + PandasWriter::type output_type = this->column_types_[i]; + switch (output_type) { + case PandasWriter::CATEGORICAL: + case PandasWriter::DATETIME_NANO_TZ: + case PandasWriter::EXTENSION: { + auto it = this->singleton_blocks_.find(i); + if (it == this->singleton_blocks_.end()) { + return Status::KeyError("No block allocated"); + } + *block = it->second; + } break; + default: + auto it = this->blocks_.find(output_type); + if (it == this->blocks_.end()) { + return Status::KeyError("No block allocated"); + } + *block = it->second; + break; } return Status::OK(); } Status WriteTableToBlocks() { auto WriteColumn = [this](int i) { - std::shared_ptr block; - RETURN_NOT_OK(this->GetBlock(i, &block)); - return block->Write(this->table_->column(i), i, this->column_block_placement_[i]); + std::shared_ptr block; + RETURN_NOT_OK(this->GetWriter(i, &block)); + // ARROW-3789 Use std::move on the array to permit self-destructing + return block->Write(std::move(arrays_[i]), i, this->column_block_placement_[i]); }; if (options_.use_threads) { - return ParallelFor(table_->num_columns(), WriteColumn); + return ParallelFor(num_columns_, WriteColumn); } else { - for (int i = 0; i < table_->num_columns(); ++i) { + for (int i = 0; i < num_columns_; ++i) { RETURN_NOT_OK(WriteColumn(i)); } return Status::OK(); } } - Status AppendBlocks(const BlockMap& blocks, PyObject* list) { - for (const auto& it : blocks) { - PyObject* item; - RETURN_NOT_OK(it.second->GetPyResult(&item)); - if (PyList_Append(list, item) < 0) { - RETURN_IF_PYERROR(); - } - - // ARROW-1017; PyList_Append increments object refcount - Py_DECREF(item); - } - return Status::OK(); - } - - Status GetResultList(PyObject** out) { - PyAcquireGIL lock; - - PyObject* result = PyList_New(0); - RETURN_IF_PYERROR(); - - RETURN_NOT_OK(AppendBlocks(blocks_, result)); - RETURN_NOT_OK(AppendBlocks(categorical_blocks_, result)); - RETURN_NOT_OK(AppendBlocks(datetimetz_blocks_, result)); - RETURN_NOT_OK(AppendBlocks(extension_blocks_, result)); - - *out = result; - return Status::OK(); - } - private: - std::shared_ptr
table_; - // column num -> block type id - std::vector column_types_; - - // column num -> relative placement within internal block - std::vector column_block_placement_; + std::vector column_types_; // block type -> type count - std::unordered_map type_counts_; - - PandasOptions options_; - std::unordered_set extension_columns_; + std::unordered_map block_sizes_; + std::unordered_map block_types_; // block type -> block - BlockMap blocks_; - - // column number -> categorical block - BlockMap categorical_blocks_; - - // column number -> datetimetz block - BlockMap datetimetz_blocks_; + WriterMap blocks_; - // column number -> extension block - BlockMap extension_blocks_; + WriterMap singleton_blocks_; }; -class ArrowDeserializer { +/// \brief Create blocks for pandas.DataFrame block manager using one block per +/// column strategy. This permits some zero-copy optimizations as well as the +/// ability for the table to "self-destruct" if selected by the user. +class SplitBlockCreator : public PandasBlockCreator { public: - ArrowDeserializer(const PandasOptions& options, - const std::shared_ptr& data, PyObject* py_ref) - : data_(data), options_(options), py_ref_(py_ref) {} - - Status AllocateOutput(int type) { - PyAcquireGIL lock; + using PandasBlockCreator::PandasBlockCreator; - npy_intp dims[1] = {data_->length()}; - PyArray_Descr* descr = internal::GetSafeNumPyDtype(type); - if (descr == nullptr) { - RETURN_IF_PYERROR(); - } - if (PyDataType_REFCHK(descr)) { - // ARROW-6876: if the array has refcounted items, let Numpy - // own the array memory so as to decref elements on array destruction - set_numpy_metadata(type, data_->type().get(), descr); - result_ = PyArray_SimpleNewFromDescr(1, dims, descr); - RETURN_IF_PYERROR(); + Status GetWriter(int i, std::shared_ptr* writer) { + PandasWriter::type output_type = PandasWriter::OBJECT; + const DataType& type = *arrays_[i]->type(); + if (options_.extension_columns.count(fields_[i]->name())) { + output_type = PandasWriter::EXTENSION; } else { - RETURN_NOT_OK(PyArray_NewFromPool(1, dims, descr, data_->type().get(), - options_.pool, &result_)); + // Null count needed to determine output type + RETURN_NOT_OK(GetPandasWriterType(*arrays_[i], options_, &output_type)); } - arr_ = reinterpret_cast(result_); - return Status::OK(); + return MakeWriter(this->options_, output_type, type, num_rows_, 1, writer); } - template - Status ConvertValuesZeroCopy(const PandasOptions& options, int npy_type, - const std::shared_ptr& arr) { - typedef typename internal::arrow_traits::T T; - - const T* in_values = GetPrimitiveValues(*arr); - - // Zero-Copy. We can pass the data pointer directly to NumPy. + Status Convert(PyObject** out) override { PyAcquireGIL lock; - PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); - npy_intp dims[1] = {arr->length()}; - set_numpy_metadata(npy_type, arr->type().get(), descr); - result_ = PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, - /*strides=*/nullptr, const_cast(in_values), - /*flags=*/0, nullptr); - arr_ = reinterpret_cast(result_); - - if (arr_ == nullptr) { - // Error occurred, trust that error set - return Status::OK(); - } - - // See ARROW-1973 for the original memory leak report. - // - // There are two scenarios: py_ref_ is nullptr or py_ref_ is not nullptr - // - // 1. py_ref_ is nullptr (it **was not** passed in to ArrowDeserializer's - // constructor) - // - // In this case, the stolen reference must not be incremented since nothing - // outside of the PyArrayObject* (the arr_ member) is holding a reference to - // it. If we increment this, then we have a memory leak. - // - // - // Here's an example of how memory can be leaked when converting an arrow Array - // of List.to a numpy array - // - // 1. Create a 1D numpy that is the flattened arrow array. - // - // There's nothing outside of the serializer that owns this new numpy array. - // - // 2. Make a capsule for the base array. - // - // The reference count of base is 1. - // - // 3. Call PyArray_SetBaseObject(arr_, base) - // - // The reference count is still 1, because the reference is stolen. - // - // 4. Increment the reference count of base (unconditionally) - // - // The reference count is now 2. This is okay if there's an object holding - // another reference. The PyArrayObject that stole the reference will - // eventually decrement the reference count, which will leaves us with a - // refcount of 1, with nothing owning that 1 reference. Memory leakage - // ensues. - // - // 2. py_ref_ is not nullptr (it **was** passed in to ArrowDeserializer's - // constructor) - // - // This case is simpler. We assume that the reference accounting is correct - // coming in. We need to preserve that accounting knowing that the - // PyArrayObject that stole the reference will eventually decref it, thus we - // increment the reference count. - - PyObject* base; - if (py_ref_ == nullptr) { - RETURN_NOT_OK(CapsulizeArray(arr, &base)); - } else { - base = py_ref_; - Py_INCREF(base); - } - - RETURN_NOT_OK(SetNdarrayBase(arr_, base)); - - // Arrow data is immutable. - PyArray_CLEARFLAGS(arr_, NPY_ARRAY_WRITEABLE); - - return Status::OK(); - } + PyObject* result = PyList_New(0); + RETURN_IF_PYERROR(); - // ---------------------------------------------------------------------- - // Allocate new array and deserialize. Can do a zero copy conversion for some - // types + for (int i = 0; i < num_columns_; ++i) { + std::shared_ptr writer; + RETURN_NOT_OK(GetWriter(i, &writer)); + // ARROW-3789 Use std::move on the array to permit self-destructing + RETURN_NOT_OK(writer->Write(std::move(arrays_[i]), i, /*rel_placement=*/0)); - template - enable_if_t::value, Status> Visit(const Type& type) { - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - - typedef typename traits::T T; - int npy_type = traits::npy_type; - - if (data_->num_chunks() == 1 && data_->null_count() == 0) { - return ConvertValuesZeroCopy(options_, npy_type, data_->chunk(0)); - } else if (options_.zero_copy_only) { - return Status::Invalid("Needed to copy ", data_->num_chunks(), " chunks with ", - data_->null_count(), " nulls, but zero_copy_only was True"); + PyObject* item; + RETURN_NOT_OK(writer->GetDataFrameResult(&item)); + if (PyList_Append(result, item) < 0) { + RETURN_IF_PYERROR(); + } + // PyList_Append increments object refcount + Py_DECREF(item); } - RETURN_NOT_OK(AllocateOutput(npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertNumericNullable(*data_, traits::na_value, out_values); - + *out = result; return Status::OK(); } - template - enable_if_t::value || is_duration_type::value, Status> - Visit(const Type& type) { - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - using c_type = typename Type::c_type; - - typedef typename traits::T T; - - if (data_->num_chunks() == 1 && data_->null_count() == 0) { - return ConvertValuesZeroCopy(options_, traits::npy_type, data_->chunk(0)); - } else if (options_.zero_copy_only) { - return Status::Invalid("Copy Needed, but zero_copy_only was True"); - } - - RETURN_NOT_OK(AllocateOutput(traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - - constexpr T na_value = traits::na_value; - constexpr int64_t kShift = traits::npy_shift; - - for (int c = 0; c < data_->num_chunks(); c++) { - const auto& arr = *data_->chunk(c); - const c_type* in_values = GetPrimitiveValues(arr); + private: + std::vector> writers_; +}; - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]) / kShift; - } +Status ConvertCategoricals(const PandasOptions& options, + std::vector>* arrays, + std::vector>* fields) { + std::vector columns_to_encode; + + // For Categorical conversions + auto EncodeColumn = [&](int j) { + int i = columns_to_encode[j]; + compute::FunctionContext ctx(options.pool); + compute::Datum out; + if (options.zero_copy_only) { + return Status::Invalid("Need to dictionary encode a column, but ", + "only zero-copy conversions allowed"); } + RETURN_NOT_OK(DictionaryEncode(&ctx, (*arrays)[i], &out)); + (*arrays)[i] = out.chunked_array(); + (*fields)[i] = (*fields)[i]->WithType((*arrays)[i]->type()); return Status::OK(); - } - - template - enable_if_date Visit(const Type& type) { - if (options_.zero_copy_only) { - return Status::Invalid("Copy Needed, but zero_copy_only was True"); - } - if (options_.date_as_object) { - return VisitObjects(ConvertDates); - } - - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - using c_type = typename Type::c_type; - - typedef typename traits::T T; - - RETURN_NOT_OK(AllocateOutput(traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - - constexpr T na_value = traits::na_value; - constexpr int64_t kShift = traits::npy_shift; - - for (int c = 0; c < data_->num_chunks(); c++) { - const auto& arr = *data_->chunk(c); - const c_type* in_values = GetPrimitiveValues(arr); + }; - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]) / kShift; + if (!options.categorical_columns.empty()) { + for (int i = 0; i < static_cast(arrays->size()); i++) { + if ((*arrays)[i]->type()->id() != Type::DICTIONARY && + options.categorical_columns.count((*fields)[i]->name())) { + columns_to_encode.push_back(i); } } - return Status::OK(); - } - - template - enable_if_time Visit(const Type& type) { - return Status::NotImplemented("Don't know how to serialize Arrow time type to NumPy"); } - - // Integer specialization - template - enable_if_integer Visit(const Type& type) { - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - - typedef typename traits::T T; - - if (data_->num_chunks() == 1 && data_->null_count() == 0) { - return ConvertValuesZeroCopy(options_, traits::npy_type, data_->chunk(0)); - } else if (options_.zero_copy_only) { - return Status::Invalid("Needed to copy ", data_->num_chunks(), " chunks with ", - data_->null_count(), " nulls, but zero_copy_only was True"); - } - - if (data_->null_count() > 0) { - if (options_.integer_object_nulls) { - return VisitObjects(ConvertIntegerObjects); - } else { - RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertIntegerWithNulls(options_, *data_, out_values); + if (options.strings_to_categorical) { + for (int i = 0; i < static_cast(arrays->size()); i++) { + if (is_base_binary_like((*arrays)[i]->type()->id())) { + columns_to_encode.push_back(i); } - } else { - RETURN_NOT_OK(AllocateOutput(traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertIntegerNoNullsSameType(options_, *data_, out_values); - } - - return Status::OK(); - } - - template - inline Status VisitObjects(FUNCTOR func) { - if (options_.zero_copy_only) { - return Status::Invalid("Object types need copies, but zero_copy_only was True"); - } - RETURN_NOT_OK(AllocateOutput(NPY_OBJECT)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - return func(options_, *data_, out_values); - } - - // Strings and binary - template - enable_if_base_binary Visit(const Type& type) { - return VisitObjects(ConvertBinaryLike); - } - - // Fixed length binary strings - Status Visit(const FixedSizeBinaryType& type) { - return VisitObjects(ConvertBinaryLike); - } - - Status Visit(const NullType& type) { return VisitObjects(ConvertNulls); } - - Status Visit(const Decimal128Type& type) { return VisitObjects(ConvertDecimals); } - - Status Visit(const Time32Type& type) { return VisitObjects(ConvertTimes); } - - Status Visit(const Time64Type& type) { return VisitObjects(ConvertTimes); } - - Status Visit(const StructType& type) { return VisitObjects(ConvertStruct); } - - // Boolean specialization - Status Visit(const BooleanType& type) { - if (options_.zero_copy_only) { - return Status::Invalid("BooleanType needs copies, but zero_copy_only was True"); - } else if (data_->null_count() > 0) { - return VisitObjects(ConvertBooleanWithNulls); - } else { - RETURN_NOT_OK(AllocateOutput(internal::arrow_traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertBooleanNoNulls(options_, *data_, out_values); } - return Status::OK(); } - - Status Visit(const ListType& type) { - if (options_.zero_copy_only) { - return Status::Invalid("ListType needs copies, but zero_copy_only was True"); + if (options.use_threads) { + return ParallelFor(columns_to_encode.size(), EncodeColumn); + } else { + for (auto i : columns_to_encode) { + RETURN_NOT_OK(EncodeColumn(i)); } -#define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \ - case Type::ArrowEnum: \ - return ConvertListsLike(options_, *data_, out_values); - - RETURN_NOT_OK(AllocateOutput(NPY_OBJECT)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - auto list_type = std::static_pointer_cast(data_->type()); - switch (list_type->value_type()->id()) { - CONVERTVALUES_LISTSLIKE_CASE(BooleanType, BOOL) - CONVERTVALUES_LISTSLIKE_CASE(UInt8Type, UINT8) - CONVERTVALUES_LISTSLIKE_CASE(Int8Type, INT8) - CONVERTVALUES_LISTSLIKE_CASE(UInt16Type, UINT16) - CONVERTVALUES_LISTSLIKE_CASE(Int16Type, INT16) - CONVERTVALUES_LISTSLIKE_CASE(UInt32Type, UINT32) - CONVERTVALUES_LISTSLIKE_CASE(Int32Type, INT32) - CONVERTVALUES_LISTSLIKE_CASE(UInt64Type, UINT64) - CONVERTVALUES_LISTSLIKE_CASE(Int64Type, INT64) - CONVERTVALUES_LISTSLIKE_CASE(Date32Type, DATE32) - CONVERTVALUES_LISTSLIKE_CASE(Date64Type, DATE64) - CONVERTVALUES_LISTSLIKE_CASE(Time32Type, TIME32) - CONVERTVALUES_LISTSLIKE_CASE(Time64Type, TIME64) - CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP) - CONVERTVALUES_LISTSLIKE_CASE(DurationType, DURATION) - CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT) - CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE) - CONVERTVALUES_LISTSLIKE_CASE(BinaryType, BINARY) - CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING) - CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL) - CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST) - default: { - return Status::NotImplemented("Not implemented type for lists: ", - list_type->value_type()->ToString()); - } - } -#undef CONVERTVALUES_LISTSLIKE_CASE - } - - Status Visit(const DictionaryType& type) { - auto block = std::make_shared(options_, data_->length()); - RETURN_NOT_OK(block->Write(data_, 0, 0)); - - PyAcquireGIL lock; - result_ = PyDict_New(); - RETURN_IF_PYERROR(); - - PyDict_SetItemString(result_, "indices", block->block_arr()); - RETURN_IF_PYERROR(); - PyDict_SetItemString(result_, "dictionary", block->dictionary()); - RETURN_IF_PYERROR(); - - PyObject* py_ordered = type.ordered() ? Py_True : Py_False; - Py_INCREF(py_ordered); - PyDict_SetItemString(result_, "ordered", py_ordered); - RETURN_IF_PYERROR(); - return Status::OK(); } +} - Status Visit(const ExtensionType& type) { - auto storage_type = type.storage_type(); +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out) { + return ConvertChunkedArrayToPandas( + options, std::make_shared(std::move(arr)), py_ref, out); +} - ArrayVector out_chunks(data_->num_chunks()); - for (int i = 0; i < data_->num_chunks(); i++) { - auto chunk = data_->chunk(i); - auto storage_data = checked_cast(*chunk).storage(); - out_chunks[i] = storage_data; +Status ConvertChunkedArrayToPandas(const PandasOptions& options, + std::shared_ptr arr, PyObject* py_ref, + PyObject** out) { + if (options.strings_to_categorical && is_base_binary_like(arr->type()->id())) { + compute::FunctionContext ctx(options.pool); + compute::Datum out; + if (options.zero_copy_only) { + return Status::Invalid("Need to dictionary encode a column, but ", + "only zero-copy conversions allowed"); } - - data_ = std::make_shared(out_chunks); - - RETURN_NOT_OK(VisitTypeInline(*data_->type(), this)); - return Status::OK(); + RETURN_NOT_OK(DictionaryEncode(&ctx, arr, &out)); + arr = out.chunked_array(); } - Status Visit(const FixedSizeListType& type) { return NotImplemented(type); } - Status Visit(const LargeListType& type) { return NotImplemented(type); } - Status Visit(const UnionType& type) { return NotImplemented(type); } - Status Visit(const DayTimeIntervalType& type) { return NotImplemented(type); } - Status Visit(const MonthIntervalType& type) { return NotImplemented(type); } + PandasOptions modified_options = options; + modified_options.strings_to_categorical = false; - Status NotImplemented(const DataType& type) { - return Status::NotImplemented( - "Conversion from arrow to pandas is not implemented for type ", type.name()); - } + PandasWriter::type output_type; + RETURN_NOT_OK(GetPandasWriterType(*arr, modified_options, &output_type)); - Status Convert(PyObject** out) { - RETURN_NOT_OK(VisitTypeInline(*data_->type(), this)); - *out = result_; - return Status::OK(); - } - - private: - std::shared_ptr data_; - PandasOptions options_; - PyObject* py_ref_; - PyArrayObject* arr_; - PyObject* result_; -}; + std::shared_ptr writer; + RETURN_NOT_OK(MakeWriter(modified_options, output_type, *arr->type(), arr->length(), + /*num_columns=*/1, &writer)); + RETURN_NOT_OK(writer->TransferSingle(std::move(arr), py_ref)); + return writer->GetSeriesResult(out); +} -Status ConvertArrayToPandas(const PandasOptions& options, - const std::shared_ptr& arr, PyObject* py_ref, +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, PyObject** out) { - auto carr = std::make_shared(arr); - return ConvertChunkedArrayToPandas(options, carr, py_ref, out); -} + std::vector> arrays = table->columns(); + std::vector> fields = table->fields(); -Status ConvertChunkedArrayToPandas(const PandasOptions& options, - const std::shared_ptr& ca, - PyObject* py_ref, PyObject** out) { - ArrowDeserializer converter(options, ca, py_ref); - return converter.Convert(out); -} + // ARROW-3789: allow "self-destructing" by releasing references to columns as + // we convert them to pandas + table = nullptr; -Status ConvertTableToPandas(const PandasOptions& options, - const std::shared_ptr
& table, PyObject** out) { - return ConvertTableToPandas(options, std::unordered_set(), table, out); -} + RETURN_NOT_OK(ConvertCategoricals(options, &arrays, &fields)); -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::shared_ptr
& table, PyObject** out) { - return ConvertTableToPandas(options, categorical_columns, - std::unordered_set(), table, out); -} + PandasOptions modified_options = options; + modified_options.strings_to_categorical = false; + modified_options.categorical_columns.clear(); -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::unordered_set& extension_columns, - const std::shared_ptr
& table, PyObject** out) { - std::shared_ptr
current_table = table; - if (!categorical_columns.empty()) { - FunctionContext ctx; - for (int i = 0; i < table->num_columns(); i++) { - std::shared_ptr col = table->column(i); - if (col->type()->id() == Type::DICTIONARY) { - // No need to dictionary encode again. Came up in ARROW-6434, - // ARROW-6435 - continue; - } - if (categorical_columns.count(table->field(i)->name())) { - Datum out; - RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out)); - std::shared_ptr array = out.chunked_array(); - auto field = table->field(i)->WithType(array->type()); - RETURN_NOT_OK(current_table->RemoveColumn(i, ¤t_table)); - RETURN_NOT_OK(current_table->AddColumn(i, field, array, ¤t_table)); - } - } + if (options.split_blocks) { + SplitBlockCreator helper(modified_options, std::move(fields), std::move(arrays)); + return helper.Convert(out); + } else { + ConsolidatedBlockCreator helper(modified_options, std::move(fields), + std::move(arrays)); + return helper.Convert(out); } - - DataFrameBlockCreator helper(options, extension_columns, current_table); - return helper.Convert(out); } } // namespace py diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h index 1196a92432d1..57e1a1da83a3 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.h +++ b/cpp/src/arrow/python/arrow_to_pandas.h @@ -18,8 +18,7 @@ // Functions for converting between pandas's NumPy-based data representation // and Arrow data structures -#ifndef ARROW_PYTHON_ADAPTERS_PANDAS_H -#define ARROW_PYTHON_ADAPTERS_PANDAS_H +#pragma once #include "arrow/python/platform.h" @@ -53,26 +52,42 @@ struct PandasOptions { bool date_as_object = false; bool use_threads = false; + /// Coerce all date and timestamp to datetime64[ns] + bool coerce_temporal_nanoseconds = false; + /// \brief If true, do not create duplicate PyObject versions of equal /// objects. This only applies to immutable objects like strings or datetime /// objects bool deduplicate_objects = false; + + /// \brief If true, create one block per column rather than consolidated + /// blocks (1 per data type). Do zero-copy wrapping when there are no + /// nulls. When using this, you may want to set pandas's consolidation policy + /// to leave the blocks split + bool split_blocks = false; + + /// \brief If true, attempt to deallocate buffers in passed Arrow object if + /// it is the only remaining shared_ptr copy of it. See ARROW-3789 for + /// original context for this feature. Only currently implemented for Table + /// conversions + bool self_destruct = false; + + // Columns that should be casted to categorical + std::unordered_set categorical_columns; + + // Columns that should be passed through to be converted to + // ExtensionArray/Block + std::unordered_set extension_columns; }; ARROW_PYTHON_EXPORT -Status ConvertArrayToPandas(const PandasOptions& options, - const std::shared_ptr& arr, PyObject* py_ref, - PyObject** out); +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out); ARROW_PYTHON_EXPORT Status ConvertChunkedArrayToPandas(const PandasOptions& options, - const std::shared_ptr& col, - PyObject* py_ref, PyObject** out); - -ARROW_PYTHON_EXPORT -Status ConvertColumnToPandas(const PandasOptions& options, - const std::shared_ptr& col, PyObject* py_ref, - PyObject** out); + std::shared_ptr col, PyObject* py_ref, + PyObject** out); // Convert a whole table as efficiently as possible to a pandas.DataFrame. // @@ -81,25 +96,8 @@ Status ConvertColumnToPandas(const PandasOptions& options, // // tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2]) ARROW_PYTHON_EXPORT -Status ConvertTableToPandas(const PandasOptions& options, - const std::shared_ptr
& table, PyObject** out); - -/// Convert a whole table as efficiently as possible to a pandas.DataFrame. -/// -/// Explicitly name columns that should be a categorical -/// This option is only used on conversions that are applied to a table. -ARROW_PYTHON_EXPORT -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::shared_ptr
& table, PyObject** out); - -ARROW_PYTHON_EXPORT -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::unordered_set& extension_columns, - const std::shared_ptr
& table, PyObject** out); +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, + PyObject** out); } // namespace py } // namespace arrow - -#endif // ARROW_PYTHON_ADAPTERS_PANDAS_H diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h index b4a6a974c258..2ce66d54fca2 100644 --- a/cpp/src/arrow/python/type_traits.h +++ b/cpp/src/arrow/python/type_traits.h @@ -32,6 +32,10 @@ namespace arrow { namespace py { + +static constexpr int64_t kPandasTimestampNull = std::numeric_limits::min(); +constexpr int64_t kNanosecondsInDay = 86400000000000LL; + namespace internal { // @@ -86,6 +90,8 @@ struct npy_traits { using TypeClass = HalfFloatType; using BuilderClass = HalfFloatBuilder; + static constexpr npy_half na_sentinel = NPY_HALF_NAN; + static constexpr bool supports_nulls = true; static inline bool isnull(npy_half v) { return v == NPY_HALF_NAN; } @@ -97,6 +103,8 @@ struct npy_traits { using TypeClass = FloatType; using BuilderClass = FloatBuilder; + static constexpr float na_sentinel = NAN; + static constexpr bool supports_nulls = true; static inline bool isnull(float v) { return v != v; } @@ -108,6 +116,8 @@ struct npy_traits { using TypeClass = DoubleType; using BuilderClass = DoubleBuilder; + static constexpr double na_sentinel = NAN; + static constexpr bool supports_nulls = true; static inline bool isnull(double v) { return v != v; } @@ -208,10 +218,6 @@ struct arrow_traits { typedef typename npy_traits::value_type T; }; -static constexpr int64_t kPandasTimestampNull = std::numeric_limits::min(); - -constexpr int64_t kNanosecondsInDay = 86400000000000LL; - template <> struct arrow_traits { static constexpr int npy_type = NPY_DATETIME; @@ -287,6 +293,21 @@ struct arrow_traits { static constexpr bool supports_nulls = true; }; +static inline NPY_DATETIMEUNIT NumPyFrequency(TimeUnit::type unit) { + switch (unit) { + case TimestampType::Unit::SECOND: + return NPY_FR_s; + case TimestampType::Unit::MILLI: + return NPY_FR_ms; + break; + case TimestampType::Unit::MICRO: + return NPY_FR_us; + default: + // NANO + return NPY_FR_ns; + } +} + static inline int NumPyTypeSize(int npy_type) { npy_type = fix_numpy_type_num(npy_type); diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc index 2690b5282968..74c1c2633819 100644 --- a/cpp/src/arrow/table.cc +++ b/cpp/src/arrow/table.cc @@ -425,6 +425,22 @@ class SimpleTable : public Table { Table::Table() : num_rows_(0) {} +std::vector> Table::columns() const { + std::vector> result; + for (int i = 0; i < this->num_columns(); ++i) { + result.emplace_back(this->column(i)); + } + return result; +} + +std::vector> Table::fields() const { + std::vector> result; + for (int i = 0; i < this->num_columns(); ++i) { + result.emplace_back(this->field(i)); + } + return result; +} + std::shared_ptr
Table::Make( const std::shared_ptr& schema, const std::vector>& columns, int64_t num_rows) { diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h index 06acdcd58b7b..b5e3ee1f135b 100644 --- a/cpp/src/arrow/table.h +++ b/cpp/src/arrow/table.h @@ -185,15 +185,21 @@ class ARROW_EXPORT Table { static Status FromChunkedStructArray(const std::shared_ptr& array, std::shared_ptr
* table); - /// Return the table schema + /// \brief Return the table schema std::shared_ptr schema() const { return schema_; } - /// Return a column by index + /// \brief Return a column by index virtual std::shared_ptr column(int i) const = 0; + /// \brief Return vector of all columns for table + std::vector> columns() const; + /// Return a column's field by index std::shared_ptr field(int i) const { return schema_->field(i); } + /// \brief Return vector of all fields for table + std::vector> fields() const; + /// \brief Construct a zero-copy slice of the table with the /// indicated offset and length /// diff --git a/cpp/src/arrow/table_test.cc b/cpp/src/arrow/table_test.cc index ac1e41218c3f..934dc69731f9 100644 --- a/cpp/src/arrow/table_test.cc +++ b/cpp/src/arrow/table_test.cc @@ -273,6 +273,29 @@ TEST_F(TestTable, InvalidColumns) { ASSERT_RAISES(Invalid, table_->ValidateFull()); } +TEST_F(TestTable, AllColumnsAndFields) { + const int length = 100; + MakeExample1(length); + table_ = Table::Make(schema_, columns_); + + auto columns = table_->columns(); + auto fields = table_->fields(); + + for (int i = 0; i < table_->num_columns(); ++i) { + AssertChunkedEqual(*table_->column(i), *columns[i]); + AssertFieldEqual(*table_->field(i), *fields[i]); + } + + // Zero length + std::vector> t2_columns; + auto t2 = Table::Make(::arrow::schema({}), t2_columns); + columns = t2->columns(); + fields = t2->fields(); + + ASSERT_EQ(0, columns.size()); + ASSERT_EQ(0, fields.size()); +} + TEST_F(TestTable, Equals) { const int length = 100; MakeExample1(length); diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index 24b58b155d3e..1d82b4c23a94 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -703,6 +703,19 @@ static inline bool is_primitive(Type::type type_id) { return false; } +static inline bool is_base_binary_like(Type::type type_id) { + switch (type_id) { + case Type::BINARY: + case Type::LARGE_BINARY: + case Type::STRING: + case Type::LARGE_STRING: + return true; + default: + break; + } + return false; +} + static inline bool is_binary_like(Type::type type_id) { switch (type_id) { case Type::BINARY: diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index c4598155d1dd..348aa5952b82 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -489,7 +489,9 @@ cdef class _PandasConvertible: bint date_as_object=True, bint use_threads=True, bint deduplicate_objects=True, - bint ignore_metadata=False + bint ignore_metadata=False, + bint split_blocks=False, + bint self_destruct=False ): """ Convert to a pandas-compatible NumPy array or DataFrame, as appropriate @@ -519,6 +521,15 @@ cdef class _PandasConvertible: ignore_metadata : boolean, default False If True, do not use the 'pandas' metadata to reconstruct the DataFrame index, if present + split_blocks : boolean, default False + If True, generate one internal "block" for each column when + creating a pandas.DataFrame from a RecordBatch or Table. While this + can temporarily reduce memory note that various pandas operations + can trigger "consolidation" which may balloon memory use + self_destruct : boolean, default False + EXPERIMENTAL: If True, attempt to deallocate memory while + converting the object to pandas. If you use the object after + calling to_pandas with this option it will crash your program Returns ------- @@ -531,7 +542,9 @@ cdef class _PandasConvertible: integer_object_nulls=integer_object_nulls, date_as_object=date_as_object, use_threads=use_threads, - deduplicate_objects=deduplicate_objects + deduplicate_objects=deduplicate_objects, + split_blocks=split_blocks, + self_destruct=self_destruct ) return self._to_pandas(options, categories=categories, ignore_metadata=ignore_metadata) @@ -546,6 +559,8 @@ cdef PandasOptions _convert_pandas_options(dict options): result.date_as_object = options['date_as_object'] result.use_threads = options['use_threads'] result.deduplicate_objects = options['deduplicate_objects'] + result.split_blocks = options['split_blocks'] + result.self_destruct = options['self_destruct'] return result @@ -974,46 +989,12 @@ cdef class Array(_PandasConvertible): return wrap_datum(out) def _to_pandas(self, options, **kwargs): - cdef: - PyObject* out - PandasOptions c_options = _convert_pandas_options(options) - Array array - - if self.type.id == _Type_TIMESTAMP and self.type.unit != 'ns': - # pandas only stores ns data - casting here is faster - array = self.cast(timestamp('ns')) - else: - array = self - - with nogil: - check_status(ConvertArrayToPandas(c_options, array.sp_array, - array, &out)) - result = pandas_api.series(wrap_array_output(out), name=self._name) - - if isinstance(self.type, TimestampType) and self.type.tz is not None: - from pyarrow.pandas_compat import make_tz_aware - - result = make_tz_aware(result, self.type.tz) - - return result + return _array_like_to_pandas(self, options) def __array__(self, dtype=None): - cdef: - PyObject* out - PandasOptions c_options - object values - - with nogil: - check_status(ConvertArrayToPandas(c_options, self.sp_array, - self, &out)) - - # wrap_array_output uses pandas to convert to Categorical, here - # always convert to numpy array - values = PyObject_to_object(out) - + values = self.to_numpy(zero_copy_only=False) if isinstance(values, dict): values = np.take(values['dictionary'], values['indices']) - if dtype is None: return values return values.astype(dtype) @@ -1057,6 +1038,7 @@ cdef class Array(_PandasConvertible): check_status(ConvertArrayToPandas(c_options, self.sp_array, self, &out)) array = PyObject_to_object(out) + if writable and not array.flags.writeable: # if the conversion already needed to a copy, writeable is True array = array.copy() @@ -1117,6 +1099,44 @@ cdef class Array(_PandasConvertible): return res +cdef _array_like_to_pandas(obj, options): + cdef: + PyObject* out + PandasOptions c_options = _convert_pandas_options(options) + + original_type = obj.type + + if obj.type.id == _Type_TIMESTAMP and obj.type.unit != 'ns': + # pandas only stores ns data - casting here is faster + obj = obj.cast(timestamp('ns')) + + # ARROW-3789(wesm): when converting to DataFrame we coerce + # date/timestamp types to nanoseconds. Not consistent but we should + # make it so in the future + c_options.coerce_temporal_nanoseconds = False + + if isinstance(obj, Array): + with nogil: + check_status(ConvertArrayToPandas(c_options, + ( obj).sp_array, + obj, &out)) + elif isinstance(obj, ChunkedArray): + with nogil: + check_status(libarrow.ConvertChunkedArrayToPandas( + c_options, + ( obj).sp_chunked_array, + obj, &out)) + + result = pandas_api.series(wrap_array_output(out), name=obj._name) + + if (isinstance(original_type, TimestampType) and + original_type.tz is not None): + from pyarrow.pandas_compat import make_tz_aware + result = make_tz_aware(result, original_type.tz) + + return result + + cdef wrap_array_output(PyObject* output): cdef object obj = PyObject_to_object(output) @@ -1957,6 +1977,18 @@ cdef class ExtensionArray(Array): result.validate() return result + def _to_pandas(self, options, **kwargs): + result = Array._to_pandas(self, options, **kwargs) + # TODO(wesm): is passing through these parameters to the storage array + # correct? + return result.to_pandas(options, **kwargs) + + def to_numpy(self, **kwargs): + """ + See Array.to_numpy + """ + return self.storage.to_numpy(**kwargs) + cdef dict _array_classes = { _Type_NA: NullArray, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 08872dab955f..95c21d7ae90e 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1513,19 +1513,16 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CSparseCSRMatrix]* out) CStatus ConvertArrayToPandas(const PandasOptions& options, - const shared_ptr[CArray]& arr, + shared_ptr[CArray] arr, object py_ref, PyObject** out) CStatus ConvertChunkedArrayToPandas(const PandasOptions& options, - const shared_ptr[CChunkedArray]& arr, + shared_ptr[CChunkedArray] arr, object py_ref, PyObject** out) - CStatus ConvertTableToPandas( - const PandasOptions& options, - const unordered_set[c_string]& categorical_columns, - const unordered_set[c_string]& extension_columns, - const shared_ptr[CTable]& table, - PyObject** out) + CStatus ConvertTableToPandas(const PandasOptions& options, + shared_ptr[CTable] table, + PyObject** out) void c_set_default_memory_pool \ " arrow::py::set_default_memory_pool"(CMemoryPool* pool)\ @@ -1555,7 +1552,12 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: c_bool integer_object_nulls c_bool date_as_object c_bool use_threads + c_bool coerce_temporal_nanoseconds c_bool deduplicate_objects + c_bool split_blocks + c_bool self_destruct + unordered_set[c_string] categorical_columns + unordered_set[c_string] extension_columns cdef cppclass CSerializedPyObject" arrow::py::SerializedPyObject": shared_ptr[CRecordBatch] batch diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index c632e8b9c81f..bab12bf1de28 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -766,8 +766,8 @@ def table_to_blockmanager(options, table, categories=None, table, [], extension_columns) _check_data_column_metadata_consistency(all_columns) - blocks = _table_to_blocks(options, table, categories, ext_columns_dtypes) columns = _deserialize_column_index(table, all_columns, column_indexes) + blocks = _table_to_blocks(options, table, categories, ext_columns_dtypes) axes = [columns, index] return BlockManager(blocks, axes) @@ -1108,11 +1108,9 @@ def _table_to_blocks(options, block_table, categories, extension_columns): # Part of table_to_blockmanager # Convert an arrow table to Block from the internal pandas API + columns = block_table.column_names result = pa.lib.table_to_blocks(options, block_table, categories, list(extension_columns.keys())) - - # Defined above - columns = block_table.column_names return [_reconstruct_block(item, columns, extension_columns) for item in result] diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5d18b349f777..2f23a24403f5 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -183,31 +183,7 @@ cdef class ChunkedArray(_PandasConvertible): return result def _to_pandas(self, options, **kwargs): - cdef: - PyObject* out - PandasOptions c_options = _convert_pandas_options(options) - ChunkedArray array - - if self.type.id == _Type_TIMESTAMP and self.type.unit != 'ns': - # pandas only stores ns data - casting here is faster - array = self.cast(timestamp('ns')) - else: - array = self - - with nogil: - check_status(libarrow.ConvertChunkedArrayToPandas( - c_options, - array.sp_chunked_array, - array, &out)) - - result = pandas_api.series(wrap_array_output(out), name=self._name) - - if isinstance(self.type, TimestampType) and self.type.tz is not None: - from pyarrow.pandas_compat import make_tz_aware - - result = make_tz_aware(result, self.type.tz) - - return result + return _array_like_to_pandas(self, options) def __array__(self, dtype=None): cdef: @@ -215,6 +191,13 @@ cdef class ChunkedArray(_PandasConvertible): PandasOptions c_options object values + if self.type.id == _Type_EXTENSION: + return ( + chunked_array( + [self.chunk(i).storage for i in range(self.num_chunks)] + ).__array__(dtype) + ) + with nogil: check_status(libarrow.ConvertChunkedArrayToPandas( c_options, @@ -874,22 +857,30 @@ def _reconstruct_record_batch(columns, schema): def table_to_blocks(options, Table table, categories, extension_columns): cdef: PyObject* result_obj - shared_ptr[CTable] c_table = table.sp_table + shared_ptr[CTable] c_table CMemoryPool* pool - unordered_set[c_string] categorical_columns PandasOptions c_options = _convert_pandas_options(options) - unordered_set[c_string] c_extension_columns if categories is not None: - categorical_columns = {tobytes(cat) for cat in categories} + c_options.categorical_columns = {tobytes(cat) for cat in categories} if extension_columns is not None: - c_extension_columns = {tobytes(col) for col in extension_columns} + c_options.extension_columns = {tobytes(col) + for col in extension_columns} + + # ARROW-3789(wesm); Convert date/timestamp types to datetime64[ns] + c_options.coerce_temporal_nanoseconds = True + + if c_options.self_destruct: + # Move the shared_ptr, table is now unsafe to use further + c_table = move(table.sp_table) + table.table = NULL + else: + c_table = table.sp_table with nogil: check_status( - libarrow.ConvertTableToPandas( - c_options, categorical_columns, c_extension_columns, c_table, - &result_obj) + libarrow.ConvertTableToPandas(c_options, move(c_table), + &result_obj) ) return PyObject_to_object(result_obj) @@ -913,6 +904,9 @@ cdef class Table(_PandasConvertible): "the `Table.from_*` functions instead.") def __repr__(self): + if self.table == NULL: + raise ValueError("Table's internal pointer is NULL, do not use " + "any methods or attributes on this object") return 'pyarrow.{}\n{}'.format(type(self).__name__, str(self.schema)) cdef void init(self, const shared_ptr[CTable]& table): diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index df64d2a1b592..61d09e2b20ba 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -75,7 +75,7 @@ def _alltypes_example(size=100): }) -def _check_pandas_roundtrip(df, expected=None, use_threads=True, +def _check_pandas_roundtrip(df, expected=None, use_threads=False, expected_schema=None, check_dtype=True, schema=None, preserve_index=False, @@ -3038,15 +3038,69 @@ def test_array_uses_memory_pool(): assert pa.total_allocated_bytes() == prior_allocation +def test_singleton_blocks_zero_copy(): + # Part of ARROW-3789 + t = pa.table([pa.array(np.arange(1000, dtype=np.int64))], ['f0']) + _check_to_pandas_memory_unchanged(t) + + +def _check_to_pandas_memory_unchanged(obj, **kwargs): + prior_allocation = pa.total_allocated_bytes() + x = obj.to_pandas(**kwargs) # noqa + + # Memory allocation unchanged -- either zero copy or self-destructing + assert pa.total_allocated_bytes() == prior_allocation + + +def test_to_pandas_split_blocks(): + # ARROW-3789 + t = pa.table([ + pa.array([1, 2, 3, 4, 5], type='i1'), + pa.array([1, 2, 3, 4, 5], type='i4'), + pa.array([1, 2, 3, 4, 5], type='i8'), + pa.array([1, 2, 3, 4, 5], type='f4'), + pa.array([1, 2, 3, 4, 5], type='f8'), + pa.array([1, 2, 3, 4, 5], type='f8'), + pa.array([1, 2, 3, 4, 5], type='f8'), + pa.array([1, 2, 3, 4, 5], type='f8'), + ], ['f{}'.format(i) for i in range(8)]) + + _check_blocks_created(t, 8) + _check_to_pandas_memory_unchanged(t, split_blocks=True) + + +def _check_blocks_created(t, number): + x = t.to_pandas(split_blocks=True) + assert len(x._data.blocks) == number + + +def test_to_pandas_self_destruct(): + K = 50 + + def _make_table(): + return pa.table([ + # Slice to force a copy + pa.array(np.random.randn(10000)[::2]) + for i in range(K) + ], ['f{}'.format(i) for i in range(K)]) + + t = _make_table() + _check_to_pandas_memory_unchanged(t, split_blocks=True, self_destruct=True) + + # Check non-split-block behavior + t = _make_table() + _check_to_pandas_memory_unchanged(t, self_destruct=True) + + def test_table_uses_memory_pool(): N = 10000 arr = pa.array(np.arange(N, dtype=np.int64)) - t = pa.table([arr], ['f0']) + t = pa.table([arr, arr, arr], ['f0', 'f1', 'f2']) prior_allocation = pa.total_allocated_bytes() x = t.to_pandas() - assert pa.total_allocated_bytes() == (prior_allocation + N * 8) + assert pa.total_allocated_bytes() == (prior_allocation + 3 * N * 8) # Check successful garbage collection x = None # noqa From c22d280dca5fc460193808dc75cf0192d2b7d320 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 6 Jan 2020 14:02:33 -0600 Subject: [PATCH 2/4] Fix one MSVC cast warning --- cpp/src/arrow/python/arrow_to_pandas.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index c64c1189f9fb..dbd5fb9b52f2 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -503,7 +503,8 @@ inline void ConvertIntegerWithNulls(const PandasOptions& options, // Upcast to double, set NaN as appropriate for (int i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? NAN : static_cast(in_values[i]); + *out_values++ = + arr.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); } } } From 8f39cce05995dbf5a0adec080ef6098e6e3a938e Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 8 Jan 2020 16:42:51 -0600 Subject: [PATCH 3/4] Add some documentation. Try fixing MSVC warnings --- cpp/src/arrow/python/arrow_to_pandas.cc | 4 +- docs/source/python/pandas.rst | 71 +++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index dbd5fb9b52f2..1b6fc89ca67a 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -1784,7 +1784,7 @@ class PandasBlockCreator { std::vector> fields_; std::vector> arrays_; - int64_t num_columns_; + int num_columns_; int64_t num_rows_; // column num -> relative placement within internal block @@ -1999,7 +1999,7 @@ Status ConvertCategoricals(const PandasOptions& options, } } if (options.use_threads) { - return ParallelFor(columns_to_encode.size(), EncodeColumn); + return ParallelFor(static_cast(columns_to_encode.size()), EncodeColumn); } else { for (auto i : columns_to_encode) { RETURN_NOT_OK(EncodeColumn(i)); diff --git a/docs/source/python/pandas.rst b/docs/source/python/pandas.rst index 83d997c86f79..20c5dd8efca9 100644 --- a/docs/source/python/pandas.rst +++ b/docs/source/python/pandas.rst @@ -221,3 +221,74 @@ Time types ~~~~~~~~~~ TODO + +Memory Usage and Zero Copy +-------------------------- + +When converting from Arrow data structures to pandas objects using various +``to_pandas`` methods, one must occasionally be mindful of issues related to +performance and memory usage. + +Since pandas's internal data representation is generally different from the +Arrow columnar format, zero copy conversions (where no memory allocation or +computation is required) are only possible in certain limited cases. + +In the worst case scenario, calling ``to_pandas`` will result in a two versions +of the data in memory, one for Arrow and one for pandas, yielding approximately +twice the memory footprint. We have implement some mitigations for this case, +particularly when creating large ``DataFrame`` objects, that we describe below. + +Zero Copy Series Conversions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Zero copy conversions from ``Array`` or ``ChunkedArray`` to NumPy arrays or +pandas Series are possible in certain narrow cases: + +* The Arrow data is stored in an integer (signed or unsigned ``int8`` through + ``int64``) or floating point type (``float16`` through ``float64``). This + includes many numeric types as well as timestamps. +* The Arrow data has no null values (since these are represented using bitmaps + which are not supported by pandas). +* For ``ChunkedArray``, the data consists of a single chunk, + i.e. ``arr.num_chunks == 1``. Multiple chunks will always require a copy + because of pandas's contiguousness requirement. + +In these scenarios, ``to_pandas`` will be zero copy. In all other scenarios, a +copy will be required. + +Reducing Memory Use in ``Table.to_pandas`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As if this writing, pandas applies a data management strategy called +"consolidation" to collect like-typed DataFrame columns in two-dimensional +NumPy arrays, referred to internally as "blocks". We have gone to great effort +to construct the precise "consolidated" blocks so that pandas will not perform +any further allocation or copies after we hand off the data to +``pandas.DataFrame``. The obvious downside of this consolidation strategy is +that it forces a "memory doubling". + +To try to limit the potential effects of "memory doubling" during +``Table.to_pandas``, we provide a couple of options: + +* ``split_blocks=True``, when enabled ``Table.to_pandas`` produces one internal + DataFrame "block" for each column, skipping the "consolidation" step. Note + that many pandas operations will trigger consolidation anyway, but the peak + memory use may be less than the worst case scenario of a full memory + doubling. As a result of this option, we are able to do zero copy conversions + of columns in the same cases where we can do zero copy with ``Array`` and + ``ChunkedArray``. +* ``self_destruct=True``, this destroys the internal Arrow memory buffers in + each column ``Table`` object as they are converted to the pandas-compatible + representation, potentially releasing memory to the operating system as soon + as a column is converted. Note that this renders the calling ``Table`` object + unsafe for further use, and any further methods called will cause your Python + process to crash. + +Used together, the call + +.. code-block:: python + + df = table.to_pandas(split_blocks=True, self_destruct=True) + +will yield significantly lower memory usage in some scenarios. Without these +options, ``to_pandas`` will always double memory. From 3b4260283107f8de07620597d6c05b3591636c96 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 14 Jan 2020 17:24:23 -0600 Subject: [PATCH 4/4] Code review comments --- cpp/src/arrow/python/arrow_to_pandas.h | 5 +++-- docs/source/python/pandas.rst | 9 +++++---- python/pyarrow/array.pxi | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h index 57e1a1da83a3..042745b06812 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.h +++ b/cpp/src/arrow/python/arrow_to_pandas.h @@ -62,8 +62,9 @@ struct PandasOptions { /// \brief If true, create one block per column rather than consolidated /// blocks (1 per data type). Do zero-copy wrapping when there are no - /// nulls. When using this, you may want to set pandas's consolidation policy - /// to leave the blocks split + /// nulls. pandas currently will consolidate the blocks on its own, causing + /// increased memory use, so keep this in mind if you are working on a + /// memory-constrained situation. bool split_blocks = false; /// \brief If true, attempt to deallocate buffers in passed Arrow object if diff --git a/docs/source/python/pandas.rst b/docs/source/python/pandas.rst index 20c5dd8efca9..5f93f6355172 100644 --- a/docs/source/python/pandas.rst +++ b/docs/source/python/pandas.rst @@ -233,7 +233,7 @@ Since pandas's internal data representation is generally different from the Arrow columnar format, zero copy conversions (where no memory allocation or computation is required) are only possible in certain limited cases. -In the worst case scenario, calling ``to_pandas`` will result in a two versions +In the worst case scenario, calling ``to_pandas`` will result in two versions of the data in memory, one for Arrow and one for pandas, yielding approximately twice the memory footprint. We have implement some mitigations for this case, particularly when creating large ``DataFrame`` objects, that we describe below. @@ -253,13 +253,13 @@ pandas Series are possible in certain narrow cases: i.e. ``arr.num_chunks == 1``. Multiple chunks will always require a copy because of pandas's contiguousness requirement. -In these scenarios, ``to_pandas`` will be zero copy. In all other scenarios, a -copy will be required. +In these scenarios, ``to_pandas`` or ``to_numpy`` will be zero copy. In all +other scenarios, a copy will be required. Reducing Memory Use in ``Table.to_pandas`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -As if this writing, pandas applies a data management strategy called +As of this writing, pandas applies a data management strategy called "consolidation" to collect like-typed DataFrame columns in two-dimensional NumPy arrays, referred to internally as "blocks". We have gone to great effort to construct the precise "consolidated" blocks so that pandas will not perform @@ -289,6 +289,7 @@ Used together, the call .. code-block:: python df = table.to_pandas(split_blocks=True, self_destruct=True) + del table # not necessary, but a good practice will yield significantly lower memory usage in some scenarios. Without these options, ``to_pandas`` will always double memory. diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 348aa5952b82..f99af9853e51 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -527,9 +527,10 @@ cdef class _PandasConvertible: can temporarily reduce memory note that various pandas operations can trigger "consolidation" which may balloon memory use self_destruct : boolean, default False - EXPERIMENTAL: If True, attempt to deallocate memory while - converting the object to pandas. If you use the object after - calling to_pandas with this option it will crash your program + EXPERIMENTAL: If True, attempt to deallocate the originating Arrow + memory while converting the Arrow object to pandas. If you use the + object after calling to_pandas with this option it will crash your + program Returns -------