diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 715de777a429..1b6fc89ca67a 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -23,9 +23,12 @@ #include #include +#include #include +#include #include #include +#include #include #include "arrow/array.h" @@ -62,16 +65,36 @@ class MemoryPool; using internal::checked_cast; using internal::ParallelFor; -namespace py { +// ---------------------------------------------------------------------- +// PyCapsule code for setting ndarray base to reference C++ object + +struct ArrayCapsule { + std::shared_ptr array; +}; + +struct BufferCapsule { + std::shared_ptr buffer; +}; + +namespace { + +void ArrayCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Array")); +} -using internal::kNanosecondsInDay; -using internal::kPandasTimestampNull; +void BufferCapsule_Destructor(PyObject* capsule) { + delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Buffer")); +} -using compute::Datum; -using compute::FunctionContext; +} // namespace // ---------------------------------------------------------------------- -// Utility code +// pandas 0.x DataFrame conversion internals + +namespace py { + +using internal::arrow_traits; +using internal::npy_traits; template struct WrapBytes {}; @@ -145,28 +168,6 @@ static inline bool ListTypeSupported(const DataType& type) { } return false; } -// ---------------------------------------------------------------------- -// PyCapsule code for setting ndarray base to reference C++ object - -struct ArrayCapsule { - std::shared_ptr array; -}; - -struct BufferCapsule { - std::shared_ptr buffer; -}; - -namespace { - -void ArrayCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Array")); -} - -void BufferCapsule_Destructor(PyObject* capsule) { - delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow::Buffer")); -} - -} // namespace Status CapsulizeArray(const std::shared_ptr& arr, PyObject** out) { auto capsule = new ArrayCapsule{{arr}}; @@ -205,68 +206,28 @@ Status SetBufferBase(PyArrayObject* arr, const std::shared_ptr& buffer) return SetNdarrayBase(arr, base); } -// ---------------------------------------------------------------------- -// pandas 0.x DataFrame conversion internals - inline void set_numpy_metadata(int type, const DataType* datatype, PyArray_Descr* out) { + auto metadata = reinterpret_cast(out->c_metadata); if (type == NPY_DATETIME) { - auto date_dtype = reinterpret_cast(out->c_metadata); if (datatype->id() == Type::TIMESTAMP) { const auto& timestamp_type = checked_cast(*datatype); - - switch (timestamp_type.unit()) { - case TimestampType::Unit::SECOND: - date_dtype->meta.base = NPY_FR_s; - break; - case TimestampType::Unit::MILLI: - date_dtype->meta.base = NPY_FR_ms; - break; - case TimestampType::Unit::MICRO: - date_dtype->meta.base = NPY_FR_us; - break; - case TimestampType::Unit::NANO: - date_dtype->meta.base = NPY_FR_ns; - break; - } + metadata->meta.base = internal::NumPyFrequency(timestamp_type.unit()); } else { - // datatype->type == Type::DATE64 - date_dtype->meta.base = NPY_FR_D; + DCHECK(false) << "NPY_DATETIME views only supported for Arrow TIMESTAMP types"; } } else if (type == NPY_TIMEDELTA) { DCHECK_EQ(datatype->id(), Type::DURATION); - auto timedelta_dtype = - reinterpret_cast(out->c_metadata); const auto& duration_type = checked_cast(*datatype); - - switch (duration_type.unit()) { - case DurationType::Unit::SECOND: - timedelta_dtype->meta.base = NPY_FR_s; - break; - case DurationType::Unit::MILLI: - timedelta_dtype->meta.base = NPY_FR_ms; - break; - case DurationType::Unit::MICRO: - timedelta_dtype->meta.base = NPY_FR_us; - break; - case DurationType::Unit::NANO: - timedelta_dtype->meta.base = NPY_FR_ns; - break; - } + metadata->meta.base = internal::NumPyFrequency(duration_type.unit()); } } -namespace { - -Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, - const DataType* arrow_type, MemoryPool* pool, PyObject** out) { +Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, MemoryPool* pool, + PyObject** out) { // ARROW-6570: Allocate memory from MemoryPool for a couple reasons // // * Track allocations // * Get better performance through custom allocators - if (arrow_type != nullptr) { - set_numpy_metadata(descr->type_num, arrow_type, descr); - } - int64_t total_size = descr->elsize; for (int i = 0; i < nd; ++i) { total_size *= dims[i]; @@ -286,9 +247,49 @@ Status PyArray_NewFromPool(int nd, npy_intp* dims, PyArray_Descr* descr, return SetBufferBase(reinterpret_cast(*out), buffer); } -} // namespace +template +inline const T* GetPrimitiveValues(const Array& arr) { + if (arr.length() == 0) { + return nullptr; + } + int elsize = checked_cast(*arr.type()).bit_width() / 8; + const auto& prim_arr = checked_cast(arr); + return reinterpret_cast(prim_arr.values()->data() + arr.offset() * elsize); +} + +Status MakeNumPyView(std::shared_ptr arr, PyObject* py_ref, int npy_type, int ndim, + npy_intp* dims, PyObject** out) { + PyAcquireGIL lock; + + PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); + set_numpy_metadata(npy_type, arr->type().get(), descr); + PyObject* result = PyArray_NewFromDescr( + &PyArray_Type, descr, ndim, dims, /*strides=*/nullptr, + const_cast(GetPrimitiveValues(*arr)), /*flags=*/0, nullptr); + PyArrayObject* np_arr = reinterpret_cast(result); + if (np_arr == nullptr) { + // Error occurred, trust that error set + return Status::OK(); + } + + PyObject* base; + if (py_ref == nullptr) { + // Capsule will be owned by the ndarray, no incref necessary. See + // ARROW-1973 + RETURN_NOT_OK(CapsulizeArray(arr, &base)); + } else { + Py_INCREF(py_ref); + base = py_ref; + } + RETURN_NOT_OK(SetNdarrayBase(np_arr, base)); + + // Do not allow Arrow data to be mutated + PyArray_CLEARFLAGS(np_arr, NPY_ARRAY_WRITEABLE); + *out = result; + return Status::OK(); +} -class PandasBlock { +class PandasWriter { public: enum type { OBJECT, @@ -304,36 +305,143 @@ class PandasBlock { FLOAT, DOUBLE, BOOL, - DATETIME, - DATETIME_WITH_TZ, - TIMEDELTA, + DATETIME_DAY, + DATETIME_SECOND, + DATETIME_MILLI, + DATETIME_MICRO, + DATETIME_NANO, + DATETIME_NANO_TZ, + TIMEDELTA_SECOND, + TIMEDELTA_MILLI, + TIMEDELTA_MICRO, + TIMEDELTA_NANO, CATEGORICAL, EXTENSION }; - PandasBlock(const PandasOptions& options, int64_t num_rows, int num_columns) - : num_rows_(num_rows), num_columns_(num_columns), options_(options) {} - virtual ~PandasBlock() {} + PandasWriter(const PandasOptions& options, int64_t num_rows, int num_columns) + : options_(options), num_rows_(num_rows), num_columns_(num_columns) {} + virtual ~PandasWriter() {} - virtual Status Allocate() = 0; - virtual Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) = 0; + void SetBlockData(PyObject* arr) { + block_arr_.reset(arr); + block_data_ = + reinterpret_cast(PyArray_DATA(reinterpret_cast(arr))); + } + + /// \brief Either copy or wrap single array to create pandas-compatible array + /// for Series or DataFrame. num_columns_ can only be 1. Will try to zero + /// copy if possible (or error if not possible and zero_copy_only=True) + virtual Status TransferSingle(std::shared_ptr data, PyObject* py_ref) = 0; + + /// \brief Copy ChunkedArray into a multi-column block + virtual Status CopyInto(std::shared_ptr data, int64_t rel_placement) = 0; + + Status EnsurePlacementAllocated() { + std::lock_guard guard(allocation_lock_); + if (placement_data_ != nullptr) { + return Status::OK(); + } + PyAcquireGIL lock; + + npy_intp placement_dims[1] = {num_columns_}; + PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); + RETURN_IF_PYERROR(); + placement_arr_.reset(placement_arr); + placement_data_ = reinterpret_cast( + PyArray_DATA(reinterpret_cast(placement_arr))); + return Status::OK(); + } + + Status EnsureAllocated() { + std::lock_guard guard(allocation_lock_); + if (block_data_ != nullptr) { + return Status::OK(); + } + RETURN_NOT_OK(Allocate()); + return Status::OK(); + } - PyObject* block_arr() const { return block_arr_.obj(); } + virtual bool CanZeroCopy(const ChunkedArray& data) const { return false; } + + virtual Status Write(std::shared_ptr data, int64_t abs_placement, + int64_t rel_placement) { + RETURN_NOT_OK(EnsurePlacementAllocated()); + if (num_columns_ == 1) { + RETURN_NOT_OK(TransferSingle(data, /*py_ref=*/nullptr)); + } else { + RETURN_NOT_OK( + CheckNoZeroCopy("Cannot do zero copy conversion into " + "multi-column DataFrame block")); + RETURN_NOT_OK(EnsureAllocated()); + RETURN_NOT_OK(CopyInto(data, rel_placement)); + } + placement_data_[rel_placement] = abs_placement; + return Status::OK(); + } - virtual Status GetPyResult(PyObject** output) { + virtual Status GetDataFrameResult(PyObject** out) { PyObject* result = PyDict_New(); RETURN_IF_PYERROR(); - PyDict_SetItemString(result, "block", block_arr_.obj()); - PyDict_SetItemString(result, "placement", placement_arr_.obj()); + PyObject* block; + RETURN_NOT_OK(GetResultBlock(&block)); - *output = result; + PyDict_SetItemString(result, "block", block); + PyDict_SetItemString(result, "placement", placement_arr_.obj()); + RETURN_NOT_OK(AddResultMetadata(result)); + *out = result; return Status::OK(); } + virtual Status GetSeriesResult(PyObject** out) { return GetBlock1D(out); } + protected: + virtual Status AddResultMetadata(PyObject* result) { return Status::OK(); } + + Status GetBlock1D(PyObject** out) { + // For Series or for certain DataFrame block types, we need to shape to a + // 1D array when there is only one column + PyAcquireGIL lock; + + DCHECK_EQ(1, num_columns_); + + npy_intp new_dims[1] = {num_rows_}; + PyArray_Dims dims; + dims.ptr = new_dims; + dims.len = 1; + + *out = PyArray_Newshape(reinterpret_cast(block_arr_.obj()), &dims, + NPY_ANYORDER); + RETURN_IF_PYERROR(); + return Status::OK(); + } + + virtual Status GetResultBlock(PyObject** out) { + *out = block_arr_.obj(); + return Status::OK(); + } + + Status CheckNoZeroCopy(const std::string& message) { + if (options_.zero_copy_only) { + return Status::Invalid(message); + } + return Status::OK(); + } + + Status CheckNotZeroCopyOnly(const ChunkedArray& data) { + if (options_.zero_copy_only) { + return Status::Invalid("Needed to copy ", data.num_chunks(), " chunks with ", + data.null_count(), " nulls, but zero_copy_only was True"); + } + return Status::OK(); + } + + virtual Status Allocate() { + return Status::NotImplemented("Override Allocate in subclasses"); + } + Status AllocateNDArray(int npy_type, int ndim = 2) { PyAcquireGIL lock; @@ -353,64 +461,50 @@ class PandasBlock { block_arr = PyArray_SimpleNewFromDescr(ndim, block_dims, descr); RETURN_IF_PYERROR(); } else { - RETURN_NOT_OK(PyArray_NewFromPool(ndim, block_dims, descr, - /*arrow_type=*/nullptr, options_.pool, - &block_arr)); + RETURN_NOT_OK( + PyArray_NewFromPool(ndim, block_dims, descr, options_.pool, &block_arr)); } - npy_intp placement_dims[1] = {num_columns_}; - PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - - RETURN_IF_PYERROR(); - - block_arr_.reset(block_arr); - placement_arr_.reset(placement_arr); + SetBlockData(block_arr); + return Status::OK(); + } - block_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(block_arr))); + void SetDatetimeUnit(NPY_DATETIMEUNIT unit) { + PyAcquireGIL lock; + auto date_dtype = reinterpret_cast( + PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); + date_dtype->meta.base = unit; + } - placement_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(placement_arr))); + PandasOptions options_; - return Status::OK(); - } + std::mutex allocation_lock_; int64_t num_rows_; int num_columns_; OwnedRefNoGIL block_arr_; - uint8_t* block_data_; - - PandasOptions options_; + uint8_t* block_data_ = nullptr; // ndarray OwnedRefNoGIL placement_arr_; - int64_t* placement_data_; + int64_t* placement_data_ = nullptr; private: - ARROW_DISALLOW_COPY_AND_ASSIGN(PandasBlock); + ARROW_DISALLOW_COPY_AND_ASSIGN(PandasWriter); }; -template -inline const T* GetPrimitiveValues(const Array& arr) { - if (arr.length() == 0) { - return nullptr; - } - const auto& prim_arr = checked_cast(arr); - const T* raw_values = reinterpret_cast(prim_arr.values()->data()); - return raw_values + arr.offset(); -} - -template +template inline void ConvertIntegerWithNulls(const PandasOptions& options, - const ChunkedArray& data, double* out_values) { + const ChunkedArray& data, OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = *data.chunk(c); - const T* in_values = GetPrimitiveValues(arr); + const InType* in_values = GetPrimitiveValues(arr); // Upcast to double, set NaN as appropriate for (int i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? NAN : static_cast(in_values[i]); + *out_values++ = + arr.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); } } } @@ -440,40 +534,6 @@ inline void ConvertIntegerNoNullsCast(const PandasOptions& options, } } -static Status ConvertBooleanWithNulls(const PandasOptions& options, - const ChunkedArray& data, PyObject** out_values) { - PyAcquireGIL lock; - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); - - for (int64_t i = 0; i < arr.length(); ++i) { - if (arr.IsNull(i)) { - Py_INCREF(Py_None); - *out_values++ = Py_None; - } else if (arr.Value(i)) { - // True - Py_INCREF(Py_True); - *out_values++ = Py_True; - } else { - // False - Py_INCREF(Py_False); - *out_values++ = Py_False; - } - } - } - return Status::OK(); -} - -static void ConvertBooleanNoNulls(const PandasOptions& options, const ChunkedArray& data, - uint8_t* out_values) { - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = static_cast(arr.Value(i)); - } - } -} - // Generic Array -> PyObject** converter that handles object deduplication, if // requested template @@ -510,7 +570,6 @@ inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArra using ArrayType = typename TypeTraits::ArrayType; using Scalar = typename MemoizationTraits::Scalar; - PyAcquireGIL lock; // TODO(fsaintjacques): propagate memory pool. ::arrow::internal::ScalarMemoTable memo_table(default_memory_pool()); std::vector unique_values; @@ -547,52 +606,8 @@ inline Status ConvertAsPyObjects(const PandasOptions& options, const ChunkedArra return Status::OK(); } -template -static Status ConvertIntegerObjects(const PandasOptions& options, - const ChunkedArray& data, PyObject** out_values) { - using T = typename Type::c_type; - auto WrapValue = [](T value, PyObject** out) { - *out = std::is_signed::value ? PyLong_FromLongLong(value) - : PyLong_FromUnsignedLongLong(value); - RETURN_IF_PYERROR(); - return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} - -template -inline Status ConvertBinaryLike(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - auto WrapValue = [](const util::string_view& view, PyObject** out) { - *out = WrapBytes::Wrap(view.data(), view.length()); - if (*out == nullptr) { - PyErr_Clear(); - return Status::UnknownError("Wrapping ", view, " failed"); - } - return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} - -inline Status ConvertNulls(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - PyAcquireGIL lock; - for (int c = 0; c < data.num_chunks(); c++) { - std::shared_ptr arr = data.chunk(c); - - for (int64_t i = 0; i < arr->length(); ++i) { - // All values are null - Py_INCREF(Py_None); - *out_values = Py_None; - ++out_values; - } - } - return Status::OK(); -} - inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& data, PyObject** out_values) { - PyAcquireGIL lock; if (data.num_chunks() == 0) { return Status::OK(); } @@ -665,16 +680,22 @@ inline Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& auto value_type = checked_cast(*data.type()).value_type(); auto flat_column = std::make_shared(value_arrays, value_type); // TODO(ARROW-489): Currently we don't have a Python reference for single columns. - // Storing a reference to the whole Array would be to expensive. + // Storing a reference to the whole Array would be too expensive. + + // ARROW-3789(wesm): During refactoring I found that unit tests assumed that + // timestamp units would be preserved on list conversions in + // Table.to_pandas. So we set the option here to not coerce things to + // nanoseconds. Bit of a hack but this seemed the simplest thing to satisfy + // the existing unit tests + PandasOptions modified_options = options; + modified_options.coerce_temporal_nanoseconds = false; OwnedRefNoGIL owned_numpy_array; - RETURN_NOT_OK(ConvertChunkedArrayToPandas(options, flat_column, nullptr, + RETURN_NOT_OK(ConvertChunkedArrayToPandas(modified_options, flat_column, nullptr, owned_numpy_array.ref())); PyObject* numpy_array = owned_numpy_array.obj(); - PyAcquireGIL lock; - int64_t chunk_offset = 0; for (int c = 0; c < data.num_chunks(); c++) { auto arr = std::static_pointer_cast(data.chunk(c)); @@ -710,300 +731,300 @@ inline Status ConvertListsLike(const PandasOptions& options, const ChunkedArray& return Status::OK(); } -template -inline void ConvertNumericNullable(const ChunkedArray& data, T na_value, T* out_values) { +template +inline void ConvertNumericNullable(const ChunkedArray& data, InType na_value, + OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = *data.chunk(c); - const T* in_values = GetPrimitiveValues(arr); + const InType* in_values = GetPrimitiveValues(arr); if (arr.null_count() > 0) { for (int64_t i = 0; i < arr.length(); ++i) { *out_values++ = arr.IsNull(i) ? na_value : in_values[i]; } } else { - memcpy(out_values, in_values, sizeof(T) * arr.length()); + memcpy(out_values, in_values, sizeof(InType) * arr.length()); out_values += arr.length(); } } } template -inline void ConvertNumericNullableCast(const ChunkedArray& data, OutType na_value, +inline void ConvertNumericNullableCast(const ChunkedArray& data, InType na_value, OutType* out_values) { for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = *data.chunk(c); const InType* in_values = GetPrimitiveValues(arr); for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]); + *out_values++ = arr.IsNull(i) ? static_cast(na_value) + : static_cast(in_values[i]); } } } -template -inline void ConvertDatetimeLikeNanos(const ChunkedArray& data, int64_t* out_values) { - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = *data.chunk(c); - const T* in_values = GetPrimitiveValues(arr); +template +class TypedPandasWriter : public PandasWriter { + public: + using T = typename npy_traits::value_type; - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? kPandasTimestampNull - : (static_cast(in_values[i]) * SHIFT); + using PandasWriter::PandasWriter; + + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + if (CanZeroCopy(*data)) { + PyObject* wrapped; + npy_intp dims[2] = {num_columns_, num_rows_}; + RETURN_NOT_OK( + MakeNumPyView(data->chunk(0), py_ref, NPY_TYPE, /*ndim=*/2, dims, &wrapped)); + SetBlockData(wrapped); + return Status::OK(); + } else { + RETURN_NOT_OK(CheckNotZeroCopyOnly(*data)); + RETURN_NOT_OK(EnsureAllocated()); + return CopyInto(data, /*rel_placement=*/0); } } -} -template -static Status ConvertDates(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - auto WrapValue = [](typename Type::c_type value, PyObject** out) { - RETURN_NOT_OK(internal::PyDate_from_int(value, Type::UNIT, out)); - RETURN_IF_PYERROR(); + Status CheckTypeExact(const DataType& type, Type::type expected) { + if (type.id() != expected) { + // TODO(wesm): stringify NumPy / pandas type + return Status::NotImplemented("Cannot write Arrow data of type ", type.ToString()); + } return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} + } -template -static Status ConvertTimes(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - const TimeUnit::type unit = checked_cast(*data.type()).unit(); + T* GetBlockColumnStart(int64_t rel_placement) { + return reinterpret_cast(block_data_) + rel_placement * num_rows_; + } - auto WrapValue = [unit](typename Type::c_type value, PyObject** out) { - RETURN_NOT_OK(internal::PyTime_from_int(value, unit, out)); - RETURN_IF_PYERROR(); - return Status::OK(); - }; - return ConvertAsPyObjects(options, data, WrapValue, out_values); -} + protected: + Status Allocate() override { return AllocateNDArray(NPY_TYPE); } +}; -static Status ConvertDecimals(const PandasOptions& options, const ChunkedArray& data, - PyObject** out_values) { - PyAcquireGIL lock; - OwnedRef decimal; - OwnedRef Decimal; - RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); - RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal)); - PyObject* decimal_constructor = Decimal.obj(); +struct ObjectWriterVisitor { + const PandasOptions& options; + const ChunkedArray& data; + PyObject** out_values; - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); + Status Visit(const NullType& type) { + for (int c = 0; c < data.num_chunks(); c++) { + std::shared_ptr arr = data.chunk(c); - for (int64_t i = 0; i < arr.length(); ++i) { - if (arr.IsNull(i)) { + for (int64_t i = 0; i < arr->length(); ++i) { + // All values are null Py_INCREF(Py_None); - *out_values++ = Py_None; - } else { - *out_values++ = - internal::DecimalFromString(decimal_constructor, arr.FormatValue(i)); - RETURN_IF_PYERROR(); + *out_values = Py_None; + ++out_values; } } + return Status::OK(); } - return Status::OK(); -} - -#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum) \ - case Type::ArrowEnum: \ - RETURN_NOT_OK((ConvertListsLike(options_, *data, out_buffer))); \ - break; - -class ObjectBlock : public PandasBlock { - public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_OBJECT); } - - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + Status Visit(const BooleanType& type) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); - PyObject** out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; - - if (type == Type::BOOL) { - RETURN_NOT_OK(ConvertBooleanWithNulls(options_, *data, out_buffer)); - } else if (type == Type::UINT8) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT8) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::UINT16) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT16) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::UINT32) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT32) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::UINT64) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::INT64) { - RETURN_NOT_OK(ConvertIntegerObjects(options_, *data, out_buffer)); - } else if (type == Type::BINARY) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::LARGE_BINARY) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::STRING) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::LARGE_STRING) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::FIXED_SIZE_BINARY) { - RETURN_NOT_OK(ConvertBinaryLike(options_, *data, out_buffer)); - } else if (type == Type::DATE32) { - RETURN_NOT_OK(ConvertDates(options_, *data, out_buffer)); - } else if (type == Type::DATE64) { - RETURN_NOT_OK(ConvertDates(options_, *data, out_buffer)); - } else if (type == Type::TIME32) { - RETURN_NOT_OK(ConvertTimes(options_, *data, out_buffer)); - } else if (type == Type::TIME64) { - RETURN_NOT_OK(ConvertTimes(options_, *data, out_buffer)); - } else if (type == Type::DECIMAL) { - RETURN_NOT_OK(ConvertDecimals(options_, *data, out_buffer)); - } else if (type == Type::NA) { - RETURN_NOT_OK(ConvertNulls(options_, *data, out_buffer)); - } else if (type == Type::LIST) { - auto list_type = std::static_pointer_cast(data->type()); - switch (list_type->value_type()->id()) { - CONVERTLISTSLIKE_CASE(BooleanType, BOOL) - CONVERTLISTSLIKE_CASE(UInt8Type, UINT8) - CONVERTLISTSLIKE_CASE(Int8Type, INT8) - CONVERTLISTSLIKE_CASE(UInt16Type, UINT16) - CONVERTLISTSLIKE_CASE(Int16Type, INT16) - CONVERTLISTSLIKE_CASE(UInt32Type, UINT32) - CONVERTLISTSLIKE_CASE(Int32Type, INT32) - CONVERTLISTSLIKE_CASE(UInt64Type, UINT64) - CONVERTLISTSLIKE_CASE(Int64Type, INT64) - CONVERTLISTSLIKE_CASE(Date32Type, DATE32) - CONVERTLISTSLIKE_CASE(Date64Type, DATE64) - CONVERTLISTSLIKE_CASE(Time32Type, TIME32) - CONVERTLISTSLIKE_CASE(Time64Type, TIME64) - CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP) - CONVERTLISTSLIKE_CASE(DurationType, DURATION) - CONVERTLISTSLIKE_CASE(FloatType, FLOAT) - CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE) - CONVERTLISTSLIKE_CASE(DecimalType, DECIMAL) - CONVERTLISTSLIKE_CASE(BinaryType, BINARY) - CONVERTLISTSLIKE_CASE(StringType, STRING) - CONVERTLISTSLIKE_CASE(ListType, LIST) - CONVERTLISTSLIKE_CASE(NullType, NA) - default: { - return Status::NotImplemented( - "Not implemented type for conversion from List to Pandas ObjectBlock: ", - list_type->value_type()->ToString()); + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values++ = Py_None; + } else if (arr.Value(i)) { + // True + Py_INCREF(Py_True); + *out_values++ = Py_True; + } else { + // False + Py_INCREF(Py_False); + *out_values++ = Py_False; } } - } else if (type == Type::STRUCT) { - RETURN_NOT_OK(ConvertStruct(options_, *data, out_buffer)); - } else { - return Status::NotImplemented("Unsupported type for object array output: ", - data->type()->ToString()); } - - placement_data_[rel_placement] = abs_placement; return Status::OK(); } -}; -template -class IntBlock : public PandasBlock { - public: - using PandasBlock::PandasBlock; - Status Allocate() override { - return AllocateNDArray(internal::arrow_traits::npy_type); + template + enable_if_integer Visit(const Type& type) { + using T = typename Type::c_type; + auto WrapValue = [](T value, PyObject** out) { + *out = std::is_signed::value ? PyLong_FromLongLong(value) + : PyLong_FromUnsignedLongLong(value); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); } - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); - - C_TYPE* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + template + enable_if_t::value || is_fixed_size_binary_type::value, + Status> + Visit(const Type& type) { + auto WrapValue = [](const util::string_view& view, PyObject** out) { + *out = WrapBytes::Wrap(view.data(), view.length()); + if (*out == nullptr) { + PyErr_Clear(); + return Status::UnknownError("Wrapping ", view, " failed"); + } + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } - if (type != ARROW_TYPE) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), " to a Pandas int", - sizeof(C_TYPE), " block"); - } + template + enable_if_date Visit(const Type& type) { + auto WrapValue = [](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyDate_from_int(value, Type::UNIT, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); + } - ConvertIntegerNoNullsSameType(options_, *data, out_buffer); - placement_data_[rel_placement] = abs_placement; - return Status::OK(); + template + enable_if_time Visit(const Type& type) { + const TimeUnit::type unit = type.unit(); + auto WrapValue = [unit](typename Type::c_type value, PyObject** out) { + RETURN_NOT_OK(internal::PyTime_from_int(value, unit, out)); + RETURN_IF_PYERROR(); + return Status::OK(); + }; + return ConvertAsPyObjects(options, data, WrapValue, out_values); } -}; -using UInt8Block = IntBlock; -using Int8Block = IntBlock; -using UInt16Block = IntBlock; -using Int16Block = IntBlock; -using UInt32Block = IntBlock; -using Int32Block = IntBlock; -using UInt64Block = IntBlock; -using Int64Block = IntBlock; + Status Visit(const Decimal128Type& type) { + OwnedRef decimal; + OwnedRef Decimal; + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); + RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal)); + PyObject* decimal_constructor = Decimal.obj(); -class Float16Block : public PandasBlock { - public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_FLOAT16); } + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = checked_cast(*data.chunk(c)); - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + for (int64_t i = 0; i < arr.length(); ++i) { + if (arr.IsNull(i)) { + Py_INCREF(Py_None); + *out_values++ = Py_None; + } else { + *out_values++ = + internal::DecimalFromString(decimal_constructor, arr.FormatValue(i)); + RETURN_IF_PYERROR(); + } + } + } - if (type != Type::HALF_FLOAT) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), - " to a Pandas float16 block"); + return Status::OK(); + } + + Status Visit(const ListType& type) { +#define CONVERTLISTSLIKE_CASE(ArrowType, ArrowEnum) \ + case Type::ArrowEnum: \ + return ConvertListsLike(options, data, out_values); + + switch (type.value_type()->id()) { + CONVERTLISTSLIKE_CASE(BooleanType, BOOL) + CONVERTLISTSLIKE_CASE(UInt8Type, UINT8) + CONVERTLISTSLIKE_CASE(Int8Type, INT8) + CONVERTLISTSLIKE_CASE(UInt16Type, UINT16) + CONVERTLISTSLIKE_CASE(Int16Type, INT16) + CONVERTLISTSLIKE_CASE(UInt32Type, UINT32) + CONVERTLISTSLIKE_CASE(Int32Type, INT32) + CONVERTLISTSLIKE_CASE(UInt64Type, UINT64) + CONVERTLISTSLIKE_CASE(Int64Type, INT64) + CONVERTLISTSLIKE_CASE(Date32Type, DATE32) + CONVERTLISTSLIKE_CASE(Date64Type, DATE64) + CONVERTLISTSLIKE_CASE(Time32Type, TIME32) + CONVERTLISTSLIKE_CASE(Time64Type, TIME64) + CONVERTLISTSLIKE_CASE(TimestampType, TIMESTAMP) + CONVERTLISTSLIKE_CASE(DurationType, DURATION) + CONVERTLISTSLIKE_CASE(FloatType, FLOAT) + CONVERTLISTSLIKE_CASE(DoubleType, DOUBLE) + CONVERTLISTSLIKE_CASE(DecimalType, DECIMAL) + CONVERTLISTSLIKE_CASE(BinaryType, BINARY) + CONVERTLISTSLIKE_CASE(StringType, STRING) + CONVERTLISTSLIKE_CASE(ListType, LIST) + CONVERTLISTSLIKE_CASE(NullType, NA) + default: { + return Status::NotImplemented( + "Not implemented type for conversion from List to Pandas: ", + type.value_type()->ToString()); + } } + } - npy_half* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + Status Visit(const StructType& type) { + return ConvertStruct(options, data, out_values); + } - ConvertNumericNullable(*data, NPY_HALF_NAN, out_buffer); - placement_data_[rel_placement] = abs_placement; - return Status::OK(); + template + enable_if_t::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_base_of::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + Status> + Visit(const Type& type) { + return Status::NotImplemented("No implemented conversion to object dtype: ", + type.ToString()); } }; -class Float32Block : public PandasBlock { +class ObjectWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_FLOAT32); } + using TypedPandasWriter::TypedPandasWriter; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + PyAcquireGIL lock; + ObjectWriterVisitor visitor{this->options_, *data, + this->GetBlockColumnStart(rel_placement)}; + return VisitTypeInline(*data->type(), &visitor); + } +}; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); +static inline bool IsNonNullContiguous(const ChunkedArray& data) { + return data.num_chunks() == 1 && data.null_count() == 0; +} - if (type != Type::FLOAT) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), - " to a Pandas float32 block"); - } +template +class IntWriter : public TypedPandasWriter { + public: + using ArrowType = typename npy_traits::TypeClass; + using TypedPandasWriter::TypedPandasWriter; - float* out_buffer = reinterpret_cast(block_data_) + rel_placement * num_rows_; + bool CanZeroCopy(const ChunkedArray& data) const override { + return IsNonNullContiguous(data); + } - ConvertNumericNullable(*data, NAN, out_buffer); - placement_data_[rel_placement] = abs_placement; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + RETURN_NOT_OK(this->CheckTypeExact(*data->type(), ArrowType::type_id)); + ConvertIntegerNoNullsSameType( + this->options_, *data, this->GetBlockColumnStart(rel_placement)); return Status::OK(); } }; -class Float64Block : public PandasBlock { +template +class FloatWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_FLOAT64); } + using ArrowType = typename npy_traits::TypeClass; + using TypedPandasWriter::TypedPandasWriter; + using T = typename ArrowType::c_type; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + bool CanZeroCopy(const ChunkedArray& data) const override { + return IsNonNullContiguous(data) && data.type()->id() == ArrowType::type_id; + } - double* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type in_type = data->type()->id(); + auto out_values = this->GetBlockColumnStart(rel_placement); -#define INTEGER_CASE(IN_TYPE) \ - ConvertIntegerWithNulls(options_, *data, out_buffer); \ +#define INTEGER_CASE(IN_TYPE) \ + ConvertIntegerWithNulls(this->options_, *data, out_values); \ break; - switch (type) { + switch (in_type) { case Type::UINT8: INTEGER_CASE(uint8_t); case Type::INT8: @@ -1020,87 +1041,178 @@ class Float64Block : public PandasBlock { INTEGER_CASE(uint64_t); case Type::INT64: INTEGER_CASE(int64_t); + case Type::HALF_FLOAT: + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); case Type::FLOAT: - ConvertNumericNullableCast(*data, NAN, out_buffer); + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); break; case Type::DOUBLE: - ConvertNumericNullable(*data, NAN, out_buffer); + ConvertNumericNullableCast(*data, npy_traits::na_sentinel, out_values); break; default: return Status::NotImplemented("Cannot write Arrow data of type ", data->type()->ToString(), - " to a Pandas float64 block"); + " to a Pandas floating point block"); } #undef INTEGER_CASE - placement_data_[rel_placement] = abs_placement; return Status::OK(); } }; -class BoolBlock : public PandasBlock { +using UInt8Writer = IntWriter; +using Int8Writer = IntWriter; +using UInt16Writer = IntWriter; +using Int16Writer = IntWriter; +using UInt32Writer = IntWriter; +using Int32Writer = IntWriter; +using UInt64Writer = IntWriter; +using Int64Writer = IntWriter; +using Float16Writer = FloatWriter; +using Float32Writer = FloatWriter; +using Float64Writer = FloatWriter; + +class BoolWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status Allocate() override { return AllocateNDArray(NPY_BOOL); } + using TypedPandasWriter::TypedPandasWriter; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - if (data->type()->id() != Type::BOOL) { - return Status::NotImplemented("Cannot write Arrow data of type ", - data->type()->ToString(), - " to a Pandas boolean block"); + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + RETURN_NOT_OK( + CheckNoZeroCopy("Zero copy conversions not possible with " + "boolean types")); + RETURN_NOT_OK(EnsureAllocated()); + return CopyInto(data, /*rel_placement=*/0); + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + RETURN_NOT_OK(this->CheckTypeExact(*data->type(), Type::BOOL)); + auto out_values = this->GetBlockColumnStart(rel_placement); + for (int c = 0; c < data->num_chunks(); c++) { + const auto& arr = checked_cast(*data->chunk(c)); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = static_cast(arr.Value(i)); + } } + return Status::OK(); + } +}; - uint8_t* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; +// ---------------------------------------------------------------------- +// Date / timestamp types - ConvertBooleanNoNulls(options_, *data, out_buffer); - placement_data_[rel_placement] = abs_placement; +template +inline void ConvertDatetimeLikeNanos(const ChunkedArray& data, int64_t* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); + + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? kPandasTimestampNull + : (static_cast(in_values[i]) * SHIFT); + } + } +} + +template +void ConvertDatesShift(const ChunkedArray& data, int64_t* out_values) { + for (int c = 0; c < data.num_chunks(); c++) { + const auto& arr = *data.chunk(c); + const T* in_values = GetPrimitiveValues(arr); + for (int64_t i = 0; i < arr.length(); ++i) { + *out_values++ = arr.IsNull(i) ? kPandasTimestampNull + : static_cast(in_values[i]) / SHIFT; + } + } +} + +class DatetimeDayWriter : public TypedPandasWriter { + public: + using TypedPandasWriter::TypedPandasWriter; + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + int64_t* out_values = this->GetBlockColumnStart(rel_placement); + const auto& type = checked_cast(*data->type()); + switch (type.unit()) { + case DateUnit::DAY: + ConvertDatesShift(*data, out_values); + break; + case DateUnit::MILLI: + ConvertDatesShift(*data, out_values); + break; + } + return Status::OK(); + } + + protected: + Status Allocate() override { + RETURN_NOT_OK(this->AllocateNDArray(NPY_DATETIME)); + SetDatetimeUnit(NPY_FR_D); return Status::OK(); } }; -class DatetimeBlock : public PandasBlock { +template +class DatetimeWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status AllocateDatetime(int ndim) { - RETURN_NOT_OK(AllocateNDArray(NPY_DATETIME, ndim)); + using TypedPandasWriter::TypedPandasWriter; - PyAcquireGIL lock; - auto date_dtype = reinterpret_cast( - PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); - date_dtype->meta.base = NPY_FR_ns; + bool CanZeroCopy(const ChunkedArray& data) const override { + if (data.type()->id() == Type::TIMESTAMP) { + const auto& type = checked_cast(*data.type()); + return IsNonNullContiguous(data) && type.unit() == UNIT; + } else { + return false; + } + } + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + const auto& ts_type = checked_cast(*data->type()); + DCHECK_EQ(UNIT, ts_type.unit()) << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, + this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } + + protected: + Status Allocate() override { + RETURN_NOT_OK(this->AllocateNDArray(NPY_DATETIME)); + SetDatetimeUnit(internal::NumPyFrequency(UNIT)); return Status::OK(); } +}; - Status Allocate() override { return AllocateDatetime(2); } +using DatetimeSecondWriter = DatetimeWriter; +using DatetimeMilliWriter = DatetimeWriter; +using DatetimeMicroWriter = DatetimeWriter; - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); +class DatetimeNanoWriter : public DatetimeWriter { + public: + using DatetimeWriter::DatetimeWriter; - int64_t* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); if (type == Type::DATE32) { // Convert from days since epoch to datetime64[ns] - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (type == Type::DATE64) { // Date64Type is millisecond timestamp stored as int64_t // TODO(wesm): Do we want to make sure to zero out the milliseconds? - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (type == Type::TIMESTAMP) { const auto& ts_type = checked_cast(*data->type()); if (ts_type.unit() == TimeUnit::NANO) { - ConvertNumericNullable(*data, kPandasTimestampNull, out_buffer); + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); } else if (ts_type.unit() == TimeUnit::MICRO) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::MILLI) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::SECOND) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else { return Status::NotImplemented("Unsupported time unit"); } @@ -1109,45 +1221,81 @@ class DatetimeBlock : public PandasBlock { data->type()->ToString(), " to a Pandas datetime block."); } + return Status::OK(); + } +}; - placement_data_[rel_placement] = abs_placement; +class DatetimeTZWriter : public DatetimeNanoWriter { + public: + DatetimeTZWriter(const PandasOptions& options, const std::string& timezone, + int64_t num_rows) + : DatetimeNanoWriter(options, num_rows, 1), timezone_(timezone) {} + + protected: + Status GetResultBlock(PyObject** out) override { return GetBlock1D(out); } + + Status AddResultMetadata(PyObject* result) override { + PyObject* py_tz = PyUnicode_FromStringAndSize( + timezone_.c_str(), static_cast(timezone_.size())); + RETURN_IF_PYERROR(); + PyDict_SetItemString(result, "timezone", py_tz); return Status::OK(); } + + private: + std::string timezone_; }; -class TimedeltaBlock : public PandasBlock { +template +class TimedeltaWriter : public TypedPandasWriter { public: - using PandasBlock::PandasBlock; - Status AllocateDatetime(int ndim) { - RETURN_NOT_OK(AllocateNDArray(NPY_TIMEDELTA, ndim)); + using TypedPandasWriter::TypedPandasWriter; - PyAcquireGIL lock; - auto date_dtype = reinterpret_cast( - PyArray_DESCR(reinterpret_cast(block_arr_.obj()))->c_metadata); - date_dtype->meta.base = NPY_FR_ns; + Status AllocateTimedelta(int ndim) { + RETURN_NOT_OK(this->AllocateNDArray(NPY_TIMEDELTA, ndim)); + SetDatetimeUnit(internal::NumPyFrequency(UNIT)); return Status::OK(); } - Status Allocate() override { return AllocateDatetime(2); } + bool CanZeroCopy(const ChunkedArray& data) const override { + const auto& type = checked_cast(*data.type()); + return IsNonNullContiguous(data) && type.unit() == UNIT; + } - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - Type::type type = data->type()->id(); + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + const auto& type = checked_cast(*data->type()); + DCHECK_EQ(UNIT, type.unit()) << "Should only call instances of this writer " + << "with arrays of the correct unit"; + ConvertNumericNullable(*data, kPandasTimestampNull, + this->GetBlockColumnStart(rel_placement)); + return Status::OK(); + } + + protected: + Status Allocate() override { return AllocateTimedelta(2); } +}; - int64_t* out_buffer = - reinterpret_cast(block_data_) + rel_placement * num_rows_; +using TimedeltaSecondWriter = TimedeltaWriter; +using TimedeltaMilliWriter = TimedeltaWriter; +using TimedeltaMicroWriter = TimedeltaWriter; + +class TimedeltaNanoWriter : public TimedeltaWriter { + public: + using TimedeltaWriter::TimedeltaWriter; + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + Type::type type = data->type()->id(); + int64_t* out_values = this->GetBlockColumnStart(rel_placement); if (type == Type::DURATION) { const auto& ts_type = checked_cast(*data->type()); - if (ts_type.unit() == TimeUnit::NANO) { - ConvertNumericNullable(*data, kPandasTimestampNull, out_buffer); + ConvertNumericNullable(*data, kPandasTimestampNull, out_values); } else if (ts_type.unit() == TimeUnit::MICRO) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::MILLI) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else if (ts_type.unit() == TimeUnit::SECOND) { - ConvertDatetimeLikeNanos(*data, out_buffer); + ConvertDatetimeLikeNanos(*data, out_values); } else { return Status::NotImplemented("Unsupported time unit"); } @@ -1156,42 +1304,10 @@ class TimedeltaBlock : public PandasBlock { data->type()->ToString(), " to a Pandas timedelta block."); } - - placement_data_[rel_placement] = abs_placement; return Status::OK(); } }; -class DatetimeTZBlock : public DatetimeBlock { - public: - DatetimeTZBlock(const PandasOptions& options, const std::string& timezone, - int64_t num_rows) - : DatetimeBlock(options, num_rows, 1), timezone_(timezone) {} - - // Like Categorical, the internal ndarray is 1-dimensional - Status Allocate() override { return AllocateDatetime(1); } - - Status GetPyResult(PyObject** output) override { - PyObject* result = PyDict_New(); - RETURN_IF_PYERROR(); - - PyObject* py_tz = PyUnicode_FromStringAndSize( - timezone_.c_str(), static_cast(timezone_.size())); - RETURN_IF_PYERROR(); - - PyDict_SetItemString(result, "block", block_arr_.obj()); - PyDict_SetItemString(result, "timezone", py_tz); - PyDict_SetItemString(result, "placement", placement_arr_.obj()); - - *output = result; - - return Status::OK(); - } - - private: - std::string timezone_; -}; - Status MakeZeroLengthArray(const std::shared_ptr& type, std::shared_ptr* out) { std::unique_ptr builder; @@ -1228,24 +1344,77 @@ Status CheckDictionaryIndices(const Array& arr, int64_t dict_length) { return Status::OK(); } -class CategoricalBlock : public PandasBlock { +template +class CategoricalWriter + : public TypedPandasWriter::npy_type> { public: - explicit CategoricalBlock(const PandasOptions& options, int64_t num_rows) - : PandasBlock(options, num_rows, 1), ordered_(false), needs_copy_(false) {} + using TRAITS = arrow_traits; + using ArrayType = typename TypeTraits::ArrayType; + using T = typename TRAITS::T; - Status Allocate() override { - return Status::NotImplemented( - "CategoricalBlock allocation happens when calling Write"); + explicit CategoricalWriter(const PandasOptions& options, int64_t num_rows) + : TypedPandasWriter(options, num_rows, 1), + ordered_(false), + needs_copy_(false) {} + + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + return Status::NotImplemented("categorical type"); } - template - Status WriteIndicesUniform(const ChunkedArray& data) { - using ArrayType = typename TypeTraits::ArrayType; - using TRAITS = internal::arrow_traits; - using T = typename TRAITS::T; + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { + const auto& dict_type = checked_cast(*data->type()); + std::shared_ptr dict; + if (data->num_chunks() == 0) { + // no dictionary values => create empty array + RETURN_NOT_OK(this->AllocateNDArray(TRAITS::npy_type, 1)); + RETURN_NOT_OK(MakeZeroLengthArray(dict_type.value_type(), &dict)); + } else { + DCHECK_EQ(IndexType::type_id, dict_type.index_type()->id()); + RETURN_NOT_OK(WriteIndices(*data, &dict)); + } - RETURN_NOT_OK(AllocateNDArray(TRAITS::npy_type, 1)); - T* out_values = reinterpret_cast(block_data_); + PyObject* pydict; + RETURN_NOT_OK(ConvertArrayToPandas(this->options_, dict, nullptr, &pydict)); + dictionary_.reset(pydict); + ordered_ = dict_type.ordered(); + return Status::OK(); + } + + Status Write(std::shared_ptr data, int64_t abs_placement, + int64_t rel_placement) override { + RETURN_NOT_OK(this->EnsurePlacementAllocated()); + RETURN_NOT_OK(TransferSingle(data, /*py_ref=*/nullptr)); + this->placement_data_[rel_placement] = abs_placement; + return Status::OK(); + } + + Status GetSeriesResult(PyObject** out) override { + PyAcquireGIL lock; + + PyObject* result = PyDict_New(); + RETURN_IF_PYERROR(); + + // Expected single array dictionary layout + PyDict_SetItemString(result, "indices", this->block_arr_.obj()); + RETURN_IF_PYERROR(); + RETURN_NOT_OK(AddResultMetadata(result)); + + *out = result; + return Status::OK(); + } + + protected: + Status AddResultMetadata(PyObject* result) override { + PyDict_SetItemString(result, "dictionary", dictionary_.obj()); + PyObject* py_ordered = ordered_ ? Py_True : Py_False; + Py_INCREF(py_ordered); + PyDict_SetItemString(result, "ordered", py_ordered); + return Status::OK(); + } + + Status WriteIndicesUniform(const ChunkedArray& data) { + RETURN_NOT_OK(this->AllocateNDArray(TRAITS::npy_type, 1)); + T* out_values = reinterpret_cast(this->block_data_); for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = checked_cast(*data.chunk(c)); @@ -1269,22 +1438,17 @@ class CategoricalBlock : public PandasBlock { return Status::OK(); } - template Status WriteIndicesVarying(const ChunkedArray& data, std::shared_ptr* out_dict) { - using ArrayType = typename TypeTraits::ArrayType; - using TRAITS = internal::arrow_traits; - using T = typename TRAITS::T; - // Yield int32 indices to allow for dictionary outgrowing the current index // type - RETURN_NOT_OK(AllocateNDArray(NPY_INT32, 1)); - auto out_values = reinterpret_cast(block_data_); + RETURN_NOT_OK(this->AllocateNDArray(NPY_INT32, 1)); + auto out_values = reinterpret_cast(this->block_data_); const auto& dict_type = checked_cast(*data.type()); std::unique_ptr unifier; RETURN_NOT_OK( - DictionaryUnifier::Make(options_.pool, dict_type.value_type(), &unifier)); + DictionaryUnifier::Make(this->options_.pool, dict_type.value_type(), &unifier)); for (int c = 0; c < data.num_chunks(); c++) { const auto& arr = checked_cast(*data.chunk(c)); const auto& indices = checked_cast(*arr.indices()); @@ -1314,203 +1478,70 @@ class CategoricalBlock : public PandasBlock { return unifier->GetResult(&unused_type, out_dict); } - template Status WriteIndices(const ChunkedArray& data, std::shared_ptr* out_dict) { - using ArrayType = typename TypeTraits::ArrayType; - using TRAITS = internal::arrow_traits; - using T = typename TRAITS::T; - DCHECK_GT(data.num_chunks(), 0); // Sniff the first chunk const auto& arr_first = checked_cast(*data.chunk(0)); const auto indices_first = std::static_pointer_cast(arr_first.indices()); - if (!needs_copy_ && data.num_chunks() == 1 && indices_first->null_count() == 0) { + if (data.num_chunks() == 1 && indices_first->null_count() == 0) { RETURN_NOT_OK(CheckDictionaryIndices(*indices_first, arr_first.dictionary()->length())); - RETURN_NOT_OK(WrapIndicesZeroCopy(TRAITS::npy_type, indices_first)); + + PyObject* wrapped; + npy_intp dims[1] = {this->num_rows_}; + RETURN_NOT_OK(MakeNumPyView(indices_first, /*py_ref=*/nullptr, TRAITS::npy_type, + /*ndim=*/1, dims, &wrapped)); + this->SetBlockData(wrapped); *out_dict = arr_first.dictionary(); } else { - if (options_.zero_copy_only) { - if (needs_copy_) { - return Status::Invalid("Need to allocate categorical memory, but ", - "only zero-copy conversions " - "allowed"); - } - - return Status::Invalid("Needed to copy ", data.num_chunks(), " chunks with ", - indices_first->null_count(), - " indices nulls, but zero_copy_only was True"); - } - + RETURN_NOT_OK(this->CheckNotZeroCopyOnly(data)); if (NeedDictionaryUnification(data)) { - RETURN_NOT_OK(WriteIndicesVarying(data, out_dict)); + RETURN_NOT_OK(WriteIndicesVarying(data, out_dict)); } else { - RETURN_NOT_OK(WriteIndicesUniform(data)); + RETURN_NOT_OK(WriteIndicesUniform(data)); *out_dict = arr_first.dictionary(); } } return Status::OK(); } - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - if (options_.strings_to_categorical && - (data->type()->id() == Type::STRING || data->type()->id() == Type::BINARY)) { - needs_copy_ = true; - compute::FunctionContext ctx(options_.pool); - - Datum out; - RETURN_NOT_OK(compute::DictionaryEncode(&ctx, data, &out)); - DCHECK_EQ(out.kind(), Datum::CHUNKED_ARRAY); - data = out.chunked_array(); - } - - const auto& dict_type = checked_cast(*data->type()); - std::shared_ptr dict; - if (data->num_chunks() == 0) { - // no dictionary values => create empty array - RETURN_NOT_OK(AllocateNDArray(/* any type */ NPY_INT32, 1)); - RETURN_NOT_OK(MakeZeroLengthArray(dict_type.value_type(), &dict)); - } else { - switch (dict_type.index_type()->id()) { - case Type::INT8: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - case Type::INT16: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - case Type::INT32: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - case Type::INT64: - RETURN_NOT_OK(WriteIndices(*data, &dict)); - break; - default: { - return Status::NotImplemented("Categorical index type not supported: ", - dict_type.index_type()->ToString()); - } - } - } - - placement_data_[rel_placement] = abs_placement; - PyObject* pydict; - RETURN_NOT_OK(ConvertArrayToPandas(options_, dict, nullptr, &pydict)); - dictionary_.reset(pydict); - ordered_ = dict_type.ordered(); - - return Status::OK(); - } - - Status GetPyResult(PyObject** output) override { - PyObject* result = PyDict_New(); - RETURN_IF_PYERROR(); - - PyDict_SetItemString(result, "block", block_arr_.obj()); - PyDict_SetItemString(result, "dictionary", dictionary_.obj()); - PyDict_SetItemString(result, "placement", placement_arr_.obj()); - - PyObject* py_ordered = ordered_ ? Py_True : Py_False; - Py_INCREF(py_ordered); - PyDict_SetItemString(result, "ordered", py_ordered); - - *output = result; - - return Status::OK(); - } - - PyObject* dictionary() const { return dictionary_.obj(); } - - protected: - template - Status WrapIndicesZeroCopy(int npy_type, - const std::shared_ptr& indices) { - const T* in_values = GetPrimitiveValues(*indices); - void* data = const_cast(in_values); - - PyAcquireGIL lock; - - PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); - if (descr == nullptr) { - // Error occurred, trust error state is set - return Status::OK(); - } - - npy_intp block_dims[1] = {num_rows_}; - PyObject* block_arr = PyArray_NewFromDescr(&PyArray_Type, descr, 1, block_dims, - nullptr, data, NPY_ARRAY_CARRAY, nullptr); - RETURN_IF_PYERROR(); - - // Add a reference to the underlying Array. Otherwise the array may be - // deleted once we leave the block conversion. - PyObject* base; - RETURN_NOT_OK(CapsulizeArray(indices, &base)); - RETURN_NOT_OK(SetNdarrayBase(reinterpret_cast(block_arr), base)); - - npy_intp placement_dims[1] = {num_columns_}; - PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - RETURN_IF_PYERROR(); - - block_arr_.reset(block_arr); - placement_arr_.reset(placement_arr); - - block_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(block_arr))); - - placement_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(placement_arr))); - - return Status::OK(); - } - OwnedRefNoGIL dictionary_; bool ordered_; bool needs_copy_; }; -class ExtensionBlock : public PandasBlock { +class ExtensionWriter : public PandasWriter { public: - using PandasBlock::PandasBlock; + using PandasWriter::PandasWriter; - // Don't create a block array here, only the placement array - Status Allocate() override { + Status TransferSingle(std::shared_ptr data, PyObject* py_ref) override { PyAcquireGIL lock; - - npy_intp placement_dims[1] = {num_columns_}; - PyObject* placement_arr = PyArray_SimpleNew(1, placement_dims, NPY_INT64); - - RETURN_IF_PYERROR(); - - placement_arr_.reset(placement_arr); - - placement_data_ = reinterpret_cast( - PyArray_DATA(reinterpret_cast(placement_arr))); - - return Status::OK(); - } - - Status Write(std::shared_ptr data, int64_t abs_placement, - int64_t rel_placement) override { - PyAcquireGIL lock; - PyObject* py_array; py_array = wrap_chunked_array(data); py_array_.reset(py_array); - placement_data_[rel_placement] = abs_placement; return Status::OK(); } - Status GetPyResult(PyObject** output) override { + Status CopyInto(std::shared_ptr data, int64_t rel_placement) override { + return TransferSingle(data, nullptr); + } + + Status GetDataFrameResult(PyObject** out) override { + PyAcquireGIL lock; PyObject* result = PyDict_New(); RETURN_IF_PYERROR(); PyDict_SetItemString(result, "py_array", py_array_.obj()); PyDict_SetItemString(result, "placement", placement_arr_.obj()); + *out = result; + return Status::OK(); + } - *output = result; - + Status GetSeriesResult(PyObject** out) override { + *out = py_array_.detach(); return Status::OK(); } @@ -1518,52 +1549,85 @@ class ExtensionBlock : public PandasBlock { OwnedRefNoGIL py_array_; }; -Status MakeBlock(const PandasOptions& options, PandasBlock::type type, int64_t num_rows, - int num_columns, std::shared_ptr* block) { -#define BLOCK_CASE(NAME, TYPE) \ - case PandasBlock::NAME: \ - *block = std::make_shared(options, num_rows, num_columns); \ +Status MakeWriter(const PandasOptions& options, PandasWriter::type writer_type, + const DataType& type, int64_t num_rows, int num_columns, + std::shared_ptr* writer) { +#define BLOCK_CASE(NAME, TYPE) \ + case PandasWriter::NAME: \ + *writer = std::make_shared(options, num_rows, num_columns); \ break; - switch (type) { - BLOCK_CASE(OBJECT, ObjectBlock); - BLOCK_CASE(UINT8, UInt8Block); - BLOCK_CASE(INT8, Int8Block); - BLOCK_CASE(UINT16, UInt16Block); - BLOCK_CASE(INT16, Int16Block); - BLOCK_CASE(UINT32, UInt32Block); - BLOCK_CASE(INT32, Int32Block); - BLOCK_CASE(UINT64, UInt64Block); - BLOCK_CASE(INT64, Int64Block); - BLOCK_CASE(HALF_FLOAT, Float16Block); - BLOCK_CASE(FLOAT, Float32Block); - BLOCK_CASE(DOUBLE, Float64Block); - BLOCK_CASE(BOOL, BoolBlock); - BLOCK_CASE(DATETIME, DatetimeBlock); - BLOCK_CASE(TIMEDELTA, TimedeltaBlock); + switch (writer_type) { + case PandasWriter::CATEGORICAL: { + const auto& index_type = *checked_cast(type).index_type(); + switch (index_type.id()) { + case Type::INT8: + *writer = std::make_shared>(options, num_rows); + break; + case Type::INT16: + *writer = std::make_shared>(options, num_rows); + break; + case Type::INT32: + *writer = std::make_shared>(options, num_rows); + break; + case Type::INT64: + *writer = std::make_shared>(options, num_rows); + break; + default: + return Status::NotImplemented("Unsupported categorical index type:", + index_type.ToString()); + } + } break; + case PandasWriter::EXTENSION: + *writer = std::make_shared(options, num_rows, num_columns); + break; + BLOCK_CASE(OBJECT, ObjectWriter); + BLOCK_CASE(UINT8, UInt8Writer); + BLOCK_CASE(INT8, Int8Writer); + BLOCK_CASE(UINT16, UInt16Writer); + BLOCK_CASE(INT16, Int16Writer); + BLOCK_CASE(UINT32, UInt32Writer); + BLOCK_CASE(INT32, Int32Writer); + BLOCK_CASE(UINT64, UInt64Writer); + BLOCK_CASE(INT64, Int64Writer); + BLOCK_CASE(HALF_FLOAT, Float16Writer); + BLOCK_CASE(FLOAT, Float32Writer); + BLOCK_CASE(DOUBLE, Float64Writer); + BLOCK_CASE(BOOL, BoolWriter); + BLOCK_CASE(DATETIME_DAY, DatetimeDayWriter); + BLOCK_CASE(DATETIME_SECOND, DatetimeSecondWriter); + BLOCK_CASE(DATETIME_MILLI, DatetimeMilliWriter); + BLOCK_CASE(DATETIME_MICRO, DatetimeMicroWriter); + BLOCK_CASE(DATETIME_NANO, DatetimeNanoWriter); + BLOCK_CASE(TIMEDELTA_SECOND, TimedeltaSecondWriter); + BLOCK_CASE(TIMEDELTA_MILLI, TimedeltaMilliWriter); + BLOCK_CASE(TIMEDELTA_MICRO, TimedeltaMicroWriter); + BLOCK_CASE(TIMEDELTA_NANO, TimedeltaNanoWriter); + case PandasWriter::DATETIME_NANO_TZ: { + const auto& ts_type = checked_cast(type); + *writer = std::make_shared(options, ts_type.timezone(), num_rows); + } break; default: return Status::NotImplemented("Unsupported block type"); } #undef BLOCK_CASE - return (*block)->Allocate(); + return Status::OK(); } -using BlockMap = std::unordered_map>; - -static Status GetPandasBlockType(const ChunkedArray& data, const PandasOptions& options, - PandasBlock::type* output_type) { -#define INTEGER_CASE(NAME) \ - *output_type = \ - data.null_count() > 0 \ - ? options.integer_object_nulls ? PandasBlock::OBJECT : PandasBlock::DOUBLE \ - : PandasBlock::NAME; \ +static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions& options, + PandasWriter::type* output_type) { +#define INTEGER_CASE(NAME) \ + *output_type = \ + data.null_count() > 0 \ + ? options.integer_object_nulls ? PandasWriter::OBJECT : PandasWriter::DOUBLE \ + : PandasWriter::NAME; \ break; switch (data.type()->id()) { case Type::BOOL: - *output_type = data.null_count() > 0 ? PandasBlock::OBJECT : PandasBlock::BOOL; + *output_type = data.null_count() > 0 ? PandasWriter::OBJECT : PandasWriter::BOOL; break; case Type::UINT8: INTEGER_CASE(UINT8); @@ -1582,55 +1646,92 @@ static Status GetPandasBlockType(const ChunkedArray& data, const PandasOptions& case Type::INT64: INTEGER_CASE(INT64); case Type::HALF_FLOAT: - *output_type = PandasBlock::HALF_FLOAT; + *output_type = PandasWriter::HALF_FLOAT; break; case Type::FLOAT: - *output_type = PandasBlock::FLOAT; + *output_type = PandasWriter::FLOAT; break; case Type::DOUBLE: - *output_type = PandasBlock::DOUBLE; + *output_type = PandasWriter::DOUBLE; break; case Type::STRING: // fall through case Type::LARGE_STRING: // fall through case Type::BINARY: // fall through case Type::LARGE_BINARY: - if (options.strings_to_categorical) { - *output_type = PandasBlock::CATEGORICAL; - break; - } // fall through case Type::NA: // fall through case Type::FIXED_SIZE_BINARY: // fall through case Type::STRUCT: // fall through case Type::TIME32: // fall through case Type::TIME64: // fall through case Type::DECIMAL: // fall through - *output_type = PandasBlock::OBJECT; + *output_type = PandasWriter::OBJECT; break; case Type::DATE32: // fall through case Type::DATE64: - *output_type = options.date_as_object ? PandasBlock::OBJECT : PandasBlock::DATETIME; + if (options.date_as_object) { + *output_type = PandasWriter::OBJECT; + } else { + *output_type = options.coerce_temporal_nanoseconds ? PandasWriter::DATETIME_NANO + : PandasWriter::DATETIME_DAY; + } break; case Type::TIMESTAMP: { const auto& ts_type = checked_cast(*data.type()); if (ts_type.timezone() != "") { - *output_type = PandasBlock::DATETIME_WITH_TZ; + *output_type = PandasWriter::DATETIME_NANO_TZ; + } else if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::DATETIME_NANO; } else { - *output_type = PandasBlock::DATETIME; + switch (ts_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::DATETIME_SECOND; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::DATETIME_MILLI; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::DATETIME_MICRO; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::DATETIME_NANO; + break; + } + } + } break; + case Type::DURATION: { + const auto& dur_type = checked_cast(*data.type()); + if (options.coerce_temporal_nanoseconds) { + *output_type = PandasWriter::TIMEDELTA_NANO; + } else { + switch (dur_type.unit()) { + case TimeUnit::SECOND: + *output_type = PandasWriter::TIMEDELTA_SECOND; + break; + case TimeUnit::MILLI: + *output_type = PandasWriter::TIMEDELTA_MILLI; + break; + case TimeUnit::MICRO: + *output_type = PandasWriter::TIMEDELTA_MICRO; + break; + case TimeUnit::NANO: + *output_type = PandasWriter::TIMEDELTA_NANO; + break; + } } } break; - case Type::DURATION: - *output_type = PandasBlock::TIMEDELTA; - break; case Type::LIST: { auto list_type = std::static_pointer_cast(data.type()); if (!ListTypeSupported(*list_type->value_type())) { return Status::NotImplemented("Not implemented type for list in DataFrameBlock: ", list_type->value_type()->ToString()); } - *output_type = PandasBlock::OBJECT; + *output_type = PandasWriter::OBJECT; } break; case Type::DICTIONARY: - *output_type = PandasBlock::CATEGORICAL; + *output_type = PandasWriter::CATEGORICAL; + break; + case Type::EXTENSION: + *output_type = PandasWriter::EXTENSION; break; default: return Status::NotImplemented( @@ -1640,66 +1741,108 @@ static Status GetPandasBlockType(const ChunkedArray& data, const PandasOptions& return Status::OK(); } -// Construct the exact pandas 0.x "BlockManager" memory layout +// Construct the exact pandas "BlockManager" memory layout // // * For each column determine the correct output pandas type // * Allocate 2D blocks (ncols x nrows) for each distinct data type in output // * Allocate block placement arrays // * Write Arrow columns out into each slice of memory; populate block // * placement arrays as we go -class DataFrameBlockCreator { +class PandasBlockCreator { public: - explicit DataFrameBlockCreator(const PandasOptions& options, - const std::unordered_set& extension_columns, - const std::shared_ptr& table) - : table_(table), options_(options), extension_columns_(extension_columns) {} + using WriterMap = std::unordered_map>; + + explicit PandasBlockCreator(const PandasOptions& options, + std::vector> fields, + std::vector> arrays) + : options_(options), fields_(std::move(fields)), arrays_(std::move(arrays)) { + num_columns_ = static_cast(arrays_.size()); + if (num_columns_ > 0) { + num_rows_ = arrays_[0]->length(); + } + column_block_placement_.resize(num_columns_); + } + + virtual Status Convert(PyObject** out) = 0; - Status Convert(PyObject** output) { - column_types_.resize(table_->num_columns()); - column_block_placement_.resize(table_->num_columns()); - type_counts_.clear(); - blocks_.clear(); + Status AppendBlocks(const WriterMap& blocks, PyObject* list) { + for (const auto& it : blocks) { + PyObject* item; + RETURN_NOT_OK(it.second->GetDataFrameResult(&item)); + if (PyList_Append(list, item) < 0) { + RETURN_IF_PYERROR(); + } + + // ARROW-1017; PyList_Append increments object refcount + Py_DECREF(item); + } + return Status::OK(); + } + protected: + PandasOptions options_; + + std::vector> fields_; + std::vector> arrays_; + int num_columns_; + int64_t num_rows_; + + // column num -> relative placement within internal block + std::vector column_block_placement_; +}; + +class ConsolidatedBlockCreator : public PandasBlockCreator { + public: + using PandasBlockCreator::PandasBlockCreator; + + Status Convert(PyObject** out) override { + column_types_.resize(num_columns_); RETURN_NOT_OK(CreateBlocks()); RETURN_NOT_OK(WriteTableToBlocks()); + PyAcquireGIL lock; + + PyObject* result = PyList_New(0); + RETURN_IF_PYERROR(); - return GetResultList(output); + RETURN_NOT_OK(AppendBlocks(blocks_, result)); + RETURN_NOT_OK(AppendBlocks(singleton_blocks_, result)); + + *out = result; + return Status::OK(); + } + + Status GetBlockType(int column_index, PandasWriter::type* out) { + if (options_.extension_columns.count(fields_[column_index]->name())) { + *out = PandasWriter::EXTENSION; + return Status::OK(); + } else { + return GetPandasWriterType(*arrays_[column_index], options_, out); + } } Status CreateBlocks() { - for (int i = 0; i < table_->num_columns(); ++i) { - std::shared_ptr col = table_->column(i); - PandasBlock::type output_type = PandasBlock::OBJECT; - if (extension_columns_.count(table_->field(i)->name())) { - output_type = PandasBlock::EXTENSION; - } else { - RETURN_NOT_OK(GetPandasBlockType(*col, options_, &output_type)); - } + for (int i = 0; i < num_columns_; ++i) { + const DataType& type = *arrays_[i]->type(); + PandasWriter::type output_type; + RETURN_NOT_OK(GetBlockType(i, &output_type)); int block_placement = 0; - std::shared_ptr block; - if (output_type == PandasBlock::CATEGORICAL) { - block = std::make_shared(options_, table_->num_rows()); - categorical_blocks_[i] = block; - } else if (output_type == PandasBlock::DATETIME_WITH_TZ) { - const auto& ts_type = checked_cast(*col->type()); - block = std::make_shared(options_, ts_type.timezone(), - table_->num_rows()); - RETURN_NOT_OK(block->Allocate()); - datetimetz_blocks_[i] = block; - } else if (output_type == PandasBlock::EXTENSION) { - block = std::make_shared(options_, table_->num_rows(), 1); - RETURN_NOT_OK(block->Allocate()); - extension_blocks_[i] = block; + std::shared_ptr writer; + if (output_type == PandasWriter::CATEGORICAL || + output_type == PandasWriter::DATETIME_NANO_TZ || + output_type == PandasWriter::EXTENSION) { + RETURN_NOT_OK(MakeWriter(options_, output_type, type, num_rows_, + /*num_columns=*/1, &writer)); + singleton_blocks_[i] = writer; } else { - auto it = type_counts_.find(output_type); - if (it != type_counts_.end()) { + auto it = block_sizes_.find(output_type); + if (it != block_sizes_.end()) { block_placement = it->second; // Increment count - it->second += 1; + ++it->second; } else { // Add key to map - type_counts_[output_type] = 1; + block_sizes_[output_type] = 1; } } column_types_[i] = output_type; @@ -1707,559 +1850,220 @@ class DataFrameBlockCreator { } // Create normal non-categorical blocks - for (const auto& it : this->type_counts_) { - PandasBlock::type type = static_cast(it.first); - std::shared_ptr block; - RETURN_NOT_OK( - MakeBlock(this->options_, type, this->table_->num_rows(), it.second, &block)); - this->blocks_[type] = block; + for (const auto& it : this->block_sizes_) { + PandasWriter::type output_type = static_cast(it.first); + std::shared_ptr block; + RETURN_NOT_OK(MakeWriter(this->options_, output_type, /*unused*/ *null(), num_rows_, + it.second, &block)); + this->blocks_[output_type] = block; } return Status::OK(); } - Status GetBlock(int i, std::shared_ptr* block) { - PandasBlock::type output_type = this->column_types_[i]; - - if (output_type == PandasBlock::CATEGORICAL) { - auto it = this->categorical_blocks_.find(i); - if (it == this->blocks_.end()) { - return Status::KeyError("No categorical block allocated"); - } - *block = it->second; - } else if (output_type == PandasBlock::DATETIME_WITH_TZ) { - auto it = this->datetimetz_blocks_.find(i); - if (it == this->datetimetz_blocks_.end()) { - return Status::KeyError("No datetimetz block allocated"); - } - *block = it->second; - } else if (output_type == PandasBlock::EXTENSION) { - auto it = this->extension_blocks_.find(i); - if (it == this->extension_blocks_.end()) { - return Status::KeyError("No extension block allocated"); - } - *block = it->second; - } else { - auto it = this->blocks_.find(output_type); - if (it == this->blocks_.end()) { - return Status::KeyError("No block allocated"); - } - *block = it->second; + Status GetWriter(int i, std::shared_ptr* block) { + PandasWriter::type output_type = this->column_types_[i]; + switch (output_type) { + case PandasWriter::CATEGORICAL: + case PandasWriter::DATETIME_NANO_TZ: + case PandasWriter::EXTENSION: { + auto it = this->singleton_blocks_.find(i); + if (it == this->singleton_blocks_.end()) { + return Status::KeyError("No block allocated"); + } + *block = it->second; + } break; + default: + auto it = this->blocks_.find(output_type); + if (it == this->blocks_.end()) { + return Status::KeyError("No block allocated"); + } + *block = it->second; + break; } return Status::OK(); } Status WriteTableToBlocks() { auto WriteColumn = [this](int i) { - std::shared_ptr block; - RETURN_NOT_OK(this->GetBlock(i, &block)); - return block->Write(this->table_->column(i), i, this->column_block_placement_[i]); + std::shared_ptr block; + RETURN_NOT_OK(this->GetWriter(i, &block)); + // ARROW-3789 Use std::move on the array to permit self-destructing + return block->Write(std::move(arrays_[i]), i, this->column_block_placement_[i]); }; if (options_.use_threads) { - return ParallelFor(table_->num_columns(), WriteColumn); + return ParallelFor(num_columns_, WriteColumn); } else { - for (int i = 0; i < table_->num_columns(); ++i) { + for (int i = 0; i < num_columns_; ++i) { RETURN_NOT_OK(WriteColumn(i)); } return Status::OK(); } } - Status AppendBlocks(const BlockMap& blocks, PyObject* list) { - for (const auto& it : blocks) { - PyObject* item; - RETURN_NOT_OK(it.second->GetPyResult(&item)); - if (PyList_Append(list, item) < 0) { - RETURN_IF_PYERROR(); - } - - // ARROW-1017; PyList_Append increments object refcount - Py_DECREF(item); - } - return Status::OK(); - } - - Status GetResultList(PyObject** out) { - PyAcquireGIL lock; - - PyObject* result = PyList_New(0); - RETURN_IF_PYERROR(); - - RETURN_NOT_OK(AppendBlocks(blocks_, result)); - RETURN_NOT_OK(AppendBlocks(categorical_blocks_, result)); - RETURN_NOT_OK(AppendBlocks(datetimetz_blocks_, result)); - RETURN_NOT_OK(AppendBlocks(extension_blocks_, result)); - - *out = result; - return Status::OK(); - } - private: - std::shared_ptr
table_; - // column num -> block type id - std::vector column_types_; - - // column num -> relative placement within internal block - std::vector column_block_placement_; + std::vector column_types_; // block type -> type count - std::unordered_map type_counts_; - - PandasOptions options_; - std::unordered_set extension_columns_; + std::unordered_map block_sizes_; + std::unordered_map block_types_; // block type -> block - BlockMap blocks_; - - // column number -> categorical block - BlockMap categorical_blocks_; + WriterMap blocks_; - // column number -> datetimetz block - BlockMap datetimetz_blocks_; - - // column number -> extension block - BlockMap extension_blocks_; + WriterMap singleton_blocks_; }; -class ArrowDeserializer { +/// \brief Create blocks for pandas.DataFrame block manager using one block per +/// column strategy. This permits some zero-copy optimizations as well as the +/// ability for the table to "self-destruct" if selected by the user. +class SplitBlockCreator : public PandasBlockCreator { public: - ArrowDeserializer(const PandasOptions& options, - const std::shared_ptr& data, PyObject* py_ref) - : data_(data), options_(options), py_ref_(py_ref) {} - - Status AllocateOutput(int type) { - PyAcquireGIL lock; + using PandasBlockCreator::PandasBlockCreator; - npy_intp dims[1] = {data_->length()}; - PyArray_Descr* descr = internal::GetSafeNumPyDtype(type); - if (descr == nullptr) { - RETURN_IF_PYERROR(); - } - if (PyDataType_REFCHK(descr)) { - // ARROW-6876: if the array has refcounted items, let Numpy - // own the array memory so as to decref elements on array destruction - set_numpy_metadata(type, data_->type().get(), descr); - result_ = PyArray_SimpleNewFromDescr(1, dims, descr); - RETURN_IF_PYERROR(); + Status GetWriter(int i, std::shared_ptr* writer) { + PandasWriter::type output_type = PandasWriter::OBJECT; + const DataType& type = *arrays_[i]->type(); + if (options_.extension_columns.count(fields_[i]->name())) { + output_type = PandasWriter::EXTENSION; } else { - RETURN_NOT_OK(PyArray_NewFromPool(1, dims, descr, data_->type().get(), - options_.pool, &result_)); + // Null count needed to determine output type + RETURN_NOT_OK(GetPandasWriterType(*arrays_[i], options_, &output_type)); } - arr_ = reinterpret_cast(result_); - return Status::OK(); + return MakeWriter(this->options_, output_type, type, num_rows_, 1, writer); } - template - Status ConvertValuesZeroCopy(const PandasOptions& options, int npy_type, - const std::shared_ptr& arr) { - typedef typename internal::arrow_traits::T T; - - const T* in_values = GetPrimitiveValues(*arr); - - // Zero-Copy. We can pass the data pointer directly to NumPy. + Status Convert(PyObject** out) override { PyAcquireGIL lock; - PyArray_Descr* descr = internal::GetSafeNumPyDtype(npy_type); - npy_intp dims[1] = {arr->length()}; - set_numpy_metadata(npy_type, arr->type().get(), descr); - result_ = PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, - /*strides=*/nullptr, const_cast(in_values), - /*flags=*/0, nullptr); - arr_ = reinterpret_cast(result_); - - if (arr_ == nullptr) { - // Error occurred, trust that error set - return Status::OK(); - } - - // See ARROW-1973 for the original memory leak report. - // - // There are two scenarios: py_ref_ is nullptr or py_ref_ is not nullptr - // - // 1. py_ref_ is nullptr (it **was not** passed in to ArrowDeserializer's - // constructor) - // - // In this case, the stolen reference must not be incremented since nothing - // outside of the PyArrayObject* (the arr_ member) is holding a reference to - // it. If we increment this, then we have a memory leak. - // - // - // Here's an example of how memory can be leaked when converting an arrow Array - // of List.to a numpy array - // - // 1. Create a 1D numpy that is the flattened arrow array. - // - // There's nothing outside of the serializer that owns this new numpy array. - // - // 2. Make a capsule for the base array. - // - // The reference count of base is 1. - // - // 3. Call PyArray_SetBaseObject(arr_, base) - // - // The reference count is still 1, because the reference is stolen. - // - // 4. Increment the reference count of base (unconditionally) - // - // The reference count is now 2. This is okay if there's an object holding - // another reference. The PyArrayObject that stole the reference will - // eventually decrement the reference count, which will leaves us with a - // refcount of 1, with nothing owning that 1 reference. Memory leakage - // ensues. - // - // 2. py_ref_ is not nullptr (it **was** passed in to ArrowDeserializer's - // constructor) - // - // This case is simpler. We assume that the reference accounting is correct - // coming in. We need to preserve that accounting knowing that the - // PyArrayObject that stole the reference will eventually decref it, thus we - // increment the reference count. - - PyObject* base; - if (py_ref_ == nullptr) { - RETURN_NOT_OK(CapsulizeArray(arr, &base)); - } else { - base = py_ref_; - Py_INCREF(base); - } - - RETURN_NOT_OK(SetNdarrayBase(arr_, base)); - - // Arrow data is immutable. - PyArray_CLEARFLAGS(arr_, NPY_ARRAY_WRITEABLE); - - return Status::OK(); - } + PyObject* result = PyList_New(0); + RETURN_IF_PYERROR(); - // ---------------------------------------------------------------------- - // Allocate new array and deserialize. Can do a zero copy conversion for some - // types + for (int i = 0; i < num_columns_; ++i) { + std::shared_ptr writer; + RETURN_NOT_OK(GetWriter(i, &writer)); + // ARROW-3789 Use std::move on the array to permit self-destructing + RETURN_NOT_OK(writer->Write(std::move(arrays_[i]), i, /*rel_placement=*/0)); - template - enable_if_t::value, Status> Visit(const Type& type) { - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - - typedef typename traits::T T; - int npy_type = traits::npy_type; - - if (data_->num_chunks() == 1 && data_->null_count() == 0) { - return ConvertValuesZeroCopy(options_, npy_type, data_->chunk(0)); - } else if (options_.zero_copy_only) { - return Status::Invalid("Needed to copy ", data_->num_chunks(), " chunks with ", - data_->null_count(), " nulls, but zero_copy_only was True"); + PyObject* item; + RETURN_NOT_OK(writer->GetDataFrameResult(&item)); + if (PyList_Append(result, item) < 0) { + RETURN_IF_PYERROR(); + } + // PyList_Append increments object refcount + Py_DECREF(item); } - RETURN_NOT_OK(AllocateOutput(npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertNumericNullable(*data_, traits::na_value, out_values); - + *out = result; return Status::OK(); } - template - enable_if_t::value || is_duration_type::value, Status> - Visit(const Type& type) { - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - using c_type = typename Type::c_type; - - typedef typename traits::T T; - - if (data_->num_chunks() == 1 && data_->null_count() == 0) { - return ConvertValuesZeroCopy(options_, traits::npy_type, data_->chunk(0)); - } else if (options_.zero_copy_only) { - return Status::Invalid("Copy Needed, but zero_copy_only was True"); - } - - RETURN_NOT_OK(AllocateOutput(traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - - constexpr T na_value = traits::na_value; - constexpr int64_t kShift = traits::npy_shift; - - for (int c = 0; c < data_->num_chunks(); c++) { - const auto& arr = *data_->chunk(c); - const c_type* in_values = GetPrimitiveValues(arr); + private: + std::vector> writers_; +}; - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]) / kShift; - } +Status ConvertCategoricals(const PandasOptions& options, + std::vector>* arrays, + std::vector>* fields) { + std::vector columns_to_encode; + + // For Categorical conversions + auto EncodeColumn = [&](int j) { + int i = columns_to_encode[j]; + compute::FunctionContext ctx(options.pool); + compute::Datum out; + if (options.zero_copy_only) { + return Status::Invalid("Need to dictionary encode a column, but ", + "only zero-copy conversions allowed"); } + RETURN_NOT_OK(DictionaryEncode(&ctx, (*arrays)[i], &out)); + (*arrays)[i] = out.chunked_array(); + (*fields)[i] = (*fields)[i]->WithType((*arrays)[i]->type()); return Status::OK(); - } - - template - enable_if_date Visit(const Type& type) { - if (options_.zero_copy_only) { - return Status::Invalid("Copy Needed, but zero_copy_only was True"); - } - if (options_.date_as_object) { - return VisitObjects(ConvertDates); - } - - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - using c_type = typename Type::c_type; - - typedef typename traits::T T; - - RETURN_NOT_OK(AllocateOutput(traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - - constexpr T na_value = traits::na_value; - constexpr int64_t kShift = traits::npy_shift; - - for (int c = 0; c < data_->num_chunks(); c++) { - const auto& arr = *data_->chunk(c); - const c_type* in_values = GetPrimitiveValues(arr); + }; - for (int64_t i = 0; i < arr.length(); ++i) { - *out_values++ = arr.IsNull(i) ? na_value : static_cast(in_values[i]) / kShift; + if (!options.categorical_columns.empty()) { + for (int i = 0; i < static_cast(arrays->size()); i++) { + if ((*arrays)[i]->type()->id() != Type::DICTIONARY && + options.categorical_columns.count((*fields)[i]->name())) { + columns_to_encode.push_back(i); } } - return Status::OK(); - } - - template - enable_if_time Visit(const Type& type) { - return Status::NotImplemented("Don't know how to serialize Arrow time type to NumPy"); } - - // Integer specialization - template - enable_if_integer Visit(const Type& type) { - constexpr int TYPE = Type::type_id; - using traits = internal::arrow_traits; - - typedef typename traits::T T; - - if (data_->num_chunks() == 1 && data_->null_count() == 0) { - return ConvertValuesZeroCopy(options_, traits::npy_type, data_->chunk(0)); - } else if (options_.zero_copy_only) { - return Status::Invalid("Needed to copy ", data_->num_chunks(), " chunks with ", - data_->null_count(), " nulls, but zero_copy_only was True"); - } - - if (data_->null_count() > 0) { - if (options_.integer_object_nulls) { - return VisitObjects(ConvertIntegerObjects); - } else { - RETURN_NOT_OK(AllocateOutput(NPY_FLOAT64)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertIntegerWithNulls(options_, *data_, out_values); + if (options.strings_to_categorical) { + for (int i = 0; i < static_cast(arrays->size()); i++) { + if (is_base_binary_like((*arrays)[i]->type()->id())) { + columns_to_encode.push_back(i); } - } else { - RETURN_NOT_OK(AllocateOutput(traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertIntegerNoNullsSameType(options_, *data_, out_values); - } - - return Status::OK(); - } - - template - inline Status VisitObjects(FUNCTOR func) { - if (options_.zero_copy_only) { - return Status::Invalid("Object types need copies, but zero_copy_only was True"); - } - RETURN_NOT_OK(AllocateOutput(NPY_OBJECT)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - return func(options_, *data_, out_values); - } - - // Strings and binary - template - enable_if_base_binary Visit(const Type& type) { - return VisitObjects(ConvertBinaryLike); - } - - // Fixed length binary strings - Status Visit(const FixedSizeBinaryType& type) { - return VisitObjects(ConvertBinaryLike); - } - - Status Visit(const NullType& type) { return VisitObjects(ConvertNulls); } - - Status Visit(const Decimal128Type& type) { return VisitObjects(ConvertDecimals); } - - Status Visit(const Time32Type& type) { return VisitObjects(ConvertTimes); } - - Status Visit(const Time64Type& type) { return VisitObjects(ConvertTimes); } - - Status Visit(const StructType& type) { return VisitObjects(ConvertStruct); } - - // Boolean specialization - Status Visit(const BooleanType& type) { - if (options_.zero_copy_only) { - return Status::Invalid("BooleanType needs copies, but zero_copy_only was True"); - } else if (data_->null_count() > 0) { - return VisitObjects(ConvertBooleanWithNulls); - } else { - RETURN_NOT_OK(AllocateOutput(internal::arrow_traits::npy_type)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - ConvertBooleanNoNulls(options_, *data_, out_values); } - return Status::OK(); } - - Status Visit(const ListType& type) { - if (options_.zero_copy_only) { - return Status::Invalid("ListType needs copies, but zero_copy_only was True"); + if (options.use_threads) { + return ParallelFor(static_cast(columns_to_encode.size()), EncodeColumn); + } else { + for (auto i : columns_to_encode) { + RETURN_NOT_OK(EncodeColumn(i)); } -#define CONVERTVALUES_LISTSLIKE_CASE(ArrowType, ArrowEnum) \ - case Type::ArrowEnum: \ - return ConvertListsLike(options_, *data_, out_values); - - RETURN_NOT_OK(AllocateOutput(NPY_OBJECT)); - auto out_values = reinterpret_cast(PyArray_DATA(arr_)); - auto list_type = std::static_pointer_cast(data_->type()); - switch (list_type->value_type()->id()) { - CONVERTVALUES_LISTSLIKE_CASE(BooleanType, BOOL) - CONVERTVALUES_LISTSLIKE_CASE(UInt8Type, UINT8) - CONVERTVALUES_LISTSLIKE_CASE(Int8Type, INT8) - CONVERTVALUES_LISTSLIKE_CASE(UInt16Type, UINT16) - CONVERTVALUES_LISTSLIKE_CASE(Int16Type, INT16) - CONVERTVALUES_LISTSLIKE_CASE(UInt32Type, UINT32) - CONVERTVALUES_LISTSLIKE_CASE(Int32Type, INT32) - CONVERTVALUES_LISTSLIKE_CASE(UInt64Type, UINT64) - CONVERTVALUES_LISTSLIKE_CASE(Int64Type, INT64) - CONVERTVALUES_LISTSLIKE_CASE(Date32Type, DATE32) - CONVERTVALUES_LISTSLIKE_CASE(Date64Type, DATE64) - CONVERTVALUES_LISTSLIKE_CASE(Time32Type, TIME32) - CONVERTVALUES_LISTSLIKE_CASE(Time64Type, TIME64) - CONVERTVALUES_LISTSLIKE_CASE(TimestampType, TIMESTAMP) - CONVERTVALUES_LISTSLIKE_CASE(DurationType, DURATION) - CONVERTVALUES_LISTSLIKE_CASE(FloatType, FLOAT) - CONVERTVALUES_LISTSLIKE_CASE(DoubleType, DOUBLE) - CONVERTVALUES_LISTSLIKE_CASE(BinaryType, BINARY) - CONVERTVALUES_LISTSLIKE_CASE(StringType, STRING) - CONVERTVALUES_LISTSLIKE_CASE(Decimal128Type, DECIMAL) - CONVERTVALUES_LISTSLIKE_CASE(ListType, LIST) - default: { - return Status::NotImplemented("Not implemented type for lists: ", - list_type->value_type()->ToString()); - } - } -#undef CONVERTVALUES_LISTSLIKE_CASE - } - - Status Visit(const DictionaryType& type) { - auto block = std::make_shared(options_, data_->length()); - RETURN_NOT_OK(block->Write(data_, 0, 0)); - - PyAcquireGIL lock; - result_ = PyDict_New(); - RETURN_IF_PYERROR(); - - PyDict_SetItemString(result_, "indices", block->block_arr()); - RETURN_IF_PYERROR(); - PyDict_SetItemString(result_, "dictionary", block->dictionary()); - RETURN_IF_PYERROR(); - - PyObject* py_ordered = type.ordered() ? Py_True : Py_False; - Py_INCREF(py_ordered); - PyDict_SetItemString(result_, "ordered", py_ordered); - RETURN_IF_PYERROR(); - return Status::OK(); } +} - Status Visit(const ExtensionType& type) { - auto storage_type = type.storage_type(); +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out) { + return ConvertChunkedArrayToPandas( + options, std::make_shared(std::move(arr)), py_ref, out); +} - ArrayVector out_chunks(data_->num_chunks()); - for (int i = 0; i < data_->num_chunks(); i++) { - auto chunk = data_->chunk(i); - auto storage_data = checked_cast(*chunk).storage(); - out_chunks[i] = storage_data; +Status ConvertChunkedArrayToPandas(const PandasOptions& options, + std::shared_ptr arr, PyObject* py_ref, + PyObject** out) { + if (options.strings_to_categorical && is_base_binary_like(arr->type()->id())) { + compute::FunctionContext ctx(options.pool); + compute::Datum out; + if (options.zero_copy_only) { + return Status::Invalid("Need to dictionary encode a column, but ", + "only zero-copy conversions allowed"); } - - data_ = std::make_shared(out_chunks); - - RETURN_NOT_OK(VisitTypeInline(*data_->type(), this)); - return Status::OK(); + RETURN_NOT_OK(DictionaryEncode(&ctx, arr, &out)); + arr = out.chunked_array(); } - Status Visit(const FixedSizeListType& type) { return NotImplemented(type); } - Status Visit(const LargeListType& type) { return NotImplemented(type); } - Status Visit(const UnionType& type) { return NotImplemented(type); } - Status Visit(const DayTimeIntervalType& type) { return NotImplemented(type); } - Status Visit(const MonthIntervalType& type) { return NotImplemented(type); } + PandasOptions modified_options = options; + modified_options.strings_to_categorical = false; - Status NotImplemented(const DataType& type) { - return Status::NotImplemented( - "Conversion from arrow to pandas is not implemented for type ", type.name()); - } + PandasWriter::type output_type; + RETURN_NOT_OK(GetPandasWriterType(*arr, modified_options, &output_type)); - Status Convert(PyObject** out) { - RETURN_NOT_OK(VisitTypeInline(*data_->type(), this)); - *out = result_; - return Status::OK(); - } - - private: - std::shared_ptr data_; - PandasOptions options_; - PyObject* py_ref_; - PyArrayObject* arr_; - PyObject* result_; -}; + std::shared_ptr writer; + RETURN_NOT_OK(MakeWriter(modified_options, output_type, *arr->type(), arr->length(), + /*num_columns=*/1, &writer)); + RETURN_NOT_OK(writer->TransferSingle(std::move(arr), py_ref)); + return writer->GetSeriesResult(out); +} -Status ConvertArrayToPandas(const PandasOptions& options, - const std::shared_ptr& arr, PyObject* py_ref, +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, PyObject** out) { - auto carr = std::make_shared(arr); - return ConvertChunkedArrayToPandas(options, carr, py_ref, out); -} + std::vector> arrays = table->columns(); + std::vector> fields = table->fields(); -Status ConvertChunkedArrayToPandas(const PandasOptions& options, - const std::shared_ptr& ca, - PyObject* py_ref, PyObject** out) { - ArrowDeserializer converter(options, ca, py_ref); - return converter.Convert(out); -} + // ARROW-3789: allow "self-destructing" by releasing references to columns as + // we convert them to pandas + table = nullptr; -Status ConvertTableToPandas(const PandasOptions& options, - const std::shared_ptr
& table, PyObject** out) { - return ConvertTableToPandas(options, std::unordered_set(), table, out); -} + RETURN_NOT_OK(ConvertCategoricals(options, &arrays, &fields)); -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::shared_ptr
& table, PyObject** out) { - return ConvertTableToPandas(options, categorical_columns, - std::unordered_set(), table, out); -} + PandasOptions modified_options = options; + modified_options.strings_to_categorical = false; + modified_options.categorical_columns.clear(); -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::unordered_set& extension_columns, - const std::shared_ptr
& table, PyObject** out) { - std::shared_ptr
current_table = table; - if (!categorical_columns.empty()) { - FunctionContext ctx; - for (int i = 0; i < table->num_columns(); i++) { - std::shared_ptr col = table->column(i); - if (col->type()->id() == Type::DICTIONARY) { - // No need to dictionary encode again. Came up in ARROW-6434, - // ARROW-6435 - continue; - } - if (categorical_columns.count(table->field(i)->name())) { - Datum out; - RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out)); - std::shared_ptr array = out.chunked_array(); - auto field = table->field(i)->WithType(array->type()); - RETURN_NOT_OK(current_table->RemoveColumn(i, ¤t_table)); - RETURN_NOT_OK(current_table->AddColumn(i, field, array, ¤t_table)); - } - } + if (options.split_blocks) { + SplitBlockCreator helper(modified_options, std::move(fields), std::move(arrays)); + return helper.Convert(out); + } else { + ConsolidatedBlockCreator helper(modified_options, std::move(fields), + std::move(arrays)); + return helper.Convert(out); } - - DataFrameBlockCreator helper(options, extension_columns, current_table); - return helper.Convert(out); } } // namespace py diff --git a/cpp/src/arrow/python/arrow_to_pandas.h b/cpp/src/arrow/python/arrow_to_pandas.h index 1196a92432d1..042745b06812 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.h +++ b/cpp/src/arrow/python/arrow_to_pandas.h @@ -18,8 +18,7 @@ // Functions for converting between pandas's NumPy-based data representation // and Arrow data structures -#ifndef ARROW_PYTHON_ADAPTERS_PANDAS_H -#define ARROW_PYTHON_ADAPTERS_PANDAS_H +#pragma once #include "arrow/python/platform.h" @@ -53,26 +52,43 @@ struct PandasOptions { bool date_as_object = false; bool use_threads = false; + /// Coerce all date and timestamp to datetime64[ns] + bool coerce_temporal_nanoseconds = false; + /// \brief If true, do not create duplicate PyObject versions of equal /// objects. This only applies to immutable objects like strings or datetime /// objects bool deduplicate_objects = false; + + /// \brief If true, create one block per column rather than consolidated + /// blocks (1 per data type). Do zero-copy wrapping when there are no + /// nulls. pandas currently will consolidate the blocks on its own, causing + /// increased memory use, so keep this in mind if you are working on a + /// memory-constrained situation. + bool split_blocks = false; + + /// \brief If true, attempt to deallocate buffers in passed Arrow object if + /// it is the only remaining shared_ptr copy of it. See ARROW-3789 for + /// original context for this feature. Only currently implemented for Table + /// conversions + bool self_destruct = false; + + // Columns that should be casted to categorical + std::unordered_set categorical_columns; + + // Columns that should be passed through to be converted to + // ExtensionArray/Block + std::unordered_set extension_columns; }; ARROW_PYTHON_EXPORT -Status ConvertArrayToPandas(const PandasOptions& options, - const std::shared_ptr& arr, PyObject* py_ref, - PyObject** out); +Status ConvertArrayToPandas(const PandasOptions& options, std::shared_ptr arr, + PyObject* py_ref, PyObject** out); ARROW_PYTHON_EXPORT Status ConvertChunkedArrayToPandas(const PandasOptions& options, - const std::shared_ptr& col, - PyObject* py_ref, PyObject** out); - -ARROW_PYTHON_EXPORT -Status ConvertColumnToPandas(const PandasOptions& options, - const std::shared_ptr& col, PyObject* py_ref, - PyObject** out); + std::shared_ptr col, PyObject* py_ref, + PyObject** out); // Convert a whole table as efficiently as possible to a pandas.DataFrame. // @@ -81,25 +97,8 @@ Status ConvertColumnToPandas(const PandasOptions& options, // // tuple item: (indices: ndarray[int32], block: ndarray[TYPE, ndim=2]) ARROW_PYTHON_EXPORT -Status ConvertTableToPandas(const PandasOptions& options, - const std::shared_ptr
& table, PyObject** out); - -/// Convert a whole table as efficiently as possible to a pandas.DataFrame. -/// -/// Explicitly name columns that should be a categorical -/// This option is only used on conversions that are applied to a table. -ARROW_PYTHON_EXPORT -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::shared_ptr
& table, PyObject** out); - -ARROW_PYTHON_EXPORT -Status ConvertTableToPandas(const PandasOptions& options, - const std::unordered_set& categorical_columns, - const std::unordered_set& extension_columns, - const std::shared_ptr
& table, PyObject** out); +Status ConvertTableToPandas(const PandasOptions& options, std::shared_ptr
table, + PyObject** out); } // namespace py } // namespace arrow - -#endif // ARROW_PYTHON_ADAPTERS_PANDAS_H diff --git a/cpp/src/arrow/python/type_traits.h b/cpp/src/arrow/python/type_traits.h index b4a6a974c258..2ce66d54fca2 100644 --- a/cpp/src/arrow/python/type_traits.h +++ b/cpp/src/arrow/python/type_traits.h @@ -32,6 +32,10 @@ namespace arrow { namespace py { + +static constexpr int64_t kPandasTimestampNull = std::numeric_limits::min(); +constexpr int64_t kNanosecondsInDay = 86400000000000LL; + namespace internal { // @@ -86,6 +90,8 @@ struct npy_traits { using TypeClass = HalfFloatType; using BuilderClass = HalfFloatBuilder; + static constexpr npy_half na_sentinel = NPY_HALF_NAN; + static constexpr bool supports_nulls = true; static inline bool isnull(npy_half v) { return v == NPY_HALF_NAN; } @@ -97,6 +103,8 @@ struct npy_traits { using TypeClass = FloatType; using BuilderClass = FloatBuilder; + static constexpr float na_sentinel = NAN; + static constexpr bool supports_nulls = true; static inline bool isnull(float v) { return v != v; } @@ -108,6 +116,8 @@ struct npy_traits { using TypeClass = DoubleType; using BuilderClass = DoubleBuilder; + static constexpr double na_sentinel = NAN; + static constexpr bool supports_nulls = true; static inline bool isnull(double v) { return v != v; } @@ -208,10 +218,6 @@ struct arrow_traits { typedef typename npy_traits::value_type T; }; -static constexpr int64_t kPandasTimestampNull = std::numeric_limits::min(); - -constexpr int64_t kNanosecondsInDay = 86400000000000LL; - template <> struct arrow_traits { static constexpr int npy_type = NPY_DATETIME; @@ -287,6 +293,21 @@ struct arrow_traits { static constexpr bool supports_nulls = true; }; +static inline NPY_DATETIMEUNIT NumPyFrequency(TimeUnit::type unit) { + switch (unit) { + case TimestampType::Unit::SECOND: + return NPY_FR_s; + case TimestampType::Unit::MILLI: + return NPY_FR_ms; + break; + case TimestampType::Unit::MICRO: + return NPY_FR_us; + default: + // NANO + return NPY_FR_ns; + } +} + static inline int NumPyTypeSize(int npy_type) { npy_type = fix_numpy_type_num(npy_type); diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc index 2690b5282968..74c1c2633819 100644 --- a/cpp/src/arrow/table.cc +++ b/cpp/src/arrow/table.cc @@ -425,6 +425,22 @@ class SimpleTable : public Table { Table::Table() : num_rows_(0) {} +std::vector> Table::columns() const { + std::vector> result; + for (int i = 0; i < this->num_columns(); ++i) { + result.emplace_back(this->column(i)); + } + return result; +} + +std::vector> Table::fields() const { + std::vector> result; + for (int i = 0; i < this->num_columns(); ++i) { + result.emplace_back(this->field(i)); + } + return result; +} + std::shared_ptr
Table::Make( const std::shared_ptr& schema, const std::vector>& columns, int64_t num_rows) { diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h index 06acdcd58b7b..b5e3ee1f135b 100644 --- a/cpp/src/arrow/table.h +++ b/cpp/src/arrow/table.h @@ -185,15 +185,21 @@ class ARROW_EXPORT Table { static Status FromChunkedStructArray(const std::shared_ptr& array, std::shared_ptr
* table); - /// Return the table schema + /// \brief Return the table schema std::shared_ptr schema() const { return schema_; } - /// Return a column by index + /// \brief Return a column by index virtual std::shared_ptr column(int i) const = 0; + /// \brief Return vector of all columns for table + std::vector> columns() const; + /// Return a column's field by index std::shared_ptr field(int i) const { return schema_->field(i); } + /// \brief Return vector of all fields for table + std::vector> fields() const; + /// \brief Construct a zero-copy slice of the table with the /// indicated offset and length /// diff --git a/cpp/src/arrow/table_test.cc b/cpp/src/arrow/table_test.cc index ac1e41218c3f..934dc69731f9 100644 --- a/cpp/src/arrow/table_test.cc +++ b/cpp/src/arrow/table_test.cc @@ -273,6 +273,29 @@ TEST_F(TestTable, InvalidColumns) { ASSERT_RAISES(Invalid, table_->ValidateFull()); } +TEST_F(TestTable, AllColumnsAndFields) { + const int length = 100; + MakeExample1(length); + table_ = Table::Make(schema_, columns_); + + auto columns = table_->columns(); + auto fields = table_->fields(); + + for (int i = 0; i < table_->num_columns(); ++i) { + AssertChunkedEqual(*table_->column(i), *columns[i]); + AssertFieldEqual(*table_->field(i), *fields[i]); + } + + // Zero length + std::vector> t2_columns; + auto t2 = Table::Make(::arrow::schema({}), t2_columns); + columns = t2->columns(); + fields = t2->fields(); + + ASSERT_EQ(0, columns.size()); + ASSERT_EQ(0, fields.size()); +} + TEST_F(TestTable, Equals) { const int length = 100; MakeExample1(length); diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index 24b58b155d3e..1d82b4c23a94 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -703,6 +703,19 @@ static inline bool is_primitive(Type::type type_id) { return false; } +static inline bool is_base_binary_like(Type::type type_id) { + switch (type_id) { + case Type::BINARY: + case Type::LARGE_BINARY: + case Type::STRING: + case Type::LARGE_STRING: + return true; + default: + break; + } + return false; +} + static inline bool is_binary_like(Type::type type_id) { switch (type_id) { case Type::BINARY: diff --git a/docs/source/python/pandas.rst b/docs/source/python/pandas.rst index 83d997c86f79..5f93f6355172 100644 --- a/docs/source/python/pandas.rst +++ b/docs/source/python/pandas.rst @@ -221,3 +221,75 @@ Time types ~~~~~~~~~~ TODO + +Memory Usage and Zero Copy +-------------------------- + +When converting from Arrow data structures to pandas objects using various +``to_pandas`` methods, one must occasionally be mindful of issues related to +performance and memory usage. + +Since pandas's internal data representation is generally different from the +Arrow columnar format, zero copy conversions (where no memory allocation or +computation is required) are only possible in certain limited cases. + +In the worst case scenario, calling ``to_pandas`` will result in two versions +of the data in memory, one for Arrow and one for pandas, yielding approximately +twice the memory footprint. We have implement some mitigations for this case, +particularly when creating large ``DataFrame`` objects, that we describe below. + +Zero Copy Series Conversions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Zero copy conversions from ``Array`` or ``ChunkedArray`` to NumPy arrays or +pandas Series are possible in certain narrow cases: + +* The Arrow data is stored in an integer (signed or unsigned ``int8`` through + ``int64``) or floating point type (``float16`` through ``float64``). This + includes many numeric types as well as timestamps. +* The Arrow data has no null values (since these are represented using bitmaps + which are not supported by pandas). +* For ``ChunkedArray``, the data consists of a single chunk, + i.e. ``arr.num_chunks == 1``. Multiple chunks will always require a copy + because of pandas's contiguousness requirement. + +In these scenarios, ``to_pandas`` or ``to_numpy`` will be zero copy. In all +other scenarios, a copy will be required. + +Reducing Memory Use in ``Table.to_pandas`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As of this writing, pandas applies a data management strategy called +"consolidation" to collect like-typed DataFrame columns in two-dimensional +NumPy arrays, referred to internally as "blocks". We have gone to great effort +to construct the precise "consolidated" blocks so that pandas will not perform +any further allocation or copies after we hand off the data to +``pandas.DataFrame``. The obvious downside of this consolidation strategy is +that it forces a "memory doubling". + +To try to limit the potential effects of "memory doubling" during +``Table.to_pandas``, we provide a couple of options: + +* ``split_blocks=True``, when enabled ``Table.to_pandas`` produces one internal + DataFrame "block" for each column, skipping the "consolidation" step. Note + that many pandas operations will trigger consolidation anyway, but the peak + memory use may be less than the worst case scenario of a full memory + doubling. As a result of this option, we are able to do zero copy conversions + of columns in the same cases where we can do zero copy with ``Array`` and + ``ChunkedArray``. +* ``self_destruct=True``, this destroys the internal Arrow memory buffers in + each column ``Table`` object as they are converted to the pandas-compatible + representation, potentially releasing memory to the operating system as soon + as a column is converted. Note that this renders the calling ``Table`` object + unsafe for further use, and any further methods called will cause your Python + process to crash. + +Used together, the call + +.. code-block:: python + + df = table.to_pandas(split_blocks=True, self_destruct=True) + del table # not necessary, but a good practice + +will yield significantly lower memory usage in some scenarios. Without these +options, ``to_pandas`` will always double memory. diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index c4598155d1dd..f99af9853e51 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -489,7 +489,9 @@ cdef class _PandasConvertible: bint date_as_object=True, bint use_threads=True, bint deduplicate_objects=True, - bint ignore_metadata=False + bint ignore_metadata=False, + bint split_blocks=False, + bint self_destruct=False ): """ Convert to a pandas-compatible NumPy array or DataFrame, as appropriate @@ -519,6 +521,16 @@ cdef class _PandasConvertible: ignore_metadata : boolean, default False If True, do not use the 'pandas' metadata to reconstruct the DataFrame index, if present + split_blocks : boolean, default False + If True, generate one internal "block" for each column when + creating a pandas.DataFrame from a RecordBatch or Table. While this + can temporarily reduce memory note that various pandas operations + can trigger "consolidation" which may balloon memory use + self_destruct : boolean, default False + EXPERIMENTAL: If True, attempt to deallocate the originating Arrow + memory while converting the Arrow object to pandas. If you use the + object after calling to_pandas with this option it will crash your + program Returns ------- @@ -531,7 +543,9 @@ cdef class _PandasConvertible: integer_object_nulls=integer_object_nulls, date_as_object=date_as_object, use_threads=use_threads, - deduplicate_objects=deduplicate_objects + deduplicate_objects=deduplicate_objects, + split_blocks=split_blocks, + self_destruct=self_destruct ) return self._to_pandas(options, categories=categories, ignore_metadata=ignore_metadata) @@ -546,6 +560,8 @@ cdef PandasOptions _convert_pandas_options(dict options): result.date_as_object = options['date_as_object'] result.use_threads = options['use_threads'] result.deduplicate_objects = options['deduplicate_objects'] + result.split_blocks = options['split_blocks'] + result.self_destruct = options['self_destruct'] return result @@ -974,46 +990,12 @@ cdef class Array(_PandasConvertible): return wrap_datum(out) def _to_pandas(self, options, **kwargs): - cdef: - PyObject* out - PandasOptions c_options = _convert_pandas_options(options) - Array array - - if self.type.id == _Type_TIMESTAMP and self.type.unit != 'ns': - # pandas only stores ns data - casting here is faster - array = self.cast(timestamp('ns')) - else: - array = self - - with nogil: - check_status(ConvertArrayToPandas(c_options, array.sp_array, - array, &out)) - result = pandas_api.series(wrap_array_output(out), name=self._name) - - if isinstance(self.type, TimestampType) and self.type.tz is not None: - from pyarrow.pandas_compat import make_tz_aware - - result = make_tz_aware(result, self.type.tz) - - return result + return _array_like_to_pandas(self, options) def __array__(self, dtype=None): - cdef: - PyObject* out - PandasOptions c_options - object values - - with nogil: - check_status(ConvertArrayToPandas(c_options, self.sp_array, - self, &out)) - - # wrap_array_output uses pandas to convert to Categorical, here - # always convert to numpy array - values = PyObject_to_object(out) - + values = self.to_numpy(zero_copy_only=False) if isinstance(values, dict): values = np.take(values['dictionary'], values['indices']) - if dtype is None: return values return values.astype(dtype) @@ -1057,6 +1039,7 @@ cdef class Array(_PandasConvertible): check_status(ConvertArrayToPandas(c_options, self.sp_array, self, &out)) array = PyObject_to_object(out) + if writable and not array.flags.writeable: # if the conversion already needed to a copy, writeable is True array = array.copy() @@ -1117,6 +1100,44 @@ cdef class Array(_PandasConvertible): return res +cdef _array_like_to_pandas(obj, options): + cdef: + PyObject* out + PandasOptions c_options = _convert_pandas_options(options) + + original_type = obj.type + + if obj.type.id == _Type_TIMESTAMP and obj.type.unit != 'ns': + # pandas only stores ns data - casting here is faster + obj = obj.cast(timestamp('ns')) + + # ARROW-3789(wesm): when converting to DataFrame we coerce + # date/timestamp types to nanoseconds. Not consistent but we should + # make it so in the future + c_options.coerce_temporal_nanoseconds = False + + if isinstance(obj, Array): + with nogil: + check_status(ConvertArrayToPandas(c_options, + ( obj).sp_array, + obj, &out)) + elif isinstance(obj, ChunkedArray): + with nogil: + check_status(libarrow.ConvertChunkedArrayToPandas( + c_options, + ( obj).sp_chunked_array, + obj, &out)) + + result = pandas_api.series(wrap_array_output(out), name=obj._name) + + if (isinstance(original_type, TimestampType) and + original_type.tz is not None): + from pyarrow.pandas_compat import make_tz_aware + result = make_tz_aware(result, original_type.tz) + + return result + + cdef wrap_array_output(PyObject* output): cdef object obj = PyObject_to_object(output) @@ -1957,6 +1978,18 @@ cdef class ExtensionArray(Array): result.validate() return result + def _to_pandas(self, options, **kwargs): + result = Array._to_pandas(self, options, **kwargs) + # TODO(wesm): is passing through these parameters to the storage array + # correct? + return result.to_pandas(options, **kwargs) + + def to_numpy(self, **kwargs): + """ + See Array.to_numpy + """ + return self.storage.to_numpy(**kwargs) + cdef dict _array_classes = { _Type_NA: NullArray, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 08872dab955f..95c21d7ae90e 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1513,19 +1513,16 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CSparseCSRMatrix]* out) CStatus ConvertArrayToPandas(const PandasOptions& options, - const shared_ptr[CArray]& arr, + shared_ptr[CArray] arr, object py_ref, PyObject** out) CStatus ConvertChunkedArrayToPandas(const PandasOptions& options, - const shared_ptr[CChunkedArray]& arr, + shared_ptr[CChunkedArray] arr, object py_ref, PyObject** out) - CStatus ConvertTableToPandas( - const PandasOptions& options, - const unordered_set[c_string]& categorical_columns, - const unordered_set[c_string]& extension_columns, - const shared_ptr[CTable]& table, - PyObject** out) + CStatus ConvertTableToPandas(const PandasOptions& options, + shared_ptr[CTable] table, + PyObject** out) void c_set_default_memory_pool \ " arrow::py::set_default_memory_pool"(CMemoryPool* pool)\ @@ -1555,7 +1552,12 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: c_bool integer_object_nulls c_bool date_as_object c_bool use_threads + c_bool coerce_temporal_nanoseconds c_bool deduplicate_objects + c_bool split_blocks + c_bool self_destruct + unordered_set[c_string] categorical_columns + unordered_set[c_string] extension_columns cdef cppclass CSerializedPyObject" arrow::py::SerializedPyObject": shared_ptr[CRecordBatch] batch diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index c632e8b9c81f..bab12bf1de28 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -766,8 +766,8 @@ def table_to_blockmanager(options, table, categories=None, table, [], extension_columns) _check_data_column_metadata_consistency(all_columns) - blocks = _table_to_blocks(options, table, categories, ext_columns_dtypes) columns = _deserialize_column_index(table, all_columns, column_indexes) + blocks = _table_to_blocks(options, table, categories, ext_columns_dtypes) axes = [columns, index] return BlockManager(blocks, axes) @@ -1108,11 +1108,9 @@ def _table_to_blocks(options, block_table, categories, extension_columns): # Part of table_to_blockmanager # Convert an arrow table to Block from the internal pandas API + columns = block_table.column_names result = pa.lib.table_to_blocks(options, block_table, categories, list(extension_columns.keys())) - - # Defined above - columns = block_table.column_names return [_reconstruct_block(item, columns, extension_columns) for item in result] diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5d18b349f777..2f23a24403f5 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -183,31 +183,7 @@ cdef class ChunkedArray(_PandasConvertible): return result def _to_pandas(self, options, **kwargs): - cdef: - PyObject* out - PandasOptions c_options = _convert_pandas_options(options) - ChunkedArray array - - if self.type.id == _Type_TIMESTAMP and self.type.unit != 'ns': - # pandas only stores ns data - casting here is faster - array = self.cast(timestamp('ns')) - else: - array = self - - with nogil: - check_status(libarrow.ConvertChunkedArrayToPandas( - c_options, - array.sp_chunked_array, - array, &out)) - - result = pandas_api.series(wrap_array_output(out), name=self._name) - - if isinstance(self.type, TimestampType) and self.type.tz is not None: - from pyarrow.pandas_compat import make_tz_aware - - result = make_tz_aware(result, self.type.tz) - - return result + return _array_like_to_pandas(self, options) def __array__(self, dtype=None): cdef: @@ -215,6 +191,13 @@ cdef class ChunkedArray(_PandasConvertible): PandasOptions c_options object values + if self.type.id == _Type_EXTENSION: + return ( + chunked_array( + [self.chunk(i).storage for i in range(self.num_chunks)] + ).__array__(dtype) + ) + with nogil: check_status(libarrow.ConvertChunkedArrayToPandas( c_options, @@ -874,22 +857,30 @@ def _reconstruct_record_batch(columns, schema): def table_to_blocks(options, Table table, categories, extension_columns): cdef: PyObject* result_obj - shared_ptr[CTable] c_table = table.sp_table + shared_ptr[CTable] c_table CMemoryPool* pool - unordered_set[c_string] categorical_columns PandasOptions c_options = _convert_pandas_options(options) - unordered_set[c_string] c_extension_columns if categories is not None: - categorical_columns = {tobytes(cat) for cat in categories} + c_options.categorical_columns = {tobytes(cat) for cat in categories} if extension_columns is not None: - c_extension_columns = {tobytes(col) for col in extension_columns} + c_options.extension_columns = {tobytes(col) + for col in extension_columns} + + # ARROW-3789(wesm); Convert date/timestamp types to datetime64[ns] + c_options.coerce_temporal_nanoseconds = True + + if c_options.self_destruct: + # Move the shared_ptr, table is now unsafe to use further + c_table = move(table.sp_table) + table.table = NULL + else: + c_table = table.sp_table with nogil: check_status( - libarrow.ConvertTableToPandas( - c_options, categorical_columns, c_extension_columns, c_table, - &result_obj) + libarrow.ConvertTableToPandas(c_options, move(c_table), + &result_obj) ) return PyObject_to_object(result_obj) @@ -913,6 +904,9 @@ cdef class Table(_PandasConvertible): "the `Table.from_*` functions instead.") def __repr__(self): + if self.table == NULL: + raise ValueError("Table's internal pointer is NULL, do not use " + "any methods or attributes on this object") return 'pyarrow.{}\n{}'.format(type(self).__name__, str(self.schema)) cdef void init(self, const shared_ptr[CTable]& table): diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index df64d2a1b592..61d09e2b20ba 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -75,7 +75,7 @@ def _alltypes_example(size=100): }) -def _check_pandas_roundtrip(df, expected=None, use_threads=True, +def _check_pandas_roundtrip(df, expected=None, use_threads=False, expected_schema=None, check_dtype=True, schema=None, preserve_index=False, @@ -3038,15 +3038,69 @@ def test_array_uses_memory_pool(): assert pa.total_allocated_bytes() == prior_allocation +def test_singleton_blocks_zero_copy(): + # Part of ARROW-3789 + t = pa.table([pa.array(np.arange(1000, dtype=np.int64))], ['f0']) + _check_to_pandas_memory_unchanged(t) + + +def _check_to_pandas_memory_unchanged(obj, **kwargs): + prior_allocation = pa.total_allocated_bytes() + x = obj.to_pandas(**kwargs) # noqa + + # Memory allocation unchanged -- either zero copy or self-destructing + assert pa.total_allocated_bytes() == prior_allocation + + +def test_to_pandas_split_blocks(): + # ARROW-3789 + t = pa.table([ + pa.array([1, 2, 3, 4, 5], type='i1'), + pa.array([1, 2, 3, 4, 5], type='i4'), + pa.array([1, 2, 3, 4, 5], type='i8'), + pa.array([1, 2, 3, 4, 5], type='f4'), + pa.array([1, 2, 3, 4, 5], type='f8'), + pa.array([1, 2, 3, 4, 5], type='f8'), + pa.array([1, 2, 3, 4, 5], type='f8'), + pa.array([1, 2, 3, 4, 5], type='f8'), + ], ['f{}'.format(i) for i in range(8)]) + + _check_blocks_created(t, 8) + _check_to_pandas_memory_unchanged(t, split_blocks=True) + + +def _check_blocks_created(t, number): + x = t.to_pandas(split_blocks=True) + assert len(x._data.blocks) == number + + +def test_to_pandas_self_destruct(): + K = 50 + + def _make_table(): + return pa.table([ + # Slice to force a copy + pa.array(np.random.randn(10000)[::2]) + for i in range(K) + ], ['f{}'.format(i) for i in range(K)]) + + t = _make_table() + _check_to_pandas_memory_unchanged(t, split_blocks=True, self_destruct=True) + + # Check non-split-block behavior + t = _make_table() + _check_to_pandas_memory_unchanged(t, self_destruct=True) + + def test_table_uses_memory_pool(): N = 10000 arr = pa.array(np.arange(N, dtype=np.int64)) - t = pa.table([arr], ['f0']) + t = pa.table([arr, arr, arr], ['f0', 'f1', 'f2']) prior_allocation = pa.total_allocated_bytes() x = t.to_pandas() - assert pa.total_allocated_bytes() == (prior_allocation + N * 8) + assert pa.total_allocated_bytes() == (prior_allocation + 3 * N * 8) # Check successful garbage collection x = None # noqa