diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index adbdfb85f29e..fa7f99e0126a 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include "arrow/flight/transport_server.h" #include "arrow/flight/types.h" #include "arrow/status.h" +#include "arrow/type.h" #include "arrow/util/io_util.h" #include "arrow/util/logging.h" #include "arrow/util/uri.h" @@ -275,78 +277,99 @@ Status FlightServerBase::GetSchema(const ServerCallContext& context, class RecordBatchStream::RecordBatchStreamImpl { public: - // Stages of the stream when producing payloads - enum class Stage { - NEW, // The stream has been created, but Next has not been called yet - DICTIONARY, // Dictionaries have been collected, and are being sent - RECORD_BATCH // Initial have been sent - }; - RecordBatchStreamImpl(const std::shared_ptr& reader, const ipc::IpcWriteOptions& options) - : reader_(reader), mapper_(*reader_->schema()), ipc_options_(options) {} + : reader_(reader), options_(options) {} std::shared_ptr schema() { return reader_->schema(); } Status GetSchemaPayload(FlightPayload* payload) { - return ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_, - &payload->ipc_message); + if (!writer_) { + // Create the IPC writer on first call + auto payload_writer = + std::make_unique(&payload_deque_); + ARROW_ASSIGN_OR_RAISE( + writer_, ipc::internal::OpenRecordBatchWriter(std::move(payload_writer), + reader_->schema(), options_)); + } + + // Return the expected schema payload. + if (payload_deque_.empty()) { + return Status::UnknownError("No schema payload generated"); + } + *payload = std::move(payload_deque_.front()); + payload_deque_.pop_front(); + return Status::OK(); } Status Next(FlightPayload* payload) { - if (stage_ == Stage::NEW) { - RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); - if (!current_batch_) { - // Signal that iteration is over + // If we have previous payloads (dictionary messages or previous record batches) + // we will return them before reading the next record batch. + if (payload_deque_.empty()) { + std::shared_ptr batch; + RETURN_NOT_OK(reader_->ReadNext(&batch)); + if (!batch) { + // End of stream + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } payload->ipc_message.metadata = nullptr; return Status::OK(); } - ARROW_ASSIGN_OR_RAISE(dictionaries_, - ipc::CollectDictionaries(*current_batch_, mapper_)); - stage_ = Stage::DICTIONARY; + if (!writer_) { + return Status::UnknownError( + "Writer should be initialized before reading Next batches"); + } + // One WriteRecordBatch call might generate multiple payloads, so we + // need to collect them in a deque. + RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); } - if (stage_ == Stage::DICTIONARY) { - if (dictionary_index_ == static_cast(dictionaries_.size())) { - stage_ = Stage::RECORD_BATCH; - return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, - &payload->ipc_message); - } else { - return GetNextDictionary(payload); - } + // There must be at least one payload generated after WriteRecordBatch or + // from previous calls to WriteRecordBatch. + if (payload_deque_.empty()) { + return Status::UnknownError("IPC writer didn't produce any payloads"); } - RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); + *payload = std::move(payload_deque_.front()); + payload_deque_.pop_front(); + return Status::OK(); + } - // TODO(ARROW-10787): Delta dictionaries - if (!current_batch_) { - // Signal that iteration is over - payload->ipc_message.metadata = nullptr; - return Status::OK(); - } else { - return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, - &payload->ipc_message); + Status Close() { + if (writer_) { + RETURN_NOT_OK(writer_->Close()); } + return reader_->Close(); } - Status Close() { return reader_->Close(); } - private: - Status GetNextDictionary(FlightPayload* payload) { - const auto& it = dictionaries_[dictionary_index_++]; - return ipc::GetDictionaryPayload(it.first, it.second, ipc_options_, - &payload->ipc_message); - } + // Simple payload writer that uses a deque to store generated payloads. + class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { + public: + explicit ServerRecordBatchPayloadWriter(std::deque* payload_deque) + : payload_deque_(payload_deque) {} - Stage stage_ = Stage::NEW; - std::shared_ptr reader_; - ipc::DictionaryFieldMapper mapper_; - ipc::IpcWriteOptions ipc_options_; - std::shared_ptr current_batch_; - std::vector>> dictionaries_; + Status Start() override { return Status::OK(); } - // Index of next dictionary to send - int dictionary_index_ = 0; + Status WritePayload(const ipc::IpcPayload& ipc_payload) override { + FlightPayload payload; + payload.ipc_message = ipc_payload; + + payload_deque_->push_back(std::move(payload)); + return Status::OK(); + } + + Status Close() override { return Status::OK(); } + + private: + std::deque* payload_deque_; + }; + + std::shared_ptr reader_; + ipc::IpcWriteOptions options_; + std::unique_ptr writer_; + std::deque payload_deque_; }; FlightMetadataWriter::~FlightMetadataWriter() = default; diff --git a/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc b/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc index 55345ad477a2..85332e6c4dfc 100644 --- a/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc +++ b/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc @@ -102,7 +102,10 @@ Status SqliteTablesWithSchemaBatchReader::ReadNext(std::shared_ptr* std::shared_ptr schema_array; ARROW_RETURN_NOT_OK(schema_builder.Finish(&schema_array)); - ARROW_ASSIGN_OR_RAISE(*batch, first_batch->AddColumn(4, "table_schema", schema_array)); + std::shared_ptr schema_field = + arrow::field("table_schema", schema_array->type(), false); + + ARROW_ASSIGN_OR_RAISE(*batch, first_batch->AddColumn(4, schema_field, schema_array)); return Status::OK(); } diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py index bd5ff12353ca..e9e99d8eb839 100644 --- a/python/pyarrow/tests/test_flight.py +++ b/python/pyarrow/tests/test_flight.py @@ -114,10 +114,12 @@ def simple_ints_table(): def simple_dicts_table(): dict_values = pa.array(["foo", "baz", "quux"], type=pa.utf8()) + new_dict_values = pa.array(["bar", "qux"], type=pa.utf8()) data = [ pa.chunked_array([ pa.DictionaryArray.from_arrays([1, 0, None], dict_values), - pa.DictionaryArray.from_arrays([2, 1], dict_values) + pa.DictionaryArray.from_arrays([2, 1], dict_values), + pa.DictionaryArray.from_arrays([0, 1], new_dict_values) ]) ] return pa.Table.from_arrays(data, names=['some_dicts'])