From 0de399e13604d81c8d0a1cf19c1154e7617c3da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 15 Jul 2025 16:19:04 +0200 Subject: [PATCH 01/13] GH-45055: [C++][Flight] Support dictionary replacement in DoGet --- cpp/src/arrow/flight/server.cc | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index adbdfb85f29e..0d7760b50765 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -32,6 +32,7 @@ #include #include +#include "arrow/array.h" #include "arrow/device.h" #include "arrow/flight/transport.h" #include "arrow/flight/transport/grpc/grpc_server.h" @@ -324,6 +325,16 @@ class RecordBatchStream::RecordBatchStreamImpl { payload->ipc_message.metadata = nullptr; return Status::OK(); } else { + std::vector>> new_dictionaries; + ARROW_ASSIGN_OR_RAISE(new_dictionaries, + ipc::CollectDictionaries(*current_batch_, mapper_)); + // Only send dictionaries if there are changes + if (ShouldSendDictionaryUpdate(new_dictionaries)) { + dictionaries_ = std::move(new_dictionaries); + dictionary_index_ = 0; + stage_ = Stage::DICTIONARY; + return GetNextDictionary(payload); + } return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, &payload->ipc_message); } @@ -338,6 +349,36 @@ class RecordBatchStream::RecordBatchStreamImpl { &payload->ipc_message); } + bool ShouldSendDictionaryUpdate( + const std::vector>>& new_dictionaries) { + const auto equal_options = EqualOptions().nans_equal(true); + + for (const auto& pair : new_dictionaries) { + int64_t dictionary_id = pair.first; + const auto& dictionary_array = pair.second; + bool found = false; + std::shared_ptr found_dictionary = nullptr; + + for (const auto& dict_pair : dictionaries_) { + if (dict_pair.first == dictionary_id) { + found = true; + found_dictionary = dict_pair.second; + break; + } + } + if (found) { + if (!dictionary_array->Equals(found_dictionary, equal_options)) { + // dictionary has changed + return true; + } + } else { + // New dictionary + return true; + } + } + return false; + } + Stage stage_ = Stage::NEW; std::shared_ptr reader_; ipc::DictionaryFieldMapper mapper_; From c1eae76a07e83434f1002d41b58ca3441355aeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 15 Jul 2025 18:59:47 +0200 Subject: [PATCH 02/13] Use custom IpcPayloadWriter to reuse IpcWriter to generate the flight payloads for RecordBatches --- cpp/src/arrow/flight/server.cc | 187 ++++++++++++++++++--------------- 1 file changed, 103 insertions(+), 84 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 0d7760b50765..f53a0784c501 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -32,7 +32,6 @@ #include #include -#include "arrow/array.h" #include "arrow/device.h" #include "arrow/flight/transport.h" #include "arrow/flight/transport/grpc/grpc_server.h" @@ -276,118 +275,138 @@ Status FlightServerBase::GetSchema(const ServerCallContext& context, class RecordBatchStream::RecordBatchStreamImpl { public: - // Stages of the stream when producing payloads - enum class Stage { - NEW, // The stream has been created, but Next has not been called yet - DICTIONARY, // Dictionaries have been collected, and are being sent - RECORD_BATCH // Initial have been sent - }; - RecordBatchStreamImpl(const std::shared_ptr& reader, const ipc::IpcWriteOptions& options) - : reader_(reader), mapper_(*reader_->schema()), ipc_options_(options) {} + : reader_(reader), options_(options) {} std::shared_ptr schema() { return reader_->schema(); } Status GetSchemaPayload(FlightPayload* payload) { - return ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_, - &payload->ipc_message); + if (!writer_) { + // Create the IPC writer on first call + auto payload_writer = std::make_unique( + &record_batch_payload_, &other_payload_); + auto writer_result = ipc::internal::OpenRecordBatchWriter( + std::move(payload_writer), reader_->schema(), options_); + + if (!writer_result.ok()) { + return writer_result.status(); + } + writer_ = std::move(writer_result).ValueOrDie(); + } + + // Return the current payload (schema) + if (other_payload_.has_value()) { + *payload = std::move(other_payload_.value()); + other_payload_.reset(); + return Status::OK(); + } + return Status::UnknownError("No schema payload generated"); } Status Next(FlightPayload* payload) { - if (stage_ == Stage::NEW) { - RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); - if (!current_batch_) { - // Signal that iteration is over - payload->ipc_message.metadata = nullptr; - return Status::OK(); - } - ARROW_ASSIGN_OR_RAISE(dictionaries_, - ipc::CollectDictionaries(*current_batch_, mapper_)); - stage_ = Stage::DICTIONARY; + if (!writer_) { + return Status::Invalid("Writer not initialized"); } - if (stage_ == Stage::DICTIONARY) { - if (dictionary_index_ == static_cast(dictionaries_.size())) { - stage_ = Stage::RECORD_BATCH; - return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, - &payload->ipc_message); - } else { - return GetNextDictionary(payload); - } + // If we have record batch payload from a previous call, return it + if (record_batch_payload_.has_value()) { + *payload = std::move(record_batch_payload_.value()); + record_batch_payload_.reset(); + return Status::OK(); } - RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); + // Read next batch + std::shared_ptr batch; + RETURN_NOT_OK(reader_->ReadNext(&batch)); - // TODO(ARROW-10787): Delta dictionaries - if (!current_batch_) { - // Signal that iteration is over + if (!batch) { + // End of stream + RETURN_NOT_OK(writer_->Close()); payload->ipc_message.metadata = nullptr; return Status::OK(); - } else { - std::vector>> new_dictionaries; - ARROW_ASSIGN_OR_RAISE(new_dictionaries, - ipc::CollectDictionaries(*current_batch_, mapper_)); - // Only send dictionaries if there are changes - if (ShouldSendDictionaryUpdate(new_dictionaries)) { - dictionaries_ = std::move(new_dictionaries); - dictionary_index_ = 0; - stage_ = Stage::DICTIONARY; - return GetNextDictionary(payload); - } - return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, - &payload->ipc_message); } - } - Status Close() { return reader_->Close(); } + RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); - private: - Status GetNextDictionary(FlightPayload* payload) { - const auto& it = dictionaries_[dictionary_index_++]; - return ipc::GetDictionaryPayload(it.first, it.second, ipc_options_, - &payload->ipc_message); + // prioritize other_payload_ for the dictionary payload + if (other_payload_.has_value()) { + *payload = std::move(other_payload_.value()); + other_payload_.reset(); + return Status::OK(); + } else if (record_batch_payload_.has_value()) { + *payload = std::move(record_batch_payload_.value()); + record_batch_payload_.reset(); + return Status::OK(); + } + + return Status::UnknownError("IPC writer didn't produce any payloads"); + } + + Status Close() { + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } + return reader_->Close(); } - bool ShouldSendDictionaryUpdate( - const std::vector>>& new_dictionaries) { - const auto equal_options = EqualOptions().nans_equal(true); - - for (const auto& pair : new_dictionaries) { - int64_t dictionary_id = pair.first; - const auto& dictionary_array = pair.second; - bool found = false; - std::shared_ptr found_dictionary = nullptr; - - for (const auto& dict_pair : dictionaries_) { - if (dict_pair.first == dictionary_id) { - found = true; - found_dictionary = dict_pair.second; - break; + private: + // Simple payload writer that uses two pointers + class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { + public: + explicit ServerRecordBatchPayloadWriter( + std::optional* record_batch_payload, + std::optional* other_payload) + : record_batch_payload_(record_batch_payload), + other_payload_(other_payload), + first_payload_(true) {} + + Status Start() override { return Status::OK(); } + + Status WritePayload(const ipc::IpcPayload& ipc_payload) override { + FlightPayload payload; + payload.ipc_message = ipc_payload; + + if (first_payload_) { + if (ipc_payload.type != ipc::MessageType::SCHEMA) { + return Status::Invalid("First IPC message should be schema"); } + first_payload_ = false; } - if (found) { - if (!dictionary_array->Equals(found_dictionary, equal_options)) { - // dictionary has changed - return true; + + if (payload.ipc_message.type == ipc::MessageType::RECORD_BATCH) { + // Store in record batch payload + if (record_batch_payload_->has_value()) { + return Status::Invalid("Record batch payload already set"); } + *record_batch_payload_ = std::move(payload); + } else if (payload.ipc_message.type == ipc::MessageType::DICTIONARY_BATCH || + payload.ipc_message.type == ipc::MessageType::SCHEMA) { + // Store in other payload + if (other_payload_->has_value()) { + return Status::Invalid("Other payload already set"); + } + *other_payload_ = std::move(payload); } else { - // New dictionary - return true; + return Status::Invalid( + "Only RECORD_BATCH, DICTIONARY_BATCH, and SCHEMA messages are supported"); } + return Status::OK(); } - return false; - } - Stage stage_ = Stage::NEW; - std::shared_ptr reader_; - ipc::DictionaryFieldMapper mapper_; - ipc::IpcWriteOptions ipc_options_; - std::shared_ptr current_batch_; - std::vector>> dictionaries_; + Status Close() override { return Status::OK(); } - // Index of next dictionary to send - int dictionary_index_ = 0; + private: + std::optional* record_batch_payload_; + std::optional* other_payload_; + bool first_payload_; + }; + + std::shared_ptr reader_; + ipc::IpcWriteOptions options_; + std::unique_ptr writer_; + std::optional record_batch_payload_; + std::optional other_payload_; }; FlightMetadataWriter::~FlightMetadataWriter() = default; From 7e2cf812dcca7eb18d7d9f2ddb9474d05e5c47c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 15 Jul 2025 19:39:20 +0200 Subject: [PATCH 03/13] Revert "Use custom IpcPayloadWriter to reuse IpcWriter to generate the flight payloads for RecordBatches" This reverts commit c1eae76a07e83434f1002d41b58ca3441355aeb9. --- cpp/src/arrow/flight/server.cc | 187 +++++++++++++++------------------ 1 file changed, 84 insertions(+), 103 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index f53a0784c501..0d7760b50765 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -32,6 +32,7 @@ #include #include +#include "arrow/array.h" #include "arrow/device.h" #include "arrow/flight/transport.h" #include "arrow/flight/transport/grpc/grpc_server.h" @@ -275,138 +276,118 @@ Status FlightServerBase::GetSchema(const ServerCallContext& context, class RecordBatchStream::RecordBatchStreamImpl { public: + // Stages of the stream when producing payloads + enum class Stage { + NEW, // The stream has been created, but Next has not been called yet + DICTIONARY, // Dictionaries have been collected, and are being sent + RECORD_BATCH // Initial have been sent + }; + RecordBatchStreamImpl(const std::shared_ptr& reader, const ipc::IpcWriteOptions& options) - : reader_(reader), options_(options) {} + : reader_(reader), mapper_(*reader_->schema()), ipc_options_(options) {} std::shared_ptr schema() { return reader_->schema(); } Status GetSchemaPayload(FlightPayload* payload) { - if (!writer_) { - // Create the IPC writer on first call - auto payload_writer = std::make_unique( - &record_batch_payload_, &other_payload_); - auto writer_result = ipc::internal::OpenRecordBatchWriter( - std::move(payload_writer), reader_->schema(), options_); - - if (!writer_result.ok()) { - return writer_result.status(); - } - writer_ = std::move(writer_result).ValueOrDie(); - } - - // Return the current payload (schema) - if (other_payload_.has_value()) { - *payload = std::move(other_payload_.value()); - other_payload_.reset(); - return Status::OK(); - } - return Status::UnknownError("No schema payload generated"); + return ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_, + &payload->ipc_message); } Status Next(FlightPayload* payload) { - if (!writer_) { - return Status::Invalid("Writer not initialized"); + if (stage_ == Stage::NEW) { + RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); + if (!current_batch_) { + // Signal that iteration is over + payload->ipc_message.metadata = nullptr; + return Status::OK(); + } + ARROW_ASSIGN_OR_RAISE(dictionaries_, + ipc::CollectDictionaries(*current_batch_, mapper_)); + stage_ = Stage::DICTIONARY; } - // If we have record batch payload from a previous call, return it - if (record_batch_payload_.has_value()) { - *payload = std::move(record_batch_payload_.value()); - record_batch_payload_.reset(); - return Status::OK(); + if (stage_ == Stage::DICTIONARY) { + if (dictionary_index_ == static_cast(dictionaries_.size())) { + stage_ = Stage::RECORD_BATCH; + return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, + &payload->ipc_message); + } else { + return GetNextDictionary(payload); + } } - // Read next batch - std::shared_ptr batch; - RETURN_NOT_OK(reader_->ReadNext(&batch)); + RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); - if (!batch) { - // End of stream - RETURN_NOT_OK(writer_->Close()); + // TODO(ARROW-10787): Delta dictionaries + if (!current_batch_) { + // Signal that iteration is over payload->ipc_message.metadata = nullptr; return Status::OK(); + } else { + std::vector>> new_dictionaries; + ARROW_ASSIGN_OR_RAISE(new_dictionaries, + ipc::CollectDictionaries(*current_batch_, mapper_)); + // Only send dictionaries if there are changes + if (ShouldSendDictionaryUpdate(new_dictionaries)) { + dictionaries_ = std::move(new_dictionaries); + dictionary_index_ = 0; + stage_ = Stage::DICTIONARY; + return GetNextDictionary(payload); + } + return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, + &payload->ipc_message); } - - RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); - - // prioritize other_payload_ for the dictionary payload - if (other_payload_.has_value()) { - *payload = std::move(other_payload_.value()); - other_payload_.reset(); - return Status::OK(); - } else if (record_batch_payload_.has_value()) { - *payload = std::move(record_batch_payload_.value()); - record_batch_payload_.reset(); - return Status::OK(); - } - - return Status::UnknownError("IPC writer didn't produce any payloads"); } - Status Close() { - if (writer_) { - RETURN_NOT_OK(writer_->Close()); - } - return reader_->Close(); - } + Status Close() { return reader_->Close(); } private: - // Simple payload writer that uses two pointers - class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { - public: - explicit ServerRecordBatchPayloadWriter( - std::optional* record_batch_payload, - std::optional* other_payload) - : record_batch_payload_(record_batch_payload), - other_payload_(other_payload), - first_payload_(true) {} - - Status Start() override { return Status::OK(); } - - Status WritePayload(const ipc::IpcPayload& ipc_payload) override { - FlightPayload payload; - payload.ipc_message = ipc_payload; - - if (first_payload_) { - if (ipc_payload.type != ipc::MessageType::SCHEMA) { - return Status::Invalid("First IPC message should be schema"); - } - first_payload_ = false; - } + Status GetNextDictionary(FlightPayload* payload) { + const auto& it = dictionaries_[dictionary_index_++]; + return ipc::GetDictionaryPayload(it.first, it.second, ipc_options_, + &payload->ipc_message); + } - if (payload.ipc_message.type == ipc::MessageType::RECORD_BATCH) { - // Store in record batch payload - if (record_batch_payload_->has_value()) { - return Status::Invalid("Record batch payload already set"); + bool ShouldSendDictionaryUpdate( + const std::vector>>& new_dictionaries) { + const auto equal_options = EqualOptions().nans_equal(true); + + for (const auto& pair : new_dictionaries) { + int64_t dictionary_id = pair.first; + const auto& dictionary_array = pair.second; + bool found = false; + std::shared_ptr found_dictionary = nullptr; + + for (const auto& dict_pair : dictionaries_) { + if (dict_pair.first == dictionary_id) { + found = true; + found_dictionary = dict_pair.second; + break; } - *record_batch_payload_ = std::move(payload); - } else if (payload.ipc_message.type == ipc::MessageType::DICTIONARY_BATCH || - payload.ipc_message.type == ipc::MessageType::SCHEMA) { - // Store in other payload - if (other_payload_->has_value()) { - return Status::Invalid("Other payload already set"); + } + if (found) { + if (!dictionary_array->Equals(found_dictionary, equal_options)) { + // dictionary has changed + return true; } - *other_payload_ = std::move(payload); } else { - return Status::Invalid( - "Only RECORD_BATCH, DICTIONARY_BATCH, and SCHEMA messages are supported"); + // New dictionary + return true; } - return Status::OK(); } + return false; + } - Status Close() override { return Status::OK(); } - - private: - std::optional* record_batch_payload_; - std::optional* other_payload_; - bool first_payload_; - }; - + Stage stage_ = Stage::NEW; std::shared_ptr reader_; - ipc::IpcWriteOptions options_; - std::unique_ptr writer_; - std::optional record_batch_payload_; - std::optional other_payload_; + ipc::DictionaryFieldMapper mapper_; + ipc::IpcWriteOptions ipc_options_; + std::shared_ptr current_batch_; + std::vector>> dictionaries_; + + // Index of next dictionary to send + int dictionary_index_ = 0; }; FlightMetadataWriter::~FlightMetadataWriter() = default; From 5ffb914e328b9ad431d273a1dcbdbab54a65fba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 15 Jul 2025 18:59:47 +0200 Subject: [PATCH 04/13] Use custom IpcPayloadWriter to reuse IpcWriter to generate the flight payloads for RecordBatches --- cpp/src/arrow/flight/server.cc | 187 ++++++++++++++++++--------------- 1 file changed, 103 insertions(+), 84 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 0d7760b50765..f53a0784c501 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -32,7 +32,6 @@ #include #include -#include "arrow/array.h" #include "arrow/device.h" #include "arrow/flight/transport.h" #include "arrow/flight/transport/grpc/grpc_server.h" @@ -276,118 +275,138 @@ Status FlightServerBase::GetSchema(const ServerCallContext& context, class RecordBatchStream::RecordBatchStreamImpl { public: - // Stages of the stream when producing payloads - enum class Stage { - NEW, // The stream has been created, but Next has not been called yet - DICTIONARY, // Dictionaries have been collected, and are being sent - RECORD_BATCH // Initial have been sent - }; - RecordBatchStreamImpl(const std::shared_ptr& reader, const ipc::IpcWriteOptions& options) - : reader_(reader), mapper_(*reader_->schema()), ipc_options_(options) {} + : reader_(reader), options_(options) {} std::shared_ptr schema() { return reader_->schema(); } Status GetSchemaPayload(FlightPayload* payload) { - return ipc::GetSchemaPayload(*reader_->schema(), ipc_options_, mapper_, - &payload->ipc_message); + if (!writer_) { + // Create the IPC writer on first call + auto payload_writer = std::make_unique( + &record_batch_payload_, &other_payload_); + auto writer_result = ipc::internal::OpenRecordBatchWriter( + std::move(payload_writer), reader_->schema(), options_); + + if (!writer_result.ok()) { + return writer_result.status(); + } + writer_ = std::move(writer_result).ValueOrDie(); + } + + // Return the current payload (schema) + if (other_payload_.has_value()) { + *payload = std::move(other_payload_.value()); + other_payload_.reset(); + return Status::OK(); + } + return Status::UnknownError("No schema payload generated"); } Status Next(FlightPayload* payload) { - if (stage_ == Stage::NEW) { - RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); - if (!current_batch_) { - // Signal that iteration is over - payload->ipc_message.metadata = nullptr; - return Status::OK(); - } - ARROW_ASSIGN_OR_RAISE(dictionaries_, - ipc::CollectDictionaries(*current_batch_, mapper_)); - stage_ = Stage::DICTIONARY; + if (!writer_) { + return Status::Invalid("Writer not initialized"); } - if (stage_ == Stage::DICTIONARY) { - if (dictionary_index_ == static_cast(dictionaries_.size())) { - stage_ = Stage::RECORD_BATCH; - return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, - &payload->ipc_message); - } else { - return GetNextDictionary(payload); - } + // If we have record batch payload from a previous call, return it + if (record_batch_payload_.has_value()) { + *payload = std::move(record_batch_payload_.value()); + record_batch_payload_.reset(); + return Status::OK(); } - RETURN_NOT_OK(reader_->ReadNext(¤t_batch_)); + // Read next batch + std::shared_ptr batch; + RETURN_NOT_OK(reader_->ReadNext(&batch)); - // TODO(ARROW-10787): Delta dictionaries - if (!current_batch_) { - // Signal that iteration is over + if (!batch) { + // End of stream + RETURN_NOT_OK(writer_->Close()); payload->ipc_message.metadata = nullptr; return Status::OK(); - } else { - std::vector>> new_dictionaries; - ARROW_ASSIGN_OR_RAISE(new_dictionaries, - ipc::CollectDictionaries(*current_batch_, mapper_)); - // Only send dictionaries if there are changes - if (ShouldSendDictionaryUpdate(new_dictionaries)) { - dictionaries_ = std::move(new_dictionaries); - dictionary_index_ = 0; - stage_ = Stage::DICTIONARY; - return GetNextDictionary(payload); - } - return ipc::GetRecordBatchPayload(*current_batch_, ipc_options_, - &payload->ipc_message); } - } - Status Close() { return reader_->Close(); } + RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); - private: - Status GetNextDictionary(FlightPayload* payload) { - const auto& it = dictionaries_[dictionary_index_++]; - return ipc::GetDictionaryPayload(it.first, it.second, ipc_options_, - &payload->ipc_message); + // prioritize other_payload_ for the dictionary payload + if (other_payload_.has_value()) { + *payload = std::move(other_payload_.value()); + other_payload_.reset(); + return Status::OK(); + } else if (record_batch_payload_.has_value()) { + *payload = std::move(record_batch_payload_.value()); + record_batch_payload_.reset(); + return Status::OK(); + } + + return Status::UnknownError("IPC writer didn't produce any payloads"); + } + + Status Close() { + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } + return reader_->Close(); } - bool ShouldSendDictionaryUpdate( - const std::vector>>& new_dictionaries) { - const auto equal_options = EqualOptions().nans_equal(true); - - for (const auto& pair : new_dictionaries) { - int64_t dictionary_id = pair.first; - const auto& dictionary_array = pair.second; - bool found = false; - std::shared_ptr found_dictionary = nullptr; - - for (const auto& dict_pair : dictionaries_) { - if (dict_pair.first == dictionary_id) { - found = true; - found_dictionary = dict_pair.second; - break; + private: + // Simple payload writer that uses two pointers + class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { + public: + explicit ServerRecordBatchPayloadWriter( + std::optional* record_batch_payload, + std::optional* other_payload) + : record_batch_payload_(record_batch_payload), + other_payload_(other_payload), + first_payload_(true) {} + + Status Start() override { return Status::OK(); } + + Status WritePayload(const ipc::IpcPayload& ipc_payload) override { + FlightPayload payload; + payload.ipc_message = ipc_payload; + + if (first_payload_) { + if (ipc_payload.type != ipc::MessageType::SCHEMA) { + return Status::Invalid("First IPC message should be schema"); } + first_payload_ = false; } - if (found) { - if (!dictionary_array->Equals(found_dictionary, equal_options)) { - // dictionary has changed - return true; + + if (payload.ipc_message.type == ipc::MessageType::RECORD_BATCH) { + // Store in record batch payload + if (record_batch_payload_->has_value()) { + return Status::Invalid("Record batch payload already set"); } + *record_batch_payload_ = std::move(payload); + } else if (payload.ipc_message.type == ipc::MessageType::DICTIONARY_BATCH || + payload.ipc_message.type == ipc::MessageType::SCHEMA) { + // Store in other payload + if (other_payload_->has_value()) { + return Status::Invalid("Other payload already set"); + } + *other_payload_ = std::move(payload); } else { - // New dictionary - return true; + return Status::Invalid( + "Only RECORD_BATCH, DICTIONARY_BATCH, and SCHEMA messages are supported"); } + return Status::OK(); } - return false; - } - Stage stage_ = Stage::NEW; - std::shared_ptr reader_; - ipc::DictionaryFieldMapper mapper_; - ipc::IpcWriteOptions ipc_options_; - std::shared_ptr current_batch_; - std::vector>> dictionaries_; + Status Close() override { return Status::OK(); } - // Index of next dictionary to send - int dictionary_index_ = 0; + private: + std::optional* record_batch_payload_; + std::optional* other_payload_; + bool first_payload_; + }; + + std::shared_ptr reader_; + ipc::IpcWriteOptions options_; + std::unique_ptr writer_; + std::optional record_batch_payload_; + std::optional other_payload_; }; FlightMetadataWriter::~FlightMetadataWriter() = default; From 7e294cbea4e61c3844d4e387ac6d5bccd3b3c45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 17 Jul 2025 18:10:56 +0200 Subject: [PATCH 05/13] Use a list to cater for more than one ipc message written per batch (multiple dictionaries) and possibly drop changes to schema --- cpp/src/arrow/flight/server.cc | 91 +++++++++++++++++----------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index f53a0784c501..36a0bd73d398 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include "arrow/flight/transport_server.h" #include "arrow/flight/types.h" #include "arrow/status.h" +#include "arrow/type.h" #include "arrow/util/io_util.h" #include "arrow/util/logging.h" #include "arrow/util/uri.h" @@ -284,8 +286,8 @@ class RecordBatchStream::RecordBatchStreamImpl { Status GetSchemaPayload(FlightPayload* payload) { if (!writer_) { // Create the IPC writer on first call - auto payload_writer = std::make_unique( - &record_batch_payload_, &other_payload_); + auto payload_writer = + std::make_unique(&payload_list_); auto writer_result = ipc::internal::OpenRecordBatchWriter( std::move(payload_writer), reader_->schema(), options_); @@ -296,9 +298,9 @@ class RecordBatchStream::RecordBatchStreamImpl { } // Return the current payload (schema) - if (other_payload_.has_value()) { - *payload = std::move(other_payload_.value()); - other_payload_.reset(); + if (!payload_list_.empty()) { + *payload = std::move(payload_list_.front()); + payload_list_.pop_front(); return Status::OK(); } return Status::UnknownError("No schema payload generated"); @@ -309,14 +311,14 @@ class RecordBatchStream::RecordBatchStreamImpl { return Status::Invalid("Writer not initialized"); } - // If we have record batch payload from a previous call, return it - if (record_batch_payload_.has_value()) { - *payload = std::move(record_batch_payload_.value()); - record_batch_payload_.reset(); + // If we have previous payloads (dictionary messages or previous record batches) + // return them first before reading next record batch. + if (!payload_list_.empty()) { + *payload = std::move(payload_list_.front()); + payload_list_.pop_front(); return Status::OK(); } - // Read next batch std::shared_ptr batch; RETURN_NOT_OK(reader_->ReadNext(&batch)); @@ -327,16 +329,37 @@ class RecordBatchStream::RecordBatchStreamImpl { return Status::OK(); } + // Check if schema has changed and recreate writer if needed + // TODO: Investigate why this is needed for a flight-sql specific test. + if (!batch->schema()->Equals(*reader_->schema())) { + // Close current writer + RETURN_NOT_OK(writer_->Close()); + + // Create new writer with the batch's schema + auto payload_writer = + std::make_unique(&payload_list_); + auto writer_result = ipc::internal::OpenRecordBatchWriter( + std::move(payload_writer), batch->schema(), options_); + + if (!writer_result.ok()) { + return writer_result.status(); + } + writer_ = std::move(writer_result).ValueOrDie(); + if (!payload_list_.empty()) { + // Drop new ipc schema message change if it was generated. + payload_list_.pop_front(); + } + } + + // One WriteRecordBatch call might generate multiple payloads, so we + // need to collect them in a list. RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); - // prioritize other_payload_ for the dictionary payload - if (other_payload_.has_value()) { - *payload = std::move(other_payload_.value()); - other_payload_.reset(); - return Status::OK(); - } else if (record_batch_payload_.has_value()) { - *payload = std::move(record_batch_payload_.value()); - record_batch_payload_.reset(); + // Return the first generated payload after WriteRecordBatch. There must be + // at least one payload generated. + if (!payload_list_.empty()) { + *payload = std::move(payload_list_.front()); + payload_list_.pop_front(); return Status::OK(); } @@ -354,12 +377,8 @@ class RecordBatchStream::RecordBatchStreamImpl { // Simple payload writer that uses two pointers class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { public: - explicit ServerRecordBatchPayloadWriter( - std::optional* record_batch_payload, - std::optional* other_payload) - : record_batch_payload_(record_batch_payload), - other_payload_(other_payload), - first_payload_(true) {} + explicit ServerRecordBatchPayloadWriter(std::list* payload_list) + : payload_list_(payload_list), first_payload_(true) {} Status Start() override { return Status::OK(); } @@ -374,39 +393,21 @@ class RecordBatchStream::RecordBatchStreamImpl { first_payload_ = false; } - if (payload.ipc_message.type == ipc::MessageType::RECORD_BATCH) { - // Store in record batch payload - if (record_batch_payload_->has_value()) { - return Status::Invalid("Record batch payload already set"); - } - *record_batch_payload_ = std::move(payload); - } else if (payload.ipc_message.type == ipc::MessageType::DICTIONARY_BATCH || - payload.ipc_message.type == ipc::MessageType::SCHEMA) { - // Store in other payload - if (other_payload_->has_value()) { - return Status::Invalid("Other payload already set"); - } - *other_payload_ = std::move(payload); - } else { - return Status::Invalid( - "Only RECORD_BATCH, DICTIONARY_BATCH, and SCHEMA messages are supported"); - } + payload_list_->push_back(std::move(payload)); return Status::OK(); } Status Close() override { return Status::OK(); } private: - std::optional* record_batch_payload_; - std::optional* other_payload_; + std::list* payload_list_; bool first_payload_; }; std::shared_ptr reader_; ipc::IpcWriteOptions options_; std::unique_ptr writer_; - std::optional record_batch_payload_; - std::optional other_payload_; + std::list payload_list_; }; FlightMetadataWriter::~FlightMetadataWriter() = default; From 9a4e28aa13431ace6ccc93c3ee5f8f6259e98b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 21 Jul 2025 15:05:57 +0200 Subject: [PATCH 06/13] Cover case were Server implementation calls Next without a generated writer --- cpp/src/arrow/flight/server.cc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 36a0bd73d398..d519ca22b8cb 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -307,10 +307,6 @@ class RecordBatchStream::RecordBatchStreamImpl { } Status Next(FlightPayload* payload) { - if (!writer_) { - return Status::Invalid("Writer not initialized"); - } - // If we have previous payloads (dictionary messages or previous record batches) // return them first before reading next record batch. if (!payload_list_.empty()) { @@ -324,16 +320,21 @@ class RecordBatchStream::RecordBatchStreamImpl { if (!batch) { // End of stream - RETURN_NOT_OK(writer_->Close()); + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } payload->ipc_message.metadata = nullptr; return Status::OK(); } - // Check if schema has changed and recreate writer if needed - // TODO: Investigate why this is needed for a flight-sql specific test. - if (!batch->schema()->Equals(*reader_->schema())) { + // Check if writer is already initialized or if schema has changed. + // To recreate the writer. + // TODO: Investigate why schema changed is needed for a flight-sql specific test. + if (!writer_ || !batch->schema()->Equals(*reader_->schema())) { // Close current writer - RETURN_NOT_OK(writer_->Close()); + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } // Create new writer with the batch's schema auto payload_writer = @@ -346,7 +347,8 @@ class RecordBatchStream::RecordBatchStreamImpl { } writer_ = std::move(writer_result).ValueOrDie(); if (!payload_list_.empty()) { - // Drop new ipc schema message change if it was generated. + // Drop Schema message if it was generated. + // We want new Dictionary or RecordBatch messages only. payload_list_.pop_front(); } } From 0c73f5f80167f7b041f5ee5f87af79b27a9d08fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 21 Jul 2025 15:30:48 +0200 Subject: [PATCH 07/13] Update simple_dicts_table used for flight test to have an updated dictionary --- python/pyarrow/tests/test_flight.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py index bd5ff12353ca..e9e99d8eb839 100644 --- a/python/pyarrow/tests/test_flight.py +++ b/python/pyarrow/tests/test_flight.py @@ -114,10 +114,12 @@ def simple_ints_table(): def simple_dicts_table(): dict_values = pa.array(["foo", "baz", "quux"], type=pa.utf8()) + new_dict_values = pa.array(["bar", "qux"], type=pa.utf8()) data = [ pa.chunked_array([ pa.DictionaryArray.from_arrays([1, 0, None], dict_values), - pa.DictionaryArray.from_arrays([2, 1], dict_values) + pa.DictionaryArray.from_arrays([2, 1], dict_values), + pa.DictionaryArray.from_arrays([0, 1], new_dict_values) ]) ] return pa.Table.from_arrays(data, names=['some_dicts']) From 4395516abce5c9308ec73c69ac06d864d029b641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 21 Jul 2025 16:21:47 +0200 Subject: [PATCH 08/13] Update code comments --- cpp/src/arrow/flight/server.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index d519ca22b8cb..1245a21d89c2 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -329,9 +329,7 @@ class RecordBatchStream::RecordBatchStreamImpl { // Check if writer is already initialized or if schema has changed. // To recreate the writer. - // TODO: Investigate why schema changed is needed for a flight-sql specific test. if (!writer_ || !batch->schema()->Equals(*reader_->schema())) { - // Close current writer if (writer_) { RETURN_NOT_OK(writer_->Close()); } @@ -376,7 +374,7 @@ class RecordBatchStream::RecordBatchStreamImpl { } private: - // Simple payload writer that uses two pointers + // Simple payload writer that uses a list of generated payloads. class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { public: explicit ServerRecordBatchPayloadWriter(std::list* payload_list) From 9f214a908e901689fdd1a441b89695e9e82a5c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Jul 2025 10:43:03 +0200 Subject: [PATCH 09/13] Review comments, mainly simplifying logic --- cpp/src/arrow/flight/server.cc | 108 +++++++++++++-------------------- 1 file changed, 42 insertions(+), 66 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 1245a21d89c2..7518c3ae3c45 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -288,13 +288,9 @@ class RecordBatchStream::RecordBatchStreamImpl { // Create the IPC writer on first call auto payload_writer = std::make_unique(&payload_list_); - auto writer_result = ipc::internal::OpenRecordBatchWriter( - std::move(payload_writer), reader_->schema(), options_); - - if (!writer_result.ok()) { - return writer_result.status(); - } - writer_ = std::move(writer_result).ValueOrDie(); + ARROW_ASSIGN_OR_RAISE( + writer_, ipc::internal::OpenRecordBatchWriter(std::move(payload_writer), + reader_->schema(), options_)); } // Return the current payload (schema) @@ -308,62 +304,50 @@ class RecordBatchStream::RecordBatchStreamImpl { Status Next(FlightPayload* payload) { // If we have previous payloads (dictionary messages or previous record batches) - // return them first before reading next record batch. - if (!payload_list_.empty()) { - *payload = std::move(payload_list_.front()); - payload_list_.pop_front(); - return Status::OK(); - } - - std::shared_ptr batch; - RETURN_NOT_OK(reader_->ReadNext(&batch)); - - if (!batch) { - // End of stream - if (writer_) { - RETURN_NOT_OK(writer_->Close()); - } - payload->ipc_message.metadata = nullptr; - return Status::OK(); - } - - // Check if writer is already initialized or if schema has changed. - // To recreate the writer. - if (!writer_ || !batch->schema()->Equals(*reader_->schema())) { - if (writer_) { - RETURN_NOT_OK(writer_->Close()); - } - - // Create new writer with the batch's schema - auto payload_writer = - std::make_unique(&payload_list_); - auto writer_result = ipc::internal::OpenRecordBatchWriter( - std::move(payload_writer), batch->schema(), options_); - - if (!writer_result.ok()) { - return writer_result.status(); + // we will return them before reading the next record batch. + if (payload_list_.empty()) { + std::shared_ptr batch; + RETURN_NOT_OK(reader_->ReadNext(&batch)); + if (!batch) { + // End of stream + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } + payload->ipc_message.metadata = nullptr; + return Status::OK(); } - writer_ = std::move(writer_result).ValueOrDie(); - if (!payload_list_.empty()) { - // Drop Schema message if it was generated. - // We want new Dictionary or RecordBatch messages only. - payload_list_.pop_front(); + // Check if writer is already initialized or if schema has changed. + // To recreate the writer. + if (!writer_ || !batch->schema()->Equals(*reader_->schema())) { + if (writer_) { + RETURN_NOT_OK(writer_->Close()); + } + // Create new writer with the batch's schema + auto payload_writer = + std::make_unique(&payload_list_); + ARROW_ASSIGN_OR_RAISE( + writer_, ipc::internal::OpenRecordBatchWriter(std::move(payload_writer), + batch->schema(), options_)); + if (!payload_list_.empty()) { + // Drop Schema message if it was generated. + // We want new Dictionary or RecordBatch messages only. + payload_list_.pop_front(); + } } + // One WriteRecordBatch call might generate multiple payloads, so we + // need to collect them in a list. + RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); } - // One WriteRecordBatch call might generate multiple payloads, so we - // need to collect them in a list. - RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); - - // Return the first generated payload after WriteRecordBatch. There must be - // at least one payload generated. - if (!payload_list_.empty()) { - *payload = std::move(payload_list_.front()); - payload_list_.pop_front(); - return Status::OK(); + // There must be at least one payload generated after WriteRecordBatch or + // from previous calls to WriteRecordBatch. + if (payload_list_.empty()) { + return Status::UnknownError("IPC writer didn't produce any payloads"); } - return Status::UnknownError("IPC writer didn't produce any payloads"); + *payload = std::move(payload_list_.front()); + payload_list_.pop_front(); + return Status::OK(); } Status Close() { @@ -378,7 +362,7 @@ class RecordBatchStream::RecordBatchStreamImpl { class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { public: explicit ServerRecordBatchPayloadWriter(std::list* payload_list) - : payload_list_(payload_list), first_payload_(true) {} + : payload_list_(payload_list) {} Status Start() override { return Status::OK(); } @@ -386,13 +370,6 @@ class RecordBatchStream::RecordBatchStreamImpl { FlightPayload payload; payload.ipc_message = ipc_payload; - if (first_payload_) { - if (ipc_payload.type != ipc::MessageType::SCHEMA) { - return Status::Invalid("First IPC message should be schema"); - } - first_payload_ = false; - } - payload_list_->push_back(std::move(payload)); return Status::OK(); } @@ -401,7 +378,6 @@ class RecordBatchStream::RecordBatchStreamImpl { private: std::list* payload_list_; - bool first_payload_; }; std::shared_ptr reader_; From 92680819ac7ef3538df2ecacb993f0c045db8435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Jul 2025 10:51:51 +0200 Subject: [PATCH 10/13] Revert logic to return schema payload to match the logic on Next() --- cpp/src/arrow/flight/server.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 7518c3ae3c45..137dfba5472e 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -293,13 +293,13 @@ class RecordBatchStream::RecordBatchStreamImpl { reader_->schema(), options_)); } - // Return the current payload (schema) - if (!payload_list_.empty()) { - *payload = std::move(payload_list_.front()); - payload_list_.pop_front(); - return Status::OK(); + // Return the expected schema payload. + if (payload_list_.empty()) { + return Status::UnknownError("No schema payload generated"); } - return Status::UnknownError("No schema payload generated"); + *payload = std::move(payload_list_.front()); + payload_list_.pop_front(); + return Status::OK(); } Status Next(FlightPayload* payload) { From d4305a8de40802db19bfaa27528bd94203219a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Jul 2025 12:01:41 +0200 Subject: [PATCH 11/13] Fix SqliteTablesWithSchemaBatchReader example to add same table_schema as flight SQL server GetTablesSchemaWithIncludedSchema --- cpp/src/arrow/flight/server.cc | 11 +++-------- .../sql/example/sqlite_tables_schema_batch_reader.cc | 5 ++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 137dfba5472e..23016fac3418 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -316,12 +316,8 @@ class RecordBatchStream::RecordBatchStreamImpl { payload->ipc_message.metadata = nullptr; return Status::OK(); } - // Check if writer is already initialized or if schema has changed. - // To recreate the writer. - if (!writer_ || !batch->schema()->Equals(*reader_->schema())) { - if (writer_) { - RETURN_NOT_OK(writer_->Close()); - } + // Check if writer is already initialized. + if (!writer_) { // Create new writer with the batch's schema auto payload_writer = std::make_unique(&payload_list_); @@ -329,8 +325,7 @@ class RecordBatchStream::RecordBatchStreamImpl { writer_, ipc::internal::OpenRecordBatchWriter(std::move(payload_writer), batch->schema(), options_)); if (!payload_list_.empty()) { - // Drop Schema message if it was generated. - // We want new Dictionary or RecordBatch messages only. + // Drop Schema message we want new Dictionary or RecordBatch messages only. payload_list_.pop_front(); } } diff --git a/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc b/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc index 55345ad477a2..85332e6c4dfc 100644 --- a/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc +++ b/cpp/src/arrow/flight/sql/example/sqlite_tables_schema_batch_reader.cc @@ -102,7 +102,10 @@ Status SqliteTablesWithSchemaBatchReader::ReadNext(std::shared_ptr* std::shared_ptr schema_array; ARROW_RETURN_NOT_OK(schema_builder.Finish(&schema_array)); - ARROW_ASSIGN_OR_RAISE(*batch, first_batch->AddColumn(4, "table_schema", schema_array)); + std::shared_ptr schema_field = + arrow::field("table_schema", schema_array->type(), false); + + ARROW_ASSIGN_OR_RAISE(*batch, first_batch->AddColumn(4, schema_field, schema_array)); return Status::OK(); } From 3608ca3f37a6f9283abfa0f0870cde8545b14b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Jul 2025 12:59:26 +0200 Subject: [PATCH 12/13] Simplify logic as Writer should be created on GetSchemaPayload and reused. Python test isn't failing locally anymore --- cpp/src/arrow/flight/server.cc | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index 23016fac3418..f65fc706c962 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -316,18 +316,9 @@ class RecordBatchStream::RecordBatchStreamImpl { payload->ipc_message.metadata = nullptr; return Status::OK(); } - // Check if writer is already initialized. if (!writer_) { - // Create new writer with the batch's schema - auto payload_writer = - std::make_unique(&payload_list_); - ARROW_ASSIGN_OR_RAISE( - writer_, ipc::internal::OpenRecordBatchWriter(std::move(payload_writer), - batch->schema(), options_)); - if (!payload_list_.empty()) { - // Drop Schema message we want new Dictionary or RecordBatch messages only. - payload_list_.pop_front(); - } + return Status::UnknownError( + "Writer should be initialized before reading Next batches"); } // One WriteRecordBatch call might generate multiple payloads, so we // need to collect them in a list. From 7b0960483dcb3b5fa9fdf1e1ac852934c765e4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 24 Jul 2025 09:37:10 +0200 Subject: [PATCH 13/13] Use deque instead of list to store payloads --- cpp/src/arrow/flight/server.cc | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index f65fc706c962..fa7f99e0126a 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include #include #include @@ -287,25 +287,25 @@ class RecordBatchStream::RecordBatchStreamImpl { if (!writer_) { // Create the IPC writer on first call auto payload_writer = - std::make_unique(&payload_list_); + std::make_unique(&payload_deque_); ARROW_ASSIGN_OR_RAISE( writer_, ipc::internal::OpenRecordBatchWriter(std::move(payload_writer), reader_->schema(), options_)); } // Return the expected schema payload. - if (payload_list_.empty()) { + if (payload_deque_.empty()) { return Status::UnknownError("No schema payload generated"); } - *payload = std::move(payload_list_.front()); - payload_list_.pop_front(); + *payload = std::move(payload_deque_.front()); + payload_deque_.pop_front(); return Status::OK(); } Status Next(FlightPayload* payload) { // If we have previous payloads (dictionary messages or previous record batches) // we will return them before reading the next record batch. - if (payload_list_.empty()) { + if (payload_deque_.empty()) { std::shared_ptr batch; RETURN_NOT_OK(reader_->ReadNext(&batch)); if (!batch) { @@ -321,18 +321,18 @@ class RecordBatchStream::RecordBatchStreamImpl { "Writer should be initialized before reading Next batches"); } // One WriteRecordBatch call might generate multiple payloads, so we - // need to collect them in a list. + // need to collect them in a deque. RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); } // There must be at least one payload generated after WriteRecordBatch or // from previous calls to WriteRecordBatch. - if (payload_list_.empty()) { + if (payload_deque_.empty()) { return Status::UnknownError("IPC writer didn't produce any payloads"); } - *payload = std::move(payload_list_.front()); - payload_list_.pop_front(); + *payload = std::move(payload_deque_.front()); + payload_deque_.pop_front(); return Status::OK(); } @@ -344,11 +344,11 @@ class RecordBatchStream::RecordBatchStreamImpl { } private: - // Simple payload writer that uses a list of generated payloads. + // Simple payload writer that uses a deque to store generated payloads. class ServerRecordBatchPayloadWriter : public ipc::internal::IpcPayloadWriter { public: - explicit ServerRecordBatchPayloadWriter(std::list* payload_list) - : payload_list_(payload_list) {} + explicit ServerRecordBatchPayloadWriter(std::deque* payload_deque) + : payload_deque_(payload_deque) {} Status Start() override { return Status::OK(); } @@ -356,20 +356,20 @@ class RecordBatchStream::RecordBatchStreamImpl { FlightPayload payload; payload.ipc_message = ipc_payload; - payload_list_->push_back(std::move(payload)); + payload_deque_->push_back(std::move(payload)); return Status::OK(); } Status Close() override { return Status::OK(); } private: - std::list* payload_list_; + std::deque* payload_deque_; }; std::shared_ptr reader_; ipc::IpcWriteOptions options_; std::unique_ptr writer_; - std::list payload_list_; + std::deque payload_deque_; }; FlightMetadataWriter::~FlightMetadataWriter() = default;