From 59da79331aefda3fc434e74eb1458cd0e195c879 Mon Sep 17 00:00:00 2001 From: kamilt Date: Thu, 12 Sep 2024 11:39:50 +0000 Subject: [PATCH 1/5] Modify asof_join_node test to provoke Issue 41706 --- cpp/src/arrow/acero/asof_join_node_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/acero/asof_join_node_test.cc b/cpp/src/arrow/acero/asof_join_node_test.cc index 5d3e9fba08bb..4a67dd8c2d6d 100644 --- a/cpp/src/arrow/acero/asof_join_node_test.cc +++ b/cpp/src/arrow/acero/asof_join_node_test.cc @@ -1571,7 +1571,7 @@ void TestSequencing(BatchesMaker maker, int num_batches, int batch_size) { "asofjoin", {l_src, r_src}, GetRepeatedOptions(2, "time", {"key"}, 1000)}; QueryOptions query_options; - query_options.use_threads = false; + query_options.use_threads = true; ASSERT_OK_AND_ASSIGN(BatchesWithCommonSchema batches, DeclarationToExecBatches(asofjoin, query_options)); @@ -1579,7 +1579,7 @@ void TestSequencing(BatchesMaker maker, int num_batches, int batch_size) { } TEST(AsofJoinTest, BatchSequencing) { - return TestSequencing(MakeIntegerBatches, /*num_batches=*/32, /*batch_size=*/1); + return TestSequencing(MakeIntegerBatches, /*num_batches=*/1000, /*batch_size=*/1); } template From c8047bb8e3d8c83f12070507f3cdc43cb6ee6152 Mon Sep 17 00:00:00 2001 From: kamilt Date: Tue, 17 Sep 2024 11:31:18 +0000 Subject: [PATCH 2/5] Fix issue 41706 using sequencer --- cpp/src/arrow/acero/asof_join_node.cc | 29 ++++++++++++++++------ cpp/src/arrow/acero/asof_join_node_test.cc | 5 ++++ cpp/src/arrow/acero/test_util_internal.cc | 4 +++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc index c4f11d01f3d5..4f57095b892c 100644 --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc @@ -18,6 +18,7 @@ #include "arrow/acero/asof_join_node.h" #include "arrow/acero/backpressure_handler.h" #include "arrow/acero/concurrent_queue_internal.h" +#include "arrow/acero/accumulation_queue.h" #include #include @@ -471,7 +472,7 @@ class BackpressureController : public BackpressureControl { std::atomic& backpressure_counter_; }; -class InputState { +class InputState: public util::SerialSequencingQueue::Processor { // InputState corresponds to an input // Input record batches are queued up in InputState until processed and // turned into output record batches. @@ -482,8 +483,9 @@ class InputState { const std::shared_ptr& schema, const col_index_t time_col_index, const std::vector& key_col_index) - : queue_(std::move(handler)), - schema_(schema), + : sequencer_(util::SerialSequencingQueue::Make(this)), + queue_(std::move(handler)), + schema_(schema), time_col_index_(time_col_index), key_col_index_(key_col_index), time_type_id_(schema_->fields()[time_col_index_]->type()->id()), @@ -699,7 +701,16 @@ class InputState { DEBUG_MANIP(std::endl)); return updated; } + Status InsertBatch(ExecBatch batch){ + return sequencer_->InsertBatch(std::move(batch)); + } + Status Process(ExecBatch batch) override { + auto rb = *batch.ToRecordBatch(schema_); + DEBUG_SYNC(node_, "received batch from input ", index_, ":", DEBUG_MANIP(std::endl), + rb->ToString(), DEBUG_MANIP(std::endl)); + return Push(rb); + } void Rehash() { DEBUG_SYNC(node_, "rehashing for input ", index_, ":", DEBUG_MANIP(std::endl)); MemoStore new_memo(DEBUG_ADD(memo_.no_future_, node_, index_)); @@ -760,6 +771,8 @@ class InputState { } private: + std::unique_ptr sequencer_; + // Pending record batches. The latest is the front. Batches cannot be empty. BackpressureConcurrentQueue> queue_; // Schema associated with the input @@ -1399,6 +1412,9 @@ class AsofJoinNode : public ExecNode { // InputReceived may be called after execution was finished. Pushing it to the // InputState is unnecessary since we're done (and anyway may cause the // BackPressureController to pause the input, causing a deadlock), so drop it. + if(::arrow::compute::kUnsequencedIndex == batch.index) + return Status::Invalid("AsofJoin requires sequenced input"); + if (process_task_.is_finished()) { DEBUG_SYNC(this, "Input received while done. Short circuiting.", DEBUG_MANIP(std::endl)); @@ -1409,12 +1425,9 @@ class AsofJoinNode : public ExecNode { ARROW_DCHECK(std_has(inputs_, input)); size_t k = std_find(inputs_, input) - inputs_.begin(); - // Put into the queue - auto rb = *batch.ToRecordBatch(input->output_schema()); - DEBUG_SYNC(this, "received batch from input ", k, ":", DEBUG_MANIP(std::endl), - rb->ToString(), DEBUG_MANIP(std::endl)); + // Put into the sequencing queue + ARROW_RETURN_NOT_OK(state_.at(k)->InsertBatch(std::move(batch))); - ARROW_RETURN_NOT_OK(state_.at(k)->Push(rb)); PushProcess(true); return Status::OK(); diff --git a/cpp/src/arrow/acero/asof_join_node_test.cc b/cpp/src/arrow/acero/asof_join_node_test.cc index 4a67dd8c2d6d..e83fbf1cb6e7 100644 --- a/cpp/src/arrow/acero/asof_join_node_test.cc +++ b/cpp/src/arrow/acero/asof_join_node_test.cc @@ -101,6 +101,7 @@ Result MakeBatchesFromNumString( BatchesWithSchema batches; batches.schema = schema; int n_fields = schema->num_fields(); + size_t batch_index=0; for (auto num_batch : num_batches.batches) { Datum two(Int32Scalar(2)); std::vector values; @@ -128,6 +129,7 @@ Result MakeBatchesFromNumString( } } ExecBatch batch(values, num_batch.length); + batch.index=batch_index++; batches.batches.push_back(batch); } return batches; @@ -185,6 +187,7 @@ Result MutateByKey(BatchesWithSchema& batches, std::string fr replace_key ? batches.schema->SetField(from_index, new_field) : batches.schema->AddField(from_index, new_field)); } + size_t batch_index = 0; for (const ExecBatch& batch : batches.batches) { std::vector new_values; for (int i = 0; i < n_fields; i++) { @@ -233,6 +236,7 @@ Result MutateByKey(BatchesWithSchema& batches, std::string fr new_values.push_back(value); } new_batches.batches.emplace_back(new_values, batch.length); + new_batches.batches.back().index = batch_index++; } return new_batches; } @@ -405,6 +409,7 @@ void DoRunUnorderedPlanTest(bool l_unordered, bool r_unordered, DoRunUnorderedPlanTest(l_unordered, r_unordered, l_schema, r_schema, GetRepeatedOptions(2, "time", {"key"}, 1000), "out-of-order on-key values"); + // "requires sequenced input"); } struct BasicTestTypes { diff --git a/cpp/src/arrow/acero/test_util_internal.cc b/cpp/src/arrow/acero/test_util_internal.cc index f50ca92238dc..9ecd5d8301d0 100644 --- a/cpp/src/arrow/acero/test_util_internal.cc +++ b/cpp/src/arrow/acero/test_util_internal.cc @@ -384,6 +384,7 @@ Result MakeIntegerBatches( int row = 0; for (int i = 0; i < num_batches; i++) { ARROW_ASSIGN_OR_RAISE(auto batch, MakeIntegerBatch(gens, schema, row, batch_size)); + batch.index = i; out.batches.push_back(std::move(batch)); row += batch_size; } @@ -410,6 +411,9 @@ BatchesWithSchema MakeBatchesFromString(const std::shared_ptr& schema, out_batches.batches.push_back(out_batches.batches[i]); } } + for(size_t batch_index=0;batch_index Date: Tue, 24 Sep 2024 11:26:03 +0000 Subject: [PATCH 3/5] Sequencing output of a scan node --- cpp/src/arrow/acero/source_node.cc | 5 +++-- cpp/src/arrow/dataset/scanner.cc | 6 ++++-- cpp/src/arrow/dataset/scanner.h | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/acero/source_node.cc b/cpp/src/arrow/acero/source_node.cc index 8060e01f074f..1337dd9c39c8 100644 --- a/cpp/src/arrow/acero/source_node.cc +++ b/cpp/src/arrow/acero/source_node.cc @@ -106,7 +106,8 @@ struct SourceNode : ExecNode, public TracedNode { RETURN_NOT_OK(ValidateExecNodeInputs(plan, inputs, 0, "SourceNode")); const auto& source_options = checked_cast(options); return plan->EmplaceNode(plan, source_options.output_schema, - source_options.generator); + source_options.generator, + Ordering::Implicit()); } const char* kind_name() const override { return "SourceNode"; } @@ -406,7 +407,7 @@ struct SchemaSourceNode : public SourceNode { struct RecordBatchReaderSourceNode : public SourceNode { RecordBatchReaderSourceNode(ExecPlan* plan, std::shared_ptr schema, arrow::AsyncGenerator> generator) - : SourceNode(plan, schema, generator) {} + : SourceNode(plan, schema, generator, Ordering::Implicit()) {} static Result Make(ExecPlan* plan, std::vector inputs, const ExecNodeOptions& options) { diff --git a/cpp/src/arrow/dataset/scanner.cc b/cpp/src/arrow/dataset/scanner.cc index a856a792a264..9eb69e8d1119 100644 --- a/cpp/src/arrow/dataset/scanner.cc +++ b/cpp/src/arrow/dataset/scanner.cc @@ -1032,10 +1032,10 @@ Result MakeScanNode(acero::ExecPlan* plan, } else { batch_gen = std::move(merged_batch_gen); } - + int64_t index=require_sequenced_output?0:compute::kUnsequencedIndex; auto gen = MakeMappedGenerator( std::move(batch_gen), - [scan_options](const EnumeratedRecordBatch& partial) + [scan_options, index](const EnumeratedRecordBatch& partial)mutable -> Result> { // TODO(ARROW-13263) fragments may be able to attach more guarantees to batches // than this, for example parquet's row group stats. Failing to do this leaves @@ -1057,6 +1057,8 @@ Result MakeScanNode(acero::ExecPlan* plan, batch->values.emplace_back(partial.record_batch.index); batch->values.emplace_back(partial.record_batch.last); batch->values.emplace_back(partial.fragment.value->ToString()); + if (index!=compute::kUnsequencedIndex) + batch->index = index++; return batch; }); diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h index d2de26789718..7817e090412a 100644 --- a/cpp/src/arrow/dataset/scanner.h +++ b/cpp/src/arrow/dataset/scanner.h @@ -563,7 +563,7 @@ class ARROW_DS_EXPORT ScanNodeOptions : public acero::ExecNodeOptions { public: explicit ScanNodeOptions(std::shared_ptr dataset, std::shared_ptr scan_options, - bool require_sequenced_output = false) + bool require_sequenced_output = true) : dataset(std::move(dataset)), scan_options(std::move(scan_options)), require_sequenced_output(require_sequenced_output) {} From f2fc33b8ed3d7d627e55dfa57315d09997a66f1f Mon Sep 17 00:00:00 2001 From: kamilt Date: Wed, 16 Oct 2024 11:52:50 +0000 Subject: [PATCH 4/5] Perform requested changes --- cpp/src/arrow/acero/accumulation_queue.h | 1 + cpp/src/arrow/acero/asof_join_node.cc | 4 ++-- cpp/src/arrow/acero/asof_join_node_test.cc | 1 - cpp/src/arrow/acero/options.h | 7 +++++-- cpp/src/arrow/acero/source_node.cc | 2 +- cpp/src/arrow/dataset/scanner.cc | 6 ++++-- cpp/src/arrow/dataset/scanner.h | 2 +- python/pyarrow/_dataset.pyx | 9 +++++++-- python/pyarrow/acero.py | 8 ++++---- python/pyarrow/includes/libarrow_dataset.pxd | 2 +- 10 files changed, 26 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/acero/accumulation_queue.h b/cpp/src/arrow/acero/accumulation_queue.h index a27b8b399ce4..a173f9840388 100644 --- a/cpp/src/arrow/acero/accumulation_queue.h +++ b/cpp/src/arrow/acero/accumulation_queue.h @@ -128,6 +128,7 @@ class SerialSequencingQueue { /// Strategy that describes how to handle items class Processor { public: + virtual ~Processor() = default; /// Process the batch /// /// This method will be called on each batch in order. Calls to this method diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc index 4f57095b892c..bcc9498dce35 100644 --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc @@ -771,8 +771,8 @@ class InputState: public util::SerialSequencingQueue::Processor { } private: - std::unique_ptr sequencer_; - + // ExecBatch Sequencer + std::unique_ptr sequencer_; // Pending record batches. The latest is the front. Batches cannot be empty. BackpressureConcurrentQueue> queue_; // Schema associated with the input diff --git a/cpp/src/arrow/acero/asof_join_node_test.cc b/cpp/src/arrow/acero/asof_join_node_test.cc index e83fbf1cb6e7..c19e34e7689c 100644 --- a/cpp/src/arrow/acero/asof_join_node_test.cc +++ b/cpp/src/arrow/acero/asof_join_node_test.cc @@ -409,7 +409,6 @@ void DoRunUnorderedPlanTest(bool l_unordered, bool r_unordered, DoRunUnorderedPlanTest(l_unordered, r_unordered, l_schema, r_schema, GetRepeatedOptions(2, "time", {"key"}, 1000), "out-of-order on-key values"); - // "requires sequenced input"); } struct BasicTestTypes { diff --git a/cpp/src/arrow/acero/options.h b/cpp/src/arrow/acero/options.h index 4447e9c67a19..b7c56a06da94 100644 --- a/cpp/src/arrow/acero/options.h +++ b/cpp/src/arrow/acero/options.h @@ -93,13 +93,16 @@ class ARROW_ACERO_EXPORT SourceNodeOptions : public ExecNodeOptions { public: /// Create an instance from values SourceNodeOptions(std::shared_ptr output_schema, - std::function>()> generator) - : output_schema(std::move(output_schema)), generator(std::move(generator)) {} + std::function>()> generator, + Ordering ordering = Ordering::Unordered()) + : output_schema(std::move(output_schema)), generator(std::move(generator)),ordering(std::move(ordering)) {} /// \brief the schema for batches that will be generated by this source std::shared_ptr output_schema; /// \brief an asynchronous stream of batches ending with std::nullopt std::function>()> generator; + + Ordering ordering = Ordering::Unordered(); }; /// \brief a node that generates data from a table already loaded in memory diff --git a/cpp/src/arrow/acero/source_node.cc b/cpp/src/arrow/acero/source_node.cc index 1337dd9c39c8..ac34e4b6a09f 100644 --- a/cpp/src/arrow/acero/source_node.cc +++ b/cpp/src/arrow/acero/source_node.cc @@ -107,7 +107,7 @@ struct SourceNode : ExecNode, public TracedNode { const auto& source_options = checked_cast(options); return plan->EmplaceNode(plan, source_options.output_schema, source_options.generator, - Ordering::Implicit()); + source_options.ordering); } const char* kind_name() const override { return "SourceNode"; } diff --git a/cpp/src/arrow/dataset/scanner.cc b/cpp/src/arrow/dataset/scanner.cc index 9eb69e8d1119..ef8712af2993 100644 --- a/cpp/src/arrow/dataset/scanner.cc +++ b/cpp/src/arrow/dataset/scanner.cc @@ -1062,16 +1062,18 @@ Result MakeScanNode(acero::ExecPlan* plan, return batch; }); + auto ordering = require_sequenced_output?Ordering::Implicit():Ordering::Unordered(); + auto fields = scan_options->dataset_schema->fields(); if (scan_options->add_augmented_fields) { for (const auto& aug_field : kAugmentedFields) { fields.push_back(aug_field); } } - + return acero::MakeExecNode( "source", plan, {}, - acero::SourceNodeOptions{schema(std::move(fields)), std::move(gen)}); + acero::SourceNodeOptions{schema(std::move(fields)), std::move(gen), ordering}); } Result MakeAugmentedProjectNode(acero::ExecPlan* plan, diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h index 7817e090412a..d2de26789718 100644 --- a/cpp/src/arrow/dataset/scanner.h +++ b/cpp/src/arrow/dataset/scanner.h @@ -563,7 +563,7 @@ class ARROW_DS_EXPORT ScanNodeOptions : public acero::ExecNodeOptions { public: explicit ScanNodeOptions(std::shared_ptr dataset, std::shared_ptr scan_options, - bool require_sequenced_output = true) + bool require_sequenced_output = false) : dataset(std::move(dataset)), scan_options(std::move(scan_options)), require_sequenced_output(require_sequenced_output) {} diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 3583a3213ccb..07024ee59fba 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -4015,11 +4015,14 @@ cdef class _ScanNodeOptions(ExecNodeOptions): def _set_options(self, Dataset dataset, dict scan_options): cdef: shared_ptr[CScanOptions] c_scan_options + bint require_sequenced_output=False c_scan_options = Scanner._make_scan_options(dataset, scan_options) + + require_sequenced_output=scan_options.get("require_sequenced_output",False) self.wrapped.reset( - new CScanNodeOptions(dataset.unwrap(), c_scan_options) + new CScanNodeOptions(dataset.unwrap(), c_scan_options, require_sequenced_output) ) @@ -4045,7 +4048,9 @@ class ScanNodeOptions(_ScanNodeOptions): dataset : pyarrow.dataset.Dataset The table which acts as the data source. **kwargs : dict, optional - Scan options. See `Scanner.from_dataset` for possible arguments. + Scan options. See `Scanner.from_dataset` for possible arguments. + require_sequenced_output : bool, default False + Assert implicit ordering on data. """ def __init__(self, Dataset dataset, **kwargs): diff --git a/python/pyarrow/acero.py b/python/pyarrow/acero.py index 77ba3ab1ce85..d1b7c909402f 100644 --- a/python/pyarrow/acero.py +++ b/python/pyarrow/acero.py @@ -56,8 +56,8 @@ class InMemoryDataset: ds = DatasetModuleStub -def _dataset_to_decl(dataset, use_threads=True): - decl = Declaration("scan", ScanNodeOptions(dataset, use_threads=use_threads)) +def _dataset_to_decl(dataset, use_threads=True, require_sequenced_output=False): + decl = Declaration("scan", ScanNodeOptions(dataset, use_threads=use_threads, require_sequenced_output=require_sequenced_output)) # Get rid of special dataset columns # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" @@ -311,13 +311,13 @@ def _perform_join_asof(left_operand, left_on, left_by, # Add the join node to the execplan if isinstance(left_operand, ds.Dataset): - left_source = _dataset_to_decl(left_operand, use_threads=use_threads) + left_source = _dataset_to_decl(left_operand, use_threads=use_threads, require_sequenced_output=True) else: left_source = Declaration( "table_source", TableSourceNodeOptions(left_operand), ) if isinstance(right_operand, ds.Dataset): - right_source = _dataset_to_decl(right_operand, use_threads=use_threads) + right_source = _dataset_to_decl(right_operand, use_threads=use_threads, require_sequenced_output=True) else: right_source = Declaration( "table_source", TableSourceNodeOptions(right_operand) diff --git a/python/pyarrow/includes/libarrow_dataset.pxd b/python/pyarrow/includes/libarrow_dataset.pxd index fe96705a54b2..51a818bdd412 100644 --- a/python/pyarrow/includes/libarrow_dataset.pxd +++ b/python/pyarrow/includes/libarrow_dataset.pxd @@ -51,7 +51,7 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil: CExpression filter cdef cppclass CScanNodeOptions "arrow::dataset::ScanNodeOptions"(CExecNodeOptions): - CScanNodeOptions(shared_ptr[CDataset] dataset, shared_ptr[CScanOptions] scan_options) + CScanNodeOptions(shared_ptr[CDataset] dataset, shared_ptr[CScanOptions] scan_options, bint require_sequenced_output) shared_ptr[CScanOptions] scan_options From d5972f2585277fe0b2ac901ad51f4cfc3916b9da Mon Sep 17 00:00:00 2001 From: kamilt Date: Wed, 16 Oct 2024 12:57:08 +0000 Subject: [PATCH 5/5] Fixed formatting --- cpp/src/arrow/acero/asof_join_node.cc | 16 ++++++++-------- cpp/src/arrow/acero/asof_join_node_test.cc | 4 ++-- cpp/src/arrow/acero/options.h | 4 +++- cpp/src/arrow/acero/test_util_internal.cc | 4 ++-- cpp/src/arrow/dataset/scanner.cc | 15 +++++++-------- python/pyarrow/_dataset.pyx | 4 ++-- python/pyarrow/acero.py | 13 ++++++++++--- 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cpp/src/arrow/acero/asof_join_node.cc b/cpp/src/arrow/acero/asof_join_node.cc index bcc9498dce35..a5a80c8805fe 100644 --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc @@ -16,9 +16,9 @@ // under the License. #include "arrow/acero/asof_join_node.h" +#include "arrow/acero/accumulation_queue.h" #include "arrow/acero/backpressure_handler.h" #include "arrow/acero/concurrent_queue_internal.h" -#include "arrow/acero/accumulation_queue.h" #include #include @@ -472,7 +472,7 @@ class BackpressureController : public BackpressureControl { std::atomic& backpressure_counter_; }; -class InputState: public util::SerialSequencingQueue::Processor { +class InputState : public util::SerialSequencingQueue::Processor { // InputState corresponds to an input // Input record batches are queued up in InputState until processed and // turned into output record batches. @@ -484,8 +484,8 @@ class InputState: public util::SerialSequencingQueue::Processor { const col_index_t time_col_index, const std::vector& key_col_index) : sequencer_(util::SerialSequencingQueue::Make(this)), - queue_(std::move(handler)), - schema_(schema), + queue_(std::move(handler)), + schema_(schema), time_col_index_(time_col_index), key_col_index_(key_col_index), time_type_id_(schema_->fields()[time_col_index_]->type()->id()), @@ -701,12 +701,12 @@ class InputState: public util::SerialSequencingQueue::Processor { DEBUG_MANIP(std::endl)); return updated; } - Status InsertBatch(ExecBatch batch){ - return sequencer_->InsertBatch(std::move(batch)); + Status InsertBatch(ExecBatch batch) { + return sequencer_->InsertBatch(std::move(batch)); } Status Process(ExecBatch batch) override { - auto rb = *batch.ToRecordBatch(schema_); + auto rb = *batch.ToRecordBatch(schema_); DEBUG_SYNC(node_, "received batch from input ", index_, ":", DEBUG_MANIP(std::endl), rb->ToString(), DEBUG_MANIP(std::endl)); return Push(rb); @@ -1412,7 +1412,7 @@ class AsofJoinNode : public ExecNode { // InputReceived may be called after execution was finished. Pushing it to the // InputState is unnecessary since we're done (and anyway may cause the // BackPressureController to pause the input, causing a deadlock), so drop it. - if(::arrow::compute::kUnsequencedIndex == batch.index) + if (::arrow::compute::kUnsequencedIndex == batch.index) return Status::Invalid("AsofJoin requires sequenced input"); if (process_task_.is_finished()) { diff --git a/cpp/src/arrow/acero/asof_join_node_test.cc b/cpp/src/arrow/acero/asof_join_node_test.cc index c19e34e7689c..2983888183a3 100644 --- a/cpp/src/arrow/acero/asof_join_node_test.cc +++ b/cpp/src/arrow/acero/asof_join_node_test.cc @@ -101,7 +101,7 @@ Result MakeBatchesFromNumString( BatchesWithSchema batches; batches.schema = schema; int n_fields = schema->num_fields(); - size_t batch_index=0; + size_t batch_index = 0; for (auto num_batch : num_batches.batches) { Datum two(Int32Scalar(2)); std::vector values; @@ -129,7 +129,7 @@ Result MakeBatchesFromNumString( } } ExecBatch batch(values, num_batch.length); - batch.index=batch_index++; + batch.index = batch_index++; batches.batches.push_back(batch); } return batches; diff --git a/cpp/src/arrow/acero/options.h b/cpp/src/arrow/acero/options.h index b7c56a06da94..2beacfe26baa 100644 --- a/cpp/src/arrow/acero/options.h +++ b/cpp/src/arrow/acero/options.h @@ -95,7 +95,9 @@ class ARROW_ACERO_EXPORT SourceNodeOptions : public ExecNodeOptions { SourceNodeOptions(std::shared_ptr output_schema, std::function>()> generator, Ordering ordering = Ordering::Unordered()) - : output_schema(std::move(output_schema)), generator(std::move(generator)),ordering(std::move(ordering)) {} + : output_schema(std::move(output_schema)), + generator(std::move(generator)), + ordering(std::move(ordering)) {} /// \brief the schema for batches that will be generated by this source std::shared_ptr output_schema; diff --git a/cpp/src/arrow/acero/test_util_internal.cc b/cpp/src/arrow/acero/test_util_internal.cc index 9ecd5d8301d0..107a20354c0e 100644 --- a/cpp/src/arrow/acero/test_util_internal.cc +++ b/cpp/src/arrow/acero/test_util_internal.cc @@ -411,8 +411,8 @@ BatchesWithSchema MakeBatchesFromString(const std::shared_ptr& schema, out_batches.batches.push_back(out_batches.batches[i]); } } - for(size_t batch_index=0;batch_index MakeScanNode(acero::ExecPlan* plan, } else { batch_gen = std::move(merged_batch_gen); } - int64_t index=require_sequenced_output?0:compute::kUnsequencedIndex; + int64_t index = require_sequenced_output ? 0 : compute::kUnsequencedIndex; auto gen = MakeMappedGenerator( std::move(batch_gen), - [scan_options, index](const EnumeratedRecordBatch& partial)mutable - -> Result> { + [scan_options, index](const EnumeratedRecordBatch& partial) mutable + -> Result> { // TODO(ARROW-13263) fragments may be able to attach more guarantees to batches // than this, for example parquet's row group stats. Failing to do this leaves // perf on the table because row group stats could be used to skip kernel execs in @@ -1057,20 +1057,19 @@ Result MakeScanNode(acero::ExecPlan* plan, batch->values.emplace_back(partial.record_batch.index); batch->values.emplace_back(partial.record_batch.last); batch->values.emplace_back(partial.fragment.value->ToString()); - if (index!=compute::kUnsequencedIndex) - batch->index = index++; + if (index != compute::kUnsequencedIndex) batch->index = index++; return batch; }); - auto ordering = require_sequenced_output?Ordering::Implicit():Ordering::Unordered(); - + auto ordering = require_sequenced_output ? Ordering::Implicit() : Ordering::Unordered(); + auto fields = scan_options->dataset_schema->fields(); if (scan_options->add_augmented_fields) { for (const auto& aug_field : kAugmentedFields) { fields.push_back(aug_field); } } - + return acero::MakeExecNode( "source", plan, {}, acero::SourceNodeOptions{schema(std::move(fields)), std::move(gen), ordering}); diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 07024ee59fba..d9891779efb3 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -4018,8 +4018,8 @@ cdef class _ScanNodeOptions(ExecNodeOptions): bint require_sequenced_output=False c_scan_options = Scanner._make_scan_options(dataset, scan_options) - - require_sequenced_output=scan_options.get("require_sequenced_output",False) + + require_sequenced_output=scan_options.get("require_sequenced_output", False) self.wrapped.reset( new CScanNodeOptions(dataset.unwrap(), c_scan_options, require_sequenced_output) diff --git a/python/pyarrow/acero.py b/python/pyarrow/acero.py index d1b7c909402f..706338bd8cdb 100644 --- a/python/pyarrow/acero.py +++ b/python/pyarrow/acero.py @@ -57,7 +57,9 @@ class InMemoryDataset: def _dataset_to_decl(dataset, use_threads=True, require_sequenced_output=False): - decl = Declaration("scan", ScanNodeOptions(dataset, use_threads=use_threads, require_sequenced_output=require_sequenced_output)) + decl = Declaration("scan", ScanNodeOptions( + dataset, use_threads=use_threads, + require_sequenced_output=require_sequenced_output)) # Get rid of special dataset columns # "__fragment_index", "__batch_index", "__last_in_fragment", "__filename" @@ -311,13 +313,18 @@ def _perform_join_asof(left_operand, left_on, left_by, # Add the join node to the execplan if isinstance(left_operand, ds.Dataset): - left_source = _dataset_to_decl(left_operand, use_threads=use_threads, require_sequenced_output=True) + left_source = _dataset_to_decl( + left_operand, + use_threads=use_threads, + require_sequenced_output=True) else: left_source = Declaration( "table_source", TableSourceNodeOptions(left_operand), ) if isinstance(right_operand, ds.Dataset): - right_source = _dataset_to_decl(right_operand, use_threads=use_threads, require_sequenced_output=True) + right_source = _dataset_to_decl( + right_operand, use_threads=use_threads, + require_sequenced_output=True) else: right_source = Declaration( "table_source", TableSourceNodeOptions(right_operand)