From 15fb55edf4be2cd35de2013dd1de5bd543fe3c33 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:15:11 +0000 Subject: [PATCH 01/17] Add `add/set/remove_column` methods to `RecordBatch` --- python/pyarrow/includes/libarrow.pxd | 6 + python/pyarrow/table.pxi | 207 +++++++++++++++++++++++++++ python/pyarrow/tests/test_table.py | 72 ++++++++++ 3 files changed, 285 insertions(+) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index bc9d05ddbbc3..8602076d092a 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -961,6 +961,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CArray] column(int i) const c_string& column_name(int i) + CResult[shared_ptr[CRecordBatch]] AddColumn( + int i, shared_ptr[CField] field, shared_ptr[CArray] column) + CResult[shared_ptr[CRecordBatch]] RemoveColumn(int i) + CResult[shared_ptr[CRecordBatch]] SetColumn( + int i, shared_ptr[CField] field, shared_ptr[CArray] column) + const vector[shared_ptr[CArray]]& columns() CResult[shared_ptr[CRecordBatch]] SelectColumns(const vector[int]&) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index d7f7895b538e..17ce8b1eb75d 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2483,6 +2483,213 @@ cdef class RecordBatch(_Tabular): def __sizeof__(self): return super(RecordBatch, self).__sizeof__() + self.nbytes + def add_column(self, int i, field_, column): + """ + Add column to RecordBatch at position. + + A new record batch is returned with the column added, the original record batch + object is left unchanged. + + Parameters + ---------- + i : int + Index to place the column at. + field_ : str or Field + If a string is passed then the type is deduced from the column + data. + column : Array or value coercible to array + Column data. + + Returns + ------- + RecordBatch + New record batch with the passed column added. + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + + Add column: + + >>> year = [2021, 2022, 2019, 2021] + >>> batch.add_column(0,"year", year) + pyarrow.RecordBatch + year: int64 + n_legs: int64 + animals: string + ---- + year: [2021,2022,2019,2021] + n_legs: [2,4,5,100] + animals: ["Flamingo","Horse","Brittle stars","Centipede"] + + Original record batch is left unchanged: + + >>> batch + pyarrow.RecordBatch + n_legs: int64 + animals: string + ---- + n_legs: [2,4,5,100] + animals: ["Flamingo","Horse","Brittle stars","Centipede"] + """ + cdef: + shared_ptr[CRecordBatch] c_batch + Field c_field + Array c_arr + + if isinstance(column, Array): + c_arr = column + else: + c_arr = array(column) + + if isinstance(field_, Field): + c_field = field_ + else: + c_field = field(field_, c_arr.type) + + with nogil: + c_batch = GetResultValue(self.batch.AddColumn( + i, c_field.sp_field, c_arr.sp_array)) + + return pyarrow_wrap_batch(c_batch) + + def append_column(self, field_, column): + """ + Append column at end of columns. + + Parameters + ---------- + field_ : str or Field + If a string is passed then the type is deduced from the column + data. + column : Array or value coercible to array + Column data. + + Returns + ------- + RecordBatch + New record batch with the passed column added. + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + + Append column at the end: + + >>> year = [2021, 2022, 2019, 2021] + >>> batch.append_column('year', year) + pyarrow.RecordBatch + n_legs: int64 + animals: string + year: int64 + ---- + n_legs: [2,4,5,100] + animals: ["Flamingo","Horse","Brittle stars","Centipede"] + year: [2021,2022,2019,2021] + """ + return self.add_column(self.num_columns, field_, column) + + def remove_column(self, int i): + """ + Create new RecordBatch with the indicated column removed. + + Parameters + ---------- + i : int + Index of column to remove. + + Returns + ------- + Table + New record batch without the column. + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + >>> batch.remove_column(1) + pyarrow.RecordBatch + n_legs: int64 + ---- + n_legs: [2,4,5,100] + """ + cdef shared_ptr[CRecordBatch] c_batch + + with nogil: + c_batch = GetResultValue(self.batch.RemoveColumn(i)) + + return pyarrow_wrap_batch(c_batch) + + def set_column(self, int i, field_, column): + """ + Replace column in RecordBatch at position. + + Parameters + ---------- + i : int + Index to place the column at. + field_ : str or Field + If a string is passed then the type is deduced from the column + data. + column : Array or value coercible to array + Column data. + + Returns + ------- + RecordBatch + New record batch with the passed column set. + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + + Replace a column: + + >>> year = [2021, 2022, 2019, 2021] + >>> batch.set_column(1,'year', year) + pyarrow.RecordBatch + n_legs: int64 + year: int64 + ---- + n_legs: [2,4,5,100] + year: [2021,2022,2019,2021] + """ + cdef: + shared_ptr[CRecordBatch] c_batch + Field c_field + Array c_arr + + if isinstance(column, Array): + c_arr = column + else: + c_arr = array(column) + + if isinstance(field_, Field): + c_field = field_ + else: + c_field = field(field_, c_arr.type) + + with nogil: + c_batch = GetResultValue(self.batch.SetColumn( + i, c_field.sp_field, c_arr.sp_array)) + + return pyarrow_wrap_batch(c_batch) + def serialize(self, memory_pool=None): """ Write RecordBatch to Buffer as encapsulated IPC message, which does not diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index f0fd5518de06..3489661101a1 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -1286,6 +1286,30 @@ def test_table_add_column(): assert t4.equals(expected) +def test_record_batch_add_column(): + data_type = pa.int64() + data = [ + range(5), + [-10, -5, 0, 5, 10], + pa.array(range(5, 10)) + ] + batch = pa.RecordBatch.from_arrays(data, names=('a', 'b', 'c')) + + new_field = pa.field('d', data_type) + t2 = batch.add_column(3, new_field, data[1]) + t3 = batch.append_column(new_field, data[1]) + + expected = pa.RecordBatch.from_arrays(data + [data[1]], + names=('a', 'b', 'c', 'd')) + assert t2.equals(expected) + assert t3.equals(expected) + + t4 = batch.add_column(0, new_field, data[1]) + expected = pa.RecordBatch.from_arrays([data[1]] + data, + names=('d', 'a', 'b', 'c')) + assert t4.equals(expected) + + def test_table_set_column(): data = [ pa.array(range(5)), @@ -1304,6 +1328,25 @@ def test_table_set_column(): assert t2.equals(expected) +def test_record_batch_set_column(): + data_type = pa.int64() + data = [ + range(5), + [-10, -5, 0, 5, 10], + pa.array(range(5, 10)) + ] + batch = pa.RecordBatch.from_arrays(data, names=('a', 'b', 'c')) + + new_field = pa.field('d', data_type) + t2 = batch.set_column(0, new_field, data[1]) + + expected_data = list(data) + expected_data[0] = data[1] + expected = pa.RecordBatch.from_arrays(expected_data, + names=('d', 'b', 'c')) + assert t2.equals(expected) + + def test_table_drop_columns(): """ drop one or more columns given labels""" a = pa.array(range(5)) @@ -1358,6 +1401,20 @@ def test_table_remove_column(): assert t2.equals(expected) +def test_record_batch_remove_column(): + data = [ + range(5), + [-10, -5, 0, 5, 10], + pa.array(range(5, 10)) + ] + batch = pa.RecordBatch.from_arrays(data, names=('a', 'b', 'c')) + + t2 = batch.remove_column(0) + t2.validate() + expected = pa.RecordBatch.from_arrays(data[1:], names=('b', 'c')) + assert t2.equals(expected) + + def test_table_remove_column_empty(): # ARROW-1865 data = [ @@ -1374,6 +1431,21 @@ def test_table_remove_column_empty(): assert t3.equals(table) +def test_record_batch_remove_column_empty(): + data = [ + range(5), + ] + batch = pa.RecordBatch.from_arrays(data, names=['a']) + + t2 = batch.remove_column(0) + t2.validate() + assert len(t2) == len(batch) + + t3 = t2.add_column(0, batch.field(0), batch[0]) + t3.validate() + assert t3.equals(batch) + + def test_empty_table_with_names(): # ARROW-13784 data = [] From c9929da76a2559ecbc81d03bb2d2643b29bbf9c9 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:08:19 +0000 Subject: [PATCH 02/17] Add `RecordBatch.drop_columns` --- python/pyarrow/table.pxi | 61 ++++++++++++++++++++++++++++++ python/pyarrow/tests/test_table.py | 20 ++++++++++ 2 files changed, 81 insertions(+) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 17ce8b1eb75d..b64f600020a1 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2690,6 +2690,67 @@ cdef class RecordBatch(_Tabular): return pyarrow_wrap_batch(c_batch) + def drop_columns(self, columns): + """ + Drop one or more columns and return a new record batch. + + Parameters + ---------- + columns : str or list[str] + Field name(s) referencing existing column(s). + + Raises + ------ + KeyError + If any of the passed column names do not exist. + + Returns + ------- + RecordBatch + New record batch without the column(s). + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + + Drop one column: + + >>> batch.drop_columns("animals") + pyarrow.RecordBatch + n_legs: int64 + ---- + n_legs: [[2,4,5,100]] + + Drop one or more columns: + + >>> batch.drop_columns(["n_legs", "animals"]) + pyarrow.RecordBatch + ... + ---- + """ + if isinstance(columns, str): + columns = [columns] + + indices = [] + for col in columns: + idx = self.schema.get_field_index(col) + if idx == -1: + raise KeyError("Column {!r} not found".format(col)) + indices.append(idx) + + indices.sort() + indices.reverse() + + batch = self + for idx in indices: + batch = batch.remove_column(idx) + + return batch + def serialize(self, memory_pool=None): """ Write RecordBatch to Buffer as encapsulated IPC message, which does not diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 3489661101a1..62f640f9e065 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -1367,6 +1367,26 @@ def test_table_drop_columns(): table.drop_columns(['d']) +def test_record_batch_drop_columns(): + """ drop one or more columns given labels""" + a = pa.array(range(5)) + b = pa.array([-10, -5, 0, 5, 10]) + c = pa.array(range(5, 10)) + + batch = pa.RecordBatch.from_arrays([a, b, c], names=('a', 'b', 'c')) + t2 = batch.drop_columns(['a', 'b']) + t3 = batch.drop_columns('a') + + exp_t2 = pa.RecordBatch.from_arrays([c], names=('c',)) + assert exp_t2.equals(t2) + exp_t3 = pa.RecordBatch.from_arrays([b, c], names=('b', 'c',)) + assert exp_t3.equals(t3) + + # -- raise KeyError if column not in RecordBatch + with pytest.raises(KeyError, match="Column 'd' not found"): + batch.drop_columns(['d']) + + def test_table_drop(): """ verify the alias of drop_columns is working""" a = pa.array(range(5)) From 0a046b1cc98c0e9cb5bd79841d89308f9c34673d Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:24:03 +0000 Subject: [PATCH 03/17] Add `RecordBatch::RenameColumns` --- cpp/src/arrow/record_batch.cc | 29 +++++++++++++++++++++++++++++ cpp/src/arrow/record_batch.h | 7 +++++++ cpp/src/arrow/record_batch_test.cc | 25 +++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index ca6b45af3d6b..4f37b701977f 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -303,6 +303,35 @@ Result> RecordBatch::ReplaceSchema( return RecordBatch::Make(std::move(schema), num_rows(), columns()); } +std::vector RecordBatch::ColumnNames() const { + std::vector names(num_columns()); + for (int i = 0; i < num_columns(); ++i) { + names[i] = schema()->field(i)->name(); + } + return names; +} + +Result> RecordBatch::RenameColumns( + const std::vector& names) const { + int n = static_cast(num_columns()); + + if (static_cast(names.size()) != n) { + return Status::Invalid("tried to rename a record batch of ", n, " columns but only ", + names.size(), " names were provided"); + } + + ArrayVector columns(n); + FieldVector fields(n); + + for (int i = 0; i < n; ++i) { + columns[i] = column(i); + fields[i] = schema()->field(i)->WithName(names[i]); + } + + return RecordBatch::Make(::arrow::schema(std::move(fields)), num_rows(), + std::move(columns)); +} + Result> RecordBatch::SelectColumns( const std::vector& indices) const { int n = static_cast(indices.size()); diff --git a/cpp/src/arrow/record_batch.h b/cpp/src/arrow/record_batch.h index 79f93a7b5997..9319ba5f67ac 100644 --- a/cpp/src/arrow/record_batch.h +++ b/cpp/src/arrow/record_batch.h @@ -219,6 +219,13 @@ class ARROW_EXPORT RecordBatch { /// \return PrettyPrint representation suitable for debugging std::string ToString() const; + /// \brief Return names of all columns + std::vector ColumnNames() const; + + /// \brief Rename columns with provided names + Result> RenameColumns( + const std::vector& names) const; + /// \brief Return new record batch with specified columns Result> SelectColumns( const std::vector& indices) const; diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index db3a2d3def73..5e100bab429e 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -314,6 +314,31 @@ TEST_F(TestRecordBatch, RemoveColumn) { AssertBatchesEqual(*new_batch, *batch4); } +TEST_F(TestRecordBatch, RenameColumns) { + const int length = 10; + + auto field1 = field("f1", int32()); + auto field2 = field("f2", uint8()); + auto field3 = field("f3", int16()); + + auto schema1 = ::arrow::schema({field1, field2, field3}); + + random::RandomArrayGenerator gen(42); + + auto array1 = gen.ArrayOf(int32(), length); + auto array2 = gen.ArrayOf(uint8(), length); + auto array3 = gen.ArrayOf(int16(), length); + + auto batch = RecordBatch::Make(schema1, length, {array1, array2, array3}); + EXPECT_THAT(batch->ColumnNames(), testing::ElementsAre("f1", "f2", "f3")); + + ASSERT_OK_AND_ASSIGN(auto renamed, batch->RenameColumns({"zero", "one", "two"})); + EXPECT_THAT(renamed->ColumnNames(), testing::ElementsAre("zero", "one", "two")); + ASSERT_OK(renamed->ValidateFull()); + + ASSERT_RAISES(Invalid, batch->RenameColumns({"hello", "world"})); +} + TEST_F(TestRecordBatch, SelectColumns) { const int length = 10; From 854e3bb74e1cf314ff08d256d5968795f062c182 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:10:08 +0000 Subject: [PATCH 04/17] Add `RecordBatch.rename_columns` --- python/pyarrow/includes/libarrow.pxd | 1 + python/pyarrow/table.pxi | 41 ++++++++++++++++++++++++++++ python/pyarrow/tests/test_table.py | 17 ++++++++++++ 3 files changed, 59 insertions(+) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8602076d092a..2da985ced038 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -969,6 +969,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const vector[shared_ptr[CArray]]& columns() + CResult[shared_ptr[CRecordBatch]] RenameColumns(const vector[c_string]&) CResult[shared_ptr[CRecordBatch]] SelectColumns(const vector[int]&) int num_columns() diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index b64f600020a1..63f4ab4e352c 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2690,6 +2690,47 @@ cdef class RecordBatch(_Tabular): return pyarrow_wrap_batch(c_batch) + def rename_columns(self, names): + """ + Create new record batch with columns renamed to provided names. + + Parameters + ---------- + names : list of str + List of new column names. + + Returns + ------- + RecordBatch + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + >>> new_names = ["n", "name"] + >>> batch.rename_columns(new_names) + pyarrow.RecordBatch + n: int64 + name: string + ---- + n: [2,4,5,100] + name: ["Flamingo","Horse","Brittle stars","Centipede"] + """ + cdef: + shared_ptr[CRecordBatch] c_batch + vector[c_string] c_names + + for name in names: + c_names.push_back(tobytes(name)) + + with nogil: + c_batch = GetResultValue(self.batch.RenameColumns(move(c_names))) + + return pyarrow_wrap_batch(c_batch) + def drop_columns(self, columns): """ Drop one or more columns and return a new record batch. diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 62f640f9e065..7b97ef28d63d 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -1500,6 +1500,23 @@ def test_table_rename_columns(): assert t2.equals(expected) +def test_record_batch_rename_columns(): + data = [ + pa.array(range(5)), + pa.array([-10, -5, 0, 5, 10]), + pa.array(range(5, 10)) + ] + batch = pa.RecordBatch.from_arrays(data, names=['a', 'b', 'c']) + assert batch.column_names == ['a', 'b', 'c'] + + t2 = batch.rename_columns(['eh', 'bee', 'sea']) + t2.validate() + assert t2.column_names == ['eh', 'bee', 'sea'] + + expected = pa.RecordBatch.from_arrays(data, names=['eh', 'bee', 'sea']) + assert t2.equals(expected) + + def test_table_flatten(): ty1 = pa.struct([pa.field('x', pa.int16()), pa.field('y', pa.float32())]) From f18aada7cd77d8abc71eebdcdbc31ece758f87a1 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:46:34 +0000 Subject: [PATCH 05/17] Move `drop_columns` implementation to `_Tabular` --- python/pyarrow/table.pxi | 188 ++++++++++++++------------------------- 1 file changed, 66 insertions(+), 122 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 63f4ab4e352c..e2804e965abb 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2150,6 +2150,72 @@ cdef class _Tabular(_PandasConvertible): pieces.append('...') return '\n'.join(pieces) + def remove_column(self, int i): + raise NotImplementedError + + def drop_columns(self, columns): + """ + Drop one or more columns and return a new Table or RecordBatch. + + Parameters + ---------- + columns : str or list[str] + Field name(s) referencing existing column(s). + + Raises + ------ + KeyError + If any of the passed column names do not exist. + + Returns + ------- + Table or RecordBatch + A tabular object without the column(s). + + Examples + -------- + Table (works similarly for RecordBatch) + + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> table = pa.Table.from_pandas(df) + + Drop one column: + + >>> table.drop_columns("animals") + pyarrow.Table + n_legs: int64 + ---- + n_legs: [[2,4,5,100]] + + Drop one or more columns: + + >>> table.drop_columns(["n_legs", "animals"]) + pyarrow.Table + ... + ---- + """ + if isinstance(columns, str): + columns = [columns] + + indices = [] + for col in columns: + idx = self.schema.get_field_index(col) + if idx == -1: + raise KeyError("Column {!r} not found".format(col)) + indices.append(idx) + + indices.sort() + indices.reverse() + + res = self + for idx in indices: + res = res.remove_column(idx) + + return res + cdef class RecordBatch(_Tabular): """ @@ -2731,67 +2797,6 @@ cdef class RecordBatch(_Tabular): return pyarrow_wrap_batch(c_batch) - def drop_columns(self, columns): - """ - Drop one or more columns and return a new record batch. - - Parameters - ---------- - columns : str or list[str] - Field name(s) referencing existing column(s). - - Raises - ------ - KeyError - If any of the passed column names do not exist. - - Returns - ------- - RecordBatch - New record batch without the column(s). - - Examples - -------- - >>> import pyarrow as pa - >>> import pandas as pd - >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], - ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) - >>> batch = pa.RecordBatch.from_pandas(df) - - Drop one column: - - >>> batch.drop_columns("animals") - pyarrow.RecordBatch - n_legs: int64 - ---- - n_legs: [[2,4,5,100]] - - Drop one or more columns: - - >>> batch.drop_columns(["n_legs", "animals"]) - pyarrow.RecordBatch - ... - ---- - """ - if isinstance(columns, str): - columns = [columns] - - indices = [] - for col in columns: - idx = self.schema.get_field_index(col) - if idx == -1: - raise KeyError("Column {!r} not found".format(col)) - indices.append(idx) - - indices.sort() - indices.reverse() - - batch = self - for idx in indices: - batch = batch.remove_column(idx) - - return batch - def serialize(self, memory_pool=None): """ Write RecordBatch to Buffer as encapsulated IPC message, which does not @@ -5133,67 +5138,6 @@ cdef class Table(_Tabular): return pyarrow_wrap_table(c_table) - def drop_columns(self, columns): - """ - Drop one or more columns and return a new table. - - Parameters - ---------- - columns : str or list[str] - Field name(s) referencing existing column(s). - - Raises - ------ - KeyError - If any of the passed column names do not exist. - - Returns - ------- - Table - New table without the column(s). - - Examples - -------- - >>> import pyarrow as pa - >>> import pandas as pd - >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], - ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) - >>> table = pa.Table.from_pandas(df) - - Drop one column: - - >>> table.drop_columns("animals") - pyarrow.Table - n_legs: int64 - ---- - n_legs: [[2,4,5,100]] - - Drop one or more columns: - - >>> table.drop_columns(["n_legs", "animals"]) - pyarrow.Table - ... - ---- - """ - if isinstance(columns, str): - columns = [columns] - - indices = [] - for col in columns: - idx = self.schema.get_field_index(col) - if idx == -1: - raise KeyError("Column {!r} not found".format(col)) - indices.append(idx) - - indices.sort() - indices.reverse() - - table = self - for idx in indices: - table = table.remove_column(idx) - - return table - def drop(self, columns): """ Drop one or more columns and return a new table. From 0c86f39e1359bcbdbc59a72a53a717e033039e73 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:57:52 +0000 Subject: [PATCH 06/17] Allow `pa.record_batch` to take a `dict` This matches the behavior of `pa.table`. --- python/pyarrow/table.pxi | 15 +++++++++++---- python/pyarrow/tests/test_table.py | 4 ++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index e2804e965abb..5c56225c2f04 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5348,10 +5348,10 @@ def record_batch(data, names=None, schema=None, metadata=None): Parameters ---------- - data : pandas.DataFrame, list, Arrow-compatible table - A DataFrame, list of arrays or chunked arrays, or a tabular object - implementing the Arrow PyCapsule Protocol (has an - ``__arrow_c_array__`` method). + data : pandas.DataFrame, dict, list, Arrow-compatible table + A DataFrame, mapping of strings to Arrays or Python lists, list of + arrays or chunked arrays, or a tabular object implementing the Arrow + PyCapsule Protocol (has an ``__arrow_c_array__`` method). names : list, default None Column names if list of arrays passed as data. Mutually exclusive with 'schema' argument. @@ -5470,6 +5470,13 @@ def record_batch(data, names=None, schema=None, metadata=None): if isinstance(data, (list, tuple)): return RecordBatch.from_arrays(data, names=names, schema=schema, metadata=metadata) + + elif isinstance(data, dict): + if names is not None: + raise ValueError( + "The 'names' argument is not valid when passing a dictionary") + return RecordBatch.from_pydict(data, schema=schema, metadata=metadata) + elif hasattr(data, "__arrow_c_array__"): if schema is not None: requested_schema = schema.__arrow_c_schema__() diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 7b97ef28d63d..40250f8192cd 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -505,12 +505,14 @@ def test_recordbatch_basics(): batch.nbytes == (5 * 2) + (5 * 4 + 1) assert sys.getsizeof(batch) >= object.__sizeof__( batch) + batch.get_total_buffer_size() + pydict = batch.to_pydict() assert pydict == OrderedDict([ ('c0', [0, 1, 2, 3, 4]), ('c1', [-10, -5, 0, None, 10]) ]) assert isinstance(pydict, dict) + assert batch == pa.record_batch(pydict, schema=batch.schema) with pytest.raises(IndexError): # bounds checking @@ -1117,12 +1119,14 @@ def test_table_basics(): assert table.nbytes == 2 * (5 * 8) assert sys.getsizeof(table) >= object.__sizeof__( table) + table.get_total_buffer_size() + pydict = table.to_pydict() assert pydict == OrderedDict([ ('a', [0, 1, 2, 3, 4]), ('b', [-10, -5, 0, 5, 10]) ]) assert isinstance(pydict, dict) + assert table == pa.table(pydict, schema=table.schema) columns = [] for col in table.itercolumns(): From 386327147f947d96edfcaf1163f2585fb57deaed Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:07:12 +0000 Subject: [PATCH 07/17] Add `RecordBatch.cast` --- python/pyarrow/table.pxi | 63 ++++++++++++++++++++ python/pyarrow/tests/test_table.py | 93 ++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 5c56225c2f04..a894e80b7f4f 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -3002,6 +3002,69 @@ cdef class RecordBatch(_Tabular): return result + def cast(self, Schema target_schema, safe=None, options=None): + """ + Cast record batch values to another schema. + + Parameters + ---------- + target_schema : Schema + Schema to cast to, the names and order of fields must match. + safe : bool, default True + Check for overflows or other unsafe conversions. + options : CastOptions, default None + Additional checks pass by CastOptions + + Returns + ------- + RecordBatch + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + >>> batch.schema + n_legs: int64 + animals: string + -- schema metadata -- + pandas: '{"index_columns": [{"kind": "range", "name": null, "start": 0, ... + + Define new schema and cast batch values: + + >>> my_schema = pa.schema([ + ... pa.field('n_legs', pa.duration('s')), + ... pa.field('animals', pa.string())] + ... ) + >>> batch.cast(target_schema=my_schema) + pyarrow.RecordBatch + n_legs: duration[s] + animals: string + ---- + n_legs: [2,4,5,100] + animals: ["Flamingo","Horse","Brittle stars","Centipede"] + """ + cdef: + Array column, casted + Field field + list newcols = [] + + if self.schema.names != target_schema.names: + raise ValueError("Target schema's field names are not matching " + "the record batch's field names: {!r}, {!r}" + .format(self.schema.names, target_schema.names)) + + for column, field in zip(self.itercolumns(), target_schema): + if not field.nullable and column.null_count > 0: + raise ValueError("Casting field {!r} with null values to non-nullable" + .format(field.name)) + casted = column.cast(field.type, safe=safe, options=options) + newcols.append(casted) + + return RecordBatch.from_arrays(newcols, schema=target_schema) + def select(self, object columns): """ Select columns of the RecordBatch. diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 40250f8192cd..39bb3eeb0168 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -1762,6 +1762,28 @@ def test_table_cast_to_incompatible_schema(): table.cast(target_schema2) +def test_record_batch_cast_to_incompatible_schema(): + data = [ + pa.array(range(5)), + pa.array([-10, -5, 0, 5, 10]), + ] + batch = pa.RecordBatch.from_arrays(data, names=tuple('ab')) + + target_schema1 = pa.schema([ + pa.field('A', pa.int32()), + pa.field('b', pa.int16()), + ]) + target_schema2 = pa.schema([ + pa.field('a', pa.int32()), + ]) + message = ("Target schema's field names are not matching the " + "record batch's field names:.*") + with pytest.raises(ValueError, match=message): + batch.cast(target_schema1) + with pytest.raises(ValueError, match=message): + batch.cast(target_schema2) + + def test_table_safe_casting(): data = [ pa.array(range(5), type=pa.int64()), @@ -1790,6 +1812,34 @@ def test_table_safe_casting(): assert casted_table.equals(expected_table) +def test_record_batch_safe_casting(): + data = [ + pa.array(range(5), type=pa.int64()), + pa.array([-10, -5, 0, 5, 10], type=pa.int32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), + pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) + ] + batch = pa.RecordBatch.from_arrays(data, names=tuple('abcd')) + + expected_data = [ + pa.array(range(5), type=pa.int32()), + pa.array([-10, -5, 0, 5, 10], type=pa.int16()), + pa.array([1, 2, 3, 4, 5], type=pa.int64()), + pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) + ] + expected_batch = pa.RecordBatch.from_arrays(expected_data, names=tuple('abcd')) + + target_schema = pa.schema([ + pa.field('a', pa.int32()), + pa.field('b', pa.int16()), + pa.field('c', pa.int64()), + pa.field('d', pa.string()) + ]) + casted_batch = batch.cast(target_schema) + + assert casted_batch.equals(expected_batch) + + def test_table_unsafe_casting(): data = [ pa.array(range(5), type=pa.int64()), @@ -1821,6 +1871,37 @@ def test_table_unsafe_casting(): assert casted_table.equals(expected_table) +def test_record_batch_unsafe_casting(): + data = [ + pa.array(range(5), type=pa.int64()), + pa.array([-10, -5, 0, 5, 10], type=pa.int32()), + pa.array([1.1, 2.2, 3.3, 4.4, 5.5], type=pa.float64()), + pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) + ] + batch = pa.RecordBatch.from_arrays(data, names=tuple('abcd')) + + expected_data = [ + pa.array(range(5), type=pa.int32()), + pa.array([-10, -5, 0, 5, 10], type=pa.int16()), + pa.array([1, 2, 3, 4, 5], type=pa.int64()), + pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) + ] + expected_batch = pa.RecordBatch.from_arrays(expected_data, names=tuple('abcd')) + + target_schema = pa.schema([ + pa.field('a', pa.int32()), + pa.field('b', pa.int16()), + pa.field('c', pa.int64()), + pa.field('d', pa.string()) + ]) + + with pytest.raises(pa.ArrowInvalid, match='truncated'): + batch.cast(target_schema) + + casted_batch = batch.cast(target_schema, safe=False) + assert casted_batch.equals(expected_batch) + + def test_invalid_table_construct(): array = np.array([0, 1], dtype=np.uint8) u8 = pa.uint8() @@ -2690,6 +2771,18 @@ def test_table_cast_invalid(): assert table.cast(new_schema).schema == new_schema +def test_record_batch_cast_invalid(): + # Casting a nullable field to non-nullable should be invalid! + batch = pa.record_batch({'a': [None, 1], 'b': [None, True]}) + new_schema = pa.schema([pa.field("a", "int64", nullable=True), + pa.field("b", "bool", nullable=False)]) + with pytest.raises(ValueError): + batch.cast(new_schema) + + batch = pa.record_batch({'a': [None, 1], 'b': [False, True]}) + assert batch.cast(new_schema).schema == new_schema + + def test_table_sort_by(): table = pa.table([ pa.array([3, 1, 4, 2, 5]), From 31c15a25f89ba69f87d0aee9eeac77ace6f0b9f2 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:43:07 +0000 Subject: [PATCH 08/17] Update docstring Co-authored-by: Joris Van den Bossche --- python/pyarrow/table.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index a894e80b7f4f..1f8c4e3fb0a0 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2551,7 +2551,7 @@ cdef class RecordBatch(_Tabular): def add_column(self, int i, field_, column): """ - Add column to RecordBatch at position. + Add column to RecordBatch at position i. A new record batch is returned with the column added, the original record batch object is left unchanged. From 559f86a3264b4427132eafef4c2651eee55517a7 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:02:23 +0000 Subject: [PATCH 09/17] Parameterize `Table` tests --- python/pyarrow/tests/test_table.py | 338 +++++++++-------------------- 1 file changed, 106 insertions(+), 232 deletions(-) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 39bb3eeb0168..3b98712b4b24 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -1267,103 +1267,81 @@ def test_table_column_with_duplicates(): table.column('a') -def test_table_add_column(): +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) + ] +) +def test_table_add_column(cls): data = [ pa.array(range(5)), pa.array([-10, -5, 0, 5, 10]), pa.array(range(5, 10)) ] - table = pa.Table.from_arrays(data, names=('a', 'b', 'c')) + table = cls.from_arrays(data, names=('a', 'b', 'c')) new_field = pa.field('d', data[1].type) t2 = table.add_column(3, new_field, data[1]) t3 = table.append_column(new_field, data[1]) - expected = pa.Table.from_arrays(data + [data[1]], - names=('a', 'b', 'c', 'd')) + expected = cls.from_arrays(data + [data[1]], + names=('a', 'b', 'c', 'd')) assert t2.equals(expected) assert t3.equals(expected) t4 = table.add_column(0, new_field, data[1]) - expected = pa.Table.from_arrays([data[1]] + data, - names=('d', 'a', 'b', 'c')) + expected = cls.from_arrays([data[1]] + data, + names=('d', 'a', 'b', 'c')) assert t4.equals(expected) -def test_record_batch_add_column(): - data_type = pa.int64() - data = [ - range(5), - [-10, -5, 0, 5, 10], - pa.array(range(5, 10)) +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) ] - batch = pa.RecordBatch.from_arrays(data, names=('a', 'b', 'c')) - - new_field = pa.field('d', data_type) - t2 = batch.add_column(3, new_field, data[1]) - t3 = batch.append_column(new_field, data[1]) - - expected = pa.RecordBatch.from_arrays(data + [data[1]], - names=('a', 'b', 'c', 'd')) - assert t2.equals(expected) - assert t3.equals(expected) - - t4 = batch.add_column(0, new_field, data[1]) - expected = pa.RecordBatch.from_arrays([data[1]] + data, - names=('d', 'a', 'b', 'c')) - assert t4.equals(expected) - - -def test_table_set_column(): +) +def test_table_set_column(cls): data = [ pa.array(range(5)), pa.array([-10, -5, 0, 5, 10]), pa.array(range(5, 10)) ] - table = pa.Table.from_arrays(data, names=('a', 'b', 'c')) + table = cls.from_arrays(data, names=('a', 'b', 'c')) new_field = pa.field('d', data[1].type) t2 = table.set_column(0, new_field, data[1]) expected_data = list(data) expected_data[0] = data[1] - expected = pa.Table.from_arrays(expected_data, - names=('d', 'b', 'c')) + expected = cls.from_arrays(expected_data, + names=('d', 'b', 'c')) assert t2.equals(expected) -def test_record_batch_set_column(): - data_type = pa.int64() - data = [ - range(5), - [-10, -5, 0, 5, 10], - pa.array(range(5, 10)) +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) ] - batch = pa.RecordBatch.from_arrays(data, names=('a', 'b', 'c')) - - new_field = pa.field('d', data_type) - t2 = batch.set_column(0, new_field, data[1]) - - expected_data = list(data) - expected_data[0] = data[1] - expected = pa.RecordBatch.from_arrays(expected_data, - names=('d', 'b', 'c')) - assert t2.equals(expected) - - -def test_table_drop_columns(): +) +def test_table_drop_columns(cls): """ drop one or more columns given labels""" a = pa.array(range(5)) b = pa.array([-10, -5, 0, 5, 10]) c = pa.array(range(5, 10)) - table = pa.Table.from_arrays([a, b, c], names=('a', 'b', 'c')) + table = cls.from_arrays([a, b, c], names=('a', 'b', 'c')) t2 = table.drop_columns(['a', 'b']) t3 = table.drop_columns('a') - exp_t2 = pa.Table.from_arrays([c], names=('c',)) + exp_t2 = cls.from_arrays([c], names=('c',)) assert exp_t2.equals(t2) - exp_t3 = pa.Table.from_arrays([b, c], names=('b', 'c',)) + exp_t3 = cls.from_arrays([b, c], names=('b', 'c',)) assert exp_t3.equals(t3) # -- raise KeyError if column not in Table @@ -1371,26 +1349,6 @@ def test_table_drop_columns(): table.drop_columns(['d']) -def test_record_batch_drop_columns(): - """ drop one or more columns given labels""" - a = pa.array(range(5)) - b = pa.array([-10, -5, 0, 5, 10]) - c = pa.array(range(5, 10)) - - batch = pa.RecordBatch.from_arrays([a, b, c], names=('a', 'b', 'c')) - t2 = batch.drop_columns(['a', 'b']) - t3 = batch.drop_columns('a') - - exp_t2 = pa.RecordBatch.from_arrays([c], names=('c',)) - assert exp_t2.equals(t2) - exp_t3 = pa.RecordBatch.from_arrays([b, c], names=('b', 'c',)) - assert exp_t3.equals(t3) - - # -- raise KeyError if column not in RecordBatch - with pytest.raises(KeyError, match="Column 'd' not found"): - batch.drop_columns(['d']) - - def test_table_drop(): """ verify the alias of drop_columns is working""" a = pa.array(range(5)) @@ -1411,40 +1369,40 @@ def test_table_drop(): table.drop(['d']) -def test_table_remove_column(): +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) + ] +) +def test_table_remove_column(cls): data = [ pa.array(range(5)), pa.array([-10, -5, 0, 5, 10]), pa.array(range(5, 10)) ] - table = pa.Table.from_arrays(data, names=('a', 'b', 'c')) + table = cls.from_arrays(data, names=('a', 'b', 'c')) t2 = table.remove_column(0) t2.validate() - expected = pa.Table.from_arrays(data[1:], names=('b', 'c')) + expected = cls.from_arrays(data[1:], names=('b', 'c')) assert t2.equals(expected) -def test_record_batch_remove_column(): - data = [ - range(5), - [-10, -5, 0, 5, 10], - pa.array(range(5, 10)) +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) ] - batch = pa.RecordBatch.from_arrays(data, names=('a', 'b', 'c')) - - t2 = batch.remove_column(0) - t2.validate() - expected = pa.RecordBatch.from_arrays(data[1:], names=('b', 'c')) - assert t2.equals(expected) - - -def test_table_remove_column_empty(): +) +def test_table_remove_column_empty(cls): # ARROW-1865 data = [ pa.array(range(5)), ] - table = pa.Table.from_arrays(data, names=['a']) + table = cls.from_arrays(data, names=['a']) t2 = table.remove_column(0) t2.validate() @@ -1455,21 +1413,6 @@ def test_table_remove_column_empty(): assert t3.equals(table) -def test_record_batch_remove_column_empty(): - data = [ - range(5), - ] - batch = pa.RecordBatch.from_arrays(data, names=['a']) - - t2 = batch.remove_column(0) - t2.validate() - assert len(t2) == len(batch) - - t3 = t2.add_column(0, batch.field(0), batch[0]) - t3.validate() - assert t3.equals(batch) - - def test_empty_table_with_names(): # ARROW-13784 data = [] @@ -1487,37 +1430,27 @@ def test_empty_table(): assert table.equals(pa.Table.from_arrays([], [])) -def test_table_rename_columns(): +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) + ] +) +def test_table_rename_columns(cls): data = [ pa.array(range(5)), pa.array([-10, -5, 0, 5, 10]), pa.array(range(5, 10)) ] - table = pa.Table.from_arrays(data, names=['a', 'b', 'c']) + table = cls.from_arrays(data, names=['a', 'b', 'c']) assert table.column_names == ['a', 'b', 'c'] t2 = table.rename_columns(['eh', 'bee', 'sea']) t2.validate() assert t2.column_names == ['eh', 'bee', 'sea'] - expected = pa.Table.from_arrays(data, names=['eh', 'bee', 'sea']) - assert t2.equals(expected) - - -def test_record_batch_rename_columns(): - data = [ - pa.array(range(5)), - pa.array([-10, -5, 0, 5, 10]), - pa.array(range(5, 10)) - ] - batch = pa.RecordBatch.from_arrays(data, names=['a', 'b', 'c']) - assert batch.column_names == ['a', 'b', 'c'] - - t2 = batch.rename_columns(['eh', 'bee', 'sea']) - t2.validate() - assert t2.column_names == ['eh', 'bee', 'sea'] - - expected = pa.RecordBatch.from_arrays(data, names=['eh', 'bee', 'sea']) + expected = cls.from_arrays(data, names=['eh', 'bee', 'sea']) assert t2.equals(expected) @@ -1740,12 +1673,19 @@ def test_table_negative_indexing(): table[4] -def test_table_cast_to_incompatible_schema(): +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) + ] +) +def test_table_cast_to_incompatible_schema(cls): data = [ pa.array(range(5)), pa.array([-10, -5, 0, 5, 10]), ] - table = pa.Table.from_arrays(data, names=tuple('ab')) + table = cls.from_arrays(data, names=tuple('ab')) target_schema1 = pa.schema([ pa.field('A', pa.int32()), @@ -1754,44 +1694,35 @@ def test_table_cast_to_incompatible_schema(): target_schema2 = pa.schema([ pa.field('a', pa.int32()), ]) - message = ("Target schema's field names are not matching the table's " - "field names:.*") + + if cls is pa.Table: + cls_name = 'table' + else: + cls_name = 'record batch' + message = ("Target schema's field names are not matching the " + f"{cls_name}'s field names:.*") + with pytest.raises(ValueError, match=message): table.cast(target_schema1) with pytest.raises(ValueError, match=message): table.cast(target_schema2) -def test_record_batch_cast_to_incompatible_schema(): - data = [ - pa.array(range(5)), - pa.array([-10, -5, 0, 5, 10]), +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) ] - batch = pa.RecordBatch.from_arrays(data, names=tuple('ab')) - - target_schema1 = pa.schema([ - pa.field('A', pa.int32()), - pa.field('b', pa.int16()), - ]) - target_schema2 = pa.schema([ - pa.field('a', pa.int32()), - ]) - message = ("Target schema's field names are not matching the " - "record batch's field names:.*") - with pytest.raises(ValueError, match=message): - batch.cast(target_schema1) - with pytest.raises(ValueError, match=message): - batch.cast(target_schema2) - - -def test_table_safe_casting(): +) +def test_table_safe_casting(cls): data = [ pa.array(range(5), type=pa.int64()), pa.array([-10, -5, 0, 5, 10], type=pa.int32()), pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] - table = pa.Table.from_arrays(data, names=tuple('abcd')) + table = cls.from_arrays(data, names=tuple('abcd')) expected_data = [ pa.array(range(5), type=pa.int32()), @@ -1799,7 +1730,7 @@ def test_table_safe_casting(): pa.array([1, 2, 3, 4, 5], type=pa.int64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] - expected_table = pa.Table.from_arrays(expected_data, names=tuple('abcd')) + expected_table = cls.from_arrays(expected_data, names=tuple('abcd')) target_schema = pa.schema([ pa.field('a', pa.int32()), @@ -1812,42 +1743,21 @@ def test_table_safe_casting(): assert casted_table.equals(expected_table) -def test_record_batch_safe_casting(): - data = [ - pa.array(range(5), type=pa.int64()), - pa.array([-10, -5, 0, 5, 10], type=pa.int32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), - pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) - ] - batch = pa.RecordBatch.from_arrays(data, names=tuple('abcd')) - - expected_data = [ - pa.array(range(5), type=pa.int32()), - pa.array([-10, -5, 0, 5, 10], type=pa.int16()), - pa.array([1, 2, 3, 4, 5], type=pa.int64()), - pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) ] - expected_batch = pa.RecordBatch.from_arrays(expected_data, names=tuple('abcd')) - - target_schema = pa.schema([ - pa.field('a', pa.int32()), - pa.field('b', pa.int16()), - pa.field('c', pa.int64()), - pa.field('d', pa.string()) - ]) - casted_batch = batch.cast(target_schema) - - assert casted_batch.equals(expected_batch) - - -def test_table_unsafe_casting(): +) +def test_table_unsafe_casting(cls): data = [ pa.array(range(5), type=pa.int64()), pa.array([-10, -5, 0, 5, 10], type=pa.int32()), pa.array([1.1, 2.2, 3.3, 4.4, 5.5], type=pa.float64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] - table = pa.Table.from_arrays(data, names=tuple('abcd')) + table = cls.from_arrays(data, names=tuple('abcd')) expected_data = [ pa.array(range(5), type=pa.int32()), @@ -1855,7 +1765,7 @@ def test_table_unsafe_casting(): pa.array([1, 2, 3, 4, 5], type=pa.int64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] - expected_table = pa.Table.from_arrays(expected_data, names=tuple('abcd')) + expected_table = cls.from_arrays(expected_data, names=tuple('abcd')) target_schema = pa.schema([ pa.field('a', pa.int32()), @@ -1871,37 +1781,6 @@ def test_table_unsafe_casting(): assert casted_table.equals(expected_table) -def test_record_batch_unsafe_casting(): - data = [ - pa.array(range(5), type=pa.int64()), - pa.array([-10, -5, 0, 5, 10], type=pa.int32()), - pa.array([1.1, 2.2, 3.3, 4.4, 5.5], type=pa.float64()), - pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) - ] - batch = pa.RecordBatch.from_arrays(data, names=tuple('abcd')) - - expected_data = [ - pa.array(range(5), type=pa.int32()), - pa.array([-10, -5, 0, 5, 10], type=pa.int16()), - pa.array([1, 2, 3, 4, 5], type=pa.int64()), - pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) - ] - expected_batch = pa.RecordBatch.from_arrays(expected_data, names=tuple('abcd')) - - target_schema = pa.schema([ - pa.field('a', pa.int32()), - pa.field('b', pa.int16()), - pa.field('c', pa.int64()), - pa.field('d', pa.string()) - ]) - - with pytest.raises(pa.ArrowInvalid, match='truncated'): - batch.cast(target_schema) - - casted_batch = batch.cast(target_schema, safe=False) - assert casted_batch.equals(expected_batch) - - def test_invalid_table_construct(): array = np.array([0, 1], dtype=np.uint8) u8 = pa.uint8() @@ -2759,30 +2638,25 @@ def test_table_join_many_columns(): }) -def test_table_cast_invalid(): +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) + ] +) +def test_table_cast_invalid(cls): # Casting a nullable field to non-nullable should be invalid! - table = pa.table({'a': [None, 1], 'b': [None, True]}) + table = cls.from_pydict({'a': [None, 1], 'b': [None, True]}) new_schema = pa.schema([pa.field("a", "int64", nullable=True), pa.field("b", "bool", nullable=False)]) with pytest.raises(ValueError): table.cast(new_schema) - table = pa.table({'a': [None, 1], 'b': [False, True]}) + table = cls.from_pydict({'a': [None, 1], 'b': [False, True]}) assert table.cast(new_schema).schema == new_schema -def test_record_batch_cast_invalid(): - # Casting a nullable field to non-nullable should be invalid! - batch = pa.record_batch({'a': [None, 1], 'b': [None, True]}) - new_schema = pa.schema([pa.field("a", "int64", nullable=True), - pa.field("b", "bool", nullable=False)]) - with pytest.raises(ValueError): - batch.cast(new_schema) - - batch = pa.record_batch({'a': [None, 1], 'b': [False, True]}) - assert batch.cast(new_schema).schema == new_schema - - def test_table_sort_by(): table = pa.table([ pa.array([3, 1, 4, 2, 5]), From 310e6e092d31059debb4bb7bdf9452b22a72819c Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:07:17 +0000 Subject: [PATCH 10/17] Don't cast unnecessarily --- cpp/src/arrow/record_batch.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index 4f37b701977f..6be74c79ca42 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -313,7 +313,7 @@ std::vector RecordBatch::ColumnNames() const { Result> RecordBatch::RenameColumns( const std::vector& names) const { - int n = static_cast(num_columns()); + int n = num_columns(); if (static_cast(names.size()) != n) { return Status::Invalid("tried to rename a record batch of ", n, " columns but only ", From 389e547a07c44ea2e755de84d2e0fe19c5c677cd Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:13:22 +0000 Subject: [PATCH 11/17] Deduplicate additions --- python/pyarrow/table.pxi | 115 ++++++++++++----------------- python/pyarrow/tests/test_table.py | 19 ----- 2 files changed, 46 insertions(+), 88 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 1f8c4e3fb0a0..cf6c1e89e8b5 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -3002,69 +3002,6 @@ cdef class RecordBatch(_Tabular): return result - def cast(self, Schema target_schema, safe=None, options=None): - """ - Cast record batch values to another schema. - - Parameters - ---------- - target_schema : Schema - Schema to cast to, the names and order of fields must match. - safe : bool, default True - Check for overflows or other unsafe conversions. - options : CastOptions, default None - Additional checks pass by CastOptions - - Returns - ------- - RecordBatch - - Examples - -------- - >>> import pyarrow as pa - >>> import pandas as pd - >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], - ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) - >>> batch = pa.RecordBatch.from_pandas(df) - >>> batch.schema - n_legs: int64 - animals: string - -- schema metadata -- - pandas: '{"index_columns": [{"kind": "range", "name": null, "start": 0, ... - - Define new schema and cast batch values: - - >>> my_schema = pa.schema([ - ... pa.field('n_legs', pa.duration('s')), - ... pa.field('animals', pa.string())] - ... ) - >>> batch.cast(target_schema=my_schema) - pyarrow.RecordBatch - n_legs: duration[s] - animals: string - ---- - n_legs: [2,4,5,100] - animals: ["Flamingo","Horse","Brittle stars","Centipede"] - """ - cdef: - Array column, casted - Field field - list newcols = [] - - if self.schema.names != target_schema.names: - raise ValueError("Target schema's field names are not matching " - "the record batch's field names: {!r}, {!r}" - .format(self.schema.names, target_schema.names)) - - for column, field in zip(self.itercolumns(), target_schema): - if not field.nullable and column.null_count > 0: - raise ValueError("Casting field {!r} with null values to non-nullable" - .format(field.name)) - casted = column.cast(field.type, safe=safe, options=options) - newcols.append(casted) - - return RecordBatch.from_arrays(newcols, schema=target_schema) - def select(self, object columns): """ Select columns of the RecordBatch. @@ -3121,7 +3058,7 @@ cdef class RecordBatch(_Tabular): def cast(self, Schema target_schema, safe=None, options=None): """ - Cast batch values to another schema. + Cast record batch values to another schema. Parameters ---------- @@ -3135,12 +3072,52 @@ cdef class RecordBatch(_Tabular): Returns ------- RecordBatch + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> batch = pa.RecordBatch.from_pandas(df) + >>> batch.schema + n_legs: int64 + animals: string + -- schema metadata -- + pandas: '{"index_columns": [{"kind": "range", "name": null, "start": 0, ... + + Define new schema and cast batch values: + + >>> my_schema = pa.schema([ + ... pa.field('n_legs', pa.duration('s')), + ... pa.field('animals', pa.string())] + ... ) + >>> batch.cast(target_schema=my_schema) + pyarrow.RecordBatch + n_legs: duration[s] + animals: string + ---- + n_legs: [2,4,5,100] + animals: ["Flamingo","Horse","Brittle stars","Centipede"] """ - # Wrap the more general Table cast implementation - tbl = Table.from_batches([self]) - casted_tbl = tbl.cast(target_schema, safe=safe, options=options) - casted_batch, = casted_tbl.to_batches() - return casted_batch + cdef: + Array column, casted + Field field + list newcols = [] + + if self.schema.names != target_schema.names: + raise ValueError("Target schema's field names are not matching " + "the record batch's field names: {!r}, {!r}" + .format(self.schema.names, target_schema.names)) + + for column, field in zip(self.itercolumns(), target_schema): + if not field.nullable and column.null_count > 0: + raise ValueError("Casting field {!r} with null values to non-nullable" + .format(field.name)) + casted = column.cast(field.type, safe=safe, options=options) + newcols.append(casted) + + return RecordBatch.from_arrays(newcols, schema=target_schema) def _to_pandas(self, options, **kwargs): return Table.from_batches([self])._to_pandas(options, **kwargs) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 3b98712b4b24..fdb62285d031 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2709,25 +2709,6 @@ def test_record_batch_sort(): assert sorted_rb_dict["c"] == ["foobar", "bar", "foo", "car"] -def test_record_batch_cast(): - rb = pa.RecordBatch.from_arrays([ - pa.array([None, 1]), - pa.array([False, True]) - ], names=["a", "b"]) - new_schema = pa.schema([pa.field("a", "int64", nullable=True), - pa.field("b", "bool", nullable=False)]) - - assert rb.cast(new_schema).schema == new_schema - - # Casting a nullable field to non-nullable is invalid - rb = pa.RecordBatch.from_arrays([ - pa.array([None, 1]), - pa.array([None, True]) - ], names=["a", "b"]) - with pytest.raises(ValueError): - rb.cast(new_schema) - - @pytest.mark.parametrize("constructor", [pa.table, pa.record_batch]) def test_numpy_asarray(constructor): table = constructor([[1, 2, 3], [4.0, 5.0, 6.0]], names=["a", "b"]) From 28014aee6667f2039b2880992ab311f16c59643b Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:14:52 +0000 Subject: [PATCH 12/17] Parameterize `test_table_sort_by` --- python/pyarrow/tests/test_table.py | 33 +++++++++--------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index fdb62285d031..3d690e1a6173 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -2657,8 +2657,15 @@ def test_table_cast_invalid(cls): assert table.cast(new_schema).schema == new_schema -def test_table_sort_by(): - table = pa.table([ +@pytest.mark.parametrize( + ('cls'), + [ + (pa.Table), + (pa.RecordBatch) + ] +) +def test_table_sort_by(cls): + table = cls.from_arrays([ pa.array([3, 1, 4, 2, 5]), pa.array(["b", "a", "b", "a", "c"]), ], names=["values", "keys"]) @@ -2673,7 +2680,7 @@ def test_table_sort_by(): "values": [5, 4, 3, 2, 1] } - tab = pa.Table.from_arrays([ + tab = cls.from_arrays([ pa.array([5, 7, 7, 35], type=pa.int64()), pa.array(["foo", "car", "bar", "foobar"]) ], names=["a", "b"]) @@ -2689,26 +2696,6 @@ def test_table_sort_by(): assert sorted_tab_dict["b"] == ["foo", "car", "bar", "foobar"] -def test_record_batch_sort(): - rb = pa.RecordBatch.from_arrays([ - pa.array([7, 35, 7, 5], type=pa.int64()), - pa.array([4, 1, 3, 2], type=pa.int64()), - pa.array(["foo", "car", "bar", "foobar"]) - ], names=["a", "b", "c"]) - - sorted_rb = rb.sort_by([("a", "descending"), ("b", "descending")]) - sorted_rb_dict = sorted_rb.to_pydict() - assert sorted_rb_dict["a"] == [35, 7, 7, 5] - assert sorted_rb_dict["b"] == [1, 4, 3, 2] - assert sorted_rb_dict["c"] == ["car", "foo", "bar", "foobar"] - - sorted_rb = rb.sort_by([("a", "ascending"), ("b", "ascending")]) - sorted_rb_dict = sorted_rb.to_pydict() - assert sorted_rb_dict["a"] == [5, 7, 7, 35] - assert sorted_rb_dict["b"] == [2, 3, 4, 1] - assert sorted_rb_dict["c"] == ["foobar", "bar", "foo", "car"] - - @pytest.mark.parametrize("constructor", [pa.table, pa.record_batch]) def test_numpy_asarray(constructor): table = constructor([[1, 2, 3], [4.0, 5.0, 6.0]], names=["a", "b"]) From 666b8e0ae7ff2fdbb72860237f14a783d1c06c22 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:17:52 +0000 Subject: [PATCH 13/17] Update `record_batch` Incorporate changes from #40292 --- python/pyarrow/table.pxi | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index cf6c1e89e8b5..b2ff64256af7 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -5388,10 +5388,10 @@ def record_batch(data, names=None, schema=None, metadata=None): Parameters ---------- - data : pandas.DataFrame, dict, list, Arrow-compatible table - A DataFrame, mapping of strings to Arrays or Python lists, list of - arrays or chunked arrays, or a tabular object implementing the Arrow - PyCapsule Protocol (has an ``__arrow_c_array__`` method). + data : dict, list, pandas.DataFrame, Arrow-compatible table + A mapping of strings to Arrays or Python lists, a list of Arrays, + a pandas DataFame, or any tabular object implementing the + Arrow PyCapsule Protocol (has an ``__arrow_c_array__`` method). names : list, default None Column names if list of arrays passed as data. Mutually exclusive with 'schema' argument. @@ -5416,16 +5416,16 @@ def record_batch(data, names=None, schema=None, metadata=None): >>> animals = pa.array(["Flamingo", "Parrot", "Dog", "Horse", "Brittle stars", "Centipede"]) >>> names = ["n_legs", "animals"] - Creating a RecordBatch from a list of arrays with names: + Construct a RecordBatch from a python dictionary: - >>> pa.record_batch([n_legs, animals], names=names) + >>> pa.record_batch({"n_legs": n_legs, "animals": animals}) pyarrow.RecordBatch n_legs: int64 animals: string ---- n_legs: [2,2,4,4,5,100] animals: ["Flamingo","Parrot","Dog","Horse","Brittle stars","Centipede"] - >>> pa.record_batch([n_legs, animals], names=["n_legs", "animals"]).to_pandas() + >>> pa.record_batch({"n_legs": n_legs, "animals": animals}).to_pandas() n_legs animals 0 2 Flamingo 1 2 Parrot @@ -5434,6 +5434,16 @@ def record_batch(data, names=None, schema=None, metadata=None): 4 5 Brittle stars 5 100 Centipede + Creating a RecordBatch from a list of arrays with names: + + >>> pa.record_batch([n_legs, animals], names=names) + pyarrow.RecordBatch + n_legs: int64 + animals: string + ---- + n_legs: [2,2,4,4,5,100] + animals: ["Flamingo","Parrot","Dog","Horse","Brittle stars","Centipede"] + Creating a RecordBatch from a list of arrays with names and metadata: >>> my_metadata={"n_legs": "How many legs does an animal have?"} @@ -5527,10 +5537,12 @@ def record_batch(data, names=None, schema=None, metadata=None): if schema is not None and batch.schema != schema: # __arrow_c_array__ coerces schema with best effort, so we might # need to cast it if the producer wasn't able to cast to exact schema. - batch = Table.from_batches([batch]).cast(schema).to_batches()[0] + batch = batch.cast(schema) return batch + elif _pandas_api.is_data_frame(data): return RecordBatch.from_pandas(data, schema=schema) + else: raise TypeError("Expected pandas DataFrame or list of arrays") From 5febf4a51f0ec9b3dab3dc97697fa8a55ba596af Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 12:26:38 +0000 Subject: [PATCH 14/17] Test that the arrays stayed the same --- cpp/src/arrow/record_batch_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/arrow/record_batch_test.cc b/cpp/src/arrow/record_batch_test.cc index 5e100bab429e..74bc234c8ea5 100644 --- a/cpp/src/arrow/record_batch_test.cc +++ b/cpp/src/arrow/record_batch_test.cc @@ -334,6 +334,7 @@ TEST_F(TestRecordBatch, RenameColumns) { ASSERT_OK_AND_ASSIGN(auto renamed, batch->RenameColumns({"zero", "one", "two"})); EXPECT_THAT(renamed->ColumnNames(), testing::ElementsAre("zero", "one", "two")); + EXPECT_THAT(renamed->columns(), testing::ElementsAre(array1, array2, array3)); ASSERT_OK(renamed->ValidateFull()); ASSERT_RAISES(Invalid, batch->RenameColumns({"hello", "world"})); From f6bde4d2fb7c61c2812513ec54cd92b596cf4d7d Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:39:36 +0000 Subject: [PATCH 15/17] Move `append_column` to `_Tabular` base class --- python/pyarrow/table.pxi | 120 +++++++++++++-------------------------- 1 file changed, 40 insertions(+), 80 deletions(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index b2ff64256af7..8b3f84ea68de 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2216,6 +2216,46 @@ cdef class _Tabular(_PandasConvertible): return res + def append_column(self, field_, column): + """ + Append column at end of columns. + + Parameters + ---------- + field_ : str or Field + If a string is passed then the type is deduced from the column + data. + column : Array or value coercible to array + Column data. + + Returns + ------- + Table or RecordBatch + New table or record batch with the passed column added. + + Examples + -------- + >>> import pyarrow as pa + >>> import pandas as pd + >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], + ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) + >>> table = pa.Table.from_pandas(df) + + Append column at the end: + + >>> year = [2021, 2022, 2019, 2021] + >>> table.append_column('year', [year]) + pyarrow.Table + n_legs: int64 + animals: string + year: int64 + ---- + n_legs: [[2,4,5,100]] + animals: [["Flamingo","Horse","Brittle stars","Centipede"]] + year: [[2021,2022,2019,2021]] + """ + return self.add_column(self.num_columns, field_, column) + cdef class RecordBatch(_Tabular): """ @@ -2623,46 +2663,6 @@ cdef class RecordBatch(_Tabular): return pyarrow_wrap_batch(c_batch) - def append_column(self, field_, column): - """ - Append column at end of columns. - - Parameters - ---------- - field_ : str or Field - If a string is passed then the type is deduced from the column - data. - column : Array or value coercible to array - Column data. - - Returns - ------- - RecordBatch - New record batch with the passed column added. - - Examples - -------- - >>> import pyarrow as pa - >>> import pandas as pd - >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], - ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) - >>> batch = pa.RecordBatch.from_pandas(df) - - Append column at the end: - - >>> year = [2021, 2022, 2019, 2021] - >>> batch.append_column('year', year) - pyarrow.RecordBatch - n_legs: int64 - animals: string - year: int64 - ---- - n_legs: [2,4,5,100] - animals: ["Flamingo","Horse","Brittle stars","Centipede"] - year: [2021,2022,2019,2021] - """ - return self.add_column(self.num_columns, field_, column) - def remove_column(self, int i): """ Create new RecordBatch with the indicated column removed. @@ -5004,46 +5004,6 @@ cdef class Table(_Tabular): return pyarrow_wrap_table(c_table) - def append_column(self, field_, column): - """ - Append column at end of columns. - - Parameters - ---------- - field_ : str or Field - If a string is passed then the type is deduced from the column - data. - column : Array, list of Array, or values coercible to arrays - Column data. - - Returns - ------- - Table - New table with the passed column added. - - Examples - -------- - >>> import pyarrow as pa - >>> import pandas as pd - >>> df = pd.DataFrame({'n_legs': [2, 4, 5, 100], - ... 'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]}) - >>> table = pa.Table.from_pandas(df) - - Append column at the end: - - >>> year = [2021, 2022, 2019, 2021] - >>> table.append_column('year', [year]) - pyarrow.Table - n_legs: int64 - animals: string - year: int64 - ---- - n_legs: [[2,4,5,100]] - animals: [["Flamingo","Horse","Brittle stars","Centipede"]] - year: [[2021,2022,2019,2021]] - """ - return self.add_column(self.num_columns, field_, column) - def remove_column(self, int i): """ Create new Table with the indicated column removed. From 87c885b9e583ffa361db8b103e6a9c72dad871eb Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 15 Mar 2024 15:33:37 +0100 Subject: [PATCH 16/17] fixup merge --- python/pyarrow/table.pxi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 33e9f2767092..2ec19db1d101 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -2151,6 +2151,7 @@ cdef class _Tabular(_PandasConvertible): return '\n'.join(pieces) def remove_column(self, int i): + # implemented in RecordBatch/Table subclasses raise NotImplementedError def drop_columns(self, columns): @@ -2216,6 +2217,10 @@ cdef class _Tabular(_PandasConvertible): return res + def add_column(self, int i, field_, column): + # implemented in RecordBatch/Table subclasses + raise NotImplementedError + def append_column(self, field_, column): """ Append column at end of columns. From 1e3711879dcff49977c36ee42dcad2c1329f16d3 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 15 Mar 2024 15:38:26 +0100 Subject: [PATCH 17/17] fixup merge submodule --- testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing b/testing index 25d16511e8d4..ad82a736c170 160000 --- a/testing +++ b/testing @@ -1 +1 @@ -Subproject commit 25d16511e8d42c2744a1d94d90169e3a36e92631 +Subproject commit ad82a736c170e97b7c8c035ebd8a801c17eec170