Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 20 additions & 22 deletions python/pyarrow/table.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -1974,6 +1974,26 @@ cdef class _Tabular(_PandasConvertible):
def num_rows(self):
raise NotImplementedError

@property
def shape(self):
"""
Dimensions of the table or record batch: (#rows, #columns).

Returns
-------
(int, int)
Number of rows and number of columns.

Examples
--------
>>> import pyarrow as pa
>>> table = pa.table({'n_legs': [None, 4, 5, None],
... 'animals': ["Flamingo", "Horse", None, "Centipede"]})
>>> table.shape
(4, 2)
"""
return (self.num_rows, self.num_columns)

@property
def schema(self):
raise NotImplementedError
Expand Down Expand Up @@ -4863,28 +4883,6 @@ cdef class Table(_Tabular):
"""
return self.table.num_rows()

@property
def shape(self):
"""
Dimensions of the table: (#rows, #columns).

Returns
-------
(int, int)
Number of rows and number of columns.

Examples
--------
>>> import pyarrow as pa
>>> import pandas as pd
>>> df = pd.DataFrame({'n_legs': [None, 4, 5, None],
... 'animals': ["Flamingo", "Horse", None, "Centipede"]})
>>> table = pa.Table.from_pandas(df)
>>> table.shape
(4, 2)
"""
return (self.num_rows, self.num_columns)

@property
def nbytes(self):
"""
Expand Down
189 changes: 109 additions & 80 deletions python/pyarrow/tests/test_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,68 +488,6 @@ def test_chunked_array_unify_dictionaries():
assert arr.to_pylist() == ["foo", "bar", None, "foo", "quux", None, "foo"]


def test_recordbatch_basics():
data = [
pa.array(range(5), type='int16'),
pa.array([-10, -5, 0, None, 10], type='int32')
]

batch = pa.record_batch(data, ['c0', 'c1'])
assert not batch.schema.metadata

assert len(batch) == 5
assert batch.num_rows == 5
assert batch.num_columns == len(data)
# (only the second array has a null bitmap)
assert batch.get_total_buffer_size() == (5 * 2) + (5 * 4 + 1)
batch.nbytes == (5 * 2) + (5 * 4 + 1)
assert sys.getsizeof(batch) >= object.__sizeof__(
batch) + batch.get_total_buffer_size()

pydict = batch.to_pydict()
assert pydict == OrderedDict([
('c0', [0, 1, 2, 3, 4]),
('c1', [-10, -5, 0, None, 10])
])
assert isinstance(pydict, dict)
assert batch == pa.record_batch(pydict, schema=batch.schema)

with pytest.raises(IndexError):
# bounds checking
batch[2]

# Schema passed explicitly
schema = pa.schema([pa.field('c0', pa.int16(),
metadata={'key': 'value'}),
pa.field('c1', pa.int32())],
metadata={b'foo': b'bar'})
batch = pa.record_batch(data, schema=schema)
assert batch.schema == schema
# schema as first positional argument
batch = pa.record_batch(data, schema)
assert batch.schema == schema
assert str(batch) == """pyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [0,1,2,3,4]
c1: [-10,-5,0,null,10]"""

assert batch.to_string(show_metadata=True) == """\
pyarrow.RecordBatch
c0: int16
-- field metadata --
key: 'value'
c1: int32
-- schema metadata --
foo: 'bar'"""

wr = weakref.ref(batch)
assert wr() is not None
del batch
assert wr() is None


def test_recordbatch_dunder_init():
with pytest.raises(TypeError, match='RecordBatch'):
pa.RecordBatch()
Expand Down Expand Up @@ -1246,47 +1184,77 @@ def test_table_to_batches():
table.to_batches(max_chunksize=0)


def test_table_basics():
@pytest.mark.parametrize(
('cls'),
[
(pa.Table),
(pa.RecordBatch)
]
)
def test_table_basics(cls):
data = [
pa.array(range(5), type='int64'),
pa.array([-10, -5, 0, 5, 10], type='int64')
pa.array(range(5), type='int16'),
pa.array([-10, -5, 0, None, 10], type='int32')
]
table = pa.table(data, names=('a', 'b'))
table = cls.from_arrays(data, names=('a', 'b'))
table.validate()

assert not table.schema.metadata
assert len(table) == 5
assert table.num_rows == 5
assert table.num_columns == 2
assert table.num_columns == len(data)
assert table.shape == (5, 2)
assert table.get_total_buffer_size() == 2 * (5 * 8)
assert table.nbytes == 2 * (5 * 8)
# (only the second array has a null bitmap)
assert table.get_total_buffer_size() == (5 * 2) + (5 * 4 + 1)
assert table.nbytes == (5 * 2) + (5 * 4 + 1)
assert sys.getsizeof(table) >= object.__sizeof__(
table) + table.get_total_buffer_size()

pydict = table.to_pydict()
assert pydict == OrderedDict([
('a', [0, 1, 2, 3, 4]),
('b', [-10, -5, 0, 5, 10])
('b', [-10, -5, 0, None, 10])
])
assert isinstance(pydict, dict)
assert table == pa.table(pydict, schema=table.schema)
assert table == cls.from_pydict(pydict, schema=table.schema)

with pytest.raises(IndexError):
# bounds checking
table[2]

columns = []
for col in table.itercolumns():
columns.append(col)
for chunk in col.iterchunks():
assert chunk is not None

with pytest.raises(IndexError):
col.chunk(-1)
if cls is pa.Table:
assert type(col) is pa.ChunkedArray

for chunk in col.iterchunks():
assert chunk is not None

with pytest.raises(IndexError):
col.chunk(col.num_chunks)
with pytest.raises(IndexError):
col.chunk(-1)

with pytest.raises(IndexError):
col.chunk(col.num_chunks)

else:
assert issubclass(type(col), pa.Array)

columns.append(col)

assert table.columns == columns
assert table == pa.table(columns, names=table.column_names)
assert table != pa.table(columns[1:], names=table.column_names[1:])
assert table == cls.from_arrays(columns, names=table.column_names)
assert table != cls.from_arrays(columns[1:], names=table.column_names[1:])
assert table != columns

# Schema passed explicitly
schema = pa.schema([pa.field('c0', pa.int16(),
metadata={'key': 'value'}),
pa.field('c1', pa.int32())],
metadata={b'foo': b'bar'})
table = cls.from_arrays(data, schema=schema)
assert table.schema == schema

wr = weakref.ref(table)
assert wr() is not None
del table
Expand Down Expand Up @@ -2354,6 +2322,67 @@ def test_table_repr_to_string_ellipsis():
c1: [[10,20,30,40,10,...,40,10,20,30,40]]"""


def test_record_batch_repr_to_string():
# Schema passed explicitly
schema = pa.schema([pa.field('c0', pa.int16(),
metadata={'key': 'value'}),
pa.field('c1', pa.int32())],
metadata={b'foo': b'bar'})

batch = pa.record_batch([pa.array([1, 2, 3, 4], type='int16'),
pa.array([10, 20, 30, 40], type='int32')],
schema=schema)
assert str(batch) == """pyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4]
c1: [10,20,30,40]"""

assert batch.to_string(show_metadata=True) == """\
pyarrow.RecordBatch
c0: int16
-- field metadata --
key: 'value'
c1: int32
-- schema metadata --
foo: 'bar'"""

assert batch.to_string(preview_cols=5) == """\
pyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4]
c1: [10,20,30,40]"""

assert batch.to_string(preview_cols=1) == """\
pyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4]
..."""


def test_record_batch_repr_to_string_ellipsis():
# Schema passed explicitly
schema = pa.schema([pa.field('c0', pa.int16(),
metadata={'key': 'value'}),
pa.field('c1', pa.int32())],
metadata={b'foo': b'bar'})

batch = pa.record_batch([pa.array([1, 2, 3, 4]*10, type='int16'),
pa.array([10, 20, 30, 40]*10, type='int32')],
schema=schema)
assert str(batch) == """pyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4,1,2,3,4,1,2,...,3,4,1,2,3,4,1,2,3,4]
c1: [10,20,30,40,10,20,30,40,10,20,...,30,40,10,20,30,40,10,20,30,40]"""


def test_table_function_unicode_schema():
col_a = "äääh"
col_b = "öööf"
Expand Down