Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 6 additions & 13 deletions cpp/src/parquet/arrow/arrow_reader_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1264,7 +1264,7 @@ TEST_F(TestInt96ParquetIO, ReadIntoTimestamp) {

using TestUInt32ParquetIO = TestParquetIO<::arrow::UInt32Type>;

TEST_F(TestUInt32ParquetIO, Parquet_2_0_Compatibility) {
TEST_F(TestUInt32ParquetIO, Parquet_2_6_Compatibility) {
// This also tests max_definition_level = 1
std::shared_ptr<Array> values;

Expand Down Expand Up @@ -2055,19 +2055,15 @@ TEST(TestArrowReadWrite, ParquetVersionTimestampDifferences) {
.version(ParquetVersion::PARQUET_1_0)
->build();
ARROW_SUPPRESS_DEPRECATION_WARNING
auto parquet_version_2_0_properties = ::parquet::WriterProperties::Builder()
.version(ParquetVersion::PARQUET_2_0)
->build();
ARROW_UNSUPPRESS_DEPRECATION_WARNING
auto parquet_version_2_4_properties = ::parquet::WriterProperties::Builder()
.version(ParquetVersion::PARQUET_2_4)
->build();
auto parquet_version_2_6_properties = ::parquet::WriterProperties::Builder()
.version(ParquetVersion::PARQUET_2_6)
->build();
const std::vector<std::shared_ptr<WriterProperties>> all_properties = {
parquet_version_1_properties, parquet_version_2_0_properties,
parquet_version_2_4_properties, parquet_version_2_6_properties};
parquet_version_1_properties, parquet_version_2_4_properties,
parquet_version_2_6_properties};

{
// Using Parquet version 1.0 and 2.4 defaults, seconds should be coerced to
Expand All @@ -2081,13 +2077,11 @@ TEST(TestArrowReadWrite, ParquetVersionTimestampDifferences) {
parquet_version_2_4_properties));
}
{
// Using Parquet version 2.0 and 2.6 defaults, seconds should be coerced to
// Using Parquet version 2.6 defaults, seconds should be coerced to
// milliseconds and nanoseconds should be retained
auto expected_schema = schema({field("ts:s", t_ms), field("ts:ms", t_ms),
field("ts:us", t_us), field("ts:ns", t_ns)});
auto expected_table = Table::Make(expected_schema, {a_ms, a_ms, a_us, a_ns});
ASSERT_NO_FATAL_FAILURE(CheckConfiguredRoundtrip(input_table, expected_table,
parquet_version_2_0_properties));
ASSERT_NO_FATAL_FAILURE(CheckConfiguredRoundtrip(input_table, expected_table,
parquet_version_2_6_properties));
}
Expand Down Expand Up @@ -2133,9 +2127,8 @@ TEST(TestArrowReadWrite, ParquetVersionTimestampDifferences) {
CreateOutputStream(), input_table->num_rows(), properties,
arrow_coerce_to_nanos_properties));
}
// Using Parquet versions "2.0" and 2.6, coercing to (int64) nanoseconds is allowed
for (const auto& properties :
{parquet_version_2_0_properties, parquet_version_2_6_properties}) {
// Using Parquet version 2.6, coercing to (int64) nanoseconds is allowed
for (const auto& properties : {parquet_version_2_6_properties}) {
ARROW_SCOPED_TRACE("format = ", ParquetVersionToString(properties->version()));
auto expected_schema = schema({field("ts:s", t_ns), field("ts:ms", t_ns),
field("ts:us", t_ns), field("ts:ns", t_ns)});
Expand Down
3 changes: 0 additions & 3 deletions cpp/src/parquet/metadata.cc
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@ std::string ParquetVersionToString(ParquetVersion::type ver) {
case ParquetVersion::PARQUET_1_0:
return "1.0";
ARROW_SUPPRESS_DEPRECATION_WARNING
case ParquetVersion::PARQUET_2_0:
return "pseudo-2.0";
ARROW_UNSUPPRESS_DEPRECATION_WARNING
case ParquetVersion::PARQUET_2_4:
return "2.4";
case ParquetVersion::PARQUET_2_6:
Expand Down
6 changes: 0 additions & 6 deletions cpp/src/parquet/type_fwd.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,6 @@ struct ParquetVersion {
/// corresponding converted type.
PARQUET_1_0,

/// DEPRECATED: Enable Parquet format 2.6 features
///
/// This misleadingly named enum value is roughly similar to PARQUET_2_6.
PARQUET_2_0 ARROW_DEPRECATED_ENUM_VALUE("use PARQUET_2_4 or PARQUET_2_6 "
"for fine-grained feature selection"),

/// Enable Parquet format 2.4 and earlier features when writing
///
/// This enables UINT32 as well as logical types which don't have
Expand Down
1 change: 0 additions & 1 deletion python/pyarrow/_parquet.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil:

enum ParquetVersion" parquet::ParquetVersion::type":
ParquetVersion_V1" parquet::ParquetVersion::PARQUET_1_0"
ParquetVersion_V2_0" parquet::ParquetVersion::PARQUET_2_0"
ParquetVersion_V2_4" parquet::ParquetVersion::PARQUET_2_4"
ParquetVersion_V2_6" parquet::ParquetVersion::PARQUET_2_6"

Expand Down
8 changes: 0 additions & 8 deletions python/pyarrow/_parquet.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -991,8 +991,6 @@ cdef class FileMetaData(_Weakrefable):
cdef ParquetVersion version = self._metadata.version()
if version == ParquetVersion_V1:
return '1.0'
elif version == ParquetVersion_V2_0:
return 'pseudo-2.0'
elif version == ParquetVersion_V2_4:
return '2.4'
elif version == ParquetVersion_V2_6:
Expand Down Expand Up @@ -1888,12 +1886,6 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
if version is not None:
if version == "1.0":
props.version(ParquetVersion_V1)
elif version in ("2.0", "pseudo-2.0"):
warnings.warn(
"Parquet format '2.0' pseudo version is deprecated, use "
"'2.4' or '2.6' for fine-grained feature selection",
FutureWarning, stacklevel=2)
props.version(ParquetVersion_V2_0)
elif version == "2.4":
props.version(ParquetVersion_V2_4)
elif version == "2.6":
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/tests/parquet/test_data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@

@pytest.mark.pandas
@pytest.mark.parametrize('chunk_size', [None, 1000])
def test_parquet_2_0_roundtrip(tempdir, chunk_size):
def test_parquet_2_6_roundtrip(tempdir, chunk_size):
df = alltypes_sample(size=10000, categorical=True)

filename = tempdir / 'pandas_roundtrip.parquet'
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/tests/parquet/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,7 @@ def test_write_metadata(tempdir):
assert b'ARROW:schema' not in schema_as_arrow.metadata

# pass through writer keyword arguments
for version in ["1.0", "2.0", "2.4", "2.6"]:
for version in ["1.0", "2.4", "2.6"]:
pq.write_metadata(schema, path, version=version)
parquet_meta = pq.read_metadata(path)
# The version is stored as a single integer in the Parquet metadata,
Expand Down
6 changes: 2 additions & 4 deletions python/pyarrow/tests/parquet/test_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def test_pandas_parquet_column_multiindex(tempdir):


@pytest.mark.pandas
def test_pandas_parquet_2_0_roundtrip_read_pandas_no_index_written(tempdir):
def test_pandas_parquet_2_roundtrip_read_pandas_no_index_written(tempdir):
df = alltypes_sample(size=10000)

filename = tempdir / 'pandas_roundtrip.parquet'
Expand Down Expand Up @@ -270,14 +270,12 @@ def test_pandas_parquet_configuration_options(tempdir):


@pytest.mark.pandas
@pytest.mark.filterwarnings("ignore:Parquet format '2.0':FutureWarning")
def test_spark_flavor_preserves_pandas_metadata():
df = _test_dataframe(size=100)
df.index = np.arange(0, 10 * len(df), 10)
df.index.name = 'foo'

result = _roundtrip_pandas_dataframe(df, {'version': '2.0',
'flavor': 'spark'})
Comment thread
AlenkaF marked this conversation as resolved.
Resolved
result = _roundtrip_pandas_dataframe(df, {'flavor': 'spark'})
tm.assert_frame_equal(result, df)


Expand Down
3 changes: 1 addition & 2 deletions python/pyarrow/tests/test_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -4892,14 +4892,13 @@ def make_df_with_timestamps():


@pytest.mark.parquet
@pytest.mark.filterwarnings("ignore:Parquet format '2.0':FutureWarning")
def test_timestamp_as_object_parquet(tempdir):
# Timestamps can be stored as Parquet and reloaded into Pandas with no loss
# of information if the timestamp_as_object option is True.
df = make_df_with_timestamps()
table = pa.Table.from_pandas(df)
filename = tempdir / "timestamps_from_pandas.parquet"
pq.write_table(table, filename, version="2.0")
Comment thread
AlenkaF marked this conversation as resolved.
Resolved
pq.write_table(table, filename)
result = pq.read_table(filename)
df2 = result.to_pandas(timestamp_as_object=True)
tm.assert_frame_equal(df, df2)
Expand Down
2 changes: 1 addition & 1 deletion r/R/enums.R
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ FileType <- enum("FileType",
#' @export
#' @rdname enums
ParquetVersionType <- enum("ParquetVersionType",
PARQUET_1_0 = 0L, PARQUET_2_0 = 1L, PARQUET_2_4 = 2L, PARQUET_2_6 = 3L
PARQUET_1_0 = 0L, PARQUET_2_4 = 2L, PARQUET_2_6 = 3L
)

#' @export
Expand Down
15 changes: 3 additions & 12 deletions r/R/parquet.R
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ read_parquet <- function(file,
#' the number of columns and number of rows), though if the data has fewer
#' than 250 million cells (rows x cols), then the total number of rows is
#' used.
#' @param version parquet version: "1.0", "2.0" (deprecated), "2.4" (default),
#' "2.6", or "latest" (currently equivalent to 2.6). Numeric values are
#' @param version parquet version: "1.0", "2.4" (default), "2.6", or
#' "latest" (currently equivalent to 2.6). Numeric values are
#' coerced to character.
#' @param compression compression algorithm. Default "snappy". See details.
#' @param compression_level compression level. Meaning depends on compression
Expand Down Expand Up @@ -232,7 +232,6 @@ ParquetArrowWriterProperties$create <- function(use_deprecated_int96_timestamps

valid_parquet_version <- c(
"1.0" = ParquetVersionType$PARQUET_1_0,
"2.0" = ParquetVersionType$PARQUET_2_0,
"2.4" = ParquetVersionType$PARQUET_2_4,
"2.6" = ParquetVersionType$PARQUET_2_6,
"latest" = ParquetVersionType$PARQUET_2_6
Expand All @@ -252,15 +251,7 @@ make_valid_parquet_version <- function(version, valid_versions = valid_parquet_v
call. = FALSE
)
}
out <- valid_versions[[arg_match(version, values = names(valid_versions))]]

if (identical(out, ParquetVersionType$PARQUET_2_0)) {
warning(
'Parquet format version "2.0" is deprecated. Use "2.4" or "2.6" to select format features.',
call. = FALSE
)
}
out
valid_versions[[arg_match(version, values = names(valid_versions))]]
}

#' @title ParquetWriterProperties class
Expand Down
2 changes: 1 addition & 1 deletion r/man/enums.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions r/man/write_parquet.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 0 additions & 9 deletions r/tests/testthat/test-parquet.R
Original file line number Diff line number Diff line change
Expand Up @@ -134,12 +134,6 @@ test_that("make_valid_parquet_version()", {
make_valid_parquet_version("1.0"),
ParquetVersionType$PARQUET_1_0
)
expect_deprecated(
expect_equal(
make_valid_parquet_version("2.0"),
ParquetVersionType$PARQUET_2_0
)
)
expect_equal(
make_valid_parquet_version("2.4"),
ParquetVersionType$PARQUET_2_4
Expand All @@ -154,9 +148,6 @@ test_that("make_valid_parquet_version()", {
)

expect_equal(make_valid_parquet_version(1), ParquetVersionType$PARQUET_1_0)
expect_deprecated(
expect_equal(make_valid_parquet_version(2), ParquetVersionType$PARQUET_2_0)
)
expect_equal(make_valid_parquet_version(1.0), ParquetVersionType$PARQUET_1_0)
expect_equal(make_valid_parquet_version(2.4), ParquetVersionType$PARQUET_2_4)
})
Expand Down