diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index e8341421d..4c638305b 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -871,9 +871,12 @@ Result> NewDataTableScan( ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); // validate options if (core_options.GetBucket() == -1) { - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented(fmt::format( - "do not support pk table bucket={} in scan process", core_options.GetBucket())); + // Batch planning of dynamic bucket primary key tables only depends on the buckets + // recorded in the manifest, and deletion vectors are looked up per (partition, bucket). + if (!table_schema->PrimaryKeys().empty() && context->IsStreamingMode()) { + return Status::NotImplemented( + fmt::format("do not support pk table bucket={} in streaming scan process", + core_options.GetBucket())); } } else if (core_options.GetBucket() < 1 && !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 17807e254..d37cf938b 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -3124,6 +3124,47 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST(ScanAndReadDynamicBucketInteTest, TestBatchReadPkDynamicBucketWithDeletionVectors) { + // Written by Java Paimon. Keys spread over several dynamic buckets, each with its own + // deletion vector index, so deletion vectors must be looked up per (partition, bucket). + const std::string table_path = + GetDataDir() + "/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/"; + const std::shared_ptr result_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), arrow::field("v", arrow::utf8())}); + const std::vector> cases = { + {"2", R"([[0, 1, "a"], [0, 2, "b"], [0, 3, "x"], [0, 4, "y"], [0, 5, "z"]])"}, + {"4", R"([[0, 1, "c"], [0, 2, "b"], [0, 3, "x"], [0, 4, "w"], [0, 5, "z"]])"}, + {"6", R"([[0, 1, "c"], [0, 3, "x"], [0, 4, "w"]])"}, + }; + for (const auto& [snapshot_id, expected_json] : cases) { + SCOPED_TRACE("snapshot " + snapshot_id); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, snapshot_id); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); + ASSERT_GT(plan->Splits().size(), 1); + + ReadContextBuilder read_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(batch_reader))); + const std::shared_ptr sort_schema = arrow::schema(result_type->fields()); + ASSERT_OK_AND_ASSIGN(actual, ReadResultCollector::SortArray(actual, sort_schema)); + const std::shared_ptr expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(result_type, expected_json).ValueOrDie()); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); + } +} + TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { std::unique_ptr dir = UniqueTestDirectory::Create("local"); ASSERT_NE(nullptr, dir); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..216346d1b 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -1730,7 +1730,7 @@ TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), - "do not support pk table bucket=-1 in scan process"); + "do not support pk table bucket=-1 in streaming scan process"); } TEST_P(ScanInteTest, TestReadWithNoSnapshot) { diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/README b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/README new file mode 100644 index 000000000..283720e93 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/README @@ -0,0 +1,21 @@ +Written by Spark 3.5.3 with Paimon 1.3.2. + +id:int not null v:string +primary key: id +bucket: -1 (dynamic bucket) +dynamic-bucket.target-row-num = 2 +deletion-vectors.enabled = true +file.format = parquet + +CREATE TABLE dv (id INT, v STRING) +TBLPROPERTIES ('primary-key' = 'id', 'bucket' = '-1', 'deletion-vectors.enabled' = 'true', + 'file.format' = 'parquet', 'dynamic-bucket.target-row-num' = '2'); + +INSERT INTO dv VALUES (1, 'a'), (2, 'b'), (3, 'x'), (4, 'y'), (5, 'z'); +-- snapshot 2: (1, a) (2, b) (3, x) (4, y) (5, z) +INSERT INTO dv VALUES (1, 'c'), (4, 'w'); +-- snapshot 4: (1, c) (2, b) (3, x) (4, w) (5, z) +DELETE FROM dv WHERE id IN (2, 5); +-- snapshot 6: (1, c) (3, x) (4, w) + +Keys are spread over several buckets, and each bucket keeps its own deletion vector index. diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-03e129c4-1fe8-43de-9fe1-13069c7aab48-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-03e129c4-1fe8-43de-9fe1-13069c7aab48-0.parquet new file mode 100644 index 000000000..717830dd4 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-03e129c4-1fe8-43de-9fe1-13069c7aab48-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-18c28124-1069-4008-856b-260c74186da9-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-18c28124-1069-4008-856b-260c74186da9-0.parquet new file mode 100644 index 000000000..626fd4940 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-18c28124-1069-4008-856b-260c74186da9-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-78d3a62b-331e-4c14-8203-a21180e669b3-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-78d3a62b-331e-4c14-8203-a21180e669b3-0.parquet new file mode 100644 index 000000000..626fd4940 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-101/data-78d3a62b-331e-4c14-8203-a21180e669b3-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-179/data-090287b9-5534-488d-9b25-72120e6c4eb8-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-179/data-090287b9-5534-488d-9b25-72120e6c4eb8-0.parquet new file mode 100644 index 000000000..877bdccb1 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-179/data-090287b9-5534-488d-9b25-72120e6c4eb8-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-179/data-8f746777-2282-4918-94c8-cf34e3ece214-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-179/data-8f746777-2282-4918-94c8-cf34e3ece214-0.parquet new file mode 100644 index 000000000..4b6156d5a Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-179/data-8f746777-2282-4918-94c8-cf34e3ece214-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-191/data-a6fa67ff-6bb8-4683-b105-7f2455bb8e6e-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-191/data-a6fa67ff-6bb8-4683-b105-7f2455bb8e6e-0.parquet new file mode 100644 index 000000000..2528c8bcd Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-191/data-a6fa67ff-6bb8-4683-b105-7f2455bb8e6e-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-191/data-e9bc1b16-1a7c-4130-ae38-420d72918dea-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-191/data-e9bc1b16-1a7c-4130-ae38-420d72918dea-0.parquet new file mode 100644 index 000000000..60264a5d9 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-191/data-e9bc1b16-1a7c-4130-ae38-420d72918dea-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-7e957b16-1260-426f-b26b-34e3aef797db-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-7e957b16-1260-426f-b26b-34e3aef797db-0.parquet new file mode 100644 index 000000000..241c96f02 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-7e957b16-1260-426f-b26b-34e3aef797db-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-8fc5ddb3-a69c-4c52-b328-66f34f95c002-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-8fc5ddb3-a69c-4c52-b328-66f34f95c002-0.parquet new file mode 100644 index 000000000..241c96f02 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-8fc5ddb3-a69c-4c52-b328-66f34f95c002-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-f3ec085a-f6e6-4db1-96b1-218cae9c74c5-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-f3ec085a-f6e6-4db1-96b1-218cae9c74c5-0.parquet new file mode 100644 index 000000000..cf7885aea Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-193/data-f3ec085a-f6e6-4db1-96b1-218cae9c74c5-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-20/data-e2c90dfb-303a-4cb6-8da5-de99a724fdf8-0.parquet b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-20/data-e2c90dfb-303a-4cb6-8da5-de99a724fdf8-0.parquet new file mode 100644 index 000000000..d45ce873f Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/bucket-20/data-e2c90dfb-303a-4cb6-8da5-de99a724fdf8-0.parquet differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-226a01b7-1ea1-410b-a0f4-c3d72dac3dfb-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-226a01b7-1ea1-410b-a0f4-c3d72dac3dfb-0 new file mode 100644 index 000000000..fd298348b --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-226a01b7-1ea1-410b-a0f4-c3d72dac3dfb-0 @@ -0,0 +1 @@ +ÒåG \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-48eb8ead-486d-4255-9698-f2fc4b10e3cb-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-48eb8ead-486d-4255-9698-f2fc4b10e3cb-0 new file mode 100644 index 000000000..834de8ef9 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-48eb8ead-486d-4255-9698-f2fc4b10e3cb-0 @@ -0,0 +1 @@ +VGpÊ \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-6edcb848-8159-4ae7-826d-8995c1a6f454-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-6edcb848-8159-4ae7-826d-8995c1a6f454-0 new file mode 100644 index 000000000..11dc1c01e --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-6edcb848-8159-4ae7-826d-8995c1a6f454-0 @@ -0,0 +1 @@ +‚Hô \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-ed55898c-8f16-432f-8fa3-95727d920a0a-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-ed55898c-8f16-432f-8fa3-95727d920a0a-0 new file mode 100644 index 000000000..599a74a8f --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-ed55898c-8f16-432f-8fa3-95727d920a0a-0 @@ -0,0 +1 @@ +Oä»ð \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-f787a693-854f-4b72-99e0-40217aaca953-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-f787a693-854f-4b72-99e0-40217aaca953-0 new file mode 100644 index 000000000..8095133ba --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/index/index-f787a693-854f-4b72-99e0-40217aaca953-0 @@ -0,0 +1 @@ +WYùž \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0 new file mode 100644 index 000000000..5c22b223d Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-2b7a0c3c-2d76-4af8-8733-095d16c3edca-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-2b7a0c3c-2d76-4af8-8733-095d16c3edca-0 new file mode 100644 index 000000000..45be5a6fe Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-2b7a0c3c-2d76-4af8-8733-095d16c3edca-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-2b7a0c3c-2d76-4af8-8733-095d16c3edca-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-2b7a0c3c-2d76-4af8-8733-095d16c3edca-1 new file mode 100644 index 000000000..9a589db62 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-2b7a0c3c-2d76-4af8-8733-095d16c3edca-1 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-4cdd7c93-4b8d-45b5-92c0-9e5bd21bc1eb-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-4cdd7c93-4b8d-45b5-92c0-9e5bd21bc1eb-0 new file mode 100644 index 000000000..fc1e1a6c6 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-4cdd7c93-4b8d-45b5-92c0-9e5bd21bc1eb-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-4cdd7c93-4b8d-45b5-92c0-9e5bd21bc1eb-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-4cdd7c93-4b8d-45b5-92c0-9e5bd21bc1eb-1 new file mode 100644 index 000000000..940814ab2 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-4cdd7c93-4b8d-45b5-92c0-9e5bd21bc1eb-1 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-bcf33eac-adb4-4446-8da1-b434b89c2705-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-bcf33eac-adb4-4446-8da1-b434b89c2705-0 new file mode 100644 index 000000000..d2ff72e11 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-bcf33eac-adb4-4446-8da1-b434b89c2705-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-bcf33eac-adb4-4446-8da1-b434b89c2705-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-bcf33eac-adb4-4446-8da1-b434b89c2705-1 new file mode 100644 index 000000000..6da91389d Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-bcf33eac-adb4-4446-8da1-b434b89c2705-1 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-0 new file mode 100644 index 000000000..0bbe62c04 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-1 new file mode 100644 index 000000000..92ac8436b Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-1 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-2 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-2 new file mode 100644 index 000000000..eb977622d Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-2 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-3 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-3 new file mode 100644 index 000000000..ded064637 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-3 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-0 new file mode 100644 index 000000000..b92b8b0d2 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-1 new file mode 100644 index 000000000..db647779d Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-1 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-2 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-2 new file mode 100644 index 000000000..75783b8a1 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-2 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-3 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-3 new file mode 100644 index 000000000..e1c3a643f Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-3 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-0 new file mode 100644 index 000000000..e38b0902d Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-0 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-1 new file mode 100644 index 000000000..138d5b51d Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-1 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-2 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-2 new file mode 100644 index 000000000..10c19be82 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-2 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-3 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-3 new file mode 100644 index 000000000..c4f747f59 Binary files /dev/null and b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/manifest/manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-3 differ diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/schema/schema-0 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/schema/schema-0 new file mode 100644 index 000000000..d82dcea1a --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/schema/schema-0 @@ -0,0 +1,24 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "v", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ "id" ], + "options" : { + "bucket" : "-1", + "owner" : "root", + "file.format" : "parquet", + "dynamic-bucket.target-row-num" : "2", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1792572933936 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/EARLIEST b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/LATEST b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/LATEST new file mode 100644 index 000000000..62f945751 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/LATEST @@ -0,0 +1 @@ +6 \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-1 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-1 new file mode 100644 index 000000000..38a0c4c45 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-1 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-1", + "deltaManifestListSize" : 990, + "changelogManifestList" : null, + "indexManifest" : "index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0", + "commitUser" : "86f3fe45-aa7d-41e3-a32b-f49670b910da", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1792572934818, + "logOffsets" : { }, + "totalRecordCount" : 5, + "deltaRecordCount" : 5, + "changelogRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-2 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-2 new file mode 100644 index 000000000..3715fb271 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-2 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-2", + "baseManifestListSize" : 990, + "deltaManifestList" : "manifest-list-0014d5c7-8244-459c-ba6c-3e66042c6b22-3", + "deltaManifestListSize" : 990, + "changelogManifestList" : null, + "indexManifest" : "index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0", + "commitUser" : "86f3fe45-aa7d-41e3-a32b-f49670b910da", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1792572934825, + "logOffsets" : { }, + "totalRecordCount" : 5, + "deltaRecordCount" : 0, + "changelogRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-3 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-3 new file mode 100644 index 000000000..7bf1cbf84 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-3 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-0", + "baseManifestListSize" : 1001, + "deltaManifestList" : "manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-1", + "deltaManifestListSize" : 991, + "changelogManifestList" : null, + "indexManifest" : "index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0", + "commitUser" : "b110fa6e-4595-43c0-b479-8756f91b126b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1792572936021, + "logOffsets" : { }, + "totalRecordCount" : 7, + "deltaRecordCount" : 2, + "changelogRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-4 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-4 new file mode 100644 index 000000000..122fa1de1 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-4 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-2", + "baseManifestListSize" : 1043, + "deltaManifestList" : "manifest-list-19ad07c9-fcfc-4be6-b685-d6918922e81b-3", + "deltaManifestListSize" : 991, + "changelogManifestList" : null, + "indexManifest" : "index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0", + "commitUser" : "b110fa6e-4595-43c0-b479-8756f91b126b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1792572936030, + "logOffsets" : { }, + "totalRecordCount" : 5, + "deltaRecordCount" : -2, + "changelogRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-5 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-5 new file mode 100644 index 000000000..f01ce92b2 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-5 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-0", + "baseManifestListSize" : 1051, + "deltaManifestList" : "manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-1", + "deltaManifestListSize" : 991, + "changelogManifestList" : null, + "indexManifest" : "index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0", + "commitUser" : "9b190bb7-7a17-4f78-9035-60a019950355", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1792572937421, + "logOffsets" : { }, + "totalRecordCount" : 7, + "deltaRecordCount" : 2, + "changelogRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-6 b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-6 new file mode 100644 index 000000000..7d9f5c370 --- /dev/null +++ b/test/test_data/parquet/pk_dynamic_bucket_dv.db/pk_dynamic_bucket_dv/snapshot/snapshot-6 @@ -0,0 +1,20 @@ +{ + "version" : 3, + "id" : 6, + "schemaId" : 0, + "baseManifestList" : "manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-2", + "baseManifestListSize" : 1091, + "deltaManifestList" : "manifest-list-51b6b5dd-86de-4a97-9a90-86dc8382f85e-3", + "deltaManifestListSize" : 991, + "changelogManifestList" : null, + "indexManifest" : "index-manifest-243d52b5-0cfc-4039-bf11-7436f05ebd99-0", + "commitUser" : "9b190bb7-7a17-4f78-9035-60a019950355", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1792572937428, + "logOffsets" : { }, + "totalRecordCount" : 3, + "deltaRecordCount" : -4, + "changelogRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file